From f267cff60baaaeb58f749a8cbfc353c331c5f6f4 Mon Sep 17 00:00:00 2001 From: winkyao Date: Wed, 10 Jul 2019 17:09:44 +0800 Subject: [PATCH 001/196] =?UTF-8?q?CHANGELOG:=20pick=20changelog=20from=20?= =?UTF-8?q?branch=20release-2.1=20and=20branch=20r=E2=80=A6=20(#11017)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 281 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55621b37ecee2..2ff200de38df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,286 @@ # TiDB Changelog All notable changes to this project will be documented in this file. See also [Release Notes](https://github.com/pingcap/docs/blob/master/releases/rn.md), [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md) and [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md). +## [2.1.14] 2019-07-04 +* Fix wrong query results caused by column pruning in some cases [#11019](https://github.com/pingcap/tidb/pull/11019) +* Fix the wrongly displayed information in `db` and `info` columns of `show processlist` [#11000](https://github.com/pingcap/tidb/pull/11000) +* Fix the issue that `MAX_EXECUTION_TIME` as a SQL hint and global variable does not work in some cases [#10999](https://github.com/pingcap/tidb/pull/10999) +* Support automatically adjust the incremental step allocated by auto-increment ID based on the load [#10997](https://github.com/pingcap/tidb/pull/10997) +* Fix the issue that the `Distsql` memory information of `MemTracker` is not correctly cleaned when a query ends [#10971](https://github.com/pingcap/tidb/pull/10971) +* Add the `MEM` column in the `information_schema.processlist` table to describe the memory usage of a query [#10896](https://github.com/pingcap/tidb/pull/10896) +* Add the `max_execution_time` global system variable to control the maximum execution time of a query [#10940](https://github.com/pingcap/tidb/pull/10940) +* Fix the panic caused by using unsupported aggregate functions [#10911](https://github.com/pingcap/tidb/pull/10911) +* Add an automatic rollback feature for the last transaction when the `load data` statement fails [#10862](https://github.com/pingcap/tidb/pull/10862) +* Fix the issue that TiDB returns a wrong result in some cases when the `OOMAction` configuration item is set to `Cancel` [#11016](https://github.com/pingcap/tidb/pull/11016) +* Disable the `TRACE` statement to avoid the TiDB panic issue [#11039](https://github.com/pingcap/tidb/pull/11039) +* Add the `mysql.expr_pushdown_blacklist` system table that dynamically enables/disables pushing down specific functions to Coprocessor [#10998](https://github.com/pingcap/tidb/pull/10998) +* Fix the issue that the `ANY_VALUE` function does not work in the `ONLY_FULL_GROUP_BY` mode [#10994](https://github.com/pingcap/tidb/pull/10994) +* Fix the incorrect evaluation caused by not doing a deep copy when evaluating the user variable of the string type [#11043](https://github.com/pingcap/tidb/pull/11043) + +## [3.0.0] 2019-06-28 +## New Features +* Support Window Functions; compatible with all window functions in MySQL 8.0, including `NTILE`, `LEAD`, `LAG`, `PERCENT_RANK`, `NTH_VALUE`, `CUME_DIST`, `FIRST_VALUE` , `LAST_VALUE`, `RANK`, `DENSE_RANK`, and `ROW_NUMBER` +* Support Views (Experimental) +* Improve Table Partition + - Support Range Partition + - Support Hash Partition +* Add the plug-in framework, supporting plugins such as IP Whitelist (Enterprise feature) and Audit Log (Enterprise feature). +* Support the SQL Plan Management function to create SQL execution plan binding to ensure query stability (Experimental) + +## SQL Optimizer +* Optimize the `NOT EXISTS` subquery and convert it to `Anti Semi Join` to improve performance +* Optimize the constant propagation on the `Outer Join`, and add the optimization rule of `Outer Join` elimination to reduce non-effective computations and improve performance +* Optimize the `IN` subquery to execute `Inner Join` after aggregation to improve performance +* Optimize `Index Join` to adapt to more scenarios +* Improve the Partition Pruning optimization rule of Range Partition +* Optimize the query logic for `_tidb_rowid`to avoid full table scan and improve performance +* Match more prefix columns of the indexes when extracting access conditions of composite indexes if there are relevant columns in the filter to improve performance +* Improve the accuracy of cost estimates by using order correlation between columns +* Optimize `Join Reorder` based on the Greedy algorithm and the dynamic planning algorithm to improve accuracy for index selection using `Join` +* Support Skyline Pruning, with some rules to prevent the execution plan from relying too heavily on statistics to improve query stability +* Improve the accuracy of row count estimation for single-column indexes with NULL values +* Support `FAST ANALYZE` that randomly samples in each Region to avoid full table scan and improve performance with statistics collection +* Support the incremental Analyze operation on monotonically increasing index columns to improve performance with statistics collection +* Support using subqueries in the `DO` statement +* Support using `Index Join` in transactions +* Optimize `prepare`/`execute` to support DDL statements with no parameters +* Modify the system behaviour to auto load statistics when the `stats-lease` variable value is 0 +* Support exporting historical statistics +* Support the `dump`/`load` correlation of histograms + +## SQL Execution Engine +* Optimize log output: `EXECUTE` outputs user variables and `COMMIT` outputs slow query logs to facilitate troubleshooting +* Support the `EXPLAIN ANALYZE` function to improve SQL tuning usability +* Support the `admin show next_row_id` command to get the ID of the next row +* Add six built-in functions: `JSON_QUOTE`, `JSON_ARRAY_APPEND`, `JSON_MERGE_PRESERVE`, `BENCHMARK` ,`COALESCE`, and `NAME_CONST` +* Optimize control logics on the chunk size to dynamically adjust based on the query context, to reduce the SQL execution time and resource consumption +* Support tracking and controlling memory usage in three operators - `TableReader`, `IndexReader` and `IndexLookupReader` +* Optimize the Merge Join operator to support an empty `ON` condition +* Optimize write performance for single tables that contains too many columns +* Improve the performance of `admin show ddl jobs` by supporting scanning data in reverse order +* Add the `split table region` statement to manually split the table Region to alleviate the hotspot issue +* Add the `split index region` statement to manually split the index Region to alleviate the hotspot issue +* Add a blacklist to prohibit pushing down expressions to Coprocessor +* Optimize the `Expensive Query` log to print the SQL query in the log when it exceeds the configured limit of execution time or memory + +## DDL +* Support migrating from character set `utf8` to `utf8mb4` +* Change the default character set from`utf8` to `utf8mb4` +* Add the `alter schema` statement to modify the character set and the collation of the database +* Support ALTER algorithm `INPLACE`/`INSTANT` +* Support `SHOW CREATE VIEW` +* Support `SHOW CREATE USER` +* Support fast recovery of mistakenly deleted tables +* Support adjusting the number of concurrencies of ADD INDEX dynamically +* Add the `pre_split_regions` option that pre-allocates Regions when creating the table using the `CREATE TABLE` statement, to relieve write hot Regions caused by lots of writes after the table creation +* Support splitting Regions by the index and range of the table specified using SQL statements to relieve hotspot issues +* Add the `ddl_error_count_limit` global variable to limit the number of DDL task retries +* Add a feature to use `SHARD_ROW_ID_BITS` to scatter row IDs when the column contains an AUTO_INCREMENT attribute to relieve the hotspot issue +* Optimize the lifetime of invalid DDL metadata to speed up recovering the normal execution of DDL operations after upgrading the TiDB cluster + +## Transactions +* Support the pessimistic transaction model (Experimental) +* Optimize transaction processing logics to adapt to more scenarios: + - Change the default value `tidb_disable_txn_auto_retry` to `on`, which means non-auto committed transactions will not be retried + - Add the `tidb_batch_commit` system variable to split a transaction into multiple ones to be executed concurrently + - Add the `tidb_low_resolution_tso` system variable to control the number of TSOs to obtain in batches and reduce the number of times that transactions request for TSOs, to improve performance in scenarios with relatively low requirement of consistency + - Add the `tidb_skip_isolation_level_check` variable to control whether to report errors when the isolation level is set to SERIALIZABLE + - Modify the `tidb_disable_txn_auto_retry` system variable to make it work on all retryable errors + +## Permission Management +* Perform permission check on the `ANALYZE`, `USE`, `SET GLOBAL`, and `SHOW PROCESSLIST` statements +* Support Role Based Access Control (RBAC) (Experimental) + +## Server +* Optimize slow query logs + - Restructure the log format + - Optimize the log content + - Optimize the log query method to support using the `INFORMATION_SCHEMA.SLOW_QUERY` and `ADMIN SHOW SLOW` statements of the memory table to query slow query logs +* Develop a unified log format specification with restructured log system to facilitate collection and analysis by tools +* Support using SQL statements to manage Binlog services, including querying status, enabling Binlog, maintaining and sending Binlog strategies. +* Support using `unix_socket` to connect to the database +* Support `Trace` for SQL statements +* Support getting information for a TiDB instance via the `/debug/zip` HTTP interface to facilitate troubleshooting. +* Optimize monitoring items to facilitate troubleshooting: + - Add the `high_error_rate_feedback_total` monitoring item to monitor the difference between the actual data volume and the estimated data volume based on statistics + - Add a QPS monitoring item in the database dimension +* Optimize the system initialization process to only allow the DDL owner to perform the initialization. This reduces the startup time for initialization or upgrading. +* Optimize the execution logic of `kill query` to improve performance and ensure resource is release properly +* Add a startup option `config-check` to check the validity of the configuration file +* Add the `tidb_back_off_weight` system variable to control the backoff time of internal error retries +* Add the `wait_timeout`and `interactive_timeout` system variables to control the maximum idle connections allowed +* Add the connection pool for TiKV to shorten the connection establishing time + +## Compatibility +* Support the `ALLOW_INVALID_DATES` SQL mode +* Support the MySQL 320 Handshake protocol +* Support manifesting unsigned BIGINT columns as auto-increment columns +* Support the `SHOW CREATE DATABASE IF NOT EXISTS` syntax +* Optimize the fault tolerance of `load data` for CSV files +* Abandon the predicate pushdown operation when the filtering condition contains a user variable to improve the compatibility with MySQL’s behavior of using user variables to simulate Window Functions + + +## [3.0.0-rc.3] 2019-06-21 +## SQL Optimizer +* Remove the feature of collecting virtual generated column statistics[#10629](https://github.com/pingcap/tidb/pull/10629) +* Fix the issue that the primary key constant overflows during point queries [#10699](https://github.com/pingcap/tidb/pull/10699) +* Fix the issue that using uninitialized information in `fast analyze` causes panic [#10691](https://github.com/pingcap/tidb/pull/10691) +* Fix the issue that executing the `create view` statement using `prepare` causes panic because of wrong column information [#10713](https://github.com/pingcap/tidb/pull/10713) +* Fix the issue that the column information is not cloned when handling window functions [#10720](https://github.com/pingcap/tidb/pull/10720) +* Fix the wrong estimation for the selectivity rate of the inner table selection in index join [#10854](https://github.com/pingcap/tidb/pull/10854) +* Support automatic loading statistics when the `stats-lease` variable value is 0 [#10811](https://github.com/pingcap/tidb/pull/10811) + +## Execution Engine +* Fix the issue that resources are not correctly released when calling the `Close` function in `StreamAggExec` [#10636](https://github.com/pingcap/tidb/pull/10636) +* Fix the issue that the order of `table_option` and `partition_options` is incorrect in the result of executing the `show create table` statement for partitioned tables [#10689](https://github.com/pingcap/tidb/pull/10689) +* Improve the performance of `admin show ddl jobs` by supporting scanning data in reverse order [#10687](https://github.com/pingcap/tidb/pull/10687) +* Fix the issue that the result of the `show grants` statement in RBAC is incompatible with that of MySQL when this statement has the `current_user` field [#10684](https://github.com/pingcap/tidb/pull/10684) +* Fix the issue that UUIDs might generate duplicate values ​​on multiple nodes [#10712](https://github.com/pingcap/tidb/pull/10712) +* Fix the issue that the `show view` privilege is not considered in `explain` [#10635](https://github.com/pingcap/tidb/pull/10635) +* Add the `split table region` statement to manually split the table Region to alleviate the hotspot issue [#10765](https://github.com/pingcap/tidb/pull/10765) +* Add the `split index region` statement to manually split the index Region to alleviate the hotspot issue [#10764](https://github.com/pingcap/tidb/pull/10764) +* Fix the incorrect execution issue when you execute multiple statements such as `create user`, `grant`, or `revoke` consecutively [#10737] (https://github.com/pingcap/tidb/pull/10737) +* Add a blacklist to prohibit pushing down expressions to Coprocessor [#10791](https://github.com/pingcap/tidb/pull/10791) +* Add the feature of printing the `expensive query` log when a query exceeds the memory configuration limit [#10849](https://github.com/pingcap/tidb/pull/10849) +* Add the `bind-info-lease` configuration item to control the update time of the modified binding execution plan [#10727](https://github.com/pingcap/tidb/pull/10727) +* Fix the OOM issue in high concurrent scenarios caused by the failure to quickly release Coprocessor resources, resulted from the `execdetails.ExecDetails` pointer [#10832] (https://github.com/pingcap/tidb/pull/10832) +* Fix the panic issue caused by the `kill` statement in some cases [#10876](https://github.com/pingcap/tidb/pull/10876) +## Server +* Fix the issue that goroutine might leak when repairing GC [#10683](https://github.com/pingcap/tidb/pull/10683) +* Support displaying the `host` information in slow queries [#10693](https://github.com/pingcap/tidb/pull/10693) +* Support reusing idle links that interact with TiKV [#10632](https://github.com/pingcap/tidb/pull/10632) +* Fix the support for enabling the `skip-grant-table` option in RBAC [#10738](https://github.com/pingcap/tidb/pull/10738) +* Fix the issue that `pessimistic-txn` configuration goes invalid [#10825](https://github.com/pingcap/tidb/pull/10825) +* Fix the issue that the actively cancelled ticlient requests are still retried [#10850](https://github.com/pingcap/tidb/pull/10850) +* Improve performance in the case where pessimistic transactions conflict with optimistic transactions [#10881](https://github.com/pingcap/tidb/pull/10881) +## DDL +* Fix the issue that modifying charset using `alter table` causes the `blob` type change [#10698](https://github.com/pingcap/tidb/pull/10698) +* Add a feature to use `SHARD_ROW_ID_BITS` to scatter row IDs when the column contains an `AUTO_INCREMENT` attribute to alleviate the hotspot issue [#10794](https://github.com/pingcap/tidb/pull/10794) +* Prohibit adding stored generated columns by using the `alter table` statement [#10808](https://github.com/pingcap/tidb/pull/10808) +* Optimize the invalid survival time of DDL metadata to shorten the period during which the DDL operation is slower after cluster upgrade [#10795](https://github.com/pingcap/tidb/pull/10795) + +## [2.1.13] 2019-06-21 +* Add a feature to use `SHARD_ROW_ID_BITS` to scatter row IDs when the column contains an `AUTO_INCREMENT` attribute to relieve the hotspot issue [#10788](https://github.com/pingcap/tidb/pull/10788) +* Optimize the lifetime of invalid DDL metadata to speed up recovering the normal execution of DDL operations after upgrading the TiDB cluster [#10789](https://github.com/pingcap/tidb/pull/10789) +* Fix the OOM issue in high concurrent scenarios caused by the failure to quickly release Coprocessor resources, resulted from the `execdetails.ExecDetails` pointer [#10833](https://github.com/pingcap/tidb/pull/10833) +* Add the `update-stats` configuration item to control whether to update statistics [#10772](https://github.com/pingcap/tidb/pull/10772) +* Add the following TiDB-specific syntax to support Region presplit to solve the hotspot issue: + - Add the `PRE_SPLIT_REGIONS` table option [#10863](https://github.com/pingcap/tidb/pull/10863) + - Add the `SPLIT TABLE table_name INDEX index_name` syntax [#10865](https://github.com/pingcap/tidb/pull/10865) + - Add the `SPLIT TABLE [table_name] BETWEEN (min_value...) AND (max_value...) REGIONS [region_num]` syntax [#10882](https://github.com/pingcap/tidb/pull/10882) +* Fix the panic issue caused by the `KILL` syntax in some cases [#10879](https://github.com/pingcap/tidb/pull/10879) +* Improve the compatibility with MySQL for `ADD_DATE` in some cases [#10718](https://github.com/pingcap/tidb/pull/10718) +* Fix the wrong estimation for the selectivity rate of the inner table selection in index join [#10856](https://github.com/pingcap/tidb/pull/10856) + +## [2.1.12] 2019-06-13 +* Fix the issue caused by unmatched data types when using the index query feedback [#10755](https://github.com/pingcap/tidb/pull/10755) +* Fix the issue that the blob column is changed to the text column caused by charset altering in some cases [#10745](https://github.com/pingcap/tidb/pull/10745) +* Fix the issue that the `GRANT` operation in the transaction mistakenly reports “Duplicate Entry” in some cases [#10739](https://github.com/pingcap/tidb/pull/10739) +* Improve the compatibility with MySQL of the following features + - The `DAYNAME` function [#10732](https://github.com/pingcap/tidb/pull/10732) + - The `MONTHNAME` function [#10733](https://github.com/pingcap/tidb/pull/10733) + - Support the 0 value for the `EXTRACT` function when processing `MONTH` [#10702](https://github.com/pingcap/tidb/pull/10702) + - The `DECIMAL` type can be converted to `TIMESTAMP` or `DATETIME` [#10734](https://github.com/pingcap/tidb/pull/10734) +* Change the column charset while changing the table charset [#10714](https://github.com/pingcap/tidb/pull/10714) +* Fix the overflow issue when converting a decimal to a float in some cases [#10730](https://github.com/pingcap/tidb/pull/10730) +* Fix the issue that some extremely large messages report the “grpc: received message larger than max” error caused by inconsistent maximum sizes of messages sent/received by gRPC of TiDB and TiKV [#10710](https://github.com/pingcap/tidb/pull/10710) +* Fix the panic issue caused by `ORDER BY` not filtering NULL in some cases [#10488](https://github.com/pingcap/tidb/pull/10488) +* Fix the issue that values returned by the `UUID` function might be duplicate when multiple nodes exist [#10711](https://github.com/pingcap/tidb/pull/10711) +* Change the value returned by `CAST(-num as datetime)` from `error` to NULL [#10703](https://github.com/pingcap/tidb/pull/10703) +* Fix the issue that an unsigned histogram meets signed ranges in some cases [#10695](https://github.com/pingcap/tidb/pull/10695) +* Fix the issue that an error is reported mistakenly for reading data when the statistics feedback meets the bigint unsigned primary key [#10307](https://github.com/pingcap/tidb/pull/10307) +* Fix the issue that the result of `Show Create Table` for partitioned tables is not correctly displayed in some cases [#10690](https://github.com/pingcap/tidb/pull/10690) +* Fix the issue that the calculation result of the `GROUP_CONCAT` aggregate function is not correct for some correlated subqueries [#10670](https://github.com/pingcap/tidb/pull/10670) +* Fix the issue that the result of `Show Create Table` for partitioned tables is not correctly displayed in some cases [#10690](https://github.com/pingcap/tidb/pull/10690) + +## [2.1.11] 2019-05-31 + +* Fix the issue that incorrect schema is used for `delete from join` [#10595](https://github.com/pingcap/tidb/pull/10595) +* Fix the issue that the built-in `CONVERT()` may return incorrect field type [#10263](https://github.com/pingcap/tidb/pull/10263) +* Merge non-overlapped feedback when updating bucket count [#10569](https://github.com/pingcap/tidb/pull/10569) +* Fix calculation errors of `unix_timestamp()-unix_timestamp(now())` [#10491](https://github.com/pingcap/tidb/pull/10491) +* Fix the incompatibility issue of `period_diff` with MySQL 8.0 [#10501](https://github.com/pingcap/tidb/pull/10501) +* Skip `Virtual Column` when collecting statistics to avoid exceptions [#10628](https://github.com/pingcap/tidb/pull/10628) +* Support the `SHOW OPEN TABLES` statement [#10374](https://github.com/pingcap/tidb/pull/10374) +* Fix the issue that goroutine leak may happen in some cases [#10656](https://github.com/pingcap/tidb/pull/10656) +* Fix the issue that setting the `tidb_snapshot` variable in some cases may cause incorrect parsing of time format [#10637](https://github.com/pingcap/tidb/pull/10637) + +## [3.0.0-rc.2] 2019-05-28 +### SQL Optimizer +* Support Index Join in more scenarios +[#10540](https://github.com/pingcap/tidb/pull/10540) +* Support exporting historical statistics [#10291](https://github.com/pingcap/tidb/pull/10291) +* Support the incremental `Analyze` operation on monotonically increasing index columns +[#10355](https://github.com/pingcap/tidb/pull/10355) +* Neglect the NULL value in the `Order By` clause [#10488](https://github.com/pingcap/tidb/pull/10488) +* Fix the wrong schema information calculation of the `UnionAll` logical operator when simplifying the column information [#10384](https://github.com/pingcap/tidb/pull/10384) +* Avoid modifying the original expression when pushing down the `Not` operator [#10363](https://github.com/pingcap/tidb/pull/10363/files) +* Support the `dump`/`load` correlation of histograms [#10573](https://github.com/pingcap/tidb/pull/10573) +### Execution Engine +* Handle virtual columns with a unique index properly when fetching duplicate rows in `batchChecker` [#10370](https://github.com/pingcap/tidb/pull/10370) +* Fix the scanning range calculation issue for the `CHAR` column [#10124](https://github.com/pingcap/tidb/pull/10124) +* Fix the issue of `PointGet` incorrectly processing negative numbers [#10113](https://github.com/pingcap/tidb/pull/10113) +* Merge `Window` functions with the same name to improve execution efficiency [#9866](https://github.com/pingcap/tidb/pull/9866) +* Allow the `RANGE` frame in a `Window` function to contain no `OrderBy` clause [#10496](https://github.com/pingcap/tidb/pull/10496) + +### Server +Fix the issue that TiDB continuously creates a new connection to TiKV when a fault occurs in TiKV [#10301](https://github.com/pingcap/tidb/pull/10301) +Make `tidb_disable_txn_auto_retry` affect all retryable errors instead of only write conflict errors [#10339](https://github.com/pingcap/tidb/pull/10339) +Allow DDL statements without parameters to be executed using `prepare`/`execute` [#10144](https://github.com/pingcap/tidb/pull/10144) +Add the `tidb_back_off_weight` variable to control the backoff time [#10266](https://github.com/pingcap/tidb/pull/10266) +Prohibit TiDB retrying non-automatically committed transactions in default conditions by setting the default value of `tidb_disable_txn_auto_retry` to `on` [#10266](https://github.com/pingcap/tidb/pull/10266) +Fix the database privilege judgment of `role` in `RBAC` [#10261](https://github.com/pingcap/tidb/pull/10261) +Support the pessimistic transaction model (experimental) [#10297](https://github.com/pingcap/tidb/pull/10297) +Reduce the wait time for handling lock conflicts in some cases [#10006](https://github.com/pingcap/tidb/pull/10006) +Make the Region cache able to visit follower nodes when a fault occurs in the leader node [#10256](https://github.com/pingcap/tidb/pull/10256) +Add the `tidb_low_resolution_tso` variable to control the number of TSOs obtained in batches and reduce the times of transactions obtaining TSO to adapt for scenarios where data consistency is not so strictly required [#10428](https://github.com/pingcap/tidb/pull/10428) + +### DDL +Fix the uppercase issue of the charset name in the storage of the old version of TiDB +[#10272](https://github.com/pingcap/tidb/pull/10272) +Support `preSplit` of table partition, which pre-allocates table Regions when creating a table to avoid write hotspots after the table is created +[#10221](https://github.com/pingcap/tidb/pull/10221) +Fix the issue that TiDB incorrectly updates the version information in PD in some cases [#10324](https://github.com/pingcap/tidb/pull/10324) +Support modifying the charset and collation using the `ALTER DATABASE` statement +[#10393](https://github.com/pingcap/tidb/pull/10393) +Support splitting Regions based on the index and range of the specified table to relieve hotspot issues +[#10203](https://github.com/pingcap/tidb/pull/10203) +Prohibit modifying the precision of the decimal column using the `alter table` statement +[#10433](https://github.com/pingcap/tidb/pull/10433) +Fix the restriction for expressions and functions in hash partition +[#10273](https://github.com/pingcap/tidb/pull/10273) +Fix the issue that adding indexes in a table that contains partitions will in some cases cause TiDB panic +[#10475](https://github.com/pingcap/tidb/pull/10475) +Validate table information before executing the DDL to avoid invalid table schemas +[#10464](https://github.com/pingcap/tidb/pull/10464) +Enable hash partition by default; and enable range columns partition when there is only one column in the partition definition +[#9936](https://github.com/pingcap/tidb/pull/9936) + + +## [2.1.10] 2019-05-21 +* Fix the issue that some abnormalities cause incorrect table schema when using `tidb_snapshot` to read the history data [#10359](https://github.com/pingcap/tidb/pull/10359) +* Fix the issue that the `NOT` function causes wrong read results in some cases [#10363](https://github.com/pingcap/tidb/pull/10363) +* Fix the wrong behavior of `Generated Column` in the `Replace` or `Insert on duplicate update` statement [#10385](https://github.com/pingcap/tidb/pull/10385) +* Fix a bug of the `BETWEEN` function in the `DATE`/`DATETIME` comparison [#10407](https://github.com/pingcap/tidb/pull/10407) +* Fix the issue that a single line of a slow log that is too long causes an error report when using the `SLOW_QUERY` table to query a slow log [#10412](https://github.com/pingcap/tidb/pull/10412) +* Fix the issue that the result of `DATETIME ` plus `INTERVAL` is not the same with that of MySQL in some cases [#10416](https://github.com/pingcap/tidb/pull/10416), [#10418](https://github.com/pingcap/tidb/pull/10418) +* Add the check for the invalid time of February in a leap year [#10417](https://github.com/pingcap/tidb/pull/10417) +* Execute the internal initialization operation limitation only in the DDL owner to avoid a large number of conflict error reports when initializing the cluster [#10426](https://github.com/pingcap/tidb/pull/10426) +* Fix the issue that `DESC` is incompatible with MySQL when the default value of the output timestamp column is `default current_timestamp on update current_timestamp` [#10337](https://github.com/pingcap/tidb/issues/10337) +* Fix the issue that an error occurs during the privilege check in the `Update` statement [#10439](https://github.com/pingcap/tidb/pull/10439) +* Fix the issue that wrong calculation of `RANGE` causes a wrong result in the `CHAR` column in some cases [#10455](https://github.com/pingcap/tidb/pull/10455) +* Fix the issue that the data might be overwritten after decreasing `SHARD_ROW_ID_BITS` [#9868](https://github.com/pingcap/tidb/pull/9868) +* Fix the issue that `ORDER BY RAND()` does not return random numbers [#10064](https://github.com/pingcap/tidb/pull/10064) +* Prohibit the `ALTER` statement modifying the precision of decimals [#10458](https://github.com/pingcap/tidb/pull/10458) +* Fix the compatibility issue of the `TIME_FORMAT` function with MySQL [#10474](https://github.com/pingcap/tidb/pull/10474) +* Check the parameter validity of `PERIOD_ADD` [#10430](https://github.com/pingcap/tidb/pull/10430) +* Fix the issue that the behavior of the invalid `YEAR` string in TiDB is incompatible with that in MySQL [#10493](https://github.com/pingcap/tidb/pull/10493) +* Support the `ALTER DATABASE` syntax [#10503](https://github.com/pingcap/tidb/pull/10503) +* Fix the issue that the `SLOW_QUERY` memory engine reports an error when no `;` exists in the slow query statement [#10536](https://github.com/pingcap/tidb/pull/10536) +* Fix the issue that the `Add index` operation in partitioned tables cannot be canceled in some cases [#10533](https://github.com/pingcap/tidb/pull/10533) +* Fix the issue that the OOM panic cannot be recovered in some cases [#10545](https://github.com/pingcap/tidb/pull/10545) +* Improve the security of the DDL operation rewriting the table metadata [#10547](https://github.com/pingcap/tidb/pull/10547) + ## [3.0.0-rc.1] 2019-05-10 ### SQL Optimizer @@ -43,6 +323,7 @@ All notable changes to this project will be documented in this file. See also [R * Fix the compatibility issue in displaying `CHARSET`/`COLLATION` descriptions in the `SHOW FULL COLUMNS` statement [#10007](https://github.com/pingcap/tidb/pull/10007) * Fix the issue that the `SHOW COLLATIONS` statement only lists collations supported by TiDB [#10186](https://github.com/pingcap/tidb/pull/10186) + ## [2.1.9] 2019-05-06 * Fix compatibility of the `MAKETIME` function when unsigned type overflows [#10089](https://github.com/pingcap/tidb/pull/10089) * Fix the stack overflow caused by constant folding in some cases [#10189](https://github.com/pingcap/tidb/pull/10189) From f409f0b4cfae416eeeaff07899c0c4080455707a Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Wed, 10 Jul 2019 17:39:38 +0800 Subject: [PATCH 002/196] expressio: fix data race of rand function (#11168) --- expression/builtin_math.go | 13 +++++++++---- expression/integration_test.go | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/expression/builtin_math.go b/expression/builtin_math.go index 4ec7a4accb4c5..88d7c9c171170 100644 --- a/expression/builtin_math.go +++ b/expression/builtin_math.go @@ -24,6 +24,7 @@ import ( "math/rand" "strconv" "strings" + "sync" "time" "github.com/cznic/mathutil" @@ -966,7 +967,7 @@ func (c *randFunctionClass) getFunction(ctx sessionctx.Context, args []Expressio bt := bf if len(args) == 0 { seed := time.Now().UnixNano() - sig = &builtinRandSig{bt, rand.New(rand.NewSource(seed))} + sig = &builtinRandSig{bt, &sync.Mutex{}, rand.New(rand.NewSource(seed))} } else if _, isConstant := args[0].(*Constant); isConstant { // According to MySQL manual: // If an integer argument N is specified, it is used as the seed value: @@ -979,7 +980,7 @@ func (c *randFunctionClass) getFunction(ctx sessionctx.Context, args []Expressio if isNull { seed = time.Now().UnixNano() } - sig = &builtinRandSig{bt, rand.New(rand.NewSource(seed))} + sig = &builtinRandSig{bt, &sync.Mutex{}, rand.New(rand.NewSource(seed))} } else { sig = &builtinRandWithSeedSig{bt} } @@ -988,11 +989,12 @@ func (c *randFunctionClass) getFunction(ctx sessionctx.Context, args []Expressio type builtinRandSig struct { baseBuiltinFunc + mu *sync.Mutex randGen *rand.Rand } func (b *builtinRandSig) Clone() builtinFunc { - newSig := &builtinRandSig{randGen: b.randGen} + newSig := &builtinRandSig{randGen: b.randGen, mu: b.mu} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } @@ -1000,7 +1002,10 @@ func (b *builtinRandSig) Clone() builtinFunc { // evalReal evals RAND(). // See https://dev.mysql.com/doc/refman/5.7/en/mathematical-functions.html#function_rand func (b *builtinRandSig) evalReal(row chunk.Row) (float64, bool, error) { - return b.randGen.Float64(), false, nil + b.mu.Lock() + res := b.randGen.Float64() + b.mu.Unlock() + return res, false, nil } type builtinRandWithSeedSig struct { diff --git a/expression/integration_test.go b/expression/integration_test.go index f8ed485f7008b..443950e434064 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -551,6 +551,8 @@ func (s *testIntegrationSuite) TestMathBuiltin(c *C) { tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") tk.MustExec("insert into t values(1),(2),(3)") + tk.Se.GetSessionVars().MaxChunkSize = 1 + tk.MustQuery("select rand(1) from t").Sort().Check(testkit.Rows("0.6046602879796196", "0.6645600532184904", "0.9405090880450124")) tk.MustQuery("select rand(a) from t").Check(testkit.Rows("0.6046602879796196", "0.16729663442585624", "0.7199826688373036")) tk.MustQuery("select rand(1), rand(2), rand(3)").Check(testkit.Rows("0.6046602879796196 0.16729663442585624 0.7199826688373036")) } From 8c20289c7de31eabf978ead77270fe6115790fa9 Mon Sep 17 00:00:00 2001 From: lysu Date: Wed, 10 Jul 2019 19:28:19 +0800 Subject: [PATCH 003/196] server: don't interrupt when nodelay executed (#11135) --- server/conn.go | 4 ++-- server/conn_test.go | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/server/conn.go b/server/conn.go index d9a780b8fe02d..c80116e01584c 100644 --- a/server/conn.go +++ b/server/conn.go @@ -1176,9 +1176,9 @@ func (cc *clientConn) handleQuery(ctx context.Context, sql string) (err error) { return err } status := atomic.LoadInt32(&cc.status) - if status == connStatusShutdown || status == connStatusWaitShutdown { + if rs != nil && (status == connStatusShutdown || status == connStatusWaitShutdown) { killConn(cc) - return errors.New("killed by another connection") + return executor.ErrQueryInterrupted } if rs != nil { if len(rs) == 1 { diff --git a/server/conn_test.go b/server/conn_test.go index ee59c01a47dee..3faed0280a994 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -26,8 +26,10 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/util/arena" "github.com/pingcap/tidb/util/testleak" @@ -38,7 +40,7 @@ type ConnTestSuite struct { store kv.Storage } -var _ = Suite(ConnTestSuite{}) +var _ = Suite(&ConnTestSuite{}) func (ts *ConnTestSuite) SetUpSuite(c *C) { testleak.BeforeTest() @@ -420,7 +422,7 @@ func (ts *ConnTestSuite) TestConnExecutionTimeout(c *C) { c.Assert(err, IsNil) err = cc.handleQuery(context.Background(), "select * FROM testTable2 WHERE SLEEP(1);") - c.Assert(err, NotNil) + c.Assert(err, IsNil) _, err = se.Execute(context.Background(), "set @@max_execution_time = 0;") c.Assert(err, IsNil) @@ -429,7 +431,33 @@ func (ts *ConnTestSuite) TestConnExecutionTimeout(c *C) { c.Assert(err, IsNil) err = cc.handleQuery(context.Background(), "select /*+ MAX_EXECUTION_TIME(100)*/ * FROM testTable2 WHERE SLEEP(1);") - c.Assert(err, NotNil) + c.Assert(err, IsNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/server/FakeClientConn"), IsNil) } + +type mockTiDBCtx struct { + TiDBContext + rs []ResultSet + err error +} + +func (c *mockTiDBCtx) Execute(ctx context.Context, sql string) ([]ResultSet, error) { + return c.rs, c.err +} + +func (c *mockTiDBCtx) GetSessionVars() *variable.SessionVars { + return &variable.SessionVars{} +} + +func (ts *ConnTestSuite) TestShutDown(c *C) { + cc := &clientConn{} + + // mock delay response + cc.ctx = &mockTiDBCtx{rs: []ResultSet{&tidbResultSet{}}, err: nil} + // set killed flag + cc.status = connStatusShutdown + // assert ErrQueryInterrupted + err := cc.handleQuery(context.Background(), "dummy") + c.Assert(err, Equals, executor.ErrQueryInterrupted) +} From b43668da82a2a8a67243acf224eca6a500e37419 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 11 Jul 2019 11:00:44 +0800 Subject: [PATCH 004/196] planner: fix union scan on partition table bug (#11187) --- executor/executor_test.go | 7 +++++++ planner/core/rule_partition_processor.go | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 2ba19bed7534f..02b80cba46a6d 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3664,6 +3664,13 @@ func (s *testSuite3) TestSelectPartition(c *C) { c.Assert(err.Error(), Equals, "[table:1735]Unknown partition 'p4' in table 'th'") err = tk.ExecToErr("select b from tr partition (r1,r4)") c.Assert(err.Error(), Equals, "[table:1735]Unknown partition 'r4' in table 'tr'") + + // test select partition table in transaction. + tk.MustExec("begin") + tk.MustExec("insert into th values (10,10),(11,11)") + tk.MustQuery("select a, b from th where b>10").Check(testkit.Rows("11 11")) + tk.MustExec("commit") + tk.MustQuery("select a, b from th where b>10").Check(testkit.Rows("11 11")) } func (s *testSuite) TestSelectView(c *C) { diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index b3a6200151ce8..4b04412e5c02a 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -60,9 +60,10 @@ func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, err // Union->(UnionScan->DataSource1), (UnionScan->DataSource2) children := make([]LogicalPlan, 0, len(ua.Children())) for _, child := range ua.Children() { - us := LogicalUnionScan{}.Init(ua.ctx) - us.SetChildren(child) - children = append(children, us) + usChild := LogicalUnionScan{}.Init(ua.ctx) + usChild.conditions = us.conditions + usChild.SetChildren(child) + children = append(children, usChild) } ua.SetChildren(children...) return ua, nil From 47f449a07d08dcd07fdc7327aa5a5b76771fc25c Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 11 Jul 2019 11:18:00 +0800 Subject: [PATCH 005/196] executor: explain analyze panic when processing `explain analyze insert ... select ...` (#11162) --- executor/adapter.go | 39 ++++++++++++++++++++++++++++----------- executor/explain_test.go | 14 ++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index 201a66525fa77..ee7bc38441d1a 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -253,17 +253,8 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { return a.handlePessimisticSelectForUpdate(ctx, e) } - // If the executor doesn't return any result to the client, we execute it without delay. - if e.Schema().Len() == 0 { - if isPessimistic { - return nil, a.handlePessimisticDML(ctx, e) - } - return a.handleNoDelayExecutor(ctx, e) - } else if proj, ok := e.(*ProjectionExec); ok && proj.calculateNoDelay { - // Currently this is only for the "DO" statement. Take "DO 1, @a=2;" as an example: - // the Projection has two expressions and two columns in the schema, but we should - // not return the result of the two expressions. - return a.handleNoDelayExecutor(ctx, e) + if handled, result, err := a.handleNoDelay(ctx, e, isPessimistic); handled { + return result, err } var txnStartTS uint64 @@ -281,6 +272,32 @@ func (a *ExecStmt) Exec(ctx context.Context) (_ sqlexec.RecordSet, err error) { }, nil } +func (a *ExecStmt) handleNoDelay(ctx context.Context, e Executor, isPessimistic bool) (bool, sqlexec.RecordSet, error) { + toCheck := e + if explain, ok := e.(*ExplainExec); ok { + if explain.analyzeExec != nil { + toCheck = explain.analyzeExec + } + } + + // If the executor doesn't return any result to the client, we execute it without delay. + if toCheck.Schema().Len() == 0 { + if isPessimistic { + return true, nil, a.handlePessimisticDML(ctx, e) + } + r, err := a.handleNoDelayExecutor(ctx, e) + return true, r, err + } else if proj, ok := toCheck.(*ProjectionExec); ok && proj.calculateNoDelay { + // Currently this is only for the "DO" statement. Take "DO 1, @a=2;" as an example: + // the Projection has two expressions and two columns in the schema, but we should + // not return the result of the two expressions. + r, err := a.handleNoDelayExecutor(ctx, e) + return true, r, err + } + + return false, nil, nil +} + // getMaxExecutionTime get the max execution timeout value. func getMaxExecutionTime(sctx sessionctx.Context, stmtNode ast.StmtNode) uint64 { ret := sctx.GetSessionVars().MaxExecutionTime diff --git a/executor/explain_test.go b/executor/explain_test.go index f1596be8e2b0c..c36179a283e52 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -60,3 +60,17 @@ func (s *testSuite1) TestExplainPriviliges(c *C) { err = tk1.ExecToErr("explain select * from v") c.Assert(err.Error(), Equals, plannercore.ErrTableaccessDenied.GenWithStackByArgs("SELECT", "explain", "%", "v").Error()) } + +func (s *testSuite1) TestExplainWrite(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int)") + tk.MustExec("explain analyze insert into t select 1") + tk.MustQuery("select * from t").Check(testkit.Rows("1")) + tk.MustExec("explain analyze update t set a=2 where a=1") + tk.MustQuery("select * from t").Check(testkit.Rows("2")) + tk.MustExec("explain insert into t select 1") + tk.MustQuery("select * from t").Check(testkit.Rows("2")) + tk.MustExec("explain analyze insert into t select 1") + tk.MustQuery("select * from t order by a").Check(testkit.Rows("1", "2")) +} diff --git a/go.mod b/go.mod index b7e02bb626a3a..31b57528e20e7 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190708123555-29973f7a22eb + github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 7757f3e24a50f..a0a8d06b00ae6 100644 --- a/go.sum +++ b/go.sum @@ -164,8 +164,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190708123555-29973f7a22eb h1:jfhJo/D1bWMF+zVaVdmixWG5EnxbnFt99GS2pdxuToo= -github.com/pingcap/parser v0.0.0-20190708123555-29973f7a22eb/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d h1:vOZjn1ami1LIjtIj0i5QunGh/sHawbhiBCb1qPx373w= +github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf h1:vmlN6DpZI5LtHd8r9YRAsyCeTU2pxRq+WlWn5CZ+ax4= github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From 7b859c1907e9e0b3d0de1c44f593b38dd8a40c72 Mon Sep 17 00:00:00 2001 From: wshwsh12 <793703860@qq.com> Date: Thu, 11 Jul 2019 11:33:22 +0800 Subject: [PATCH 006/196] executor: fix a bug for 'int column non-int constant' (#11050) --- expression/builtin_compare.go | 110 ++++++++++++++++++++-------- expression/builtin_compare_test.go | 4 + expression/integration_test.go | 14 ++++ expression/simple_rewriter.go | 9 ++- planner/core/expression_rewriter.go | 8 +- 5 files changed, 111 insertions(+), 34 deletions(-) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index a6e12a6b2474c..d9b8b45a86ec4 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -1070,7 +1070,13 @@ func isTemporalColumn(expr Expression) bool { } // tryToConvertConstantInt tries to convert a constant with other type to a int constant. -func tryToConvertConstantInt(ctx sessionctx.Context, isUnsigned bool, con *Constant) (_ *Constant, isAlwaysFalse bool) { +// isExceptional indicates whether the 'int column [cmp] const' might be true/false. +// If isExceptional is true, ExecptionalVal is returned. Or, CorrectVal is returned. +// CorrectVal: The computed result. If the constant can be converted to int without exception, return the val. Else return 'con'(the input). +// ExceptionalVal : It is used to get more information to check whether 'int column [cmp] const' is true/false +// If the op == LT,LE,GT,GE and it gets an Overflow when converting, return inf/-inf. +// If the op == EQ,NullEQ and the constant can never be equal to the int column, return ‘con’(the input, a non-int constant). +func tryToConvertConstantInt(ctx sessionctx.Context, targetFieldType *types.FieldType, con *Constant) (_ *Constant, isExceptional bool) { if con.GetType().EvalType() == types.ETInt { return con, false } @@ -1079,37 +1085,48 @@ func tryToConvertConstantInt(ctx sessionctx.Context, isUnsigned bool, con *Const return con, false } sc := ctx.GetSessionVars().StmtCtx - fieldType := types.NewFieldType(mysql.TypeLonglong) - if isUnsigned { - fieldType.Flag |= mysql.UnsignedFlag - } - dt, err = dt.ConvertTo(sc, fieldType) + + dt, err = dt.ConvertTo(sc, targetFieldType) if err != nil { - return con, terror.ErrorEqual(err, types.ErrOverflow) + if terror.ErrorEqual(err, types.ErrOverflow) { + return &Constant{ + Value: dt, + RetType: targetFieldType, + }, true + } + return con, false } return &Constant{ Value: dt, - RetType: fieldType, + RetType: targetFieldType, DeferredExpr: con.DeferredExpr, }, false } -// RefineComparedConstant changes an non-integer constant argument to its ceiling or floor result by the given op. -// isAlwaysFalse indicates whether the int column "con" is false. -func RefineComparedConstant(ctx sessionctx.Context, isUnsigned bool, con *Constant, op opcode.Op) (_ *Constant, isAlwaysFalse bool) { +// RefineComparedConstant changes a non-integer constant argument to its ceiling or floor result by the given op. +// isExceptional indicates whether the 'int column [cmp] const' might be true/false. +// If isExceptional is true, ExecptionalVal is returned. Or, CorrectVal is returned. +// CorrectVal: The computed result. If the constant can be converted to int without exception, return the val. Else return 'con'(the input). +// ExceptionalVal : It is used to get more information to check whether 'int column [cmp] const' is true/false +// If the op == LT,LE,GT,GE and it gets an Overflow when converting, return inf/-inf. +// If the op == EQ,NullEQ and the constant can never be equal to the int column, return ‘con’(the input, a non-int constant). +func RefineComparedConstant(ctx sessionctx.Context, targetFieldType types.FieldType, con *Constant, op opcode.Op) (_ *Constant, isExceptional bool) { dt, err := con.Eval(chunk.Row{}) if err != nil { return con, false } sc := ctx.GetSessionVars().StmtCtx - intFieldType := types.NewFieldType(mysql.TypeLonglong) - if isUnsigned { - intFieldType.Flag |= mysql.UnsignedFlag - } + var intDatum types.Datum - intDatum, err = dt.ConvertTo(sc, intFieldType) + intDatum, err = dt.ConvertTo(sc, &targetFieldType) if err != nil { - return con, terror.ErrorEqual(err, types.ErrOverflow) + if terror.ErrorEqual(err, types.ErrOverflow) { + return &Constant{ + Value: intDatum, + RetType: &targetFieldType, + }, true + } + return con, false } c, err := intDatum.CompareDatum(sc, &con.Value) if err != nil { @@ -1118,7 +1135,7 @@ func RefineComparedConstant(ctx sessionctx.Context, isUnsigned bool, con *Consta if c == 0 { return &Constant{ Value: intDatum, - RetType: intFieldType, + RetType: &targetFieldType, DeferredExpr: con.DeferredExpr, }, false } @@ -1126,12 +1143,12 @@ func RefineComparedConstant(ctx sessionctx.Context, isUnsigned bool, con *Consta case opcode.LT, opcode.GE: resultExpr := NewFunctionInternal(ctx, ast.Ceil, types.NewFieldType(mysql.TypeUnspecified), con) if resultCon, ok := resultExpr.(*Constant); ok { - return tryToConvertConstantInt(ctx, isUnsigned, resultCon) + return tryToConvertConstantInt(ctx, &targetFieldType, resultCon) } case opcode.LE, opcode.GT: resultExpr := NewFunctionInternal(ctx, ast.Floor, types.NewFieldType(mysql.TypeUnspecified), con) if resultCon, ok := resultExpr.(*Constant); ok { - return tryToConvertConstantInt(ctx, isUnsigned, resultCon) + return tryToConvertConstantInt(ctx, &targetFieldType, resultCon) } case opcode.NullEQ, opcode.EQ: switch con.RetType.EvalType() { @@ -1159,7 +1176,7 @@ func RefineComparedConstant(ctx sessionctx.Context, isUnsigned bool, con *Consta } return &Constant{ Value: intDatum, - RetType: intFieldType, + RetType: &targetFieldType, DeferredExpr: con.DeferredExpr, }, false } @@ -1175,27 +1192,60 @@ func (c *compareFunctionClass) refineArgs(ctx sessionctx.Context, args []Express arg1IsInt := arg1Type.EvalType() == types.ETInt arg0, arg0IsCon := args[0].(*Constant) arg1, arg1IsCon := args[1].(*Constant) - isAlways, finalArg0, finalArg1 := false, args[0], args[1] + isExceptional, finalArg0, finalArg1 := false, args[0], args[1] + isPositiveInfinite, isNegativeInfinite := false, false // int non-constant [cmp] non-int constant if arg0IsInt && !arg0IsCon && !arg1IsInt && arg1IsCon { - finalArg1, isAlways = RefineComparedConstant(ctx, mysql.HasUnsignedFlag(arg0Type.Flag), arg1, c.op) + arg1, isExceptional = RefineComparedConstant(ctx, *arg0Type, arg1, c.op) + finalArg1 = arg1 + if isExceptional && arg1.RetType.EvalType() == types.ETInt { + // Judge it is inf or -inf + // For int: + // inf: 01111111 & 1 == 1 + // -inf: 10000000 & 1 == 0 + // For uint: + // inf: 11111111 & 1 == 1 + // -inf: 00000000 & 0 == 0 + if arg1.Value.GetInt64()&1 == 1 { + isPositiveInfinite = true + } else { + isNegativeInfinite = true + } + } } // non-int constant [cmp] int non-constant if arg1IsInt && !arg1IsCon && !arg0IsInt && arg0IsCon { - finalArg0, isAlways = RefineComparedConstant(ctx, mysql.HasUnsignedFlag(arg1Type.Flag), arg0, symmetricOp[c.op]) + arg0, isExceptional = RefineComparedConstant(ctx, *arg1Type, arg0, symmetricOp[c.op]) + finalArg0 = arg0 + if isExceptional && arg0.RetType.EvalType() == types.ETInt { + if arg0.Value.GetInt64()&1 == 1 { + isNegativeInfinite = true + } else { + isPositiveInfinite = true + } + } } - if !isAlways { - return []Expression{finalArg0, finalArg1} + + if isExceptional && (c.op == opcode.EQ || c.op == opcode.NullEQ) { + // This will always be false. + return []Expression{Zero.Clone(), One.Clone()} } - switch c.op { - case opcode.LT, opcode.LE: + if isPositiveInfinite { + // If the op is opcode.LT, opcode.LE // This will always be true. + // If the op is opcode.GT, opcode.GE + // This will always be false. return []Expression{Zero.Clone(), One.Clone()} - case opcode.EQ, opcode.NullEQ, opcode.GT, opcode.GE: + } + if isNegativeInfinite { + // If the op is opcode.GT, opcode.GE + // This will always be true. + // If the op is opcode.LT, opcode.LE // This will always be false. return []Expression{One.Clone(), Zero.Clone()} } - return args + + return []Expression{finalArg0, finalArg1} } // getFunction sets compare built-in function signatures for various types. diff --git a/expression/builtin_compare_test.go b/expression/builtin_compare_test.go index 3aee49ddaec77..ad09320fde511 100644 --- a/expression/builtin_compare_test.go +++ b/expression/builtin_compare_test.go @@ -63,6 +63,10 @@ func (s *testEvaluatorSuite) TestCompareFunctionWithRefine(c *C) { {"'1.1' != a", "ne(1.1, cast(a))"}, {"'123456789123456711111189' = a", "0"}, {"123456789123456789.12345 = a", "0"}, + {"123456789123456789123456789.12345 > a", "1"}, + {"-123456789123456789123456789.12345 > a", "0"}, + {"123456789123456789123456789.12345 < a", "0"}, + {"-123456789123456789123456789.12345 < a", "1"}, // This cast can not be eliminated, // since converting "aaaa" to an int will cause DataTruncate error. {"'aaaa'=a", "eq(cast(aaaa), cast(a))"}, diff --git a/expression/integration_test.go b/expression/integration_test.go index 443950e434064..20348d8a226b2 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4435,3 +4435,17 @@ func (s *testIntegrationSuite) TestInvalidEndingStatement(c *C) { assertParseErr(`drop table if exists t'`) assertParseErr(`drop table if exists t"`) } + +func (s *testIntegrationSuite) TestIssue10675(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(a int);`) + tk.MustExec(`insert into t values(1);`) + tk.MustQuery(`select * from t where a < -184467440737095516167.1;`).Check(testkit.Rows()) + tk.MustQuery(`select * from t where a > -184467440737095516167.1;`).Check( + testkit.Rows("1")) + tk.MustQuery(`select * from t where a < 184467440737095516167.1;`).Check( + testkit.Rows("1")) + tk.MustQuery(`select * from t where a > 184467440737095516167.1;`).Check(testkit.Rows()) +} diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go index e768be64bf18e..28e2c38e46f7f 100644 --- a/expression/simple_rewriter.go +++ b/expression/simple_rewriter.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/parser/opcode" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util" ) @@ -468,10 +468,15 @@ func (sr *simpleRewriter) inToExpression(lLen int, not bool, tp *types.FieldType return } leftEt := leftFt.EvalType() + if leftEt == types.ETInt { for i := 0; i < len(elems); i++ { if c, ok := elems[i].(*Constant); ok { - elems[i], _ = RefineComparedConstant(sr.ctx, mysql.HasUnsignedFlag(leftFt.Flag), c, opcode.EQ) + var isExceptional bool + elems[i], isExceptional = RefineComparedConstant(sr.ctx, *leftFt, c, opcode.EQ) + if isExceptional { + elems[i] = c + } } } } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index efff3275d9f75..d6b6a680498cf 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -29,7 +29,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/stringutil" ) @@ -1127,7 +1127,11 @@ func (er *expressionRewriter) inToExpression(lLen int, not bool, tp *types.Field if leftEt == types.ETInt { for i := 1; i < len(args); i++ { if c, ok := args[i].(*expression.Constant); ok { - args[i], _ = expression.RefineComparedConstant(er.ctx, mysql.HasUnsignedFlag(leftFt.Flag), c, opcode.EQ) + var isExceptional bool + args[i], isExceptional = expression.RefineComparedConstant(er.ctx, *leftFt, c, opcode.EQ) + if isExceptional { + args[i] = c + } } } } From 5664dd2abc2d80ab7a86b9e631e6207ee7e631ea Mon Sep 17 00:00:00 2001 From: lysu Date: Thu, 11 Jul 2019 12:44:55 +0800 Subject: [PATCH 007/196] infochema, server: export region_id in hot_region table (#10982) --- infoschema/tables.go | 27 ++++++++++++++-------- server/http_handler.go | 13 ++--------- store/helper/helper.go | 52 +++++++++++++++++++----------------------- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index d5c1ef3ce81a0..5282438b19549 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -572,6 +572,7 @@ var tableTiDBHotRegionsCols = []columnInfo{ {"DB_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, {"TABLE_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, {"INDEX_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"REGION_ID", mysql.TypeLonglong, 21, 0, nil, nil}, {"TYPE", mysql.TypeVarchar, 64, 0, nil, nil}, {"MAX_HOT_DEGREE", mysql.TypeLonglong, 21, 0, nil, nil}, {"REGION_COUNT", mysql.TypeLonglong, 21, 0, nil, nil}, @@ -694,18 +695,18 @@ func dataForTikVRegionPeers(ctx sessionctx.Context) (records [][]types.Datum, er row[0].SetInt64(regionStat.ID) row[1].SetInt64(peer.ID) row[2].SetInt64(peer.StoreID) - if peer.ID == regionStat.Leader.ID { + if peer.IsLearner { row[3].SetInt64(1) } else { row[3].SetInt64(0) } - if peer.IsLearner { + if peer.ID == regionStat.Leader.ID { row[4].SetInt64(1) } else { - row[4].SetInt64(0) + row[3].SetInt64(0) } if pendingPeerIDSet.Exist(peer.ID) { - row[5].SetString(pendingPeer) + row[4].SetString(pendingPeer) } else if downSec, ok := downPeerMap[peer.ID]; ok { row[5].SetString(downPeer) row[6].SetInt64(downSec) @@ -1672,9 +1673,9 @@ func dataForTiDBHotRegions(ctx sessionctx.Context) (records [][]types.Datum, err return records, nil } -func dataForHotRegionByMetrics(metrics map[helper.TblIndex]helper.RegionMetric, tp string) [][]types.Datum { +func dataForHotRegionByMetrics(metrics []helper.HotTableIndex, tp string) [][]types.Datum { rows := make([][]types.Datum, 0, len(metrics)) - for tblIndex, regionMetric := range metrics { + for _, tblIndex := range metrics { row := make([]types.Datum, len(tableTiDBHotRegionsCols)) if tblIndex.IndexName != "" { row[1].SetInt64(tblIndex.IndexID) @@ -1686,10 +1687,16 @@ func dataForHotRegionByMetrics(metrics map[helper.TblIndex]helper.RegionMetric, row[0].SetInt64(tblIndex.TableID) row[2].SetString(tblIndex.DbName) row[3].SetString(tblIndex.TableName) - row[5].SetString(tp) - row[6].SetInt64(int64(regionMetric.MaxHotDegree)) - row[7].SetInt64(int64(regionMetric.Count)) - row[8].SetUint64(regionMetric.FlowBytes) + row[5].SetUint64(tblIndex.RegionID) + row[6].SetString(tp) + if tblIndex.RegionMetric == nil { + row[7].SetNull() + row[8].SetNull() + } else { + row[7].SetInt64(int64(tblIndex.RegionMetric.MaxHotDegree)) + row[8].SetInt64(int64(tblIndex.RegionMetric.Count)) + } + row[9].SetUint64(tblIndex.RegionMetric.FlowBytes) rows = append(rows, row) } return rows diff --git a/server/http_handler.go b/server/http_handler.go index a925c5a653c1d..3e14f6e044685 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -23,7 +23,6 @@ import ( "math" "net/http" "net/url" - "sort" "strconv" "strings" "sync/atomic" @@ -1068,17 +1067,9 @@ func (h regionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { writeError(w, err) return } - asSortedEntry := func(metric map[helper.TblIndex]helper.RegionMetric) hotRegions { - hs := make(hotRegions, 0, len(metric)) - for key, value := range metric { - hs = append(hs, hotRegion{key, value}) - } - sort.Sort(hs) - return hs - } writeData(w, map[string]interface{}{ - "write": asSortedEntry(hotWrite), - "read": asSortedEntry(hotRead), + "write": hotWrite, + "read": hotRead, }) return } diff --git a/store/helper/helper.go b/store/helper/helper.go index 0e827b40537bf..f36616eade9d6 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -99,7 +99,7 @@ type RegionMetric struct { } // ScrapeHotInfo gets the needed hot region information by the url given. -func (h *Helper) ScrapeHotInfo(rw string, allSchemas []*model.DBInfo) (map[TblIndex]RegionMetric, error) { +func (h *Helper) ScrapeHotInfo(rw string, allSchemas []*model.DBInfo) ([]HotTableIndex, error) { regionMetrics, err := h.FetchHotRegion(rw) if err != nil { return nil, err @@ -121,9 +121,7 @@ func (h *Helper) FetchHotRegion(rw string) (map[uint64]RegionMetric, error) { if err != nil { return nil, errors.Trace(err) } - timeout, cancelFunc := context.WithTimeout(context.Background(), 50*time.Millisecond) - resp, err := http.DefaultClient.Do(req.WithContext(timeout)) - cancelFunc() + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, errors.Trace(err) } @@ -175,10 +173,22 @@ type RegionFrameRange struct { region *tikv.KeyLocation // the region } -// FetchRegionTableIndex constructs a map that maps a table to its hot region information by the given raw hot region metrics. -func (h *Helper) FetchRegionTableIndex(metrics map[uint64]RegionMetric, allSchemas []*model.DBInfo) (map[TblIndex]RegionMetric, error) { - idxMetrics := make(map[TblIndex]RegionMetric) +// HotTableIndex contains region and its table/index info. +type HotTableIndex struct { + RegionID uint64 `json:"region_id"` + RegionMetric *RegionMetric `json:"region_metric"` + DbName string `json:"db_name"` + TableName string `json:"table_name"` + TableID int64 `json:"table_id"` + IndexName string `json:"index_name"` + IndexID int64 `json:"index_id"` +} + +// FetchRegionTableIndex constructs a map that maps a table to its hot region information by the given raw hot RegionMetric metrics. +func (h *Helper) FetchRegionTableIndex(metrics map[uint64]RegionMetric, allSchemas []*model.DBInfo) ([]HotTableIndex, error) { + hotTables := make([]HotTableIndex, 0, len(metrics)) for regionID, regionMetric := range metrics { + t := HotTableIndex{RegionID: regionID, RegionMetric: ®ionMetric} region, err := h.RegionCache.LocateRegionByID(tikv.NewBackoffer(context.Background(), 500), regionID) if err != nil { logutil.BgLogger().Error("locate region failed", zap.Error(err)) @@ -189,32 +199,18 @@ func (h *Helper) FetchRegionTableIndex(metrics map[uint64]RegionMetric, allSchem if err != nil { return nil, err } - f := h.FindTableIndexOfRegion(allSchemas, hotRange) if f != nil { - idx := TblIndex{ - DbName: f.DBName, - TableName: f.TableName, - TableID: f.TableID, - IndexName: f.IndexName, - IndexID: f.IndexID, - } - metric, exists := idxMetrics[idx] - if !exists { - metric = regionMetric - metric.Count++ - idxMetrics[idx] = metric - } else { - metric.FlowBytes += regionMetric.FlowBytes - if metric.MaxHotDegree < regionMetric.MaxHotDegree { - metric.MaxHotDegree = regionMetric.MaxHotDegree - } - metric.Count++ - } + t.DbName = f.DBName + t.TableName = f.TableName + t.TableID = f.TableID + t.IndexName = f.IndexName + t.IndexID = f.IndexID } + hotTables = append(hotTables, t) } - return idxMetrics, nil + return hotTables, nil } // FindTableIndexOfRegion finds what table is involved in this hot region. And constructs the new frame item for future use. From 2ea24e02015c92cbcc0595df5554167ae4494339 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 11 Jul 2019 13:26:09 +0800 Subject: [PATCH 008/196] ddl: do not split index when creating table (#11183) Signed-off-by: Shuaipeng Yu --- ddl/split_region.go | 93 ++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/ddl/split_region.go b/ddl/split_region.go index 189a93fdbd7cf..1c1a6263be69f 100644 --- a/ddl/split_region.go +++ b/ddl/split_region.go @@ -28,58 +28,65 @@ func splitPartitionTableRegion(store kv.SplitableStore, pi *model.PartitionInfo, regionIDs = append(regionIDs, splitRecordRegion(store, def.ID, scatter)) } if scatter { - waitScatterRegionFinish(store, regionIDs) + waitScatterRegionFinish(store, regionIDs...) } } func splitTableRegion(store kv.SplitableStore, tbInfo *model.TableInfo, scatter bool) { - regionIDs := make([]uint64, 0, len(tbInfo.Indices)+1) if tbInfo.ShardRowIDBits > 0 && tbInfo.PreSplitRegions > 0 { - // Example: - // ShardRowIDBits = 5 - // PreSplitRegions = 3 - // - // then will pre-split 2^(3-1) = 4 regions. - // - // in this code: - // max = 1 << (tblInfo.ShardRowIDBits - 1) = 1 << (5-1) = 16 - // step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions)) = 1 << (5-3) = 4; - // - // then split regionID is below: - // 4 << 59 = 2305843009213693952 - // 8 << 59 = 4611686018427387904 - // 12 << 59 = 6917529027641081856 - // - // The 4 pre-split regions range is below: - // 0 ~ 2305843009213693952 - // 2305843009213693952 ~ 4611686018427387904 - // 4611686018427387904 ~ 6917529027641081856 - // 6917529027641081856 ~ 9223372036854775807 ( (1 << 63) - 1 ) - // - // And the max _tidb_rowid is 9223372036854775807, it won't be negative number. + splitPreSplitedTable(store, tbInfo, scatter) + } else { + regionID := splitRecordRegion(store, tbInfo.ID, scatter) + if scatter { + waitScatterRegionFinish(store, regionID) + } + } +} - // Split table region. - step := int64(1 << (tbInfo.ShardRowIDBits - tbInfo.PreSplitRegions)) - // The highest bit is the symbol bit,and alloc _tidb_rowid will always be positive number. - // So we only need to split the region for the positive number. - max := int64(1 << (tbInfo.ShardRowIDBits - 1)) - for p := int64(step); p < max; p += step { - recordID := p << (64 - tbInfo.ShardRowIDBits) - recordPrefix := tablecodec.GenTableRecordPrefix(tbInfo.ID) - key := tablecodec.EncodeRecordKey(recordPrefix, recordID) - regionID, err := store.SplitRegion(key, scatter) - if err != nil { - logutil.BgLogger().Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) - } else { - regionIDs = append(regionIDs, regionID) - } +func splitPreSplitedTable(store kv.SplitableStore, tbInfo *model.TableInfo, scatter bool) { + // Example: + // ShardRowIDBits = 5 + // PreSplitRegions = 3 + // + // then will pre-split 2^(3-1) = 4 regions. + // + // in this code: + // max = 1 << (tblInfo.ShardRowIDBits - 1) = 1 << (5-1) = 16 + // step := int64(1 << (tblInfo.ShardRowIDBits - tblInfo.PreSplitRegions)) = 1 << (5-3) = 4; + // + // then split regionID is below: + // 4 << 59 = 2305843009213693952 + // 8 << 59 = 4611686018427387904 + // 12 << 59 = 6917529027641081856 + // + // The 4 pre-split regions range is below: + // 0 ~ 2305843009213693952 + // 2305843009213693952 ~ 4611686018427387904 + // 4611686018427387904 ~ 6917529027641081856 + // 6917529027641081856 ~ 9223372036854775807 ( (1 << 63) - 1 ) + // + // And the max _tidb_rowid is 9223372036854775807, it won't be negative number. + + // Split table region. + regionIDs := make([]uint64, 0, 1<<(tbInfo.PreSplitRegions-1)+len(tbInfo.Indices)) + step := int64(1 << (tbInfo.ShardRowIDBits - tbInfo.PreSplitRegions)) + // The highest bit is the symbol bit,and alloc _tidb_rowid will always be positive number. + // So we only need to split the region for the positive number. + max := int64(1 << (tbInfo.ShardRowIDBits - 1)) + for p := int64(step); p < max; p += step { + recordID := p << (64 - tbInfo.ShardRowIDBits) + recordPrefix := tablecodec.GenTableRecordPrefix(tbInfo.ID) + key := tablecodec.EncodeRecordKey(recordPrefix, recordID) + regionID, err := store.SplitRegion(key, scatter) + if err != nil { + logutil.BgLogger().Warn("[ddl] pre split table region failed", zap.Int64("recordID", recordID), zap.Error(err)) + } else { + regionIDs = append(regionIDs, regionID) } - } else { - regionIDs = append(regionIDs, splitRecordRegion(store, tbInfo.ID, scatter)) } regionIDs = append(regionIDs, splitIndexRegion(store, tbInfo, scatter)...) if scatter { - waitScatterRegionFinish(store, regionIDs) + waitScatterRegionFinish(store, regionIDs...) } } @@ -109,7 +116,7 @@ func splitIndexRegion(store kv.SplitableStore, tblInfo *model.TableInfo, scatter return regionIDs } -func waitScatterRegionFinish(store kv.SplitableStore, regionIDs []uint64) { +func waitScatterRegionFinish(store kv.SplitableStore, regionIDs ...uint64) { for _, regionID := range regionIDs { err := store.WaitScatterRegionFinish(regionID) if err != nil { From 53f8a2c3664891ebe2f3f3dcaa8e5e2bebb7e772 Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Thu, 11 Jul 2019 14:36:28 +0800 Subject: [PATCH 009/196] executor, metrics: add metrics for fast analyze. (#11079) --- executor/analyze.go | 41 +++++++++++++++++++++++++++++++---------- metrics/metrics.go | 1 + metrics/stats.go | 9 +++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 130ad4597e36b..d37a27a32fc5d 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -526,6 +526,14 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis return hists, cms, nil } +var ( + fastAnalyzeHistogramSample = metrics.FastAnalyzeHistogram.WithLabelValues(metrics.LblGeneral, "sample") + fastAnalyzeHistogramAccessRegions = metrics.FastAnalyzeHistogram.WithLabelValues(metrics.LblGeneral, "access_regions") + fastAnalyzeHistogramRegionError = metrics.FastAnalyzeHistogram.WithLabelValues(metrics.LblGeneral, "region_error") + fastAnalyzeHistogramSeekKeys = metrics.FastAnalyzeHistogram.WithLabelValues(metrics.LblGeneral, "seek_keys") + fastAnalyzeHistogramScanKeys = metrics.FastAnalyzeHistogram.WithLabelValues(metrics.LblGeneral, "scan_keys") +) + func analyzeFastExec(exec *AnalyzeFastExec) []analyzeResult { hists, cms, err := exec.buildStats() if err != nil { @@ -624,6 +632,7 @@ func (e *AnalyzeFastExec) getSampRegionsRowCount(bo *tikv.Backoffer, needRebuild if *err != nil { return } + ctx := context.Background() resp, *err = client.SendRequest(ctx, rpcCtx.Addr, req, tikv.ReadTimeoutMedium) if *err != nil { @@ -740,6 +749,7 @@ func (e *AnalyzeFastExec) buildSampTask() (needRebuild bool, err error) { task.EndOffset = e.rowCount + cnt e.rowCount += cnt } + accessRegionsCounter := 0 for { // Search for the region which contains the targetKey. loc, err := e.cache.LocateKey(bo, targetKey) @@ -749,6 +759,8 @@ func (e *AnalyzeFastExec) buildSampTask() (needRebuild bool, err error) { if bytes.Compare(endKey, loc.StartKey) < 0 { break } + accessRegionsCounter++ + // Set the next search key. targetKey = loc.EndKey @@ -767,6 +779,7 @@ func (e *AnalyzeFastExec) buildSampTask() (needRebuild bool, err error) { break } } + fastAnalyzeHistogramAccessRegions.Observe(float64(accessRegionsCounter)) return false, nil } @@ -881,7 +894,7 @@ func (e *AnalyzeFastExec) handleBatchSeekResponse(kvMap map[string][]byte) (err return nil } -func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (err error) { +func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (scanKeysSize int, err error) { hasPKInfo := 0 if e.pkInfo != nil { hasPKInfo = 1 @@ -890,6 +903,7 @@ func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (err error) { for ; iter.Valid() && err == nil; err = iter.Next() { // reservoir sampling e.rowCount++ + scanKeysSize++ randNum := rander.Int63n(int64(e.rowCount)) if randNum > int64(MaxSampleSize) && e.sampCursor == int32(MaxSampleSize) { continue @@ -903,28 +917,29 @@ func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (err error) { err = e.updateCollectorSamples(iter.Value(), iter.Key(), p, hasPKInfo) if err != nil { - return err + return } } - return err + return } -func (e *AnalyzeFastExec) handleScanTasks(bo *tikv.Backoffer) error { +func (e *AnalyzeFastExec) handleScanTasks(bo *tikv.Backoffer) (keysSize int, err error) { snapshot, err := e.ctx.GetStore().(tikv.Storage).GetSnapshot(kv.MaxVersion) if err != nil { - return err + return 0, err } for _, t := range e.scanTasks { iter, err := snapshot.Iter(t.StartKey, t.EndKey) if err != nil { - return err + return keysSize, err } - err = e.handleScanIter(iter) + size, err := e.handleScanIter(iter) + keysSize += size if err != nil { - return err + return keysSize, err } } - return nil + return keysSize, nil } func (e *AnalyzeFastExec) handleSampTasks(bo *tikv.Backoffer, workID int, err *error) { @@ -969,6 +984,8 @@ func (e *AnalyzeFastExec) handleSampTasks(bo *tikv.Backoffer, workID int, err *e kvMap[string(iter.Key())] = iter.Value() } } + fastAnalyzeHistogramSeekKeys.Observe(float64(len(keys))) + fastAnalyzeHistogramSample.Observe(float64(len(kvMap))) *err = e.handleBatchSeekResponse(kvMap) if *err != nil { @@ -1060,7 +1077,8 @@ func (e *AnalyzeFastExec) runTasks() ([]*statistics.Histogram, []*statistics.CMS } } - err := e.handleScanTasks(bo) + scanKeysSize, err := e.handleScanTasks(bo) + fastAnalyzeHistogramScanKeys.Observe(float64(scanKeysSize)) if err != nil { return nil, nil, err } @@ -1107,12 +1125,15 @@ func (e *AnalyzeFastExec) buildStats() (hists []*statistics.Histogram, cms []*st // Only four rebuilds for sample task are allowed. needRebuild, maxBuildTimes := true, 5 + regionErrorCounter := 0 for counter := maxBuildTimes; needRebuild && counter > 0; counter-- { + regionErrorCounter++ needRebuild, err = e.buildSampTask() if err != nil { return nil, nil, err } } + fastAnalyzeHistogramRegionError.Observe(float64(regionErrorCounter)) if needRebuild { errMsg := "build fast analyze task failed, exceed maxBuildTimes: %v" return nil, nil, errors.Errorf(errMsg, maxBuildTimes) diff --git a/metrics/metrics.go b/metrics/metrics.go index 7b4b97ca1859c..a21fd394c4d63 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -76,6 +76,7 @@ func RegisterMetrics() { prometheus.MustRegister(HandShakeErrorCounter) prometheus.MustRegister(HandleJobHistogram) prometheus.MustRegister(SignificantFeedbackCounter) + prometheus.MustRegister(FastAnalyzeHistogram) prometheus.MustRegister(JobsGauge) prometheus.MustRegister(KeepAliveCounter) prometheus.MustRegister(LoadPrivilegeCounter) diff --git a/metrics/stats.go b/metrics/stats.go index e5450a63b403e..779bb88023d41 100644 --- a/metrics/stats.go +++ b/metrics/stats.go @@ -84,4 +84,13 @@ var ( Name: "high_error_rate_feedback_total", Help: "Counter of query feedback whose actual count is much different than calculated by current statistics", }) + + FastAnalyzeHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "statistics", + Name: "fast_analyze_status", + Help: "Bucketed histogram of some stats in fast analyze.", + Buckets: prometheus.ExponentialBuckets(1, 2, 16), + }, []string{LblSQLType, LblType}) ) From 9385c6eca03f95590ddd3f7e390994f42af2e800 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 11 Jul 2019 15:24:27 +0800 Subject: [PATCH 010/196] executor: fix a bug of 'insert on duplicate update' statement on partitioned table (#11204) In this statement: insert into t1 set a=1,b=1 on duplicate key update a=1,b=1 Batch checker find a=1,b=1 is duplicated, then in the "on duplicate update" step, it uses the non-partitioned table to get the old row, which is a bug. getOldRow returns this error: (1105, u'can not be duplicated row, due to old row not found. handle 1 not found') --- executor/insert.go | 2 +- executor/insert_test.go | 14 ++++++++++++++ executor/simple_test.go | 9 +++++++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/executor/insert.go b/executor/insert.go index 133b882a81d2a..c541ddce08a1b 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -162,7 +162,7 @@ func (e *InsertExec) Open(ctx context.Context) error { // updateDupRow updates a duplicate row to a new row. func (e *InsertExec) updateDupRow(row toBeCheckedRow, handle int64, onDuplicate []*expression.Assignment) error { - oldRow, err := e.getOldRow(e.ctx, e.Table, handle, e.GenExprs) + oldRow, err := e.getOldRow(e.ctx, row.t, handle, e.GenExprs) if err != nil { logutil.BgLogger().Error("get old row failed when insert on dup", zap.Int64("handle", handle), zap.String("toBeInsertedRow", types.DatumsToStrNoErr(row.row))) return err diff --git a/executor/insert_test.go b/executor/insert_test.go index 8c3510b044126..e26834572f12b 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -286,3 +286,17 @@ func (s *testSuite3) TestAllowInvalidDates(c *C) { runWithMode("STRICT_TRANS_TABLES,ALLOW_INVALID_DATES") runWithMode("ALLOW_INVALID_DATES") } + +func (s *testSuite3) TestPartitionInsertOnDuplicate(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec(`create table t1 (a int,b int,primary key(a,b)) partition by range(a) (partition p0 values less than (100),partition p1 values less than (1000))`) + tk.MustExec(`insert into t1 set a=1, b=1`) + tk.MustExec(`insert into t1 set a=1,b=1 on duplicate key update a=1,b=1`) + tk.MustQuery(`select * from t1`).Check(testkit.Rows("1 1")) + + tk.MustExec(`create table t2 (a int,b int,primary key(a,b)) partition by hash(a) partitions 4`) + tk.MustExec(`insert into t2 set a=1,b=1;`) + tk.MustExec(`insert into t2 set a=1,b=1 on duplicate key update a=1,b=1`) + tk.MustQuery(`select * from t2`).Check(testkit.Rows("1 1")) +} diff --git a/executor/simple_test.go b/executor/simple_test.go index 440224cc494e5..0de9f78d714a0 100644 --- a/executor/simple_test.go +++ b/executor/simple_test.go @@ -413,14 +413,19 @@ func (s *testFlushSuite) TestFlushPrivilegesPanic(c *C) { c.Assert(err, IsNil) defer store.Close() - config.GetGlobalConfig().Security.SkipGrantTable = true + saveConf := config.GetGlobalConfig() + conf := config.NewConfig() + conf.Security.SkipGrantTable = true + config.StoreGlobalConfig(conf) + dom, err := session.BootstrapSession(store) c.Assert(err, IsNil) defer dom.Close() tk := testkit.NewTestKit(c, store) tk.MustExec("FLUSH PRIVILEGES") - config.GetGlobalConfig().Security.SkipGrantTable = false + + config.StoreGlobalConfig(saveConf) } func (s *testSuite3) TestDropStats(c *C) { From 4a1374d93a9450c08bfa8a268dc53fc918d9c440 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 11 Jul 2019 20:26:03 +0800 Subject: [PATCH 011/196] planner: fix bug when pruning columns for TableDual (#11054) --- executor/executor_test.go | 3 +++ planner/core/rule_column_pruning.go | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 02b80cba46a6d..b32b405277d0e 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -1941,6 +1941,9 @@ func (s *testSuite) TestTableDual(c *C) { result.Check(testkit.Rows("1")) result = tk.MustQuery("Select 1 from dual where 1") result.Check(testkit.Rows("1")) + + tk.MustExec("create table t(a int primary key)") + tk.MustQuery("select t1.* from t t1, t t2 where t1.a=t2.a and 1=0").Check(testkit.Rows()) } func (s *testSuite) TestTableScan(c *C) { diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index 6a5a278921388..1f4873618203f 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -262,8 +262,15 @@ func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column) err } } for k, cols := range p.schema.TblID2Handle { - if p.schema.ColumnIndex(cols[0]) == -1 { + for i := len(cols) - 1; i >= 0; i-- { + if p.schema.ColumnIndex(cols[i]) == -1 { + cols = append(cols[:i], cols[i+1:]...) + } + } + if len(cols) == 0 { delete(p.schema.TblID2Handle, k) + } else { + p.schema.TblID2Handle[k] = cols } } return nil From 5c436bb1d6e359a3640a39ea02a7752ab24290d6 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Thu, 11 Jul 2019 23:56:51 +0800 Subject: [PATCH 012/196] planner: support subquery in `SHOW` statement (#10942) --- executor/builder.go | 9 +---- planner/core/common_plans.go | 4 +- planner/core/find_best_task.go | 5 ++- planner/core/initialize.go | 4 +- planner/core/integration_test.go | 61 +++++++++++++++++++++++++++++++ planner/core/logical_plan_test.go | 2 +- planner/core/logical_plans.go | 4 ++ planner/core/physical_plans.go | 4 ++ planner/core/plan.go | 8 ++++ planner/core/planbuilder.go | 45 +++++++++++++++++------ planner/core/point_get_plan.go | 3 ++ planner/core/resolve_indices.go | 11 ------ planner/core/stringer.go | 6 +-- 13 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 planner/core/integration_test.go diff --git a/executor/builder.go b/executor/builder.go index d83c4302742fa..fa98c390f4c01 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -576,14 +576,7 @@ func (b *executorBuilder) buildShow(v *plannercore.Show) Executor { b.err = err } } - if len(v.Conditions) == 0 { - return e - } - sel := &SelectionExec{ - baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID(), e), - filters: v.Conditions, - } - return sel + return e } func (b *executorBuilder) buildSimple(v *plannercore.Simple) Executor { diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index a6768d02bc9eb..96a33507287f2 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -357,7 +357,7 @@ type Deallocate struct { // Show represents a show plan. type Show struct { - baseSchemaProducer + physicalSchemaProducer Tp ast.ShowStmtType // Databases/Tables/Columns/.... DBName string @@ -370,8 +370,6 @@ type Show struct { Roles []*auth.RoleIdentity // Used for show grants. IfNotExists bool // Used for `show create database if not exists` - Conditions []expression.Expression - GlobalScope bool // Used by show variables } diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 98b2468f5d193..40e20ea43287d 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -69,7 +69,10 @@ func (p *LogicalTableDual) findBestTask(prop *property.PhysicalProperty) (task, if !prop.IsEmpty() { return invalidTask, nil } - dual := PhysicalTableDual{RowCount: p.RowCount}.Init(p.ctx, p.stats) + dual := PhysicalTableDual{ + RowCount: p.RowCount, + placeHolder: p.placeHolder, + }.Init(p.ctx, p.stats) dual.SetSchema(p.schema) return &rootTask{p: dual}, nil } diff --git a/planner/core/initialize.go b/planner/core/initialize.go index ff2a3dbc7b059..f0f1bddb393bd 100644 --- a/planner/core/initialize.go +++ b/planner/core/initialize.go @@ -267,7 +267,9 @@ func (p Insert) Init(ctx sessionctx.Context) *Insert { // Init initializes Show. func (p Show) Init(ctx sessionctx.Context) *Show { - p.basePlan = newBasePlan(ctx, TypeShow) + p.basePhysicalPlan = newBasePhysicalPlan(ctx, TypeShow, &p) + // Just use pseudo stats to avoid panic. + p.stats = &property.StatsInfo{RowCount: 1} return &p } diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go new file mode 100644 index 0000000000000..ff7ca01e6dd92 --- /dev/null +++ b/planner/core/integration_test.go @@ -0,0 +1,61 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/tidb/util/testkit" +) + +var _ = Suite(&testIntegrationSuite{}) + +type testIntegrationSuite struct { +} + +func (s *testIntegrationSuite) TestShowSubquery(c *C) { + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a varchar(10), b int, c int)") + tk.MustQuery("show columns from t where true").Check(testkit.Rows( + "a varchar(10) YES ", + "b int(11) YES ", + "c int(11) YES ", + )) + tk.MustQuery("show columns from t where field = 'b'").Check(testkit.Rows( + "b int(11) YES ", + )) + tk.MustQuery("show columns from t where field in (select 'b')").Check(testkit.Rows( + "b int(11) YES ", + )) + tk.MustQuery("show columns from t where field in (select 'b') and true").Check(testkit.Rows( + "b int(11) YES ", + )) + tk.MustQuery("show columns from t where field in (select 'b') and false").Check(testkit.Rows()) + tk.MustExec("insert into t values('c', 0, 0)") + tk.MustQuery("show columns from t where field < all (select a from t)").Check(testkit.Rows( + "a varchar(10) YES ", + "b int(11) YES ", + )) + tk.MustExec("insert into t values('b', 0, 0)") + tk.MustQuery("show columns from t where field < all (select a from t)").Check(testkit.Rows( + "a varchar(10) YES ", + )) +} diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 536ac04908964..25d94c1cf516b 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -829,7 +829,7 @@ func (s *testPlanSuite) TestPlanBuilder(c *C) { }, { sql: "show columns from t where `Key` = 'pri' like 't*'", - plan: "Show([eq(cast(key), 0)])", + plan: "Show->Sel([eq(cast(key), 0)])", }, { sql: "do sleep(5)", diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 5fc8b28d66ae4..cd8a7e611c1c4 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -308,6 +308,10 @@ type LogicalTableDual struct { logicalSchemaProducer RowCount int + // placeHolder indicates if this dual plan is a place holder in query optimization + // for data sources like `Show`, if true, the dual plan would be substituted by + // `Show` in the final plan. + placeHolder bool } // LogicalUnionScan is only used in non read-only txn. diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 32d257090270d..4d89ec7b1af6e 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -378,6 +378,10 @@ type PhysicalTableDual struct { physicalSchemaProducer RowCount int + // placeHolder indicates if this dual plan is a place holder in query optimization + // for data sources like `Show`, if true, the dual plan would be substituted by + // `Show` in the final plan. + placeHolder bool } // PhysicalWindow is the physical operator of window function. diff --git a/planner/core/plan.go b/planner/core/plan.go index a141115e36140..12fd4b402e67e 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -141,6 +141,9 @@ type PhysicalPlan interface { // SetChildren sets the children for the plan. SetChildren(...PhysicalPlan) + // SetChild sets the ith child for the plan. + SetChild(i int, child PhysicalPlan) + // ResolveIndices resolves the indices for columns. After doing this, the columns can evaluate the rows by their indices. ResolveIndices() error } @@ -296,6 +299,11 @@ func (p *basePhysicalPlan) SetChildren(children ...PhysicalPlan) { p.children = children } +// SetChild implements PhysicalPlan SetChild interface. +func (p *basePhysicalPlan) SetChild(i int, child PhysicalPlan) { + p.children[i] = child +} + func (p *basePlan) context() sessionctx.Context { return p.ctx } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index ee4ec9c9d0e8a..ebae4fc1b4dee 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1121,35 +1121,58 @@ func (b *PlanBuilder) buildShow(show *ast.ShowStmt) (Plan, error) { for _, col := range p.schema.Columns { col.UniqueID = b.ctx.GetSessionVars().AllocPlanColumnID() } - mockTablePlan := LogicalTableDual{}.Init(b.ctx) + mockTablePlan := LogicalTableDual{placeHolder: true}.Init(b.ctx) mockTablePlan.SetSchema(p.schema) + var err error + var np LogicalPlan + np = mockTablePlan if show.Pattern != nil { show.Pattern.Expr = &ast.ColumnNameExpr{ Name: &ast.ColumnName{Name: p.Schema().Columns[0].ColName}, } - expr, _, err := b.rewrite(show.Pattern, mockTablePlan, nil, false) + np, err = b.buildSelection(np, show.Pattern, nil) if err != nil { return nil, err } - p.Conditions = append(p.Conditions, expr) } if show.Where != nil { - conds := splitWhere(show.Where) - for _, cond := range conds { - expr, _, err := b.rewrite(cond, mockTablePlan, nil, false) - if err != nil { - return nil, err - } - p.Conditions = append(p.Conditions, expr) + np, err = b.buildSelection(np, show.Where, nil) + if err != nil { + return nil, err } - err := p.ResolveIndices() + } + if np != mockTablePlan { + fieldsLen := len(mockTablePlan.schema.Columns) + proj := LogicalProjection{Exprs: make([]expression.Expression, 0, fieldsLen)}.Init(b.ctx) + schema := expression.NewSchema(make([]*expression.Column, 0, fieldsLen)...) + for _, col := range mockTablePlan.schema.Columns { + proj.Exprs = append(proj.Exprs, col) + newCol := col.Clone().(*expression.Column) + newCol.UniqueID = b.ctx.GetSessionVars().AllocPlanColumnID() + schema.Append(newCol) + } + proj.SetSchema(schema) + proj.SetChildren(np) + physical, err := DoOptimize(b.optFlag|flagEliminateProjection, proj) if err != nil { return nil, err } + return substitutePlaceHolderDual(physical, p), nil } return p, nil } +func substitutePlaceHolderDual(src PhysicalPlan, dst PhysicalPlan) PhysicalPlan { + if dual, ok := src.(*PhysicalTableDual); ok && dual.placeHolder { + return dst + } + for i, child := range src.Children() { + newChild := substitutePlaceHolderDual(child, dst) + src.SetChild(i, newChild) + } + return src +} + func (b *PlanBuilder) buildSimple(node ast.StmtNode) (Plan, error) { p := &Simple{Statement: node} diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 6bf2b6e197066..7be3ba1e49b07 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -124,6 +124,9 @@ func (p *PointGetPlan) Children() []PhysicalPlan { // SetChildren sets the children for the plan. func (p *PointGetPlan) SetChildren(...PhysicalPlan) {} +// SetChild sets a specific child for the plan. +func (p *PointGetPlan) SetChild(i int, child PhysicalPlan) {} + // ResolveIndices resolves the indices for columns. After doing this, the columns can evaluate the rows by their indices. func (p *PointGetPlan) ResolveIndices() error { return nil diff --git a/planner/core/resolve_indices.go b/planner/core/resolve_indices.go index def6cea1f32f5..2340e747bb731 100644 --- a/planner/core/resolve_indices.go +++ b/planner/core/resolve_indices.go @@ -480,17 +480,6 @@ func (p *Insert) ResolveIndices() (err error) { return } -// ResolveIndices implements Plan interface. -func (p *Show) ResolveIndices() (err error) { - for i, expr := range p.Conditions { - p.Conditions[i], err = expr.ResolveIndices(p.schema) - if err != nil { - return err - } - } - return err -} - func (p *physicalSchemaProducer) ResolveIndices() (err error) { err = p.basePhysicalPlan.ResolveIndices() if err != nil { diff --git a/planner/core/stringer.go b/planner/core/stringer.go index beba86d8cd527..f4178509bc742 100644 --- a/planner/core/stringer.go +++ b/planner/core/stringer.go @@ -114,11 +114,7 @@ func toString(in Plan, strs []string, idxs []int) ([]string, []int) { case *ShowDDL: str = "ShowDDL" case *Show: - if len(x.Conditions) == 0 { - str = "Show" - } else { - str = fmt.Sprintf("Show(%s)", x.Conditions) - } + str = "Show" case *LogicalSort, *PhysicalSort: str = "Sort" case *LogicalJoin: From d36d66393467de7af319d23f8a805d6cbf27420e Mon Sep 17 00:00:00 2001 From: Tanner Date: Fri, 12 Jul 2019 12:10:53 +0800 Subject: [PATCH 013/196] ddl: refine error messages in unsupported column options (#11065) --- ddl/db_integration_test.go | 27 +++++++++++++++++++++++++++ ddl/ddl_api.go | 16 +++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 4943c00432b6e..367549acc13cb 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -770,6 +770,33 @@ func (s *testIntegrationSuite4) TestChangingTableCharset(c *C) { } +func (s *testIntegrationSuite5) TestModifyingColumnOption(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") + + errMsg := "[ddl:203]" // unsupported modify column with references + assertErrCode := func(sql string, errCodeStr string) { + _, err := tk.Exec(sql) + c.Assert(err, NotNil) + c.Assert(err.Error()[:len(errCodeStr)], Equals, errCodeStr) + } + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (b char(1) default null) engine=InnoDB default charset=utf8mb4 collate=utf8mb4_general_ci") + tk.MustExec("alter table t1 modify column b char(1) character set utf8mb4 collate utf8mb4_general_ci") + + tk.MustExec("drop table t1") + tk.MustExec("create table t1 (b char(1) collate utf8mb4_general_ci)") + tk.MustExec("alter table t1 modify b char(1) character set utf8mb4 collate utf8mb4_general_ci") + + tk.MustExec("drop table t1") + tk.MustExec("drop table if exists t2") + tk.MustExec("create table t1 (a int)") + tk.MustExec("create table t2 (b int, c int)") + assertErrCode("alter table t2 modify column c int references t1(a)", errMsg) +} + func (s *testIntegrationSuite2) TestCaseInsensitiveCharsetAndCollate(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index a279c501dd0ca..5a930f6aafcf6 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2520,9 +2520,14 @@ func processColumnOptions(ctx sessionctx.Context, col *table.Column, options []* for _, colName := range findColumnNamesInExpr(opt.Expr) { col.Dependences[colName.Name.L] = struct{}{} } + case ast.ColumnOptionCollate: + col.Collate = opt.StrValue + case ast.ColumnOptionReference: + return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("with references")) + case ast.ColumnOptionFulltext: + return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs("with full text")) default: - // TODO: Support other types. - return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs(opt.Tp)) + return errors.Trace(errUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp))) } } @@ -2604,11 +2609,12 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or if err != nil { return nil, errors.Trace(err) } - err = modifiable(&col.FieldType, &newCol.FieldType) - if err != nil { + + if err = processColumnOptions(ctx, newCol, specNewColumn.Options); err != nil { return nil, errors.Trace(err) } - if err = processColumnOptions(ctx, newCol, specNewColumn.Options); err != nil { + + if err = modifiable(&col.FieldType, &newCol.FieldType); err != nil { return nil, errors.Trace(err) } From 7f3bff8afef001c4d93f4c7632ad9a9595cded8b Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Fri, 12 Jul 2019 12:42:33 +0800 Subject: [PATCH 014/196] session: do not keep history when the transaction retry is disabled (#11192) --- ddl/db_test.go | 2 +- session/session.go | 61 ++++++++++++++++++++------ session/session_test.go | 78 +++++++++++++++++++++++++++++++++- session/tidb.go | 17 +++++--- sessionctx/variable/session.go | 3 +- 5 files changed, 137 insertions(+), 24 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index a374c893ca4e5..0f4e890238aae 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -3098,9 +3098,9 @@ func (s *testDBSuite2) TestLockTables(c *C) { // Test lock table by other session in transaction and commit with retry. tk.MustExec("unlock tables") tk2.MustExec("unlock tables") + tk.MustExec("set @@session.tidb_disable_txn_auto_retry=0") tk.MustExec("begin") tk.MustExec("insert into t1 set a=1") - tk.MustExec("set @@session.tidb_disable_txn_auto_retry=0") tk2.MustExec("lock tables t1 write") _, err = tk.Exec("commit") c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue, Commentf("err: %v\n", err)) diff --git a/session/session.go b/session/session.go index 0d63d84bf7b16..37b5fab958b94 100644 --- a/session/session.go +++ b/session/session.go @@ -134,10 +134,8 @@ var ( ) type stmtRecord struct { - stmtID uint32 st sqlexec.Statement stmtCtx *stmtctx.StatementContext - params []interface{} } // StmtHistory holds all histories of statements in a txn. @@ -146,12 +144,10 @@ type StmtHistory struct { } // Add appends a stmt to history list. -func (h *StmtHistory) Add(stmtID uint32, st sqlexec.Statement, stmtCtx *stmtctx.StatementContext, params ...interface{}) { +func (h *StmtHistory) Add(st sqlexec.Statement, stmtCtx *stmtctx.StatementContext) { s := &stmtRecord{ - stmtID: stmtID, st: st, stmtCtx: stmtCtx, - params: append(([]interface{})(nil), params...), } h.history = append(h.history, s) } @@ -451,14 +447,8 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { err := s.doCommit(ctx) if err != nil { commitRetryLimit := s.sessionVars.RetryLimit - if s.sessionVars.DisableTxnAutoRetry && !s.sessionVars.InRestrictedSQL { - // Do not retry non-autocommit transactions. - // For autocommit single statement transactions, the history count is always 1. - // For explicit transactions, the statement count is more than 1. - history := GetHistory(s) - if history.Count() > 1 { - commitRetryLimit = 0 - } + if !s.sessionVars.TxnCtx.CouldRetry { + commitRetryLimit = 0 } // Don't retry in BatchInsert mode. As a counter-example, insert into t1 select * from t2, // BatchInsert already commit the first batch 1000 rows, then it commit 1000-2000 and retry the statement, @@ -517,6 +507,13 @@ func (s *session) CommitTxn(ctx context.Context) error { if commitDetail != nil { s.sessionVars.StmtCtx.MergeExecDetails(nil, commitDetail) } + + failpoint.Inject("keepHistory", func(val failpoint.Value) { + if val.(bool) { + failpoint.Return(err) + } + }) + s.sessionVars.TxnCtx.Cleanup() s.recordTransactionCounter(err) return err @@ -1247,10 +1244,48 @@ func (s *session) Txn(active bool) (kv.Transaction, error) { if !s.sessionVars.IsAutocommit() { s.sessionVars.SetStatusFlag(mysql.ServerStatusInTrans, true) } + s.sessionVars.TxnCtx.CouldRetry = s.isTxnRetryable() } return &s.txn, nil } +// isTxnRetryable (if returns true) means the transaction could retry. +// If the transaction is in pessimistic mode, do not retry. +// If the session is already in transaction, enable retry or internal SQL could retry. +// If not, the transaction could always retry, because it should be auto committed transaction. +// Anyway the retry limit is 0, the transaction could not retry. +func (s *session) isTxnRetryable() bool { + sessVars := s.sessionVars + + // The pessimistic transaction no need to retry. + if sessVars.TxnCtx.IsPessimistic { + return false + } + + // If retry limit is 0, the transaction could not retry. + if sessVars.RetryLimit == 0 { + return false + } + + // If the session is not InTxn, it is an auto-committed transaction. + // The auto-committed transaction could always retry. + if !sessVars.InTxn() { + return true + } + + // The internal transaction could always retry. + if sessVars.InRestrictedSQL { + return true + } + + // If the retry is enabled, the transaction could retry. + if !sessVars.DisableTxnAutoRetry { + return true + } + + return false +} + func (s *session) NewTxn(ctx context.Context) error { if s.txn.Valid() { txnID := s.txn.StartTS() diff --git a/session/session_test.go b/session/session_test.go index 514e374ee99c4..e7c01d368c0ce 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -545,7 +545,7 @@ func (s *testSessionSuite) TestRetryCleanTxn(c *C) { c.Assert(err, IsNil) stmt, _ := session.Compile(context.TODO(), tk.Se, stmtNode) executor.ResetContextOfStmt(tk.Se, stmtNode) - history.Add(0, stmt, tk.Se.GetSessionVars().StmtCtx) + history.Add(stmt, tk.Se.GetSessionVars().StmtCtx) _, err = tk.Exec("commit") c.Assert(err, NotNil) txn, err := tk.Se.Txn(false) @@ -559,6 +559,7 @@ func (s *testSessionSuite) TestReadOnlyNotInHistory(c *C) { tk.MustExec("create table history (a int)") tk.MustExec("insert history values (1), (2), (3)") tk.MustExec("set @@autocommit = 0") + tk.MustExec("set tidb_disable_txn_auto_retry = 0") tk.MustQuery("select * from history") history := session.GetHistory(tk.Se) c.Assert(history.Count(), Equals, 0) @@ -572,6 +573,76 @@ func (s *testSessionSuite) TestReadOnlyNotInHistory(c *C) { c.Assert(history.Count(), Equals, 0) } +func (s *testSessionSuite) TestNoHistoryWhenDisableRetry(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table history (a int)") + tk.MustExec("set @@autocommit = 0") + + // retry_limit = 0 will not add history. + tk.MustExec("set @@tidb_retry_limit = 0") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) + + // Disable auto_retry will add history for auto committed only + tk.MustExec("set @@autocommit = 1") + tk.MustExec("set @@tidb_retry_limit = 10") + tk.MustExec("set @@tidb_disable_txn_auto_retry = 1") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `1*return(true)->return(false)`), IsNil) + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) + tk.MustExec("begin") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) + tk.MustExec("commit") + + // Enable auto_retry will add history for both. + tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/keepHistory", `1*return(true)->return(false)`), IsNil) + tk.MustExec("insert history values (1)") + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/keepHistory"), IsNil) + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 1) + tk.MustExec("begin") + tk.MustExec("insert history values (1)") + c.Assert(session.GetHistory(tk.Se).Count(), Equals, 2) + tk.MustExec("commit") +} + +func (s *testSessionSuite) TestNoRetryForCurrentTxn(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk1 := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table history (a int)") + tk.MustExec("insert history values (1)") + + // Firstly, disable retry. + tk.MustExec("set tidb_disable_txn_auto_retry = 1") + tk.MustExec("begin") + tk.MustExec("update history set a = 2") + // Enable retry now. + tk.MustExec("set tidb_disable_txn_auto_retry = 0") + + tk1.MustExec("update history set a = 3") + c.Assert(tk.ExecToErr("commit"), NotNil) +} + +func (s *testSessionSuite) TestRetryForCurrentTxn(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk1 := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create table history (a int)") + tk.MustExec("insert history values (1)") + + // Firstly, enable retry. + tk.MustExec("set tidb_disable_txn_auto_retry = 0") + tk.MustExec("begin") + tk.MustExec("update history set a = 2") + // Disable retry now. + tk.MustExec("set tidb_disable_txn_auto_retry = 1") + + tk1.MustExec("update history set a = 3") + tk.MustExec("commit") + tk.MustQuery("select * from history").Check(testkit.Rows("2")) +} + // TestTruncateAlloc tests that the auto_increment ID does not reuse the old table's allocator. func (s *testSessionSuite) TestTruncateAlloc(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) @@ -990,6 +1061,7 @@ func (s *testSessionSuite) TestBinaryReadOnly(c *C) { id2, _, _, err := tk.Se.PrepareStmt("insert into t values (?)") c.Assert(err, IsNil) tk.MustExec("set autocommit = 0") + tk.MustExec("set tidb_disable_txn_auto_retry = 0") _, err = tk.Se.ExecutePreparedStmt(context.Background(), id, []types.Datum{types.NewDatum(1)}) c.Assert(err, IsNil) c.Assert(session.GetHistory(tk.Se).Count(), Equals, 0) @@ -2177,6 +2249,7 @@ func (s *testSessionSuite) TestStatementCountLimit(c *C) { defer func() { config.GetGlobalConfig().Performance.StmtCountLimit = saved }() + tk.MustExec("set tidb_disable_txn_auto_retry = 0") tk.MustExec("begin") tk.MustExec("insert into stmt_count_limit values (1)") tk.MustExec("insert into stmt_count_limit values (2)") @@ -2195,6 +2268,7 @@ func (s *testSessionSuite) TestStatementCountLimit(c *C) { func (s *testSessionSuite) TestBatchCommit(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("set tidb_batch_commit = 1") + tk.MustExec("set tidb_disable_txn_auto_retry = 0") tk.MustExec("create table t (id int)") saved := config.GetGlobalConfig().Performance config.GetGlobalConfig().Performance.StmtCountLimit = 3 @@ -2440,7 +2514,7 @@ func (s *testSchemaSuite) TestDisableTxnAutoRetry(c *C) { // session 1 starts a transaction early. // execute a select statement to clear retry history. tk1.MustExec("select 1") - tk1.Se.NewTxn(context.Background()) + tk1.Se.PrepareTxnCtx(context.Background()) // session 2 update the value. tk2.MustExec("update no_retry set id = 4") // AutoCommit update will retry, so it would not fail. diff --git a/session/tidb.go b/session/tidb.go index 284a9b9430ccd..03009b5b96b53 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -176,13 +176,14 @@ func finishStmt(ctx context.Context, sctx sessionctx.Context, se *session, sessV return se.CommitTxn(ctx) } - return checkStmtLimit(ctx, sctx, se, sessVars) + return checkStmtLimit(ctx, sctx, se) } -func checkStmtLimit(ctx context.Context, sctx sessionctx.Context, se *session, sessVars *variable.SessionVars) error { +func checkStmtLimit(ctx context.Context, sctx sessionctx.Context, se *session) error { // If the user insert, insert, insert ... but never commit, TiDB would OOM. // So we limit the statement count in a transaction here. var err error + sessVars := se.GetSessionVars() history := GetHistory(sctx) if history.Count() > int(config.GetGlobalConfig().Performance.StmtCountLimit) { if !sessVars.BatchCommit { @@ -195,7 +196,7 @@ func checkStmtLimit(ctx context.Context, sctx sessionctx.Context, se *session, s // The last history could not be "commit"/"rollback" statement. // It means it is impossible to start a new transaction at the end of the transaction. // Because after the server executed "commit"/"rollback" statement, the session is out of the transaction. - se.sessionVars.SetStatusFlag(mysql.ServerStatusInTrans, true) + sessVars.SetStatusFlag(mysql.ServerStatusInTrans, true) } return err } @@ -222,12 +223,14 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement) } rs, err = s.Exec(ctx) sessVars := se.GetSessionVars() - // All the history should be added here. sessVars.TxnCtx.StatementCount++ if !s.IsReadOnly(sessVars) { - if err == nil && !sessVars.TxnCtx.IsPessimistic { - GetHistory(sctx).Add(0, s, se.sessionVars.StmtCtx) + // All the history should be added here. + if err == nil && sessVars.TxnCtx.CouldRetry { + GetHistory(sctx).Add(s, sessVars.StmtCtx) } + + // Handle the stmt commit/rollback. if txn, err1 := sctx.Txn(false); err1 == nil { if txn.Valid() { if err != nil { @@ -240,8 +243,8 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement) logutil.BgLogger().Error("get txn error", zap.Error(err1)) } } - err = finishStmt(ctx, sctx, se, sessVars, err) + if se.txn.pending() { // After run statement finish, txn state is still pending means the // statement never need a Txn(), such as: diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 8bc0914a5dc8a..940c20a6e7863 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -104,6 +104,7 @@ type TransactionContext struct { DirtyDB interface{} Binlog interface{} InfoSchema interface{} + CouldRetry bool History interface{} SchemaVersion int64 StartTS uint64 @@ -135,7 +136,7 @@ func (tc *TransactionContext) UpdateDeltaForTable(tableID int64, delta int64, co // Cleanup clears up transaction info that no longer use. func (tc *TransactionContext) Cleanup() { - //tc.InfoSchema = nil; we cannot do it now, because some operation like handleFieldList depend on this. + // tc.InfoSchema = nil; we cannot do it now, because some operation like handleFieldList depend on this. tc.DirtyDB = nil tc.Binlog = nil tc.History = nil From 6cd002761b581fa08654385c892b2bc6d180ab24 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 12 Jul 2019 13:08:43 +0800 Subject: [PATCH 015/196] planner: some cleanup (#11164) --- planner/core/expression_rewriter.go | 8 ++--- planner/core/indexmerge_test.go | 7 +--- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plan_test.go | 43 ++++-------------------- planner/core/optimizer.go | 6 +--- planner/core/planbuilder_test.go | 6 ++-- planner/core/rule_partition_processor.go | 18 +++++----- 7 files changed, 23 insertions(+), 67 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index d6b6a680498cf..a241b1133a27a 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -42,13 +42,11 @@ func evalAstExpr(ctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) if val, ok := expr.(*driver.ValueExpr); ok { return val.Datum, nil } - b := &PlanBuilder{ - ctx: ctx, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + var is infoschema.InfoSchema if ctx.GetSessionVars().TxnCtx.InfoSchema != nil { - b.is = ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + is = ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) } + b := NewPlanBuilder(ctx, is) fakePlan := LogicalTableDual{}.Init(ctx) newExpr, _, err := b.rewrite(expr, fakePlan, nil, true) if err != nil { diff --git a/planner/core/indexmerge_test.go b/planner/core/indexmerge_test.go index 2ff5f09c91ef0..07ed20fb2ac69 100644 --- a/planner/core/indexmerge_test.go +++ b/planner/core/indexmerge_test.go @@ -16,7 +16,6 @@ package core import ( . "github.com/pingcap/check" "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/sessionctx" @@ -108,11 +107,7 @@ func (s *testIndexMergeSuite) TestIndexMergePathGenerateion(c *C) { stmt, err := s.ParseOneStmt(tc.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) if err != nil { c.Assert(err.Error(), Equals, tc.idxMergeDigest, comment) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index fd502d0b5ad20..d22ee24afa1c0 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2293,7 +2293,7 @@ func (b *PlanBuilder) buildDataSource(tn *ast.TableName) (LogicalPlan, error) { if err != nil { return nil, err } - if txn.Valid() && !txn.IsReadOnly() { + if txn.Valid() && !txn.IsReadOnly() && !isMemDB { us := LogicalUnionScan{}.Init(b.ctx) us.SetChildren(ds) result = us diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 25d94c1cf516b..5c1586e2548e9 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -21,7 +21,6 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser" - "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -1714,11 +1713,7 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - colMapper: make(map[*ast.ColumnNameExpr]int), - ctx: MockContext(), - is: s.is, - } + builder := NewPlanBuilder(MockContext(), s.is) builder.ctx.GetSessionVars().HashJoinConcurrency = 1 _, err = builder.Build(stmt) c.Assert(err, IsNil, comment) @@ -1832,11 +1827,7 @@ func (s *testPlanSuite) TestUnion(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) plan, err := builder.Build(stmt) if tt.err { c.Assert(err, NotNil) @@ -1964,11 +1955,7 @@ func (s *testPlanSuite) TestTopNPushDown(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) c.Assert(err, IsNil) p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) @@ -2075,11 +2062,7 @@ func (s *testPlanSuite) TestOuterJoinEliminator(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) c.Assert(err, IsNil) p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) @@ -2106,11 +2089,7 @@ func (s *testPlanSuite) TestSelectView(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) c.Assert(err, IsNil) p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) @@ -2333,11 +2312,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) if err != nil { c.Assert(err.Error(), Equals, tt.result, comment) @@ -2418,11 +2393,7 @@ func (s *testPlanSuite) TestSkylinePruning(c *C) { stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - builder := &PlanBuilder{ - ctx: MockContext(), - is: s.is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(MockContext(), s.is) p, err := builder.Build(stmt) if err != nil { c.Assert(err.Error(), Equals, tt.result, comment) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index d41dadde11710..ab7d09117fe19 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -74,11 +74,7 @@ type logicalOptRule interface { func BuildLogicalPlan(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (Plan, error) { ctx.GetSessionVars().PlanID = 0 ctx.GetSessionVars().PlanColumnID = 0 - builder := &PlanBuilder{ - ctx: ctx, - is: is, - colMapper: make(map[*ast.ColumnNameExpr]int), - } + builder := NewPlanBuilder(ctx, is) p, err := builder.Build(node) if err != nil { return nil, err diff --git a/planner/core/planbuilder_test.go b/planner/core/planbuilder_test.go index 3c1487d1a3637..282e95a2cd9ff 100644 --- a/planner/core/planbuilder_test.go +++ b/planner/core/planbuilder_test.go @@ -94,9 +94,7 @@ func (s *testPlanBuilderSuite) TestGetPathByIndexName(c *C) { } func (s *testPlanBuilderSuite) TestRewriterPool(c *C) { - builder := &PlanBuilder{ - ctx: MockContext(), - } + builder := NewPlanBuilder(MockContext(), nil) // Make sure PlanBuilder.getExpressionRewriter() provides clean rewriter from pool. // First, pick one rewriter from the pool and make it dirty. @@ -149,7 +147,7 @@ func (s *testPlanBuilderSuite) TestDisableFold(c *C) { stmt := st.(*ast.SelectStmt) expr := stmt.Fields.Fields[0].Expr - builder := &PlanBuilder{ctx: ctx} + builder := NewPlanBuilder(ctx, nil) builder.rewriterCounter++ rewriter := builder.getExpressionRewriter(nil) c.Assert(rewriter, NotNil) diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 4b04412e5c02a..f2c3ebc59db63 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -45,12 +45,11 @@ func (s *partitionProcessor) optimize(lp LogicalPlan) (LogicalPlan, error) { func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, error) { // Assert there will not be sel -> sel in the ast. - switch lp.(type) { + switch p := lp.(type) { case *DataSource: - return s.prune(lp.(*DataSource)) + return s.prune(p) case *LogicalUnionScan: - us := lp.(*LogicalUnionScan) - ds := us.Children()[0] + ds := p.Children()[0] ds, err := s.prune(ds.(*DataSource)) if err != nil { return nil, err @@ -60,17 +59,16 @@ func (s *partitionProcessor) rewriteDataSource(lp LogicalPlan) (LogicalPlan, err // Union->(UnionScan->DataSource1), (UnionScan->DataSource2) children := make([]LogicalPlan, 0, len(ua.Children())) for _, child := range ua.Children() { - usChild := LogicalUnionScan{}.Init(ua.ctx) - usChild.conditions = us.conditions - usChild.SetChildren(child) - children = append(children, usChild) + us := LogicalUnionScan{conditions: p.conditions}.Init(ua.ctx) + us.SetChildren(child) + children = append(children, us) } ua.SetChildren(children...) return ua, nil } // Only one partition, no union all. - us.SetChildren(ds) - return us, nil + p.SetChildren(ds) + return p, nil default: children := lp.Children() for i, child := range children { From 84432823b3d3a79b6bffc493e300bf87d9cf3afb Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Fri, 12 Jul 2019 13:36:58 +0800 Subject: [PATCH 016/196] refactor the logic of load data batch insert, make batchCheckGet happen once per transaction (#11132) --- executor/executor_test.go | 2 ++ executor/load_data.go | 23 +++++++++++++++++------ executor/write_test.go | 4 ++++ server/conn.go | 9 ++++++++- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index b32b405277d0e..1aab28019c041 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -443,6 +443,8 @@ func checkCases(tests []testCase, ld *executor.LoadDataInfo, data, reachLimit, err1 := ld.InsertData(context.Background(), tt.data1, tt.data2) c.Assert(err1, IsNil) c.Assert(reachLimit, IsFalse) + err1 = ld.CheckAndInsertOneBatch() + c.Assert(err1, IsNil) if tt.restData == nil { c.Assert(data, HasLen, 0, Commentf("data1:%v, data2:%v, data:%v", string(tt.data1), string(tt.data2), string(data))) diff --git a/executor/load_data.go b/executor/load_data.go index df49b777cfc34..98ef7f3e62d59 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -106,11 +106,13 @@ type LoadDataInfo struct { LinesInfo *ast.LinesClause IgnoreLines uint64 Ctx sessionctx.Context + rows [][]types.Datum } // SetMaxRowsInBatch sets the max number of rows to insert in a batch. func (e *LoadDataInfo) SetMaxRowsInBatch(limit uint64) { e.maxRowsInBatch = limit + e.rows = make([][]types.Datum, 0, limit) } // getValidData returns prevData and curData that starts from starting symbol. @@ -223,7 +225,6 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) isEOF = true prevData, curData = curData, prevData } - rows := make([][]types.Datum, 0, e.maxRowsInBatch) for len(curData) > 0 { line, curData, hasStarting = e.getLine(prevData, curData) prevData = nil @@ -252,7 +253,7 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) if err != nil { return nil, false, err } - rows = append(rows, e.colsToRow(ctx, cols)) + e.rows = append(e.rows, e.colsToRow(ctx, cols)) e.rowCount++ if e.maxRowsInBatch != 0 && e.rowCount%e.maxRowsInBatch == 0 { reachLimit = true @@ -261,12 +262,22 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) break } } - e.ctx.GetSessionVars().StmtCtx.AddRecordRows(uint64(len(rows))) - err := e.batchCheckAndInsert(rows, e.addRecordLD) + return curData, reachLimit, nil +} + +// CheckAndInsertOneBatch is used to commit one transaction batch full filled data +func (e *LoadDataInfo) CheckAndInsertOneBatch() error { + var err error + if len(e.rows) == 0 { + return err + } + e.ctx.GetSessionVars().StmtCtx.AddRecordRows(uint64(len(e.rows))) + err = e.batchCheckAndInsert(e.rows, e.addRecordLD) if err != nil { - return nil, reachLimit, err + return err } - return curData, reachLimit, nil + e.rows = e.rows[:0] + return err } // SetMessage sets info message(ERR_LOAD_INFO) generated by LOAD statement, it is public because of the special way that diff --git a/executor/write_test.go b/executor/write_test.go index a98f7729451b0..dfa1d936bbe86 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -1807,6 +1807,8 @@ func (s *testSuite4) TestLoadData(c *C) { _, reachLimit, err := ld.InsertData(context.Background(), nil, nil) c.Assert(err, IsNil) c.Assert(reachLimit, IsFalse) + err = ld.CheckAndInsertOneBatch() + c.Assert(err, IsNil) r := tk.MustQuery(selectSQL) r.Check(nil) @@ -2056,6 +2058,8 @@ func (s *testSuite4) TestLoadDataIntoPartitionedTable(c *C) { _, _, err := ld.InsertData(context.Background(), nil, []byte("1,2\n3,4\n5,6\n7,8\n9,10\n")) c.Assert(err, IsNil) + err = ld.CheckAndInsertOneBatch() + c.Assert(err, IsNil) ld.SetMessage() err = ctx.StmtCommit() c.Assert(err, IsNil) diff --git a/server/conn.go b/server/conn.go index c80116e01584c..391cd779f4da7 100644 --- a/server/conn.go +++ b/server/conn.go @@ -1049,6 +1049,10 @@ func insertDataWithCommit(ctx context.Context, prevData, curData []byte, loadDat if !reachLimit { break } + err := loadDataInfo.CheckAndInsertOneBatch() + if err != nil { + return nil, err + } if err = loadDataInfo.Ctx.StmtCommit(); err != nil { return nil, err } @@ -1113,7 +1117,10 @@ func (cc *clientConn) handleLoadData(ctx context.Context, loadDataInfo *executor if err != nil { loadDataInfo.Ctx.StmtRollback() } else { - err = loadDataInfo.Ctx.StmtCommit() + err = loadDataInfo.CheckAndInsertOneBatch() + if err == nil { + err = loadDataInfo.Ctx.StmtCommit() + } } var txn kv.Transaction From 0ba4d0b41b0234a8bd1d70282dfd1bb28b9f7a30 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Fri, 12 Jul 2019 14:11:14 +0800 Subject: [PATCH 017/196] executor: fix bug of point get when meet null values (#11219) --- executor/point_get.go | 51 +++++++++++++------------------------- executor/point_get_test.go | 2 ++ 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/executor/point_get.go b/executor/point_get.go index 955d68ede1032..6e5f78d97e226 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -196,60 +196,43 @@ func (e *PointGetExecutor) get(key kv.Key) (val []byte, err error) { } func (e *PointGetExecutor) decodeRowValToChunk(rowVal []byte, chk *chunk.Chunk) error { - // One column could be filled for multi-times in the schema. e.g. select b, b, c, c from t where a = 1. - // We need to set the positions in the schema for the same column. - colID2DecodedPos := make(map[int64]int, e.schema.Len()) - decodedPos2SchemaPos := make([][]int, 0, e.schema.Len()) - for schemaPos, col := range e.schema.Columns { - if decodedPos, ok := colID2DecodedPos[col.ID]; !ok { - colID2DecodedPos[col.ID] = len(colID2DecodedPos) - decodedPos2SchemaPos = append(decodedPos2SchemaPos, []int{schemaPos}) - } else { - decodedPos2SchemaPos[decodedPos] = append(decodedPos2SchemaPos[decodedPos], schemaPos) + colID2CutPos := make(map[int64]int, e.schema.Len()) + for _, col := range e.schema.Columns { + if _, ok := colID2CutPos[col.ID]; !ok { + colID2CutPos[col.ID] = len(colID2CutPos) } } - decodedVals, err := tablecodec.CutRowNew(rowVal, colID2DecodedPos) + cutVals, err := tablecodec.CutRowNew(rowVal, colID2CutPos) if err != nil { return err } - if decodedVals == nil { - decodedVals = make([][]byte, len(colID2DecodedPos)) + if cutVals == nil { + cutVals = make([][]byte, len(colID2CutPos)) } decoder := codec.NewDecoder(chk, e.ctx.GetSessionVars().Location()) - for id, decodedPos := range colID2DecodedPos { - schemaPoses := decodedPos2SchemaPos[decodedPos] - firstPos := schemaPoses[0] - if e.tblInfo.PKIsHandle && mysql.HasPriKeyFlag(e.schema.Columns[firstPos].RetType.Flag) { - chk.AppendInt64(firstPos, e.handle) - // Fill other positions. - for i := 1; i < len(schemaPoses); i++ { - chk.MakeRef(firstPos, schemaPoses[i]) - } + for i, col := range e.schema.Columns { + if e.tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.RetType.Flag) { + chk.AppendInt64(i, e.handle) continue } - // ExtraHandleID is added when building plan, we can make sure that there's only one column's ID is this. - if id == model.ExtraHandleID { - chk.AppendInt64(firstPos, e.handle) + if col.ID == model.ExtraHandleID { + chk.AppendInt64(i, e.handle) continue } - if len(decodedVals[decodedPos]) == 0 { - // This branch only entered for updating and deleting. It won't have one column in multiple positions. - colInfo := getColInfoByID(e.tblInfo, id) + cutPos := colID2CutPos[col.ID] + if len(cutVals[cutPos]) == 0 { + colInfo := getColInfoByID(e.tblInfo, col.ID) d, err1 := table.GetColOriginDefaultValue(e.ctx, colInfo) if err1 != nil { return err1 } - chk.AppendDatum(firstPos, &d) + chk.AppendDatum(i, &d) continue } - _, err = decoder.DecodeOne(decodedVals[decodedPos], firstPos, e.schema.Columns[firstPos].RetType) + _, err = decoder.DecodeOne(cutVals[cutPos], i, col.RetType) if err != nil { return err } - // Fill other positions. - for i := 1; i < len(schemaPoses); i++ { - chk.MakeRef(firstPos, schemaPoses[i]) - } } return nil } diff --git a/executor/point_get_test.go b/executor/point_get_test.go index 048bddb6bc7f9..b14a7de0cadaf 100644 --- a/executor/point_get_test.go +++ b/executor/point_get_test.go @@ -101,6 +101,8 @@ func (s *testPointGetSuite) TestPointGet(c *C) { tk.MustQuery(`select a, a, b, a, b, c, b, c, c from t where a = 5;`).Check(testkit.Rows( `5 5 6 5 6 7 6 7 7`, )) + tk.MustQuery(`select b, b from t where a = 1`).Check(testkit.Rows( + " ")) } func (s *testPointGetSuite) TestPointGetCharPK(c *C) { From e0fc847c8bc6c5fe04fcce98f3e11ecc59e1fe42 Mon Sep 17 00:00:00 2001 From: Lingyu Song Date: Fri, 12 Jul 2019 14:56:55 +0800 Subject: [PATCH 018/196] executor: handle missing timestamp value for load data (#11093) --- executor/load_data.go | 6 ++++++ executor/write_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/executor/load_data.go b/executor/load_data.go index 98ef7f3e62d59..16d55d5f36114 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -293,8 +293,14 @@ func (e *LoadDataInfo) SetMessage() { } func (e *LoadDataInfo) colsToRow(ctx context.Context, cols []field) []types.Datum { + totalCols := e.Table.Cols() for i := 0; i < len(e.row); i++ { if i >= len(cols) { + // If some columns is missing and their type is time and has not null flag, they should be set as current time. + if types.IsTypeTime(totalCols[i].Tp) && mysql.HasNotNullFlag(totalCols[i].Flag) { + e.row[i].SetMysqlTime(types.CurrentTime(totalCols[i].Tp)) + continue + } e.row[i].SetNull() continue } diff --git a/executor/write_test.go b/executor/write_test.go index dfa1d936bbe86..69874534a57bd 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -21,6 +21,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/core" @@ -1779,6 +1780,43 @@ func (s *testSuite4) TestQualifiedDelete(c *C) { c.Assert(err, NotNil) } +func (s *testSuite4) TestLoadDataMissingColumn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + createSQL := `create table load_data_missing (id int, t timestamp not null)` + tk.MustExec(createSQL) + tk.MustExec("load data local infile '/tmp/nonexistence.csv' ignore into table load_data_missing") + ctx := tk.Se.(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataInfo) + c.Assert(ok, IsTrue) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + c.Assert(ld, NotNil) + + deleteSQL := "delete from load_data_missing" + selectSQL := "select * from load_data_missing;" + _, reachLimit, err := ld.InsertData(context.Background(), nil, nil) + c.Assert(err, IsNil) + c.Assert(reachLimit, IsFalse) + r := tk.MustQuery(selectSQL) + r.Check(nil) + + curTime := types.CurrentTime(mysql.TypeTimestamp) + timeStr := curTime.String() + tests := []testCase{ + {nil, []byte("12\n"), []string{fmt.Sprintf("12|%v", timeStr)}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + checkCases(tests, ld, c, tk, ctx, selectSQL, deleteSQL) + + tk.MustExec("alter table load_data_missing add column t2 timestamp null") + curTime = types.CurrentTime(mysql.TypeTimestamp) + timeStr = curTime.String() + tests = []testCase{ + {nil, []byte("12\n"), []string{fmt.Sprintf("12|%v|", timeStr)}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + checkCases(tests, ld, c, tk, ctx, selectSQL, deleteSQL) + +} + func (s *testSuite4) TestLoadData(c *C) { trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" tk := testkit.NewTestKit(c, s.store) From d98fb747c01ce36fe732faaa5f64b797a7ba9f20 Mon Sep 17 00:00:00 2001 From: baishen Date: Fri, 12 Jul 2019 15:10:31 +0800 Subject: [PATCH 019/196] types: fix incorrect `weekday` for `ALLOW_INVALID_DATES` mode (#10864) --- types/format_test.go | 2 +- types/mytime.go | 3 ++- types/mytime_test.go | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/types/format_test.go b/types/format_test.go index 493ac31e75ccf..d4434dc301a8c 100644 --- a/types/format_test.go +++ b/types/format_test.go @@ -62,7 +62,7 @@ func (s *testTimeSuite) TestTimeFormatMethod(c *C) { // %W %w %a is not compatible in this case because Week() use GoTime() currently. "0000-01-00 00:00:00.123456", `%b %M %m %c %D %d %e %j %k %h %i %p %r %T %s %f %U %u %V %v %a %W %w %X %x %Y %y %%`, - `Jan January 01 1 0th 00 0 000 0 12 00 AM 12:00:00 AM 00:00:00 00 123456 00 00 00 52 Sun Sunday 0 4294967295 4294967295 0000 00 %`, + `Jan January 01 1 0th 00 0 000 0 12 00 AM 12:00:00 AM 00:00:00 00 123456 00 00 00 52 Fri Friday 5 4294967295 4294967295 0000 00 %`, }, } for i, t := range tblDate { diff --git a/types/mytime.go b/types/mytime.go index 5ee12ec20aa78..733571f0239a1 100644 --- a/types/mytime.go +++ b/types/mytime.go @@ -70,8 +70,9 @@ func (t MysqlTime) Microsecond() int { func (t MysqlTime) Weekday() gotime.Weekday { // TODO: Consider time_zone variable. t1, err := t.GoTime(gotime.Local) + // allow invalid dates if err != nil { - return 0 + return t1.Weekday() } return t1.Weekday() } diff --git a/types/mytime_test.go b/types/mytime_test.go index f2897c878908b..76a32f4b40a94 100644 --- a/types/mytime_test.go +++ b/types/mytime_test.go @@ -271,3 +271,19 @@ func (s *testMyTimeSuite) TestAddDate(c *C) { c.Assert(res.Year(), Equals, t.year+t.ot.Year()) } } + +func (s *testMyTimeSuite) TestWeekday(c *C) { + tests := []struct { + Input MysqlTime + Expect string + }{ + {MysqlTime{2019, 01, 01, 0, 0, 0, 0}, "Tuesday"}, + {MysqlTime{2019, 02, 31, 0, 0, 0, 0}, "Sunday"}, + {MysqlTime{2019, 04, 31, 0, 0, 0, 0}, "Wednesday"}, + } + + for _, tt := range tests { + weekday := tt.Input.Weekday() + c.Check(weekday.String(), Equals, tt.Expect) + } +} From ad404bb645f9d42b04358e92aef3d0c089592131 Mon Sep 17 00:00:00 2001 From: tp <544016459@qq.com> Date: Fri, 12 Jul 2019 02:19:57 -0500 Subject: [PATCH 020/196] *: avoid `tidb-server -h` output test flags (#11209) --- Makefile | 4 ++-- server/sql_info_fetcher.go | 7 +++---- session/tidb.go | 30 ++++++++++++++++++++++++++++++ util/testkit/ctestkit.go | 2 ++ util/testkit/fake.go | 3 +++ util/testkit/testkit.go | 34 +++------------------------------- util/testutil/testutil.go | 2 ++ 7 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 util/testkit/fake.go diff --git a/Makefile b/Makefile index 2af63a547d3b9..a4d0195433e3c 100644 --- a/Makefile +++ b/Makefile @@ -12,14 +12,14 @@ path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH))):$(PWD)/tools/bin export PATH := $(path_to_add):$(PATH) GO := GO111MODULE=on go -GOBUILD := CGO_ENABLED=1 $(GO) build $(BUILD_FLAG) +GOBUILD := CGO_ENABLED=1 $(GO) build $(BUILD_FLAG) -tags codes GOTEST := CGO_ENABLED=1 $(GO) test -p 4 OVERALLS := CGO_ENABLED=1 GO111MODULE=on overalls ARCH := "`uname -s`" LINUX := "Linux" MAC := "Darwin" -PACKAGE_LIST := go list ./...| grep -vE "cmd" +PACKAGE_LIST := go list ./...| grep -vE "cmd" | grep -vE "test" PACKAGES := $$($(PACKAGE_LIST)) PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)/||' FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go") diff --git a/server/sql_info_fetcher.go b/server/sql_info_fetcher.go index 8f07a340c13d6..204233d8acd42 100644 --- a/server/sql_info_fetcher.go +++ b/server/sql_info_fetcher.go @@ -34,7 +34,6 @@ import ( "github.com/pingcap/tidb/statistics/handle" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/util/sqlexec" - "github.com/pingcap/tidb/util/testkit" ) type sqlInfoFetcher struct { @@ -171,7 +170,7 @@ func (sh *sqlInfoFetcher) zipInfoForSQL(w http.ResponseWriter, r *http.Request) terror.Log(err) return } - sRows, err := testkit.ResultSetToStringSlice(reqCtx, sh.s, recordSets[0]) + sRows, err := session.ResultSetToStringSlice(reqCtx, sh.s, recordSets[0]) if err != nil { err = sh.writeErrFile(zw, "explain.err.txt", err) terror.Log(err) @@ -248,7 +247,7 @@ func (sh *sqlInfoFetcher) getExplainAnalyze(ctx context.Context, sql string, res resultChan <- &explainAnalyzeResult{err: err} return } - rows, err := testkit.ResultSetToStringSlice(ctx, sh.s, recordSets[0]) + rows, err := session.ResultSetToStringSlice(ctx, sh.s, recordSets[0]) if err != nil { terror.Log(err) rows = nil @@ -292,7 +291,7 @@ func (sh *sqlInfoFetcher) getShowCreateTable(pair tableNamePair, zw *zip.Writer) if err != nil { return err } - sRows, err := testkit.ResultSetToStringSlice(context.Background(), sh.s, recordSets[0]) + sRows, err := session.ResultSetToStringSlice(context.Background(), sh.s, recordSets[0]) if err != nil { terror.Log(err) return nil diff --git a/session/tidb.go b/session/tidb.go index 03009b5b96b53..70c3f9a9665c8 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -297,6 +297,36 @@ func GetRows4Test(ctx context.Context, sctx sessionctx.Context, rs sqlexec.Recor return rows, nil } +// ResultSetToStringSlice changes the RecordSet to [][]string. +func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet) ([][]string, error) { + rows, err := GetRows4Test(ctx, s, rs) + if err != nil { + return nil, err + } + err = rs.Close() + if err != nil { + return nil, err + } + sRows := make([][]string, len(rows)) + for i := range rows { + row := rows[i] + iRow := make([]string, row.Len()) + for j := 0; j < row.Len(); j++ { + if row.IsNull(j) { + iRow[j] = "" + } else { + d := row.GetDatum(j, &rs.Fields()[j].Column.FieldType) + iRow[j], err = d.ToString() + if err != nil { + return nil, err + } + } + } + sRows[i] = iRow + } + return sRows, nil +} + var ( errForUpdateCantRetry = terror.ClassSession.New(codeForUpdateCantRetry, mysql.MySQLErrName[mysql.ErrForUpdateCantRetry]) diff --git a/util/testkit/ctestkit.go b/util/testkit/ctestkit.go index a7c23070baabe..d32091aff44ef 100644 --- a/util/testkit/ctestkit.go +++ b/util/testkit/ctestkit.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !codes + package testkit import ( diff --git a/util/testkit/fake.go b/util/testkit/fake.go new file mode 100644 index 0000000000000..20321b82ae7d9 --- /dev/null +++ b/util/testkit/fake.go @@ -0,0 +1,3 @@ +// +build codes + +package testkit diff --git a/util/testkit/testkit.go b/util/testkit/testkit.go index 6d936246707e3..947afe70961de 100644 --- a/util/testkit/testkit.go +++ b/util/testkit/testkit.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !codes + package testkit import ( @@ -241,39 +243,9 @@ func (tk *TestKit) ResultSetToResult(rs sqlexec.RecordSet, comment check.Comment return tk.ResultSetToResultWithCtx(context.Background(), rs, comment) } -// ResultSetToStringSlice changes the RecordSet to [][]string. -func ResultSetToStringSlice(ctx context.Context, s session.Session, rs sqlexec.RecordSet) ([][]string, error) { - rows, err := session.GetRows4Test(ctx, s, rs) - if err != nil { - return nil, err - } - err = rs.Close() - if err != nil { - return nil, err - } - sRows := make([][]string, len(rows)) - for i := range rows { - row := rows[i] - iRow := make([]string, row.Len()) - for j := 0; j < row.Len(); j++ { - if row.IsNull(j) { - iRow[j] = "" - } else { - d := row.GetDatum(j, &rs.Fields()[j].Column.FieldType) - iRow[j], err = d.ToString() - if err != nil { - return nil, err - } - } - } - sRows[i] = iRow - } - return sRows, nil -} - // ResultSetToResultWithCtx converts sqlexec.RecordSet to testkit.Result. func (tk *TestKit) ResultSetToResultWithCtx(ctx context.Context, rs sqlexec.RecordSet, comment check.CommentInterface) *Result { - sRows, err := ResultSetToStringSlice(ctx, tk.Se, rs) + sRows, err := session.ResultSetToStringSlice(ctx, tk.Se, rs) tk.c.Check(err, check.IsNil, comment) return &Result{rows: sRows, c: tk.c, comment: comment} } diff --git a/util/testutil/testutil.go b/util/testutil/testutil.go index 003756b0fd405..e93c1f71cd949 100644 --- a/util/testutil/testutil.go +++ b/util/testutil/testutil.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !codes + package testutil import ( From f496b775fd6dc6b25c24ed7b7d955c1bcf923235 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 12 Jul 2019 16:25:01 +0800 Subject: [PATCH 021/196] planner: unsigned pk cannot be pushed as index column (#11094) --- executor/executor_test.go | 5 +++ planner/cascades/optimize_test.go | 2 +- planner/core/indexmerge_test.go | 2 +- planner/core/logical_plan_test.go | 2 +- planner/core/logical_plans.go | 2 +- planner/core/mock.go | 51 +++++++++++++++++++++++++++-- planner/core/physical_plan_test.go | 10 +++++- planner/core/preprocess_test.go | 2 +- planner/implementation/base_test.go | 2 +- planner/memo/group_test.go | 2 +- 10 files changed, 69 insertions(+), 11 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 1aab28019c041..e860a6e200d52 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3046,6 +3046,11 @@ func (s *testSuite) TestUnsignedPk(c *C) { num2Str := strconv.FormatUint(num2, 10) tk.MustQuery("select * from t order by id").Check(testkit.Rows("1", "2", num1Str, num2Str)) tk.MustQuery("select * from t where id not in (2)").Check(testkit.Rows(num1Str, num2Str, "1")) + tk.MustExec("drop table t") + tk.MustExec("create table t(a bigint unsigned primary key, b int, index idx(b))") + tk.MustExec("insert into t values(9223372036854775808, 1), (1, 1)") + tk.MustQuery("select * from t use index(idx) where b = 1 and a < 2").Check(testkit.Rows("1 1")) + tk.MustQuery("select * from t use index(idx) where b = 1 order by b, a").Check(testkit.Rows("1 1", "9223372036854775808 1")) } func (s *testSuite) TestIssue5666(c *C) { diff --git a/planner/cascades/optimize_test.go b/planner/cascades/optimize_test.go index e0f9660c78238..4841b32c271fc 100644 --- a/planner/cascades/optimize_test.go +++ b/planner/cascades/optimize_test.go @@ -42,7 +42,7 @@ type testCascadesSuite struct { func (s *testCascadesSuite) SetUpSuite(c *C) { testleak.BeforeTest() - s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockSignedTable()}) s.sctx = plannercore.MockContext() s.Parser = parser.New() } diff --git a/planner/core/indexmerge_test.go b/planner/core/indexmerge_test.go index 07ed20fb2ac69..a3600667d2458 100644 --- a/planner/core/indexmerge_test.go +++ b/planner/core/indexmerge_test.go @@ -32,7 +32,7 @@ type testIndexMergeSuite struct { } func (s *testIndexMergeSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockTable(), MockView()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockView()}) s.ctx = MockContext() s.Parser = parser.New() } diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 5c1586e2548e9..450db19a241de 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -46,7 +46,7 @@ type testPlanSuite struct { } func (s *testPlanSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockTable(), MockView()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{MockSignedTable(), MockUnsignedTable(), MockView()}) s.ctx = MockContext() s.Parser = parser.New() } diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index cd8a7e611c1c4..25094e5d0028b 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -494,7 +494,7 @@ func (ds *DataSource) deriveIndexPathStats(path *accessPath, conds []expression. path.idxCols, path.idxColLens = expression.IndexInfo2Cols(ds.schema.Columns, path.index) if !path.index.Unique && !path.index.Primary && len(path.index.Columns) == len(path.idxCols) { handleCol := ds.getHandleCol() - if handleCol != nil { + if handleCol != nil && !mysql.HasUnsignedFlag(handleCol.RetType.Flag) { path.idxCols = append(path.idxCols, handleCol) path.idxColLens = append(path.idxColLens, types.UnspecifiedLength) } diff --git a/planner/core/mock.go b/planner/core/mock.go index e91146078a389..51f94de536cbb 100644 --- a/planner/core/mock.go +++ b/planner/core/mock.go @@ -39,8 +39,8 @@ func newDateType() types.FieldType { return *ft } -// MockTable is only used for plan related tests. -func MockTable() *model.TableInfo { +// MockSignedTable is only used for plan related tests. +func MockSignedTable() *model.TableInfo { // column: a, b, c, d, e, c_str, d_str, e_str, f, g // PK: a // indeices: c_d_e, e, f, g, f_g, c_d_e_str, c_d_e_str_prefix @@ -263,6 +263,51 @@ func MockTable() *model.TableInfo { return table } +// MockUnsignedTable is only used for plan related tests. +func MockUnsignedTable() *model.TableInfo { + // column: a, b + // PK: a + // indeices: b + indices := []*model.IndexInfo{ + { + Name: model.NewCIStr("b"), + Columns: []*model.IndexColumn{ + { + Name: model.NewCIStr("b"), + Length: types.UnspecifiedLength, + Offset: 4, + }, + }, + State: model.StatePublic, + Unique: true, + }, + } + pkColumn := &model.ColumnInfo{ + State: model.StatePublic, + Offset: 0, + Name: model.NewCIStr("a"), + FieldType: newLongType(), + ID: 1, + } + col0 := &model.ColumnInfo{ + State: model.StatePublic, + Offset: 1, + Name: model.NewCIStr("b"), + FieldType: newLongType(), + ID: 2, + } + pkColumn.Flag = mysql.PriKeyFlag | mysql.NotNullFlag | mysql.UnsignedFlag + // Column 'b', 'c', 'd', 'f', 'g' is not null. + col0.Flag = mysql.NotNullFlag + table := &model.TableInfo{ + Columns: []*model.ColumnInfo{pkColumn, col0}, + Indices: indices, + Name: model.NewCIStr("t2"), + PKIsHandle: true, + } + return table +} + // MockView is only used for plan related tests. func MockView() *model.TableInfo { selectStmt := "select b,c,d from t" @@ -308,7 +353,7 @@ func MockContext() sessionctx.Context { // MockPartitionInfoSchema mocks an info schema for partition table. func MockPartitionInfoSchema(definitions []model.PartitionDefinition) infoschema.InfoSchema { - tableInfo := MockTable() + tableInfo := MockSignedTable() cols := make([]*model.ColumnInfo, 0, len(tableInfo.Columns)) cols = append(cols, tableInfo.Columns...) last := tableInfo.Columns[len(tableInfo.Columns)-1] diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 6ef50a36b16cd..12354f17d9eba 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -39,7 +39,7 @@ type testPlanSuite struct { } func (s *testPlanSuite) SetUpSuite(c *C) { - s.is = infoschema.MockInfoSchema([]*model.TableInfo{core.MockTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable()}) s.Parser = parser.New() s.Parser.EnableWindowFunc(true) } @@ -210,6 +210,14 @@ func (s *testPlanSuite) TestDAGPlanBuilderSimpleCase(c *C) { sql: "select lead(a, 1) over (partition by null) as c from t", best: "TableReader(Table(t))->Window(lead(test.t.a, 1) over())->Projection", }, + { + sql: "select * from t use index(f) where f = 1 and a = 1", + best: "IndexLookUp(Index(t.f)[[1,1]]->Sel([eq(test.t.a, 1)]), Table(t))", + }, + { + sql: "select * from t2 use index(b) where b = 1 and a = 1", + best: "IndexReader(Index(t2.b)[[1,1]]->Sel([eq(test.t2.a, 1)]))", + }, } for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index 31fb17cf7f38f..fc652943c3821 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -219,7 +219,7 @@ func (s *testValidatorSuite) TestValidator(c *C) { _, err = se.Execute(context.Background(), "use test") c.Assert(err, IsNil) ctx := se.(sessionctx.Context) - is := infoschema.MockInfoSchema([]*model.TableInfo{core.MockTable()}) + is := infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable()}) for _, tt := range tests { stmts, err1 := session.Parse(ctx, tt.sql) c.Assert(err1, IsNil) diff --git a/planner/implementation/base_test.go b/planner/implementation/base_test.go index a27b92df96df8..589d8e65207a3 100644 --- a/planner/implementation/base_test.go +++ b/planner/implementation/base_test.go @@ -40,7 +40,7 @@ type testImplSuite struct { func (s *testImplSuite) SetUpSuite(c *C) { testleak.BeforeTest() - s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockSignedTable()}) s.sctx = plannercore.MockContext() s.Parser = parser.New() } diff --git a/planner/memo/group_test.go b/planner/memo/group_test.go index 46fe680be1d48..51ccf1f029096 100644 --- a/planner/memo/group_test.go +++ b/planner/memo/group_test.go @@ -42,7 +42,7 @@ type testMemoSuite struct { func (s *testMemoSuite) SetUpSuite(c *C) { testleak.BeforeTest() - s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockTable()}) + s.is = infoschema.MockInfoSchema([]*model.TableInfo{plannercore.MockSignedTable()}) s.sctx = plannercore.MockContext() s.Parser = parser.New() } From 8d2c9be4618bff4ce5a4d2901ea1daebf2c1c46c Mon Sep 17 00:00:00 2001 From: zhaoyanxing Date: Fri, 12 Jul 2019 16:33:01 +0800 Subject: [PATCH 022/196] CONVERT_TZ should return NULL if its arguments are invalid. (#11176) --- expression/builtin_time.go | 19 ++++++++++--------- expression/builtin_time_test.go | 19 ++++++++++++++----- expression/integration_test.go | 13 ++++++++++--- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index c41d232914262..75cfa8acacad1 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -5088,21 +5088,22 @@ func (b *builtinConvertTzSig) Clone() builtinFunc { } // evalTime evals CONVERT_TZ(dt,from_tz,to_tz). +// `CONVERT_TZ` function returns NULL if the arguments are invalid. // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_convert-tz func (b *builtinConvertTzSig) evalTime(row chunk.Row) (types.Time, bool, error) { dt, isNull, err := b.args[0].EvalTime(b.ctx, row) if isNull || err != nil { - return types.Time{}, true, err + return types.Time{}, true, nil } fromTzStr, isNull, err := b.args[1].EvalString(b.ctx, row) - if isNull || err != nil { - return types.Time{}, true, err + if isNull || err != nil || fromTzStr == "" { + return types.Time{}, true, nil } toTzStr, isNull, err := b.args[2].EvalString(b.ctx, row) - if isNull || err != nil { - return types.Time{}, true, err + if isNull || err != nil || toTzStr == "" { + return types.Time{}, true, nil } fromTzMatched := b.timezoneRegex.MatchString(fromTzStr) @@ -5111,17 +5112,17 @@ func (b *builtinConvertTzSig) evalTime(row chunk.Row) (types.Time, bool, error) if !fromTzMatched && !toTzMatched { fromTz, err := time.LoadLocation(fromTzStr) if err != nil { - return types.Time{}, true, err + return types.Time{}, true, nil } toTz, err := time.LoadLocation(toTzStr) if err != nil { - return types.Time{}, true, err + return types.Time{}, true, nil } t, err := dt.Time.GoTime(fromTz) if err != nil { - return types.Time{}, true, err + return types.Time{}, true, nil } return types.Time{ @@ -5133,7 +5134,7 @@ func (b *builtinConvertTzSig) evalTime(row chunk.Row) (types.Time, bool, error) if fromTzMatched && toTzMatched { t, err := dt.Time.GoTime(time.Local) if err != nil { - return types.Time{}, true, err + return types.Time{}, true, nil } return types.Time{ diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index 60dbbf504523b..4c570ffa15c6c 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -2437,8 +2437,8 @@ func (s *testEvaluatorSuite) TestSecToTime(c *C) { func (s *testEvaluatorSuite) TestConvertTz(c *C) { tests := []struct { t interface{} - fromTz string - toTz string + fromTz interface{} + toTz interface{} Success bool expect string }{ @@ -2450,11 +2450,20 @@ func (s *testEvaluatorSuite) TestConvertTz(c *C) { {"2004-01-01 12:00:00", "-00:00", "+13:00", true, "2004-01-02 01:00:00"}, {"2004-01-01 12:00:00", "-00:00", "-13:00", true, ""}, {"2004-01-01 12:00:00", "-00:00", "-12:88", true, ""}, - {"2004-01-01 12:00:00", "+10:82", "GMT", false, ""}, + {"2004-01-01 12:00:00", "+10:82", "GMT", true, ""}, {"2004-01-01 12:00:00", "+00:00", "GMT", true, ""}, {"2004-01-01 12:00:00", "GMT", "+00:00", true, ""}, {20040101, "+00:00", "+10:32", true, "2004-01-01 10:32:00"}, {3.14159, "+00:00", "+10:32", true, ""}, + {"2004-01-01 12:00:00", "", "GMT", true, ""}, + {"2004-01-01 12:00:00", "GMT", "", true, ""}, + {"2004-01-01 12:00:00", "a", "GMT", true, ""}, + {"2004-01-01 12:00:00", "0", "GMT", true, ""}, + {"2004-01-01 12:00:00", "GMT", "a", true, ""}, + {"2004-01-01 12:00:00", "GMT", "0", true, ""}, + {nil, "GMT", "+00:00", true, ""}, + {"2004-01-01 12:00:00", nil, "+00:00", true, ""}, + {"2004-01-01 12:00:00", "GMT", nil, true, ""}, } fc := funcs[ast.ConvertTz] for _, test := range tests { @@ -2462,8 +2471,8 @@ func (s *testEvaluatorSuite) TestConvertTz(c *C) { s.datumsToConstants( []types.Datum{ types.NewDatum(test.t), - types.NewStringDatum(test.fromTz), - types.NewStringDatum(test.toTz)})) + types.NewDatum(test.fromTz), + types.NewDatum(test.toTz)})) c.Assert(err, IsNil) d, err := evalBuiltinFunc(f, chunk.Row{}) if test.Success { diff --git a/expression/integration_test.go b/expression/integration_test.go index 20348d8a226b2..58926c0af2de2 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -1798,9 +1798,16 @@ func (s *testIntegrationSuite) TestTimeBuiltin(c *C) { // for convert_tz result = tk.MustQuery(`select convert_tz("2004-01-01 12:00:00", "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00.01", "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00.01234567", "+00:00", "+10:32");`) result.Check(testkit.Rows("2004-01-01 22:32:00 2004-01-01 22:32:00.01 2004-01-01 22:32:00.012346")) - // TODO: release the following test after fix #4462 - //result = tk.MustQuery(`select convert_tz(20040101, "+00:00", "+10:32"), convert_tz(20040101.01, "+00:00", "+10:32"), convert_tz(20040101.01234567, "+00:00", "+10:32");`) - //result.Check(testkit.Rows("2004-01-01 10:32:00 2004-01-01 10:32:00.00 2004-01-01 10:32:00.000000")) + result = tk.MustQuery(`select convert_tz(20040101, "+00:00", "+10:32"), convert_tz(20040101.01, "+00:00", "+10:32"), convert_tz(20040101.01234567, "+00:00", "+10:32");`) + result.Check(testkit.Rows("2004-01-01 10:32:00 2004-01-01 10:32:00.00 2004-01-01 10:32:00.000000")) + result = tk.MustQuery(`select convert_tz(NULL, "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00", NULL, "+10:32"), convert_tz("2004-01-01 12:00:00", "+00:00", NULL);`) + result.Check(testkit.Rows(" ")) + result = tk.MustQuery(`select convert_tz("a", "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00", "a", "+10:32"), convert_tz("2004-01-01 12:00:00", "+00:00", "a");`) + result.Check(testkit.Rows(" ")) + result = tk.MustQuery(`select convert_tz("", "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00", "", "+10:32"), convert_tz("2004-01-01 12:00:00", "+00:00", "");`) + result.Check(testkit.Rows(" ")) + result = tk.MustQuery(`select convert_tz("0", "+00:00", "+10:32"), convert_tz("2004-01-01 12:00:00", "0", "+10:32"), convert_tz("2004-01-01 12:00:00", "+00:00", "0");`) + result.Check(testkit.Rows(" ")) // for from_unixtime tk.MustExec(`set @@session.time_zone = "+08:00"`) From bdec3414e75966dcc5de73fa70097a003350151a Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 12 Jul 2019 17:48:35 +0800 Subject: [PATCH 023/196] executor: fix the behavior when index join meet prefix index (#11081) --- executor/builder.go | 9 +++++++++ executor/index_lookup_join.go | 12 ++++++++++++ executor/index_lookup_join_test.go | 8 ++++++++ planner/core/exhaust_physical_plans.go | 18 +++++++++++++----- planner/core/physical_plans.go | 2 ++ util/ranger/ranger.go | 18 +++++++++--------- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index fa98c390f4c01..a31d958aafabc 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1731,6 +1731,13 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) if defaultValues == nil { defaultValues = make([]types.Datum, len(innerTypes)) } + hasPrefixCol := false + for _, l := range v.IdxColLens { + if l != types.UnspecifiedLength { + hasPrefixCol = true + break + } + } e := &IndexLookUpJoin{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID(), outerExec), outerCtx: outerCtx{ @@ -1740,6 +1747,8 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) innerCtx: innerCtx{ readerBuilder: &dataReaderBuilder{Plan: innerPlan, executorBuilder: b}, rowTypes: innerTypes, + colLens: v.IdxColLens, + hasPrefixCol: hasPrefixCol, }, workerWg: new(sync.WaitGroup), joiner: newJoiner(b.ctx, v.JoinType, v.OuterIndex == 1, defaultValues, v.OtherConditions, leftTypes, rightTypes), diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 0604403125f26..7f90e85303fe5 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -89,6 +89,8 @@ type innerCtx struct { readerBuilder *dataReaderBuilder rowTypes []*types.FieldType keyCols []int + colLens []int + hasPrefixCol bool } type lookUpJoinTask struct { @@ -490,6 +492,16 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi } // Store the encoded lookup key in chunk, so we can use it to lookup the matched inners directly. task.encodedLookUpKeys.AppendBytes(0, keyBuf) + if iw.hasPrefixCol { + for i := range iw.outerCtx.keyCols { + // If it's a prefix column. Try to fix it. + if iw.colLens[i] != types.UnspecifiedLength { + ranger.CutDatumByPrefixLen(&dLookUpKey[i], iw.colLens[i], iw.rowTypes[iw.keyCols[i]]) + } + } + // dLookUpKey is sorted and deduplicated at sortAndDedupLookUpContents. + // So we don't need to do it here. + } lookUpContents = append(lookUpContents, &indexJoinLookUpContent{keys: dLookUpKey, row: task.outerResult.GetRow(i)}) } diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 0d1c00e381e73..faa4230e22590 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -152,3 +152,11 @@ func (s *testSuite) TestIndexJoinOverflow(c *C) { tk.MustExec(`create table t2(a int unsigned, index idx(a));`) tk.MustQuery(`select /*+ TIDB_INLJ(t2) */ * from t1 join t2 on t1.a = t2.a;`).Check(testkit.Rows()) } + +func (s *testSuite2) TestIssue11061(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t1, t2") + tk.MustExec("create table t1(c varchar(30), index ix_c(c(10)))") + tk.MustExec("insert into t1 (c) values('7_chars'), ('13_characters')") + tk.MustQuery("SELECT /*+ TIDB_INLJ(t1) */ SUM(LENGTH(c)) FROM t1 WHERE c IN (SELECT t1.c FROM t1)").Check(testkit.Rows("20")) +} diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index aa3ca1fb44dc7..8f856003f80c3 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -321,8 +321,15 @@ func joinKeysMatchIndex(keys, indexCols []*expression.Column, colLengths []int) // When inner plan is TableReader, the parameter `ranges` will be nil. Because pk only have one column. So all of its range // is generated during execution time. -func (p *LogicalJoin) constructIndexJoin(prop *property.PhysicalProperty, outerIdx int, innerPlan PhysicalPlan, - ranges []*ranger.Range, keyOff2IdxOff []int, compareFilters *ColWithCmpFuncManager) []PhysicalPlan { +func (p *LogicalJoin) constructIndexJoin( + prop *property.PhysicalProperty, + outerIdx int, + innerPlan PhysicalPlan, + ranges []*ranger.Range, + keyOff2IdxOff []int, + lens []int, + compareFilters *ColWithCmpFuncManager, +) []PhysicalPlan { joinType := p.JoinType outerSchema := p.children[outerIdx].Schema() var ( @@ -372,6 +379,7 @@ func (p *LogicalJoin) constructIndexJoin(prop *property.PhysicalProperty, outerI DefaultValues: p.DefaultValues, innerPlan: innerPlan, KeyOff2IdxOff: newKeyOff, + IdxColLens: lens, Ranges: ranges, CompareFilters: compareFilters, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), chReqProps...) @@ -430,7 +438,7 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou innerPlan := p.constructInnerTableScan(ds, pkCol, outerJoinKeys, us) // Since the primary key means one value corresponding to exact one row, this will always be a no worse one // comparing to other index. - return p.constructIndexJoin(prop, outerIdx, innerPlan, nil, keyOff2IdxOff, nil) + return p.constructIndexJoin(prop, outerIdx, innerPlan, nil, keyOff2IdxOff, nil, nil) } } helper := &indexJoinBuildHelper{join: p} @@ -454,10 +462,10 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou keyOff2IdxOff[keyOff] = idxOff } } - idxCols, _ := expression.IndexInfo2Cols(ds.schema.Columns, helper.chosenIndexInfo) + idxCols, lens := expression.IndexInfo2Cols(ds.schema.Columns, helper.chosenIndexInfo) rangeInfo := helper.buildRangeDecidedByInformation(idxCols, outerJoinKeys) innerPlan := p.constructInnerIndexScan(ds, helper.chosenIndexInfo, helper.chosenRemained, outerJoinKeys, us, rangeInfo) - return p.constructIndexJoin(prop, outerIdx, innerPlan, helper.chosenRanges, keyOff2IdxOff, helper.lastColManager) + return p.constructIndexJoin(prop, outerIdx, innerPlan, helper.chosenRanges, keyOff2IdxOff, lens, helper.lastColManager) } return nil } diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 4d89ec7b1af6e..67048b70f7e7c 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -232,6 +232,8 @@ type PhysicalIndexJoin struct { Ranges []*ranger.Range // KeyOff2IdxOff maps the offsets in join key to the offsets in the index. KeyOff2IdxOff []int + // IdxColLens stores the length of each index column. + IdxColLens []int // CompareFilters stores the filters for last column if those filters need to be evaluated during execution. // e.g. select * from t where t.a = t1.a and t.b > t1.b and t.b < t1.b+10 // If there's index(t.a, t.b). All the filters can be used to construct index range but t.b > t1.b and t.b < t1.b=10 diff --git a/util/ranger/ranger.go b/util/ranger/ranger.go index 015f6ea447eb7..4b9b12e0b6e13 100644 --- a/util/ranger/ranger.go +++ b/util/ranger/ranger.go @@ -263,10 +263,10 @@ func buildColumnRange(accessConditions []expression.Expression, sc *stmtctx.Stat } if colLen != types.UnspecifiedLength { for _, ran := range ranges { - if fixRangeDatum(&ran.LowVal[0], colLen, tp) { + if CutDatumByPrefixLen(&ran.LowVal[0], colLen, tp) { ran.LowExclude = false } - if fixRangeDatum(&ran.HighVal[0], colLen, tp) { + if CutDatumByPrefixLen(&ran.HighVal[0], colLen, tp) { ran.HighExclude = false } } @@ -425,17 +425,17 @@ func fixPrefixColRange(ranges []*Range, lengths []int, tp []*types.FieldType) bo for _, ran := range ranges { lowTail := len(ran.LowVal) - 1 for i := 0; i < lowTail; i++ { - fixRangeDatum(&ran.LowVal[i], lengths[i], tp[i]) + CutDatumByPrefixLen(&ran.LowVal[i], lengths[i], tp[i]) } - lowCut := fixRangeDatum(&ran.LowVal[lowTail], lengths[lowTail], tp[lowTail]) + lowCut := CutDatumByPrefixLen(&ran.LowVal[lowTail], lengths[lowTail], tp[lowTail]) if lowCut { ran.LowExclude = false } highTail := len(ran.HighVal) - 1 for i := 0; i < highTail; i++ { - fixRangeDatum(&ran.HighVal[i], lengths[i], tp[i]) + CutDatumByPrefixLen(&ran.HighVal[i], lengths[i], tp[i]) } - highCut := fixRangeDatum(&ran.HighVal[highTail], lengths[highTail], tp[highTail]) + highCut := CutDatumByPrefixLen(&ran.HighVal[highTail], lengths[highTail], tp[highTail]) if highCut { ran.HighExclude = false } @@ -444,9 +444,9 @@ func fixPrefixColRange(ranges []*Range, lengths []int, tp []*types.FieldType) bo return hasCut } -func fixRangeDatum(v *types.Datum, length int, tp *types.FieldType) bool { - // If this column is prefix and the prefix length is smaller than the range, cut it. - // In case of UTF8, prefix should be cut by characters rather than bytes +// CutDatumByPrefixLen cuts the datum according to the prefix length. +// If it's UTF8 encoded, we will cut it by characters rather than bytes. +func CutDatumByPrefixLen(v *types.Datum, length int, tp *types.FieldType) bool { if v.Kind() == types.KindString || v.Kind() == types.KindBytes { colCharset := tp.Charset colValue := v.GetBytes() From c8ed781d5133b855199e604f59561f0cfa9f3e10 Mon Sep 17 00:00:00 2001 From: Maxwell Date: Sun, 14 Jul 2019 17:58:49 +0800 Subject: [PATCH 024/196] executor: push-down LoadDataStmt in distsql (#11067) moving function `statementContextToFlags` to `StatementContext.PushDownFlags()` Co-authored-by: Lonng Co-authored-by: Foreyes --- executor/admin.go | 6 +-- executor/builder.go | 8 ++-- executor/distsql.go | 31 -------------- go.sum | 3 -- sessionctx/stmtctx/stmtctx.go | 34 +++++++++++++++ sessionctx/stmtctx/stmtctx_test.go | 66 ++++++++++++++++++++++-------- 6 files changed, 91 insertions(+), 57 deletions(-) diff --git a/executor/admin.go b/executor/admin.go index fbb1d4e21fde5..5f8ca71e2680c 100644 --- a/executor/admin.go +++ b/executor/admin.go @@ -135,7 +135,7 @@ func (e *CheckIndexRangeExec) buildDAGPB() (*tipb.DAGRequest, error) { dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx - dagReq.Flags = statementContextToFlags(sc) + dagReq.Flags = sc.PushDownFlags() for i := range e.schema.Columns { dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i)) } @@ -233,7 +233,7 @@ func (e *RecoverIndexExec) buildDAGPB(txn kv.Transaction, limitCnt uint64) (*tip dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx - dagReq.Flags = statementContextToFlags(sc) + dagReq.Flags = sc.PushDownFlags() for i := range e.columns { dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i)) } @@ -670,7 +670,7 @@ func (e *CleanupIndexExec) buildIdxDAGPB(txn kv.Transaction) (*tipb.DAGRequest, dagReq.StartTs = txn.StartTS() dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(e.ctx.GetSessionVars().Location()) sc := e.ctx.GetSessionVars().StmtCtx - dagReq.Flags = statementContextToFlags(sc) + dagReq.Flags = sc.PushDownFlags() for i := range e.idxCols { dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i)) } diff --git a/executor/builder.go b/executor/builder.go index a31d958aafabc..20c4f9525e92a 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1400,6 +1400,7 @@ func (b *executorBuilder) buildDelete(v *plannercore.Delete) Executor { func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeIndexTask, maxNumBuckets uint64, autoAnalyze string) *analyzeTask { _, offset := timeutil.Zone(b.ctx.GetSessionVars().Location()) + sc := b.ctx.GetSessionVars().StmtCtx e := &AnalyzeIndexExec{ ctx: b.ctx, physicalTableID: task.PhysicalTableID, @@ -1408,7 +1409,7 @@ func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeInde analyzePB: &tipb.AnalyzeReq{ Tp: tipb.AnalyzeType_TypeIndex, StartTs: math.MaxUint64, - Flags: statementContextToFlags(b.ctx.GetSessionVars().StmtCtx), + Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, maxNumBuckets: maxNumBuckets, @@ -1466,6 +1467,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo } _, offset := timeutil.Zone(b.ctx.GetSessionVars().Location()) + sc := b.ctx.GetSessionVars().StmtCtx e := &AnalyzeColumnsExec{ ctx: b.ctx, physicalTableID: task.PhysicalTableID, @@ -1475,7 +1477,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo analyzePB: &tipb.AnalyzeReq{ Tp: tipb.AnalyzeType_TypeColumn, StartTs: math.MaxUint64, - Flags: statementContextToFlags(b.ctx.GetSessionVars().StmtCtx), + Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, maxNumBuckets: maxNumBuckets, @@ -1658,7 +1660,7 @@ func (b *executorBuilder) constructDAGReq(plans []plannercore.PhysicalPlan) (dag } dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(b.ctx.GetSessionVars().Location()) sc := b.ctx.GetSessionVars().StmtCtx - dagReq.Flags = statementContextToFlags(sc) + dagReq.Flags = sc.PushDownFlags() dagReq.Executors, streaming, err = constructDistExec(b.ctx, plans) return dagReq, streaming, err } diff --git a/executor/distsql.go b/executor/distsql.go index 946705aa3196a..fcfbc110fcad3 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/tidb/kv" plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" @@ -123,36 +122,6 @@ func closeAll(objs ...Closeable) error { return nil } -// statementContextToFlags converts StatementContext to tipb.SelectRequest.Flags. -func statementContextToFlags(sc *stmtctx.StatementContext) uint64 { - var flags uint64 - if sc.InInsertStmt { - flags |= model.FlagInInsertStmt - } else if sc.InUpdateStmt || sc.InDeleteStmt { - flags |= model.FlagInUpdateOrDeleteStmt - } else if sc.InSelectStmt { - flags |= model.FlagInSelectStmt - } - if sc.IgnoreTruncate { - flags |= model.FlagIgnoreTruncate - } else if sc.TruncateAsWarning { - flags |= model.FlagTruncateAsWarning - } - if sc.OverflowAsWarning { - flags |= model.FlagOverflowAsWarning - } - if sc.IgnoreZeroInDate { - flags |= model.FlagIgnoreZeroInDate - } - if sc.DividedByZeroAsWarning { - flags |= model.FlagDividedByZeroAsWarning - } - if sc.PadCharToFullLength { - flags |= model.FlagPadCharToFullLength - } - return flags -} - // handleIsExtra checks whether this column is a extra handle column generated during plan building phase. func handleIsExtra(col *expression.Column) bool { if col != nil && col.ID == model.ExtraHandleID { diff --git a/go.sum b/go.sum index a0a8d06b00ae6..914c267d1ff5f 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,6 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mo github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -192,7 +191,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFd github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -297,7 +295,6 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index 4068487c05df6..fdbc0f9ec6653 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/parser" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/memory" @@ -438,6 +439,39 @@ func (sc *StatementContext) ShouldIgnoreOverflowError() bool { return false } +// PushDownFlags converts StatementContext to tipb.SelectRequest.Flags. +func (sc *StatementContext) PushDownFlags() uint64 { + var flags uint64 + if sc.InInsertStmt { + flags |= model.FlagInInsertStmt + } else if sc.InUpdateStmt || sc.InDeleteStmt { + flags |= model.FlagInUpdateOrDeleteStmt + } else if sc.InSelectStmt { + flags |= model.FlagInSelectStmt + } + if sc.IgnoreTruncate { + flags |= model.FlagIgnoreTruncate + } else if sc.TruncateAsWarning { + flags |= model.FlagTruncateAsWarning + } + if sc.OverflowAsWarning { + flags |= model.FlagOverflowAsWarning + } + if sc.IgnoreZeroInDate { + flags |= model.FlagIgnoreZeroInDate + } + if sc.DividedByZeroAsWarning { + flags |= model.FlagDividedByZeroAsWarning + } + if sc.PadCharToFullLength { + flags |= model.FlagPadCharToFullLength + } + if sc.InLoadDataStmt { + flags |= model.FlagInLoadDataStmt + } + return flags +} + // CopTasksDetails returns some useful information of cop-tasks during execution. func (sc *StatementContext) CopTasksDetails() *CopTasksDetails { sc.mu.Lock() diff --git a/sessionctx/stmtctx/stmtctx_test.go b/sessionctx/stmtctx/stmtctx_test.go index 9fb77aaa3ee2e..6ba2e862f27dc 100644 --- a/sessionctx/stmtctx/stmtctx_test.go +++ b/sessionctx/stmtctx/stmtctx_test.go @@ -11,18 +11,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package stmtctx +package stmtctx_test import ( "fmt" "testing" "time" + . "github.com/pingcap/check" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/util/execdetails" ) -func TestCopTasksDetails(t *testing.T) { - ctx := new(StatementContext) +func TestT(t *testing.T) { + TestingT(t) +} + +type stmtctxSuit struct{} + +var _ = Suite(&stmtctxSuit{}) + +func (s *stmtctxSuit) TestCopTasksDetails(c *C) { + ctx := new(stmtctx.StatementContext) for i := 0; i < 100; i++ { d := &execdetails.ExecDetails{ CalleeAddress: fmt.Sprintf("%v", i+1), @@ -31,20 +41,42 @@ func TestCopTasksDetails(t *testing.T) { } ctx.MergeExecDetails(d, nil) } - c := ctx.CopTasksDetails() - if c.NumCopTasks != 100 || - c.AvgProcessTime != time.Second*101/2 || - c.P90ProcessTime != time.Second*91 || - c.MaxProcessTime != time.Second*100 || - c.MaxProcessAddress != "100" || - c.AvgWaitTime != time.Millisecond*101/2 || - c.P90WaitTime != time.Millisecond*91 || - c.MaxWaitTime != time.Millisecond*100 || - c.MaxWaitAddress != "100" { - t.Fatal(c) + d := ctx.CopTasksDetails() + c.Assert(d.NumCopTasks, Equals, 100) + c.Assert(d.AvgProcessTime, Equals, time.Second*101/2) + c.Assert(d.P90ProcessTime, Equals, time.Second*91) + c.Assert(d.MaxProcessTime, Equals, time.Second*100) + c.Assert(d.MaxProcessAddress, Equals, "100") + c.Assert(d.AvgWaitTime, Equals, time.Millisecond*101/2) + c.Assert(d.P90WaitTime, Equals, time.Millisecond*91) + c.Assert(d.MaxWaitTime, Equals, time.Millisecond*100) + c.Assert(d.MaxWaitAddress, Equals, "100") + fields := d.ToZapFields() + c.Assert(len(fields), Equals, 9) +} + +func (s *stmtctxSuit) TestStatementContextPushDownFLags(c *C) { + testCases := []struct { + in *stmtctx.StatementContext + out uint64 + }{ + {&stmtctx.StatementContext{InInsertStmt: true}, 8}, + {&stmtctx.StatementContext{InUpdateStmt: true}, 16}, + {&stmtctx.StatementContext{InDeleteStmt: true}, 16}, + {&stmtctx.StatementContext{InSelectStmt: true}, 32}, + {&stmtctx.StatementContext{IgnoreTruncate: true}, 1}, + {&stmtctx.StatementContext{TruncateAsWarning: true}, 2}, + {&stmtctx.StatementContext{OverflowAsWarning: true}, 64}, + {&stmtctx.StatementContext{IgnoreZeroInDate: true}, 128}, + {&stmtctx.StatementContext{DividedByZeroAsWarning: true}, 256}, + {&stmtctx.StatementContext{PadCharToFullLength: true}, 4}, + {&stmtctx.StatementContext{InLoadDataStmt: true}, 1024}, + {&stmtctx.StatementContext{InSelectStmt: true, TruncateAsWarning: true}, 34}, + {&stmtctx.StatementContext{DividedByZeroAsWarning: true, IgnoreTruncate: true}, 257}, + {&stmtctx.StatementContext{InUpdateStmt: true, IgnoreZeroInDate: true, InLoadDataStmt: true}, 1168}, } - fields := c.ToZapFields() - if len(fields) != 9 { - t.Fatal(c) + for _, tt := range testCases { + got := tt.in.PushDownFlags() + c.Assert(got, Equals, tt.out, Commentf("get %v, want %v", got, tt.out)) } } From 9c3997720b99130c698bd6c023eebb803464f5f6 Mon Sep 17 00:00:00 2001 From: goroutine Date: Mon, 15 Jul 2019 10:16:45 +0800 Subject: [PATCH 025/196] chunk: use native go style to do initialization (#11242) --- util/chunk/chunk.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 5a375814b7a18..df331d049bfb7 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -55,9 +55,16 @@ func NewChunkWithCapacity(fields []*types.FieldType, cap int) *Chunk { // cap: the limit for the max number of rows. // maxChunkSize: the max limit for the number of rows. func New(fields []*types.FieldType, cap, maxChunkSize int) *Chunk { - chk := new(Chunk) - chk.columns = make([]*column, 0, len(fields)) - chk.capacity = mathutil.Min(cap, maxChunkSize) + chk := &Chunk{ + columns: make([]*column, 0, len(fields)), + capacity: mathutil.Min(cap, maxChunkSize), + // set the default value of requiredRows to maxChunkSize to let chk.IsFull() behave + // like how we judge whether a chunk is full now, then the statement + // "chk.NumRows() < maxChunkSize" + // equals to "!chk.IsFull()". + requiredRows: maxChunkSize, + } + for _, f := range fields { elemLen := getFixedLen(f) if elemLen == varElemLen { @@ -66,14 +73,7 @@ func New(fields []*types.FieldType, cap, maxChunkSize int) *Chunk { chk.columns = append(chk.columns, newFixedLenColumn(elemLen, chk.capacity)) } } - chk.numVirtualRows = 0 - - // set the default value of requiredRows to maxChunkSize to let chk.IsFull() behave - // like how we judge whether a chunk is full now, then the statement - // "chk.NumRows() < maxChunkSize" - // is equal to - // "!chk.IsFull()". - chk.requiredRows = maxChunkSize + return chk } From 8f61bfaa829046b31fc05b967ed6f0c5681d95cb Mon Sep 17 00:00:00 2001 From: goroutine Date: Mon, 15 Jul 2019 10:20:35 +0800 Subject: [PATCH 026/196] planner: tiny refactor and more go style (#11244) --- planner/core/logical_plan_builder.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index d22ee24afa1c0..fb19ca7f6169f 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -42,7 +42,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/types/parser_driver" + driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" ) @@ -2374,7 +2374,7 @@ func (b *PlanBuilder) BuildDataSourceFromView(dbName model.CIStr, tableInfo *mod // we add a projection on the original DataSource, and calculate those columns in the projection // so that plans above it can reference generated columns by their name. func (b *PlanBuilder) projectVirtualColumns(ds *DataSource, columns []*table.Column) (*LogicalProjection, error) { - var hasVirtualGeneratedColumn = false + hasVirtualGeneratedColumn := false for _, column := range columns { if column.IsGenerated() && !column.GeneratedStored { hasVirtualGeneratedColumn = true @@ -2384,7 +2384,7 @@ func (b *PlanBuilder) projectVirtualColumns(ds *DataSource, columns []*table.Col if !hasVirtualGeneratedColumn { return nil, nil } - var proj = LogicalProjection{ + proj := LogicalProjection{ Exprs: make([]expression.Expression, 0, len(columns)), calculateGenCols: true, }.Init(b.ctx) @@ -2429,9 +2429,7 @@ func (b *PlanBuilder) projectVirtualColumns(ds *DataSource, columns []*table.Col // buildApplyWithJoinType builds apply plan with outerPlan and innerPlan, which apply join with particular join type for // every row from outerPlan and the whole innerPlan. func (b *PlanBuilder) buildApplyWithJoinType(outerPlan, innerPlan LogicalPlan, tp JoinType) LogicalPlan { - b.optFlag = b.optFlag | flagPredicatePushDown - b.optFlag = b.optFlag | flagBuildKeyInfo - b.optFlag = b.optFlag | flagDecorrelate + b.optFlag = b.optFlag | flagPredicatePushDown | flagBuildKeyInfo | flagDecorrelate ap := LogicalApply{LogicalJoin: LogicalJoin{JoinType: tp}}.Init(b.ctx) ap.SetChildren(outerPlan, innerPlan) ap.SetSchema(expression.MergeSchema(outerPlan.Schema(), innerPlan.Schema())) @@ -2443,9 +2441,7 @@ func (b *PlanBuilder) buildApplyWithJoinType(outerPlan, innerPlan LogicalPlan, t // buildSemiApply builds apply plan with outerPlan and innerPlan, which apply semi-join for every row from outerPlan and the whole innerPlan. func (b *PlanBuilder) buildSemiApply(outerPlan, innerPlan LogicalPlan, condition []expression.Expression, asScalar, not bool) (LogicalPlan, error) { - b.optFlag = b.optFlag | flagPredicatePushDown - b.optFlag = b.optFlag | flagBuildKeyInfo - b.optFlag = b.optFlag | flagDecorrelate + b.optFlag = b.optFlag | flagPredicatePushDown | flagBuildKeyInfo | flagDecorrelate join, err := b.buildSemiJoin(outerPlan, innerPlan, condition, asScalar, not) if err != nil { From cb984ad326c0817432f0fecedf1cd51244177870 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 15 Jul 2019 10:33:04 +0800 Subject: [PATCH 027/196] go.mod: update go.mod to use the latest pd (#11224) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31b57528e20e7..0fd738a958d1f 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d - github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf + github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 github.com/prometheus/client_golang v0.9.0 diff --git a/go.sum b/go.sum index 914c267d1ff5f..2bf2b6d4c859c 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3F github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d h1:vOZjn1ami1LIjtIj0i5QunGh/sHawbhiBCb1qPx373w= github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= -github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf h1:vmlN6DpZI5LtHd8r9YRAsyCeTU2pxRq+WlWn5CZ+ax4= -github.com/pingcap/pd v0.0.0-20190617100349-293d4b5189bf/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= +github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= +github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw= From a090e6be2991bf85b18fcdb096f84d41f4f6bd85 Mon Sep 17 00:00:00 2001 From: zhaoyanxing Date: Mon, 15 Jul 2019 11:33:06 +0800 Subject: [PATCH 028/196] Function INSERT should be NULL if any argument is NULL (#11237) --- expression/builtin_string.go | 18 ++++++------ expression/builtin_string_test.go | 49 +++++++++++++++++++++++++++++++ expression/integration_test.go | 4 +++ 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/expression/builtin_string.go b/expression/builtin_string.go index 7088bf5af7302..ed553476a74a1 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -3325,15 +3325,11 @@ func (b *builtinInsertBinarySig) evalString(row chunk.Row) (string, bool, error) if isNull || err != nil { return "", true, err } - strLength := int64(len(str)) pos, isNull, err := b.args[1].EvalInt(b.ctx, row) if isNull || err != nil { return "", true, err } - if pos < 1 || pos > strLength { - return str, false, nil - } length, isNull, err := b.args[2].EvalInt(b.ctx, row) if isNull || err != nil { @@ -3345,6 +3341,10 @@ func (b *builtinInsertBinarySig) evalString(row chunk.Row) (string, bool, error) return "", true, err } + strLength := int64(len(str)) + if pos < 1 || pos > strLength { + return str, false, nil + } if length > strLength-pos+1 || length < 0 { length = strLength - pos + 1 } @@ -3376,16 +3376,11 @@ func (b *builtinInsertSig) evalString(row chunk.Row) (string, bool, error) { if isNull || err != nil { return "", true, err } - runes := []rune(str) - runeLength := int64(len(runes)) pos, isNull, err := b.args[1].EvalInt(b.ctx, row) if isNull || err != nil { return "", true, err } - if pos < 1 || pos > runeLength { - return str, false, nil - } length, isNull, err := b.args[2].EvalInt(b.ctx, row) if isNull || err != nil { @@ -3397,6 +3392,11 @@ func (b *builtinInsertSig) evalString(row chunk.Row) (string, bool, error) { return "", true, err } + runes := []rune(str) + runeLength := int64(len(runes)) + if pos < 1 || pos > runeLength { + return str, false, nil + } if length > runeLength-pos+1 || length < 0 { length = runeLength - pos + 1 } diff --git a/expression/builtin_string_test.go b/expression/builtin_string_test.go index c4fdf1cc5e2b6..5c9b6debb62e4 100644 --- a/expression/builtin_string_test.go +++ b/expression/builtin_string_test.go @@ -1474,12 +1474,32 @@ func (s *testEvaluatorSuite) TestInsertBinarySig(c *C) { input := chunk.NewChunkWithCapacity(colTypes, 2) input.AppendString(0, "abc") input.AppendString(0, "abc") + input.AppendString(0, "abc") + input.AppendNull(0) + input.AppendString(0, "abc") + input.AppendString(0, "abc") + input.AppendString(0, "abc") + input.AppendInt64(1, 3) input.AppendInt64(1, 3) + input.AppendInt64(1, 0) input.AppendInt64(1, 3) + input.AppendNull(1) + input.AppendInt64(1, 3) + input.AppendInt64(1, 3) + input.AppendInt64(2, -1) + input.AppendInt64(2, -1) input.AppendInt64(2, -1) input.AppendInt64(2, -1) + input.AppendInt64(2, -1) + input.AppendNull(2) + input.AppendInt64(2, -1) input.AppendString(3, "d") input.AppendString(3, "de") + input.AppendString(3, "d") + input.AppendString(3, "d") + input.AppendString(3, "d") + input.AppendString(3, "d") + input.AppendNull(3) res, isNull, err := insert.evalString(input.GetRow(0)) c.Assert(res, Equals, "abd") @@ -1491,6 +1511,31 @@ func (s *testEvaluatorSuite) TestInsertBinarySig(c *C) { c.Assert(isNull, IsTrue) c.Assert(err, IsNil) + res, isNull, err = insert.evalString(input.GetRow(2)) + c.Assert(res, Equals, "abc") + c.Assert(isNull, IsFalse) + c.Assert(err, IsNil) + + res, isNull, err = insert.evalString(input.GetRow(3)) + c.Assert(res, Equals, "") + c.Assert(isNull, IsTrue) + c.Assert(err, IsNil) + + res, isNull, err = insert.evalString(input.GetRow(4)) + c.Assert(res, Equals, "") + c.Assert(isNull, IsTrue) + c.Assert(err, IsNil) + + res, isNull, err = insert.evalString(input.GetRow(5)) + c.Assert(res, Equals, "") + c.Assert(isNull, IsTrue) + c.Assert(err, IsNil) + + res, isNull, err = insert.evalString(input.GetRow(6)) + c.Assert(res, Equals, "") + c.Assert(isNull, IsTrue) + c.Assert(err, IsNil) + warnings := s.ctx.GetSessionVars().StmtCtx.GetWarnings() c.Assert(len(warnings), Equals, 1) lastWarn := warnings[len(warnings)-1] @@ -1855,6 +1900,8 @@ func (s *testEvaluatorSuite) TestInsert(c *C) { {[]interface{}{"Quadratic", 3, 4, nil}, nil}, {[]interface{}{"Quadratic", 3, -1, "What"}, "QuWhat"}, {[]interface{}{"Quadratic", 3, 1, "What"}, "QuWhatdratic"}, + {[]interface{}{"Quadratic", -1, nil, "What"}, nil}, + {[]interface{}{"Quadratic", -1, 4, nil}, nil}, {[]interface{}{"我叫小雨呀", 3, 2, "王雨叶"}, "我叫王雨叶呀"}, {[]interface{}{"我叫小雨呀", -1, 2, "王雨叶"}, "我叫小雨呀"}, @@ -1865,6 +1912,8 @@ func (s *testEvaluatorSuite) TestInsert(c *C) { {[]interface{}{"我叫小雨呀", 3, 4, nil}, nil}, {[]interface{}{"我叫小雨呀", 3, -1, "王雨叶"}, "我叫王雨叶"}, {[]interface{}{"我叫小雨呀", 3, 1, "王雨叶"}, "我叫王雨叶雨呀"}, + {[]interface{}{"我叫小雨呀", -1, nil, "王雨叶"}, nil}, + {[]interface{}{"我叫小雨呀", -1, 2, nil}, nil}, } fc := funcs[ast.InsertFunc] for _, test := range tests { diff --git a/expression/integration_test.go b/expression/integration_test.go index 58926c0af2de2..d1f7272f8eb68 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -920,6 +920,10 @@ func (s *testIntegrationSuite) TestStringBuiltin(c *C) { result.Check(testkit.Rows("aaa文 ba aaa ba")) result = tk.MustQuery(`select insert("bb", NULL, 1, "aa"), insert("bb", 1, NULL, "aa"), insert(NULL, 1, 1, "aaa"), insert("bb", 1, 1, NULL);`) result.Check(testkit.Rows(" ")) + result = tk.MustQuery(`SELECT INSERT("bb", 0, 1, NULL), INSERT("bb", 0, NULL, "aaa");`) + result.Check(testkit.Rows(" ")) + result = tk.MustQuery(`SELECT INSERT("中文", 0, 1, NULL), INSERT("中文", 0, NULL, "aaa");`) + result.Check(testkit.Rows(" ")) // for export_set result = tk.MustQuery(`select export_set(7, "1", "0", ",", 65);`) From 9b824457c132ed2e7dadc3ba20944540bc6168a7 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Mon, 15 Jul 2019 13:05:54 +0800 Subject: [PATCH 029/196] *: improve column size calculation of statistics (#11091) --- executor/analyze.go | 2 +- executor/show_stats.go | 2 +- infoschema/tables_test.go | 8 +++--- planner/core/plan_to_pb.go | 2 +- statistics/handle/ddl_test.go | 4 +-- statistics/handle/handle_test.go | 16 ++++++------ statistics/handle/update_test.go | 33 +++++++++++++------------ statistics/histogram.go | 37 +++++++++++++++++----------- table/tables/tables.go | 42 ++++++++++++++++++++++---------- tablecodec/tablecodec.go | 7 +++--- tablecodec/tablecodec_test.go | 27 +++++++++++++++++--- 11 files changed, 113 insertions(+), 67 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index d37a27a32fc5d..b6a9389af9df3 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -1002,7 +1002,7 @@ func (e *AnalyzeFastExec) buildColumnStats(ID int64, collector *statistics.Sampl collector.NullCount++ continue } - bytes, err := tablecodec.EncodeValue(e.ctx.GetSessionVars().StmtCtx, sample.Value) + bytes, err := tablecodec.EncodeValue(e.ctx.GetSessionVars().StmtCtx, nil, sample.Value) if err != nil { return nil, nil, err } diff --git a/executor/show_stats.go b/executor/show_stats.go index 9bbf358a26d39..690a95c1a73da 100644 --- a/executor/show_stats.go +++ b/executor/show_stats.go @@ -86,7 +86,7 @@ func (e *ShowExec) appendTableForStatsHistograms(dbName, tblName, partitionName if col.IsInvalid(nil, false) { continue } - e.histogramToRow(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count)) + e.histogramToRow(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count, false)) } for _, idx := range statsTbl.Indices { e.histogramToRow(dbName, tblName, partitionName, idx.Info.Name.O, 1, idx.Histogram, 0) diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 85033dd359d89..df0c95b2ab5a0 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -143,22 +143,22 @@ func (s *testTableSuite) TestDataForTableStatsField(c *C) { c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) c.Assert(h.Update(is), IsNil) tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( - testkit.Rows("3 17 51 3")) + testkit.Rows("3 18 54 6")) tk.MustExec(`insert into t(c, d, e) values(4, 5, "f")`) c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) c.Assert(h.Update(is), IsNil) tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( - testkit.Rows("4 17 68 4")) + testkit.Rows("4 18 72 8")) tk.MustExec("delete from t where c >= 3") c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) c.Assert(h.Update(is), IsNil) tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( - testkit.Rows("2 17 34 2")) + testkit.Rows("2 18 36 4")) tk.MustExec("delete from t where c=3") c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) c.Assert(h.Update(is), IsNil) tk.MustQuery("select table_rows, avg_row_length, data_length, index_length from information_schema.tables where table_name='t'").Check( - testkit.Rows("2 17 34 2")) + testkit.Rows("2 18 36 4")) } func (s *testTableSuite) TestCharacterSetCollations(c *C) { diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index 4694f7bcc5c14..789d1f0a132b3 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -152,7 +152,7 @@ func SetPBColumnsDefaultValue(ctx sessionctx.Context, pbColumns []*tipb.ColumnIn return err } - pbColumns[i].DefaultVal, err = tablecodec.EncodeValue(ctx.GetSessionVars().StmtCtx, d) + pbColumns[i].DefaultVal, err = tablecodec.EncodeValue(sessVars.StmtCtx, nil, d) if err != nil { return err } diff --git a/statistics/handle/ddl_test.go b/statistics/handle/ddl_test.go index a85d40e154bef..4baab71c6c87c 100644 --- a/statistics/handle/ddl_test.go +++ b/statistics/handle/ddl_test.go @@ -160,7 +160,7 @@ func (s *testStatsSuite) TestDDLHistogram(c *C) { tableInfo = tbl.Meta() statsTbl = do.StatsHandle().GetTableStats(tableInfo) c.Assert(statsTbl.Pseudo, IsFalse) - c.Check(statsTbl.Columns[tableInfo.Columns[5].ID].AvgColSize(statsTbl.Count), Equals, 3.0) + c.Check(statsTbl.Columns[tableInfo.Columns[5].ID].AvgColSize(statsTbl.Count, false), Equals, 3.0) testKit.MustExec("create index i on t(c2, c1)") testKit.MustExec("analyze table t") @@ -212,6 +212,6 @@ PARTITION BY RANGE ( a ) ( for _, def := range pi.Definitions { statsTbl := h.GetPartitionStats(tableInfo, def.ID) c.Assert(statsTbl.Pseudo, IsFalse) - c.Check(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count), Equals, 3.0) + c.Check(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count, false), Equals, 3.0) } } diff --git a/statistics/handle/handle_test.go b/statistics/handle/handle_test.go index 1d53e53ed2e16..7a23de43a73f1 100644 --- a/statistics/handle/handle_test.go +++ b/statistics/handle/handle_test.go @@ -208,19 +208,19 @@ func (s *testStatsSuite) TestAvgColLen(c *C) { c.Assert(err, IsNil) tableInfo := tbl.Meta() statsTbl := do.StatsHandle().GetTableStats(tableInfo) - c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSize(statsTbl.Count), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSize(statsTbl.Count, false), Equals, 1.0) // The size of varchar type is LEN + BYTE, here is 1 + 7 = 8 - c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count), Equals, 8.0) - c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count), Equals, 4.0) - c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count), Equals, 16.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) testKit.MustExec("insert into t values(132, '123456789112', 1232.3, '2018-03-07 19:17:29')") testKit.MustExec("analyze table t") statsTbl = do.StatsHandle().GetTableStats(tableInfo) - c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSize(statsTbl.Count), Equals, 8.0) - c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count), Equals, 10.5) - c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count), Equals, 4.0) - c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count), Equals, 16.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[0].ID].AvgColSize(statsTbl.Count, false), Equals, 1.5) + c.Assert(statsTbl.Columns[tableInfo.Columns[1].ID].AvgColSize(statsTbl.Count, false), Equals, 10.5) + c.Assert(statsTbl.Columns[tableInfo.Columns[2].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) + c.Assert(statsTbl.Columns[tableInfo.Columns[3].ID].AvgColSize(statsTbl.Count, false), Equals, 8.0) } func (s *testStatsSuite) TestDurationToTS(c *C) { diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index b15b8f47a5788..84e428ee22f5a 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -160,8 +160,8 @@ func (s *testStatsSuite) TestSingleSessionInsert(c *C) { rs := testKit.MustQuery("select modify_count from mysql.stats_meta") rs.Check(testkit.Rows("40", "70")) - rs = testKit.MustQuery("select tot_col_size from mysql.stats_histograms") - rs.Check(testkit.Rows("0", "0", "10", "10")) + rs = testKit.MustQuery("select tot_col_size from mysql.stats_histograms").Sort() + rs.Check(testkit.Rows("0", "0", "20", "20")) // test dump delta only when `modify count / count` is greater than the ratio. originValue := handle.DumpStatsDeltaRatio @@ -343,7 +343,7 @@ func (s *testStatsSuite) TestUpdatePartition(c *C) { statsTbl := h.GetPartitionStats(tableInfo, def.ID) c.Assert(statsTbl.ModifyCount, Equals, int64(1)) c.Assert(statsTbl.Count, Equals, int64(1)) - c.Assert(statsTbl.Columns[bColID].TotColSize, Equals, int64(1)) + c.Assert(statsTbl.Columns[bColID].TotColSize, Equals, int64(2)) } testKit.MustExec(`update t set a = a + 1, b = "aa"`) @@ -353,7 +353,7 @@ func (s *testStatsSuite) TestUpdatePartition(c *C) { statsTbl := h.GetPartitionStats(tableInfo, def.ID) c.Assert(statsTbl.ModifyCount, Equals, int64(2)) c.Assert(statsTbl.Count, Equals, int64(1)) - c.Assert(statsTbl.Columns[bColID].TotColSize, Equals, int64(2)) + c.Assert(statsTbl.Columns[bColID].TotColSize, Equals, int64(3)) } testKit.MustExec("delete from t") @@ -442,7 +442,7 @@ func (s *testStatsSuite) TestAutoUpdate(c *C) { c.Assert(stats.ModifyCount, Equals, int64(1)) for _, item := range stats.Columns { // TotColSize = 6, because the table has not been analyzed, and insert statement will add 3(length of 'eee') to TotColSize. - c.Assert(item.TotColSize, Equals, int64(14)) + c.Assert(item.TotColSize, Equals, int64(15)) break } @@ -1307,7 +1307,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: "select * from t use index(idx_ac) where a = 1 and c < 21", - hist: "column:3 ndv:20 totColSize:20\n" + + hist: "column:3 ndv:20 totColSize:40\n" + "num: 13 lower_bound: -9223372036854775808 upper_bound: 6 repeats: 0\n" + "num: 13 lower_bound: 7 upper_bound: 13 repeats: 0\n" + "num: 12 lower_bound: 14 upper_bound: 21 repeats: 0", @@ -1318,7 +1318,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: "select * from t use index(idx_ad) where a = 1 and d < 21", - hist: "column:4 ndv:20 totColSize:160\n" + + hist: "column:4 ndv:20 totColSize:320\n" + "num: 13 lower_bound: -10000000000000 upper_bound: 6 repeats: 0\n" + "num: 12 lower_bound: 7 upper_bound: 13 repeats: 0\n" + "num: 10 lower_bound: 14 upper_bound: 21 repeats: 0", @@ -1329,7 +1329,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: "select * from t use index(idx_ae) where a = 1 and e < 21", - hist: "column:5 ndv:20 totColSize:160\n" + + hist: "column:5 ndv:20 totColSize:320\n" + "num: 13 lower_bound: -100000000000000000000000 upper_bound: 6 repeats: 0\n" + "num: 12 lower_bound: 7 upper_bound: 13 repeats: 0\n" + "num: 10 lower_bound: 14 upper_bound: 21 repeats: 0", @@ -1340,7 +1340,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: "select * from t use index(idx_af) where a = 1 and f < 21", - hist: "column:6 ndv:20 totColSize:200\n" + + hist: "column:6 ndv:20 totColSize:400\n" + "num: 13 lower_bound: -999999999999999.99 upper_bound: 6.00 repeats: 0\n" + "num: 12 lower_bound: 7.00 upper_bound: 13.00 repeats: 0\n" + "num: 10 lower_bound: 14.00 upper_bound: 21.00 repeats: 0", @@ -1351,7 +1351,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: "select * from t use index(idx_ag) where a = 1 and g < 21", - hist: "column:7 ndv:20 totColSize:98\n" + + hist: "column:7 ndv:20 totColSize:196\n" + "num: 13 lower_bound: -838:59:59 upper_bound: 00:00:06 repeats: 0\n" + "num: 11 lower_bound: 00:00:07 upper_bound: 00:00:13 repeats: 0\n" + "num: 10 lower_bound: 00:00:14 upper_bound: 00:00:21 repeats: 0", @@ -1362,7 +1362,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { }, { sql: `select * from t use index(idx_ah) where a = 1 and h < "1000-01-21"`, - hist: "column:8 ndv:20 totColSize:180\n" + + hist: "column:8 ndv:20 totColSize:360\n" + "num: 13 lower_bound: 1000-01-01 upper_bound: 1000-01-07 repeats: 0\n" + "num: 11 lower_bound: 1000-01-08 upper_bound: 1000-01-14 repeats: 0\n" + "num: 10 lower_bound: 1000-01-15 upper_bound: 1000-01-21 repeats: 0", @@ -1504,7 +1504,7 @@ func (s *testStatsSuite) TestFeedbackRanges(c *C) { }, { sql: "select * from t use index(idx) where a = 1 and (b <= 50 or (b > 130 and b < 140))", - hist: "column:2 ndv:20 totColSize:20\n" + + hist: "column:2 ndv:20 totColSize:30\n" + "num: 7 lower_bound: -128 upper_bound: 6 repeats: 0\n" + "num: 7 lower_bound: 7 upper_bound: 13 repeats: 1\n" + "num: 6 lower_bound: 14 upper_bound: 19 repeats: 1", @@ -1561,7 +1561,7 @@ func (s *testStatsSuite) TestUnsignedFeedbackRanges(c *C) { }{ { sql: "select * from t where a <= 50", - hist: "column:1 ndv:30 totColSize:0\n" + + hist: "column:1 ndv:30 totColSize:10\n" + "num: 8 lower_bound: 0 upper_bound: 7 repeats: 0\n" + "num: 8 lower_bound: 8 upper_bound: 15 repeats: 0\n" + "num: 14 lower_bound: 16 upper_bound: 50 repeats: 0", @@ -1569,7 +1569,7 @@ func (s *testStatsSuite) TestUnsignedFeedbackRanges(c *C) { }, { sql: "select count(*) from t", - hist: "column:1 ndv:30 totColSize:0\n" + + hist: "column:1 ndv:30 totColSize:10\n" + "num: 8 lower_bound: 0 upper_bound: 7 repeats: 0\n" + "num: 8 lower_bound: 8 upper_bound: 15 repeats: 0\n" + "num: 14 lower_bound: 16 upper_bound: 255 repeats: 0", @@ -1577,7 +1577,7 @@ func (s *testStatsSuite) TestUnsignedFeedbackRanges(c *C) { }, { sql: "select * from t1 where a <= 50", - hist: "column:1 ndv:30 totColSize:0\n" + + hist: "column:1 ndv:30 totColSize:10\n" + "num: 8 lower_bound: 0 upper_bound: 7 repeats: 0\n" + "num: 8 lower_bound: 8 upper_bound: 15 repeats: 0\n" + "num: 14 lower_bound: 16 upper_bound: 50 repeats: 0", @@ -1585,7 +1585,7 @@ func (s *testStatsSuite) TestUnsignedFeedbackRanges(c *C) { }, { sql: "select count(*) from t1", - hist: "column:1 ndv:30 totColSize:0\n" + + hist: "column:1 ndv:30 totColSize:10\n" + "num: 8 lower_bound: 0 upper_bound: 7 repeats: 0\n" + "num: 8 lower_bound: 8 upper_bound: 15 repeats: 0\n" + "num: 14 lower_bound: 16 upper_bound: 18446744073709551615 repeats: 0", @@ -1593,6 +1593,7 @@ func (s *testStatsSuite) TestUnsignedFeedbackRanges(c *C) { }, } is := s.do.InfoSchema() + c.Assert(h.Update(is), IsNil) for i, t := range tests { table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr(t.tblName)) c.Assert(err, IsNil) diff --git a/statistics/histogram.go b/statistics/histogram.go index f00873ce3a1dc..2ffbd933909a0 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -108,25 +108,34 @@ func (hg *Histogram) GetUpper(idx int) *types.Datum { return &d } -// AvgColSize is the average column size of the histogram. -func (c *Column) AvgColSize(count int64) float64 { +// AvgColSize is the average column size of the histogram. These sizes are derived from function `encode` +// and `Datum::ConvertTo`, so we need to update them if those 2 functions are changed. +func (c *Column) AvgColSize(count int64, isKey bool) float64 { if count == 0 { return 0 } - switch c.Histogram.Tp.Tp { - case mysql.TypeFloat: - return 4 - case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, - mysql.TypeDouble, mysql.TypeYear: + // Note that, if the handle column is encoded as value, instead of key, i.e, + // when the handle column is in a unique index, the real column size may be + // smaller than 8 because it is encoded using `EncodeVarint`. Since we don't + // know the exact value size now, use 8 as approximation. + if c.IsHandle { return 8 - case mysql.TypeDuration, mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: - return 16 - case mysql.TypeNewDecimal: - return types.MyDecimalStructSize - default: - // Keep two decimal place. - return math.Round(float64(c.TotColSize)/float64(count)*100) / 100 } + histCount := c.TotalRowCount() + notNullRatio := 1.0 + if histCount > 0 { + notNullRatio = 1.0 - float64(c.NullCount)/histCount + } + switch c.Histogram.Tp.Tp { + case mysql.TypeFloat, mysql.TypeDouble, mysql.TypeDuration, mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: + return 8 * notNullRatio + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear, mysql.TypeEnum, mysql.TypeBit, mysql.TypeSet: + if isKey { + return 8 * notNullRatio + } + } + // Keep two decimal place. + return math.Round(float64(c.TotColSize)/float64(count)*100) / 100 } // AppendBucket appends a bucket into `hg`. diff --git a/table/tables/tables.go b/table/tables/tables.go index 5a1aa172482bd..e6b04136e0e3c 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -314,7 +314,9 @@ func (t *tableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, new } key := t.RecordKey(h) - value, err := tablecodec.EncodeRow(ctx.GetSessionVars().StmtCtx, row, colIDs, nil, nil) + sessVars := ctx.GetSessionVars() + sc := sessVars.StmtCtx + value, err := tablecodec.EncodeRow(sc, row, colIDs, nil, nil) if err != nil { return err } @@ -338,13 +340,21 @@ func (t *tableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, new } } colSize := make(map[int64]int64) + encodedCol := make([]byte, 0, 16) for id, col := range t.Cols() { - val := int64(len(newData[id].GetBytes()) - len(oldData[id].GetBytes())) - if val != 0 { - colSize[col.ID] = val + encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], newData[id]) + if err != nil { + continue } + newLen := len(encodedCol) - 1 + encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], oldData[id]) + if err != nil { + continue + } + oldLen := len(encodedCol) - 1 + colSize[col.ID] = int64(newLen - oldLen) } - ctx.GetSessionVars().TxnCtx.UpdateDeltaForTable(t.physicalTableID, 0, 1, colSize) + sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 0, 1, colSize) return nil } @@ -504,7 +514,8 @@ func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts .. writeBufs := sessVars.GetWriteStmtBufs() adjustRowValuesBuf(writeBufs, len(row)) key := t.RecordKey(recordID) - writeBufs.RowValBuf, err = tablecodec.EncodeRow(ctx.GetSessionVars().StmtCtx, row, colIDs, writeBufs.RowValBuf, writeBufs.AddRowValues) + sc := sessVars.StmtCtx + writeBufs.RowValBuf, err = tablecodec.EncodeRow(sc, row, colIDs, writeBufs.RowValBuf, writeBufs.AddRowValues) if err != nil { return 0, err } @@ -532,13 +543,15 @@ func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts .. return 0, err } } - sessVars.StmtCtx.AddAffectedRows(1) + sc.AddAffectedRows(1) colSize := make(map[int64]int64) + encodedCol := make([]byte, 0, 16) for id, col := range t.Cols() { - val := int64(len(r[id].GetBytes())) - if val != 0 { - colSize[col.ID] = val + encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], r[id]) + if err != nil { + continue } + colSize[col.ID] = int64(len(encodedCol) - 1) } sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 1, 1, colSize) return recordID, nil @@ -714,11 +727,14 @@ func (t *tableCommon) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Da err = t.addDeleteBinlog(ctx, binlogRow, colIDs) } colSize := make(map[int64]int64) + encodedCol := make([]byte, 0, 16) + sc := ctx.GetSessionVars().StmtCtx for id, col := range t.Cols() { - val := -int64(len(r[id].GetBytes())) - if val != 0 { - colSize[col.ID] = val + encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], r[id]) + if err != nil { + continue } + colSize[col.ID] = -int64(len(encodedCol) - 1) } ctx.GetSessionVars().TxnCtx.UpdateDeltaForTable(t.physicalTableID, -1, 1, colSize) return err diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index 62adacc54a9e8..0666427666fe0 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -209,14 +209,13 @@ func DecodeRowKey(key kv.Key) (int64, error) { } // EncodeValue encodes a go value to bytes. -func EncodeValue(sc *stmtctx.StatementContext, raw types.Datum) ([]byte, error) { +func EncodeValue(sc *stmtctx.StatementContext, b []byte, raw types.Datum) ([]byte, error) { var v types.Datum err := flatten(sc, raw, &v) if err != nil { - return nil, errors.Trace(err) + return nil, err } - b, err := codec.EncodeValue(sc, nil, v) - return b, errors.Trace(err) + return codec.EncodeValue(sc, b, v) } // EncodeRow encode row data and column ids into a slice of byte. diff --git a/tablecodec/tablecodec_test.go b/tablecodec/tablecodec_test.go index 7c339d8b7ef14..4d4175282ee28 100644 --- a/tablecodec/tablecodec_test.go +++ b/tablecodec/tablecodec_test.go @@ -243,11 +243,11 @@ func (s *testTableCodecSuite) TestCutRow(c *C) { sc := &stmtctx.StatementContext{TimeZone: time.UTC} data := make([][]byte, 3) - data[0], err = EncodeValue(sc, row[0]) + data[0], err = EncodeValue(sc, nil, row[0]) c.Assert(err, IsNil) - data[1], err = EncodeValue(sc, row[1]) + data[1], err = EncodeValue(sc, nil, row[1]) c.Assert(err, IsNil) - data[2], err = EncodeValue(sc, row[2]) + data[2], err = EncodeValue(sc, nil, row[2]) c.Assert(err, IsNil) // Encode colIDs := make([]int64, 0, 3) @@ -490,3 +490,24 @@ func BenchmarkHasTablePrefixBuiltin(b *testing.B) { k.HasPrefix(tablePrefix) } } + +// Bench result: +// BenchmarkEncodeValue 5000000 368 ns/op +func BenchmarkEncodeValue(b *testing.B) { + row := make([]types.Datum, 7) + row[0] = types.NewIntDatum(100) + row[1] = types.NewBytesDatum([]byte("abc")) + row[2] = types.NewDecimalDatum(types.NewDecFromInt(1)) + row[3] = types.NewMysqlEnumDatum(types.Enum{Name: "a", Value: 0}) + row[4] = types.NewDatum(types.Set{Name: "a", Value: 0}) + row[5] = types.NewDatum(types.BinaryLiteral{100}) + row[6] = types.NewFloat32Datum(1.5) + b.ResetTimer() + encodedCol := make([]byte, 0, 16) + for i := 0; i < b.N; i++ { + for _, d := range row { + encodedCol = encodedCol[:0] + EncodeValue(nil, encodedCol, d) + } + } +} From d381d84d1bf9b11734bc96753fcbfe397d09177a Mon Sep 17 00:00:00 2001 From: Du Chuan Date: Mon, 15 Jul 2019 15:47:17 +0800 Subject: [PATCH 030/196] executor,sessionctx: refine output format for tidb-specific variables of boolean type (#11239) --- executor/set_test.go | 20 ++++++++++++++++++-- sessionctx/variable/varsutil.go | 21 +++++++++------------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/executor/set_test.go b/executor/set_test.go index 421b17071f293..fb370b1e855da 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -429,9 +429,25 @@ func (s *testSuite2) TestValidateSetVar(c *C) { c.Assert(terror.ErrorEqual(err, variable.ErrWrongValueForVar), IsTrue, Commentf("err %v", err)) tk.MustExec("set @@tidb_batch_delete='On';") + tk.MustQuery("select @@tidb_batch_delete;").Check(testkit.Rows("1")) tk.MustExec("set @@tidb_batch_delete='oFf';") + tk.MustQuery("select @@tidb_batch_delete;").Check(testkit.Rows("0")) tk.MustExec("set @@tidb_batch_delete=1;") + tk.MustQuery("select @@tidb_batch_delete;").Check(testkit.Rows("1")) tk.MustExec("set @@tidb_batch_delete=0;") + tk.MustQuery("select @@tidb_batch_delete;").Check(testkit.Rows("0")) + + tk.MustExec("set @@tidb_opt_agg_push_down=off;") + tk.MustQuery("select @@tidb_opt_agg_push_down;").Check(testkit.Rows("0")) + + tk.MustExec("set @@tidb_constraint_check_in_place=on;") + tk.MustQuery("select @@tidb_constraint_check_in_place;").Check(testkit.Rows("1")) + + tk.MustExec("set @@tidb_general_log=0;") + tk.MustQuery("select @@tidb_general_log;").Check(testkit.Rows("0")) + + tk.MustExec("set @@tidb_enable_streaming=1;") + tk.MustQuery("select @@tidb_enable_streaming;").Check(testkit.Rows("1")) _, err = tk.Exec("set @@tidb_batch_delete=3;") c.Assert(terror.ErrorEqual(err, variable.ErrWrongValueForVar), IsTrue, Commentf("err %v", err)) @@ -789,9 +805,9 @@ func (s *testSuite2) TestEnableNoopFunctionsVar(c *C) { _, err = tk.Exec(`set tidb_enable_noop_functions=11`) c.Assert(err, NotNil) tk.MustExec(`set tidb_enable_noop_functions="off";`) - tk.MustQuery(`select @@tidb_enable_noop_functions;`).Check(testkit.Rows("off")) + tk.MustQuery(`select @@tidb_enable_noop_functions;`).Check(testkit.Rows("0")) tk.MustExec(`set tidb_enable_noop_functions="on";`) - tk.MustQuery(`select @@tidb_enable_noop_functions;`).Check(testkit.Rows("on")) + tk.MustQuery(`select @@tidb_enable_noop_functions;`).Check(testkit.Rows("1")) tk.MustExec(`set tidb_enable_noop_functions=0;`) tk.MustQuery(`select @@tidb_enable_noop_functions;`).Check(testkit.Rows("0")) } diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 65f69300ae61d..47a60f1a7b7ed 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -374,9 +374,16 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, return "1", nil } return value, ErrWrongValueForVar.GenWithStackByArgs(name, value) - case GeneralLog, TiDBGeneralLog, AvoidTemporalUpgrade, BigTables, CheckProxyUsers, LogBin, + case TiDBSkipUTF8Check, TiDBOptAggPushDown, + TiDBOptInSubqToJoinAndAgg, TiDBEnableFastAnalyze, + TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, + TiDBBatchDelete, TiDBBatchCommit, TiDBEnableCascadesPlanner, TiDBEnableWindowFunction, + TiDBCheckMb4ValueInUTF8, TiDBLowResolutionTSO, TiDBEnableIndexMerge, TiDBEnableNoopFuncs, + TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace: + fallthrough + case GeneralLog, AvoidTemporalUpgrade, BigTables, CheckProxyUsers, LogBin, CoreFile, EndMakersInJSON, SQLLogBin, OfflineMode, PseudoSlaveMode, LowPriorityUpdates, - SkipNameResolve, SQLSafeUpdates, TiDBConstraintCheckInPlace, serverReadOnly, SlaveAllowBatching, + SkipNameResolve, SQLSafeUpdates, serverReadOnly, SlaveAllowBatching, Flush, PerformanceSchema, LocalInFile, ShowOldTemporals, KeepFilesOnCreate, AutoCommit, SQLWarnings, UniqueChecks, OldAlterTable, LogBinTrustFunctionCreators, SQLBigSelects, BinlogDirectNonTransactionalUpdates, SQLQuoteShowCreate, AutomaticSpPrivileges, @@ -415,16 +422,6 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, } } return value, ErrWrongValueForVar.GenWithStackByArgs(name, value) - case TiDBSkipUTF8Check, TiDBOptAggPushDown, - TiDBOptInSubqToJoinAndAgg, TiDBEnableFastAnalyze, - TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, - TiDBBatchDelete, TiDBBatchCommit, TiDBEnableCascadesPlanner, TiDBEnableWindowFunction, - TiDBCheckMb4ValueInUTF8, TiDBLowResolutionTSO, TiDBEnableIndexMerge, TiDBEnableNoopFuncs, - TiDBScatterRegion: - if strings.EqualFold(value, "ON") || value == "1" || strings.EqualFold(value, "OFF") || value == "0" { - return value, nil - } - return value, ErrWrongValueForVar.GenWithStackByArgs(name, value) case MaxExecutionTime: return checkUInt64SystemVar(name, value, 0, math.MaxUint64, vars) case TiDBEnableTablePartition: From d420a1f212dccea7125ac00087c3d3a11e278fd2 Mon Sep 17 00:00:00 2001 From: lysu Date: Mon, 15 Jul 2019 16:11:14 +0800 Subject: [PATCH 031/196] *: reduce tikvrpc request size (#11055) --- executor/analyze.go | 9 +- executor/analyze_test.go | 2 +- server/http_handler.go | 9 +- store/helper/helper.go | 7 +- store/mockstore/mocktikv/rpc.go | 54 ++-- store/tikv/2pc.go | 93 +++---- store/tikv/2pc_test.go | 15 +- store/tikv/client_fail_test.go | 5 +- store/tikv/coprocessor.go | 26 +- store/tikv/delete_range.go | 13 +- store/tikv/gcworker/gc_worker.go | 33 +-- store/tikv/gcworker/gc_worker_test.go | 8 +- store/tikv/lock_resolver.go | 30 +-- store/tikv/lock_test.go | 11 +- store/tikv/rawkv.go | 91 +++---- store/tikv/region_request_test.go | 66 ++--- store/tikv/scan.go | 29 +-- store/tikv/snapshot.go | 29 +-- store/tikv/split_region.go | 12 +- store/tikv/tikvrpc/tikvrpc.go | 346 ++++++++++++++++++-------- 20 files changed, 438 insertions(+), 450 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index b6a9389af9df3..cb2840e3d05b1 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -620,12 +620,9 @@ func (e *AnalyzeFastExec) getSampRegionsRowCount(bo *tikv.Backoffer, needRebuild if !ok { return } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdDebugGetRegionProperties, - DebugGetRegionProperties: &debugpb.GetRegionPropertiesRequest{ - RegionId: loc.Region.GetID(), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdDebugGetRegionProperties, &debugpb.GetRegionPropertiesRequest{ + RegionId: loc.Region.GetID(), + }) var resp *tikvrpc.Response var rpcCtx *tikv.RPCContext rpcCtx, *err = e.cache.GetRPCContext(bo, loc.Region) diff --git a/executor/analyze_test.go b/executor/analyze_test.go index ec3d98e188d42..50577ca7c50ba 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -400,7 +400,7 @@ func (c *regionProperityClient) SendRequest(ctx context.Context, addr string, re defer c.mu.Unlock() c.mu.count++ // Mock failure once. - if req.DebugGetRegionProperties.RegionId == c.mu.regionID { + if req.DebugGetRegionProperties().RegionId == c.mu.regionID { c.mu.regionID = 0 return &tikvrpc.Response{}, nil } diff --git a/server/http_handler.go b/server/http_handler.go index 3e14f6e044685..deaf0b7c316cb 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -151,12 +151,9 @@ func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey []by return nil, errors.Trace(err) } - tikvReq := &tikvrpc.Request{ - Type: tikvrpc.CmdMvccGetByStartTs, - MvccGetByStartTs: &kvrpcpb.MvccGetByStartTsRequest{ - StartTs: startTS, - }, - } + tikvReq := tikvrpc.NewRequest(tikvrpc.CmdMvccGetByStartTs, &kvrpcpb.MvccGetByStartTsRequest{ + StartTs: startTS, + }) tikvReq.Context.Priority = kvrpcpb.CommandPri_Low kvResp, err := t.Store.SendReq(bo, tikvReq, curRegion.Region, time.Hour) if err != nil { diff --git a/store/helper/helper.go b/store/helper/helper.go index f36616eade9d6..43cbcf0177e0c 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -50,12 +50,7 @@ func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyRe return nil, errors.Trace(err) } - tikvReq := &tikvrpc.Request{ - Type: tikvrpc.CmdMvccGetByKey, - MvccGetByKey: &kvrpcpb.MvccGetByKeyRequest{ - Key: encodedKey, - }, - } + tikvReq := tikvrpc.NewRequest(tikvrpc.CmdMvccGetByKey, &kvrpcpb.MvccGetByKeyRequest{Key: encodedKey}) kvResp, err := h.Store.SendReq(tikv.NewBackoffer(context.Background(), 500), tikvReq, keyLocation.Region, time.Minute) if err != nil { logutil.BgLogger().Info("get MVCC by encoded key failed", diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index 5e9d253cc4ffe..fb9f104b7a3c0 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -671,14 +671,14 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.Type = req.Type switch req.Type { case tikvrpc.CmdGet: - r := req.Get + r := req.Get() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.Get = &kvrpcpb.GetResponse{RegionError: err} return resp, nil } resp.Get = handler.handleKvGet(r) case tikvrpc.CmdScan: - r := req.Scan + r := req.Scan() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.Scan = &kvrpcpb.ScanResponse{RegionError: err} return resp, nil @@ -686,21 +686,21 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.Scan = handler.handleKvScan(r) case tikvrpc.CmdPrewrite: - r := req.Prewrite + r := req.Prewrite() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.Prewrite = &kvrpcpb.PrewriteResponse{RegionError: err} return resp, nil } resp.Prewrite = handler.handleKvPrewrite(r) case tikvrpc.CmdPessimisticLock: - r := req.PessimisticLock + r := req.PessimisticLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.PessimisticLock = &kvrpcpb.PessimisticLockResponse{RegionError: err} return resp, nil } resp.PessimisticLock = handler.handleKvPessimisticLock(r) case tikvrpc.CmdPessimisticRollback: - r := req.PessimisticRollback + r := req.PessimisticRollback() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.PessimisticRollback = &kvrpcpb.PessimisticRollbackResponse{RegionError: err} return resp, nil @@ -724,7 +724,7 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } }) - r := req.Commit + r := req.Commit() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.Commit = &kvrpcpb.CommitResponse{RegionError: err} return resp, nil @@ -736,104 +736,104 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } }) case tikvrpc.CmdCleanup: - r := req.Cleanup + r := req.Cleanup() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.Cleanup = &kvrpcpb.CleanupResponse{RegionError: err} return resp, nil } resp.Cleanup = handler.handleKvCleanup(r) case tikvrpc.CmdBatchGet: - r := req.BatchGet + r := req.BatchGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.BatchGet = &kvrpcpb.BatchGetResponse{RegionError: err} return resp, nil } resp.BatchGet = handler.handleKvBatchGet(r) case tikvrpc.CmdBatchRollback: - r := req.BatchRollback + r := req.BatchRollback() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.BatchRollback = &kvrpcpb.BatchRollbackResponse{RegionError: err} return resp, nil } resp.BatchRollback = handler.handleKvBatchRollback(r) case tikvrpc.CmdScanLock: - r := req.ScanLock + r := req.ScanLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.ScanLock = &kvrpcpb.ScanLockResponse{RegionError: err} return resp, nil } resp.ScanLock = handler.handleKvScanLock(r) case tikvrpc.CmdResolveLock: - r := req.ResolveLock + r := req.ResolveLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.ResolveLock = &kvrpcpb.ResolveLockResponse{RegionError: err} return resp, nil } resp.ResolveLock = handler.handleKvResolveLock(r) case tikvrpc.CmdGC: - r := req.GC + r := req.GC() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.GC = &kvrpcpb.GCResponse{RegionError: err} return resp, nil } resp.GC = &kvrpcpb.GCResponse{} case tikvrpc.CmdDeleteRange: - r := req.DeleteRange + r := req.DeleteRange() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.DeleteRange = &kvrpcpb.DeleteRangeResponse{RegionError: err} return resp, nil } resp.DeleteRange = handler.handleKvDeleteRange(r) case tikvrpc.CmdRawGet: - r := req.RawGet + r := req.RawGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawGet = &kvrpcpb.RawGetResponse{RegionError: err} return resp, nil } resp.RawGet = handler.handleKvRawGet(r) case tikvrpc.CmdRawBatchGet: - r := req.RawBatchGet + r := req.RawBatchGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawBatchGet = &kvrpcpb.RawBatchGetResponse{RegionError: err} return resp, nil } resp.RawBatchGet = handler.handleKvRawBatchGet(r) case tikvrpc.CmdRawPut: - r := req.RawPut + r := req.RawPut() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawPut = &kvrpcpb.RawPutResponse{RegionError: err} return resp, nil } resp.RawPut = handler.handleKvRawPut(r) case tikvrpc.CmdRawBatchPut: - r := req.RawBatchPut + r := req.RawBatchPut() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawBatchPut = &kvrpcpb.RawBatchPutResponse{RegionError: err} return resp, nil } resp.RawBatchPut = handler.handleKvRawBatchPut(r) case tikvrpc.CmdRawDelete: - r := req.RawDelete + r := req.RawDelete() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawDelete = &kvrpcpb.RawDeleteResponse{RegionError: err} return resp, nil } resp.RawDelete = handler.handleKvRawDelete(r) case tikvrpc.CmdRawBatchDelete: - r := req.RawBatchDelete + r := req.RawBatchDelete() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawBatchDelete = &kvrpcpb.RawBatchDeleteResponse{RegionError: err} } resp.RawBatchDelete = handler.handleKvRawBatchDelete(r) case tikvrpc.CmdRawDeleteRange: - r := req.RawDeleteRange + r := req.RawDeleteRange() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawDeleteRange = &kvrpcpb.RawDeleteRangeResponse{RegionError: err} return resp, nil } resp.RawDeleteRange = handler.handleKvRawDeleteRange(r) case tikvrpc.CmdRawScan: - r := req.RawScan + r := req.RawScan() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.RawScan = &kvrpcpb.RawScanResponse{RegionError: err} return resp, nil @@ -842,7 +842,7 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R case tikvrpc.CmdUnsafeDestroyRange: panic("unimplemented") case tikvrpc.CmdCop: - r := req.Cop + r := req.Cop() if err := handler.checkRequestContext(reqCtx); err != nil { resp.Cop = &coprocessor.Response{RegionError: err} return resp, nil @@ -862,7 +862,7 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } resp.Cop = res case tikvrpc.CmdCopStream: - r := req.Cop + r := req.Cop() if err := handler.checkRequestContext(reqCtx); err != nil { resp.CopStream = &tikvrpc.CopStreamResponse{ Tikv_CoprocessorStreamClient: &mockCopStreamErrClient{Error: err}, @@ -895,21 +895,21 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R streamResp.Response = first resp.CopStream = streamResp case tikvrpc.CmdMvccGetByKey: - r := req.MvccGetByKey + r := req.MvccGetByKey() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.MvccGetByKey = &kvrpcpb.MvccGetByKeyResponse{RegionError: err} return resp, nil } resp.MvccGetByKey = handler.handleMvccGetByKey(r) case tikvrpc.CmdMvccGetByStartTs: - r := req.MvccGetByStartTs + r := req.MvccGetByStartTs() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.MvccGetByStartTS = &kvrpcpb.MvccGetByStartTsResponse{RegionError: err} return resp, nil } resp.MvccGetByStartTS = handler.handleMvccGetByStartTS(r) case tikvrpc.CmdSplitRegion: - r := req.SplitRegion + r := req.SplitRegion() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { resp.SplitRegion = &kvrpcpb.SplitRegionResponse{RegionError: err} return resp, nil @@ -917,7 +917,7 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.SplitRegion = handler.handleSplitRegion(r) // DebugGetRegionProperties is for fast analyze in mock tikv. case tikvrpc.CmdDebugGetRegionProperties: - r := req.DebugGetRegionProperties + r := req.DebugGetRegionProperties() region, _ := c.Cluster.GetRegion(r.RegionId) scanResp := handler.handleKvScan(&kvrpcpb.ScanRequest{StartKey: MvccKey(region.StartKey).Raw(), EndKey: MvccKey(region.EndKey).Raw(), Version: math.MaxUint64, Limit: math.MaxUint32}) resp.DebugGetRegionProperties = &debugpb.GetRegionPropertiesResponse{ diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 04b2e7f2b2c55..5db1f24307e06 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -473,22 +473,16 @@ func (c *twoPhaseCommitter) buildPrewriteRequest(batch batchKeys) *tikvrpc.Reque isPessimisticLock[i] = true } } - return &tikvrpc.Request{ - Type: tikvrpc.CmdPrewrite, - Prewrite: &pb.PrewriteRequest{ - Mutations: mutations, - PrimaryLock: c.primary(), - StartVersion: c.startTS, - LockTtl: c.lockTTL, - IsPessimisticLock: isPessimisticLock, - ForUpdateTs: c.forUpdateTS, - TxnSize: uint64(len(batch.keys)), - }, - Context: pb.Context{ - Priority: c.priority, - SyncLog: c.syncLog, - }, + req := &pb.PrewriteRequest{ + Mutations: mutations, + PrimaryLock: c.primary(), + StartVersion: c.startTS, + LockTtl: c.lockTTL, + IsPessimisticLock: isPessimisticLock, + ForUpdateTs: c.forUpdateTS, + TxnSize: uint64(len(batch.keys)), } + return tikvrpc.NewRequest(tikvrpc.CmdPrewrite, req, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) } func (c *twoPhaseCommitter) prewriteSingleBatch(bo *Backoffer, batch batchKeys) error { @@ -572,21 +566,14 @@ func (c *twoPhaseCommitter) pessimisticLockSingleBatch(bo *Backoffer, batch batc mutations[i] = mut } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdPessimisticLock, - PessimisticLock: &pb.PessimisticLockRequest{ - Mutations: mutations, - PrimaryLock: c.primary(), - StartVersion: c.startTS, - ForUpdateTs: c.forUpdateTS, - LockTtl: PessimisticLockTTL, - IsFirstLock: c.isFirstLock, - }, - Context: pb.Context{ - Priority: c.priority, - SyncLog: c.syncLog, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticLock, &pb.PessimisticLockRequest{ + Mutations: mutations, + PrimaryLock: c.primary(), + StartVersion: c.startTS, + ForUpdateTs: c.forUpdateTS, + LockTtl: PessimisticLockTTL, + IsFirstLock: c.isFirstLock, + }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) for { resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort) if err != nil { @@ -643,14 +630,11 @@ func (c *twoPhaseCommitter) pessimisticLockSingleBatch(bo *Backoffer, batch batc } func (c *twoPhaseCommitter) pessimisticRollbackSingleBatch(bo *Backoffer, batch batchKeys) error { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdPessimisticRollback, - PessimisticRollback: &pb.PessimisticRollbackRequest{ - StartVersion: c.startTS, - ForUpdateTs: c.forUpdateTS, - Keys: batch.keys, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdPessimisticRollback, &pb.PessimisticRollbackRequest{ + StartVersion: c.startTS, + ForUpdateTs: c.forUpdateTS, + Keys: batch.keys, + }) for { resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort) if err != nil { @@ -709,19 +693,11 @@ func (c *twoPhaseCommitter) getUndeterminedErr() error { } func (c *twoPhaseCommitter) commitSingleBatch(bo *Backoffer, batch batchKeys) error { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdCommit, - Commit: &pb.CommitRequest{ - StartVersion: c.startTS, - Keys: batch.keys, - CommitVersion: c.commitTS, - }, - Context: pb.Context{ - Priority: c.priority, - SyncLog: c.syncLog, - }, - } - req.Context.Priority = c.priority + req := tikvrpc.NewRequest(tikvrpc.CmdCommit, &pb.CommitRequest{ + StartVersion: c.startTS, + Keys: batch.keys, + CommitVersion: c.commitTS, + }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) sender := NewRegionRequestSender(c.store.regionCache, c.store.client) resp, err := sender.SendReq(bo, req, batch.region, readTimeoutShort) @@ -789,17 +765,10 @@ func (c *twoPhaseCommitter) commitSingleBatch(bo *Backoffer, batch batchKeys) er } func (c *twoPhaseCommitter) cleanupSingleBatch(bo *Backoffer, batch batchKeys) error { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdBatchRollback, - BatchRollback: &pb.BatchRollbackRequest{ - Keys: batch.keys, - StartVersion: c.startTS, - }, - Context: pb.Context{ - Priority: c.priority, - SyncLog: c.syncLog, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdBatchRollback, &pb.BatchRollbackRequest{ + Keys: batch.keys, + StartVersion: c.startTS, + }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) resp, err := c.store.SendReq(bo, req, batch.region, readTimeoutShort) if err != nil { return errors.Trace(err) diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index b1fc5d69a9e5e..52883f24247cf 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -237,13 +237,10 @@ func (s *testCommitterSuite) isKeyLocked(c *C, key []byte) bool { ver, err := s.store.CurrentVersion() c.Assert(err, IsNil) bo := NewBackoffer(context.Background(), getMaxBackoff) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdGet, - Get: &kvrpcpb.GetRequest{ - Key: key, - Version: ver.Ver, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdGet, &kvrpcpb.GetRequest{ + Key: key, + Version: ver.Ver, + }) loc, err := s.store.regionCache.LocateKey(bo, key) c.Assert(err, IsNil) resp, err := s.store.SendReq(bo, req, loc.Region, readTimeoutShort) @@ -455,8 +452,8 @@ func (s *testCommitterSuite) TestPessimisticPrewriteRequest(c *C) { batch.keys = append(batch.keys, []byte("t1")) batch.region = RegionVerID{1, 1, 1} req := commiter.buildPrewriteRequest(batch) - c.Assert(len(req.Prewrite.IsPessimisticLock), Greater, 0) - c.Assert(req.Prewrite.ForUpdateTs, Equals, uint64(100)) + c.Assert(len(req.Prewrite().IsPessimisticLock), Greater, 0) + c.Assert(req.Prewrite().ForUpdateTs, Equals, uint64(100)) } func (s *testCommitterSuite) TestUnsetPrimaryKey(c *C) { diff --git a/store/tikv/client_fail_test.go b/store/tikv/client_fail_test.go index 5fc73b6ff1152..74c67ab25b0a5 100644 --- a/store/tikv/client_fail_test.go +++ b/store/tikv/client_fail_test.go @@ -52,10 +52,7 @@ func (s *testClientSuite) TestPanicInRecvLoop(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop"), IsNil) time.Sleep(time.Second) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdEmpty, - Empty: &tikvpb.BatchCommandsEmptyRequest{}, - } + req := tikvrpc.NewRequest(tikvrpc.CmdEmpty, &tikvpb.BatchCommandsEmptyRequest{}) _, err = rpcClient.SendRequest(context.Background(), addr, req, time.Second) c.Assert(err, IsNil) server.Stop() diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index e0b74f113b8c8..b3e96859a5f7b 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -612,21 +612,17 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch }) sender := NewRegionRequestSender(worker.store.regionCache, worker.store.client) - req := &tikvrpc.Request{ - Type: task.cmdType, - Cop: &coprocessor.Request{ - Tp: worker.req.Tp, - Data: worker.req.Data, - Ranges: task.ranges.toPBRanges(), - }, - Context: kvrpcpb.Context{ - IsolationLevel: pbIsolationLevel(worker.req.IsolationLevel), - Priority: kvPriorityToCommandPri(worker.req.Priority), - NotFillCache: worker.req.NotFillCache, - HandleTime: true, - ScanDetail: true, - }, - } + req := tikvrpc.NewRequest(task.cmdType, &coprocessor.Request{ + Tp: worker.req.Tp, + Data: worker.req.Data, + Ranges: task.ranges.toPBRanges(), + }, kvrpcpb.Context{ + IsolationLevel: pbIsolationLevel(worker.req.IsolationLevel), + Priority: kvPriorityToCommandPri(worker.req.Priority), + NotFillCache: worker.req.NotFillCache, + HandleTime: true, + ScanDetail: true, + }) startTime := time.Now() resp, rpcCtx, err := sender.SendReqCtx(bo, req, task.region, ReadTimeoutMedium) if err != nil { diff --git a/store/tikv/delete_range.go b/store/tikv/delete_range.go index f721ae6305030..aa719c8977c81 100644 --- a/store/tikv/delete_range.go +++ b/store/tikv/delete_range.go @@ -104,14 +104,11 @@ func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (Ra endKey = rangeEndKey } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdDeleteRange, - DeleteRange: &kvrpcpb.DeleteRangeRequest{ - StartKey: startKey, - EndKey: endKey, - NotifyOnly: t.notifyOnly, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdDeleteRange, &kvrpcpb.DeleteRangeRequest{ + StartKey: startKey, + EndKey: endKey, + NotifyOnly: t.notifyOnly, + }) resp, err := t.store.SendReq(bo, req, loc.Region, ReadTimeoutMedium) if err != nil { diff --git a/store/tikv/gcworker/gc_worker.go b/store/tikv/gcworker/gc_worker.go index e26cf40b5c19e..522b9226ced39 100644 --- a/store/tikv/gcworker/gc_worker.go +++ b/store/tikv/gcworker/gc_worker.go @@ -658,13 +658,10 @@ func (w *GCWorker) doUnsafeDestroyRangeRequest(ctx context.Context, startKey []b return errors.Trace(err) } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdUnsafeDestroyRange, - UnsafeDestroyRange: &kvrpcpb.UnsafeDestroyRangeRequest{ - StartKey: startKey, - EndKey: endKey, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdUnsafeDestroyRange, &kvrpcpb.UnsafeDestroyRangeRequest{ + StartKey: startKey, + EndKey: endKey, + }) var wg sync.WaitGroup errChan := make(chan error, len(stores)) @@ -821,13 +818,10 @@ func (w *GCWorker) resolveLocksForRange(ctx context.Context, safePoint uint64, s // for scan lock request, we must return all locks even if they are generated // by the same transaction. because gc worker need to make sure all locks have been // cleaned. - req := &tikvrpc.Request{ - Type: tikvrpc.CmdScanLock, - ScanLock: &kvrpcpb.ScanLockRequest{ - MaxVersion: safePoint, - Limit: gcScanLockLimit, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdScanLock, &kvrpcpb.ScanLockRequest{ + MaxVersion: safePoint, + Limit: gcScanLockLimit, + }) var stat tikv.RangeTaskStat key := startKey @@ -840,7 +834,7 @@ func (w *GCWorker) resolveLocksForRange(ctx context.Context, safePoint uint64, s bo := tikv.NewBackoffer(ctx, tikv.GcResolveLockMaxBackoff) - req.ScanLock.StartKey = key + req.ScanLock().StartKey = key loc, err := w.store.GetRegionCache().LocateKey(bo, key) if err != nil { return stat, errors.Trace(err) @@ -986,12 +980,9 @@ func (w *GCWorker) doGCForRange(ctx context.Context, startKey []byte, endKey []b // doGCForRegion used for gc for region. // these two errors should not return together, for more, see the func 'doGC' func (w *GCWorker) doGCForRegion(bo *tikv.Backoffer, safePoint uint64, region tikv.RegionVerID) (*errorpb.Error, error) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdGC, - GC: &kvrpcpb.GCRequest{ - SafePoint: safePoint, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdGC, &kvrpcpb.GCRequest{ + SafePoint: safePoint, + }) resp, err := w.store.SendReq(bo, req, region, tikv.GCTimeout) if err != nil { diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index fc2f14c9bca54..781b6be23b689 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -402,7 +402,7 @@ func (s *testGCWorkerSuite) testDeleteRangesFailureImpl(c *C, failType int) { Type: tikvrpc.CmdUnsafeDestroyRange, UnsafeDestroyRange: &kvrpcpb.UnsafeDestroyRangeResponse{}, } - if bytes.Equal(req.UnsafeDestroyRange.GetStartKey(), failKey) && addr == failStore.GetAddress() { + if bytes.Equal(req.UnsafeDestroyRange().GetStartKey(), failKey) && addr == failStore.GetAddress() { if failType == failRPCErr { return nil, errors.New("error") } else if failType == failNilResp { @@ -488,7 +488,7 @@ Loop: } sort.Slice(sentReq, func(i, j int) bool { - cmp := bytes.Compare(sentReq[i].req.UnsafeDestroyRange.StartKey, sentReq[j].req.UnsafeDestroyRange.StartKey) + cmp := bytes.Compare(sentReq[i].req.UnsafeDestroyRange().StartKey, sentReq[j].req.UnsafeDestroyRange().StartKey) return cmp < 0 || (cmp == 0 && sentReq[i].addr < sentReq[j].addr) }) @@ -501,8 +501,8 @@ Loop: for storeIndex := range expectedStores { i := rangeIndex*len(expectedStores) + storeIndex c.Assert(sentReq[i].addr, Equals, expectedStores[storeIndex].Address) - c.Assert(sentReq[i].req.UnsafeDestroyRange.GetStartKey(), DeepEquals, sortedRanges[rangeIndex].StartKey) - c.Assert(sentReq[i].req.UnsafeDestroyRange.GetEndKey(), DeepEquals, sortedRanges[rangeIndex].EndKey) + c.Assert(sentReq[i].req.UnsafeDestroyRange().GetStartKey(), DeepEquals, sortedRanges[rangeIndex].StartKey) + c.Assert(sentReq[i].req.UnsafeDestroyRange().GetEndKey(), DeepEquals, sortedRanges[rangeIndex].EndKey) } } } diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index ebabeb90621fc..d0fc33122a1dc 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -220,12 +220,7 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi }) } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdResolveLock, - ResolveLock: &kvrpcpb.ResolveLockRequest{ - TxnInfos: listTxnInfos, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdResolveLock, &kvrpcpb.ResolveLockRequest{TxnInfos: listTxnInfos}) startTime = time.Now() resp, err := lr.store.SendReq(bo, req, loc, readTimeoutShort) if err != nil { @@ -341,13 +336,10 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte tikvLockResolverCountWithQueryTxnStatus.Inc() var status TxnStatus - req := &tikvrpc.Request{ - Type: tikvrpc.CmdCleanup, - Cleanup: &kvrpcpb.CleanupRequest{ - Key: primary, - StartVersion: txnID, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdCleanup, &kvrpcpb.CleanupRequest{ + Key: primary, + StartVersion: txnID, + }) for { loc, err := lr.store.GetRegionCache().LocateKey(bo, primary) if err != nil { @@ -399,21 +391,19 @@ func (lr *LockResolver) resolveLock(bo *Backoffer, l *Lock, status TxnStatus, cl if _, ok := cleanRegions[loc.Region]; ok { return nil } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdResolveLock, - ResolveLock: &kvrpcpb.ResolveLockRequest{ - StartVersion: l.TxnID, - }, + lreq := &kvrpcpb.ResolveLockRequest{ + StartVersion: l.TxnID, } if status.IsCommitted() { - req.ResolveLock.CommitVersion = status.CommitTS() + lreq.CommitVersion = status.CommitTS() } if l.TxnSize < bigTxnThreshold { // Only resolve specified keys when it is a small transaction, // prevent from scanning the whole region in this case. tikvLockResolverCountWithResolveLockLite.Inc() - req.ResolveLock.Keys = [][]byte{l.Key} + lreq.Keys = [][]byte{l.Key} } + req := tikvrpc.NewRequest(tikvrpc.CmdResolveLock, lreq) resp, err := lr.store.SendReq(bo, req, loc.Region, readTimeoutShort) if err != nil { return errors.Trace(err) diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index 3b9ed28f3fded..ea0aaf6aff78f 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -213,13 +213,10 @@ func (s *testLockSuite) mustGetLock(c *C, key []byte) *Lock { ver, err := s.store.CurrentVersion() c.Assert(err, IsNil) bo := NewBackoffer(context.Background(), getMaxBackoff) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdGet, - Get: &kvrpcpb.GetRequest{ - Key: key, - Version: ver.Ver, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdGet, &kvrpcpb.GetRequest{ + Key: key, + Version: ver.Ver, + }) loc, err := s.store.regionCache.LocateKey(bo, key) c.Assert(err, IsNil) resp, err := s.store.SendReq(bo, req, loc.Region, readTimeoutShort) diff --git a/store/tikv/rawkv.go b/store/tikv/rawkv.go index aea8c9abf52dd..9a43eb0437d68 100644 --- a/store/tikv/rawkv.go +++ b/store/tikv/rawkv.go @@ -104,12 +104,7 @@ func (c *RawKVClient) Get(key []byte) ([]byte, error) { start := time.Now() defer func() { tikvRawkvCmdHistogramWithGet.Observe(time.Since(start).Seconds()) }() - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawGet, - RawGet: &kvrpcpb.RawGetRequest{ - Key: key, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawGet, &kvrpcpb.RawGetRequest{Key: key}) resp, _, err := c.sendReq(key, req, false) if err != nil { return nil, errors.Trace(err) @@ -168,13 +163,10 @@ func (c *RawKVClient) Put(key, value []byte) error { return errors.New("empty value is not supported") } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: key, - Value: value, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: key, + Value: value, + }) resp, _, err := c.sendReq(key, req, false) if err != nil { return errors.Trace(err) @@ -214,12 +206,9 @@ func (c *RawKVClient) Delete(key []byte) error { start := time.Now() defer func() { tikvRawkvCmdHistogramWithDelete.Observe(time.Since(start).Seconds()) }() - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawDelete, - RawDelete: &kvrpcpb.RawDeleteRequest{ - Key: key, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawDelete, &kvrpcpb.RawDeleteRequest{ + Key: key, + }) resp, _, err := c.sendReq(key, req, false) if err != nil { return errors.Trace(err) @@ -303,14 +292,11 @@ func (c *RawKVClient) Scan(startKey, endKey []byte, limit int) (keys [][]byte, v } for len(keys) < limit { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawScan, - RawScan: &kvrpcpb.RawScanRequest{ - StartKey: startKey, - EndKey: endKey, - Limit: uint32(limit - len(keys)), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawScan, &kvrpcpb.RawScanRequest{ + StartKey: startKey, + EndKey: endKey, + Limit: uint32(limit - len(keys)), + }) resp, loc, err := c.sendReq(startKey, req, false) if err != nil { return nil, nil, errors.Trace(err) @@ -349,15 +335,12 @@ func (c *RawKVClient) ReverseScan(startKey, endKey []byte, limit int) (keys [][] } for len(keys) < limit { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawScan, - RawScan: &kvrpcpb.RawScanRequest{ - StartKey: startKey, - EndKey: endKey, - Limit: uint32(limit - len(keys)), - Reverse: true, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawScan, &kvrpcpb.RawScanRequest{ + StartKey: startKey, + EndKey: endKey, + Limit: uint32(limit - len(keys)), + Reverse: true, + }) resp, loc, err := c.sendReq(startKey, req, true) if err != nil { return nil, nil, errors.Trace(err) @@ -462,19 +445,13 @@ func (c *RawKVClient) doBatchReq(bo *Backoffer, batch batch, cmdType tikvrpc.Cmd var req *tikvrpc.Request switch cmdType { case tikvrpc.CmdRawBatchGet: - req = &tikvrpc.Request{ - Type: cmdType, - RawBatchGet: &kvrpcpb.RawBatchGetRequest{ - Keys: batch.keys, - }, - } + req = tikvrpc.NewRequest(cmdType, &kvrpcpb.RawBatchGetRequest{ + Keys: batch.keys, + }) case tikvrpc.CmdRawBatchDelete: - req = &tikvrpc.Request{ - Type: cmdType, - RawBatchDelete: &kvrpcpb.RawBatchDeleteRequest{ - Keys: batch.keys, - }, - } + req = tikvrpc.NewRequest(cmdType, &kvrpcpb.RawBatchDeleteRequest{ + Keys: batch.keys, + }) } sender := NewRegionRequestSender(c.regionCache, c.rpcClient) @@ -538,13 +515,10 @@ func (c *RawKVClient) sendDeleteRangeReq(startKey []byte, endKey []byte) (*tikvr actualEndKey = loc.EndKey } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawDeleteRange, - RawDeleteRange: &kvrpcpb.RawDeleteRangeRequest{ - StartKey: startKey, - EndKey: actualEndKey, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawDeleteRange, &kvrpcpb.RawDeleteRangeRequest{ + StartKey: startKey, + EndKey: actualEndKey, + }) resp, err := sender.SendReq(bo, req, loc.Region, readTimeoutShort) if err != nil { @@ -648,12 +622,7 @@ func (c *RawKVClient) doBatchPut(bo *Backoffer, batch batch) error { kvPair = append(kvPair, &kvrpcpb.KvPair{Key: key, Value: batch.values[i]}) } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawBatchPut, - RawBatchPut: &kvrpcpb.RawBatchPutRequest{ - Pairs: kvPair, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawBatchPut, &kvrpcpb.RawBatchPutRequest{Pairs: kvPair}) sender := NewRegionRequestSender(c.regionCache, c.rpcClient) resp, err := sender.SendReq(bo, req, batch.regionID, readTimeoutShort) diff --git a/store/tikv/region_request_test.go b/store/tikv/region_request_test.go index 52cc1636bef7e..5ee508e817d08 100644 --- a/store/tikv/region_request_test.go +++ b/store/tikv/region_request_test.go @@ -60,13 +60,10 @@ func (s *testRegionRequestSuite) TearDownTest(c *C) { } func (s *testRegionRequestSuite) TestOnSendFailedWithStoreRestart(c *C) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) @@ -93,13 +90,10 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithStoreRestart(c *C) { } func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOne(c *C) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) @@ -131,13 +125,10 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOn } func (s *testRegionRequestSuite) TestSendReqCtx(c *C) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) @@ -148,13 +139,10 @@ func (s *testRegionRequestSuite) TestSendReqCtx(c *C) { } func (s *testRegionRequestSuite) TestOnSendFailedWithCancelled(c *C) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) @@ -181,13 +169,10 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCancelled(c *C) { } func (s *testRegionRequestSuite) TestNoReloadRegionWhenCtxCanceled(c *C) { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) c.Assert(region, NotNil) @@ -340,13 +325,10 @@ func (s *testRegionRequestSuite) TestNoReloadRegionForGrpcWhenCtxCanceled(c *C) client := newRPCClient(config.Security{}) sender := NewRegionRequestSender(s.cache, client) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdRawPut, - RawPut: &kvrpcpb.RawPutRequest{ - Key: []byte("key"), - Value: []byte("value"), - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdRawPut, &kvrpcpb.RawPutRequest{ + Key: []byte("key"), + Value: []byte("value"), + }) region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) diff --git a/store/tikv/scan.go b/store/tikv/scan.go index e0220f5e63563..c50313ae91be5 100644 --- a/store/tikv/scan.go +++ b/store/tikv/scan.go @@ -185,25 +185,22 @@ func (s *Scanner) getData(bo *Backoffer) error { } } - req := &tikvrpc.Request{ - Type: tikvrpc.CmdScan, - Scan: &pb.ScanRequest{ - StartKey: s.nextStartKey, - EndKey: reqEndKey, - Limit: uint32(s.batchSize), - Version: s.startTS(), - KeyOnly: s.snapshot.keyOnly, - }, - Context: pb.Context{ - Priority: s.snapshot.priority, - NotFillCache: s.snapshot.notFillCache, - }, + sreq := &pb.ScanRequest{ + StartKey: s.nextStartKey, + EndKey: reqEndKey, + Limit: uint32(s.batchSize), + Version: s.startTS(), + KeyOnly: s.snapshot.keyOnly, } if s.reverse { - req.Scan.StartKey = s.nextEndKey - req.Scan.EndKey = reqStartKey - req.Scan.Reverse = true + sreq.StartKey = s.nextEndKey + sreq.EndKey = reqStartKey + sreq.Reverse = true } + req := tikvrpc.NewRequest(tikvrpc.CmdScan, sreq, pb.Context{ + Priority: s.snapshot.priority, + NotFillCache: s.snapshot.notFillCache, + }) resp, err := sender.SendReq(bo, req, loc.Region, ReadTimeoutMedium) if err != nil { return errors.Trace(err) diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index f940448a6caa0..f4fbe811ec605 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -154,17 +154,13 @@ func (s *tikvSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, coll pending := batch.keys for { - req := &tikvrpc.Request{ - Type: tikvrpc.CmdBatchGet, - BatchGet: &pb.BatchGetRequest{ - Keys: pending, - Version: s.version.Ver, - }, - Context: pb.Context{ - Priority: s.priority, - NotFillCache: s.notFillCache, - }, - } + req := tikvrpc.NewRequest(tikvrpc.CmdBatchGet, &pb.BatchGetRequest{ + Keys: pending, + Version: s.version.Ver, + }, pb.Context{ + Priority: s.priority, + NotFillCache: s.notFillCache, + }) resp, err := sender.SendReq(bo, req, batch.region, ReadTimeoutMedium) if err != nil { return errors.Trace(err) @@ -236,17 +232,14 @@ func (s *tikvSnapshot) Get(k kv.Key) ([]byte, error) { func (s *tikvSnapshot) get(bo *Backoffer, k kv.Key) ([]byte, error) { sender := NewRegionRequestSender(s.store.regionCache, s.store.client) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdGet, - Get: &pb.GetRequest{ + req := tikvrpc.NewRequest(tikvrpc.CmdGet, + &pb.GetRequest{ Key: k, Version: s.version.Ver, - }, - Context: pb.Context{ + }, pb.Context{ Priority: s.priority, NotFillCache: s.notFillCache, - }, - } + }) for { loc, err := s.store.regionCache.LocateKey(bo, k) if err != nil { diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index e949153ae5846..89986bad22b87 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -33,13 +33,11 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key, scatter bool) (regionID uint64, zap.Binary("at", splitKey)) bo := NewBackoffer(context.Background(), splitRegionBackoff) sender := NewRegionRequestSender(s.regionCache, s.client) - req := &tikvrpc.Request{ - Type: tikvrpc.CmdSplitRegion, - SplitRegion: &kvrpcpb.SplitRegionRequest{ - SplitKey: splitKey, - }, - } - req.Context.Priority = kvrpcpb.CommandPri_Normal + req := tikvrpc.NewRequest(tikvrpc.CmdSplitRegion, &kvrpcpb.SplitRegionRequest{ + SplitKey: splitKey, + }, kvrpcpb.Context{ + Priority: kvrpcpb.CommandPri_Normal, + }) for { loc, err := s.regionCache.LocateKey(bo, splitKey) if err != nil { diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index b3cf78ee5e82e..2f9101a480135 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -134,90 +134,215 @@ func (t CmdType) String() string { // Request wraps all kv/coprocessor requests. type Request struct { + Type CmdType + req interface{} kvrpcpb.Context - Type CmdType - Get *kvrpcpb.GetRequest - Scan *kvrpcpb.ScanRequest - Prewrite *kvrpcpb.PrewriteRequest - Commit *kvrpcpb.CommitRequest - Cleanup *kvrpcpb.CleanupRequest - BatchGet *kvrpcpb.BatchGetRequest - BatchRollback *kvrpcpb.BatchRollbackRequest - ScanLock *kvrpcpb.ScanLockRequest - ResolveLock *kvrpcpb.ResolveLockRequest - GC *kvrpcpb.GCRequest - DeleteRange *kvrpcpb.DeleteRangeRequest - RawGet *kvrpcpb.RawGetRequest - RawBatchGet *kvrpcpb.RawBatchGetRequest - RawPut *kvrpcpb.RawPutRequest - RawBatchPut *kvrpcpb.RawBatchPutRequest - RawDelete *kvrpcpb.RawDeleteRequest - RawBatchDelete *kvrpcpb.RawBatchDeleteRequest - RawDeleteRange *kvrpcpb.RawDeleteRangeRequest - RawScan *kvrpcpb.RawScanRequest - UnsafeDestroyRange *kvrpcpb.UnsafeDestroyRangeRequest - Cop *coprocessor.Request - MvccGetByKey *kvrpcpb.MvccGetByKeyRequest - MvccGetByStartTs *kvrpcpb.MvccGetByStartTsRequest - SplitRegion *kvrpcpb.SplitRegionRequest - - PessimisticLock *kvrpcpb.PessimisticLockRequest - PessimisticRollback *kvrpcpb.PessimisticRollbackRequest - - DebugGetRegionProperties *debugpb.GetRegionPropertiesRequest - - Empty *tikvpb.BatchCommandsEmptyRequest +} + +// NewRequest returns new kv rpc request. +func NewRequest(typ CmdType, pointer interface{}, ctxs ...kvrpcpb.Context) *Request { + if len(ctxs) > 0 { + return &Request{ + Type: typ, + req: pointer, + Context: ctxs[0], + } + } + return &Request{ + Type: typ, + req: pointer, + } +} + +// Get returns GetRequest in request. +func (req *Request) Get() *kvrpcpb.GetRequest { + return req.req.(*kvrpcpb.GetRequest) +} + +// Scan returns ScanRequest in request. +func (req *Request) Scan() *kvrpcpb.ScanRequest { + return req.req.(*kvrpcpb.ScanRequest) +} + +// Prewrite returns PrewriteRequest in request. +func (req *Request) Prewrite() *kvrpcpb.PrewriteRequest { + return req.req.(*kvrpcpb.PrewriteRequest) +} + +// Commit returns CommitRequest in request. +func (req *Request) Commit() *kvrpcpb.CommitRequest { + return req.req.(*kvrpcpb.CommitRequest) +} + +// Cleanup returns CleanupRequest in request. +func (req *Request) Cleanup() *kvrpcpb.CleanupRequest { + return req.req.(*kvrpcpb.CleanupRequest) +} + +// BatchGet returns BatchGetRequest in request. +func (req *Request) BatchGet() *kvrpcpb.BatchGetRequest { + return req.req.(*kvrpcpb.BatchGetRequest) +} + +// BatchRollback returns BatchRollbackRequest in request. +func (req *Request) BatchRollback() *kvrpcpb.BatchRollbackRequest { + return req.req.(*kvrpcpb.BatchRollbackRequest) +} + +// ScanLock returns ScanLockRequest in request. +func (req *Request) ScanLock() *kvrpcpb.ScanLockRequest { + return req.req.(*kvrpcpb.ScanLockRequest) +} + +// ResolveLock returns ResolveLockRequest in request. +func (req *Request) ResolveLock() *kvrpcpb.ResolveLockRequest { + return req.req.(*kvrpcpb.ResolveLockRequest) +} + +// GC returns GCRequest in request. +func (req *Request) GC() *kvrpcpb.GCRequest { + return req.req.(*kvrpcpb.GCRequest) +} + +// DeleteRange returns DeleteRangeRequest in request. +func (req *Request) DeleteRange() *kvrpcpb.DeleteRangeRequest { + return req.req.(*kvrpcpb.DeleteRangeRequest) +} + +// RawGet returns RawGetRequest in request. +func (req *Request) RawGet() *kvrpcpb.RawGetRequest { + return req.req.(*kvrpcpb.RawGetRequest) +} + +// RawBatchGet returns RawBatchGetRequest in request. +func (req *Request) RawBatchGet() *kvrpcpb.RawBatchGetRequest { + return req.req.(*kvrpcpb.RawBatchGetRequest) +} + +// RawPut returns RawPutRequest in request. +func (req *Request) RawPut() *kvrpcpb.RawPutRequest { + return req.req.(*kvrpcpb.RawPutRequest) +} + +// RawBatchPut returns RawBatchPutRequest in request. +func (req *Request) RawBatchPut() *kvrpcpb.RawBatchPutRequest { + return req.req.(*kvrpcpb.RawBatchPutRequest) +} + +// RawDelete returns PrewriteRequest in request. +func (req *Request) RawDelete() *kvrpcpb.RawDeleteRequest { + return req.req.(*kvrpcpb.RawDeleteRequest) +} + +// RawBatchDelete returns RawBatchDeleteRequest in request. +func (req *Request) RawBatchDelete() *kvrpcpb.RawBatchDeleteRequest { + return req.req.(*kvrpcpb.RawBatchDeleteRequest) +} + +// RawDeleteRange returns RawDeleteRangeRequest in request. +func (req *Request) RawDeleteRange() *kvrpcpb.RawDeleteRangeRequest { + return req.req.(*kvrpcpb.RawDeleteRangeRequest) +} + +// RawScan returns RawScanRequest in request. +func (req *Request) RawScan() *kvrpcpb.RawScanRequest { + return req.req.(*kvrpcpb.RawScanRequest) +} + +// UnsafeDestroyRange returns UnsafeDestroyRangeRequest in request. +func (req *Request) UnsafeDestroyRange() *kvrpcpb.UnsafeDestroyRangeRequest { + return req.req.(*kvrpcpb.UnsafeDestroyRangeRequest) +} + +// Cop returns coprocessor request in request. +func (req *Request) Cop() *coprocessor.Request { + return req.req.(*coprocessor.Request) +} + +// MvccGetByKey returns MvccGetByKeyRequest in request. +func (req *Request) MvccGetByKey() *kvrpcpb.MvccGetByKeyRequest { + return req.req.(*kvrpcpb.MvccGetByKeyRequest) +} + +// MvccGetByStartTs returns MvccGetByStartTsRequest in request. +func (req *Request) MvccGetByStartTs() *kvrpcpb.MvccGetByStartTsRequest { + return req.req.(*kvrpcpb.MvccGetByStartTsRequest) +} + +// SplitRegion returns SplitRegionRequest in request. +func (req *Request) SplitRegion() *kvrpcpb.SplitRegionRequest { + return req.req.(*kvrpcpb.SplitRegionRequest) +} + +// PessimisticLock returns PessimisticLockRequest in request. +func (req *Request) PessimisticLock() *kvrpcpb.PessimisticLockRequest { + return req.req.(*kvrpcpb.PessimisticLockRequest) +} + +// PessimisticRollback returns PessimisticRollbackRequest in request. +func (req *Request) PessimisticRollback() *kvrpcpb.PessimisticRollbackRequest { + return req.req.(*kvrpcpb.PessimisticRollbackRequest) +} + +// DebugGetRegionProperties returns GetRegionPropertiesRequest in request. +func (req *Request) DebugGetRegionProperties() *debugpb.GetRegionPropertiesRequest { + return req.req.(*debugpb.GetRegionPropertiesRequest) +} + +// Empty returns BatchCommandsEmptyRequest in request +func (req *Request) Empty() *tikvpb.BatchCommandsEmptyRequest { + return req.req.(*tikvpb.BatchCommandsEmptyRequest) } // ToBatchCommandsRequest converts the request to an entry in BatchCommands request. func (req *Request) ToBatchCommandsRequest() *tikvpb.BatchCommandsRequest_Request { switch req.Type { case CmdGet: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Get{Get: req.Get}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Get{Get: req.Get()}} case CmdScan: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Scan{Scan: req.Scan}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Scan{Scan: req.Scan()}} case CmdPrewrite: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Prewrite{Prewrite: req.Prewrite}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Prewrite{Prewrite: req.Prewrite()}} case CmdCommit: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Commit{Commit: req.Commit}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Commit{Commit: req.Commit()}} case CmdCleanup: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Cleanup{Cleanup: req.Cleanup}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Cleanup{Cleanup: req.Cleanup()}} case CmdBatchGet: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_BatchGet{BatchGet: req.BatchGet}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_BatchGet{BatchGet: req.BatchGet()}} case CmdBatchRollback: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_BatchRollback{BatchRollback: req.BatchRollback}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_BatchRollback{BatchRollback: req.BatchRollback()}} case CmdScanLock: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_ScanLock{ScanLock: req.ScanLock}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_ScanLock{ScanLock: req.ScanLock()}} case CmdResolveLock: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_ResolveLock{ResolveLock: req.ResolveLock}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_ResolveLock{ResolveLock: req.ResolveLock()}} case CmdGC: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_GC{GC: req.GC}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_GC{GC: req.GC()}} case CmdDeleteRange: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_DeleteRange{DeleteRange: req.DeleteRange}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_DeleteRange{DeleteRange: req.DeleteRange()}} case CmdRawGet: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawGet{RawGet: req.RawGet}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawGet{RawGet: req.RawGet()}} case CmdRawBatchGet: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchGet{RawBatchGet: req.RawBatchGet}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchGet{RawBatchGet: req.RawBatchGet()}} case CmdRawPut: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawPut{RawPut: req.RawPut}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawPut{RawPut: req.RawPut()}} case CmdRawBatchPut: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchPut{RawBatchPut: req.RawBatchPut}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchPut{RawBatchPut: req.RawBatchPut()}} case CmdRawDelete: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawDelete{RawDelete: req.RawDelete}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawDelete{RawDelete: req.RawDelete()}} case CmdRawBatchDelete: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchDelete{RawBatchDelete: req.RawBatchDelete}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawBatchDelete{RawBatchDelete: req.RawBatchDelete()}} case CmdRawDeleteRange: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawDeleteRange{RawDeleteRange: req.RawDeleteRange}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawDeleteRange{RawDeleteRange: req.RawDeleteRange()}} case CmdRawScan: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawScan{RawScan: req.RawScan}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_RawScan{RawScan: req.RawScan()}} case CmdCop: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Coprocessor{Coprocessor: req.Cop}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Coprocessor{Coprocessor: req.Cop()}} case CmdPessimisticLock: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_PessimisticLock{PessimisticLock: req.PessimisticLock}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_PessimisticLock{PessimisticLock: req.PessimisticLock()}} case CmdPessimisticRollback: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_PessimisticRollback{PessimisticRollback: req.PessimisticRollback}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_PessimisticRollback{PessimisticRollback: req.PessimisticRollback()}} case CmdEmpty: - return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Empty{Empty: req.Empty}} + return &tikvpb.BatchCommandsRequest_Request{Cmd: &tikvpb.BatchCommandsRequest_Request_Empty{Empty: req.Empty()}} } return nil } @@ -340,60 +465,61 @@ func SetContext(req *Request, region *metapb.Region, peer *metapb.Peer) error { switch req.Type { case CmdGet: - req.Get.Context = ctx + req.Get().Context = ctx case CmdScan: - req.Scan.Context = ctx + req.Scan().Context = ctx case CmdPrewrite: - req.Prewrite.Context = ctx + req.Prewrite().Context = ctx case CmdPessimisticLock: - req.PessimisticLock.Context = ctx + req.PessimisticLock().Context = ctx case CmdPessimisticRollback: - req.PessimisticRollback.Context = ctx + req.PessimisticRollback().Context = ctx case CmdCommit: - req.Commit.Context = ctx + req.Commit().Context = ctx case CmdCleanup: - req.Cleanup.Context = ctx + req.Cleanup().Context = ctx case CmdBatchGet: - req.BatchGet.Context = ctx + req.BatchGet().Context = ctx case CmdBatchRollback: - req.BatchRollback.Context = ctx + req.BatchRollback().Context = ctx case CmdScanLock: - req.ScanLock.Context = ctx + req.ScanLock().Context = ctx case CmdResolveLock: - req.ResolveLock.Context = ctx + req.ResolveLock().Context = ctx case CmdGC: - req.GC.Context = ctx + req.GC().Context = ctx case CmdDeleteRange: - req.DeleteRange.Context = ctx + req.DeleteRange().Context = ctx case CmdRawGet: - req.RawGet.Context = ctx + req.RawGet().Context = ctx case CmdRawBatchGet: - req.RawBatchGet.Context = ctx + req.RawBatchGet().Context = ctx case CmdRawPut: - req.RawPut.Context = ctx + req.RawPut().Context = ctx case CmdRawBatchPut: - req.RawBatchPut.Context = ctx + req.RawBatchPut().Context = ctx case CmdRawDelete: - req.RawDelete.Context = ctx + req.RawDelete().Context = ctx case CmdRawBatchDelete: - req.RawBatchDelete.Context = ctx + req.RawBatchDelete().Context = ctx case CmdRawDeleteRange: - req.RawDeleteRange.Context = ctx + req.RawDeleteRange().Context = ctx case CmdRawScan: - req.RawScan.Context = ctx + req.RawScan().Context = ctx case CmdUnsafeDestroyRange: - req.UnsafeDestroyRange.Context = ctx + req.UnsafeDestroyRange().Context = ctx case CmdCop: - req.Cop.Context = ctx + req.Cop().Context = ctx case CmdCopStream: - req.Cop.Context = ctx + req.Cop().Context = ctx case CmdMvccGetByKey: - req.MvccGetByKey.Context = ctx + req.MvccGetByKey().Context = ctx case CmdMvccGetByStartTs: - req.MvccGetByStartTs.Context = ctx + req.MvccGetByStartTs().Context = ctx case CmdSplitRegion: - req.SplitRegion.Context = ctx + req.SplitRegion().Context = ctx case CmdEmpty: + req.SplitRegion().Context = ctx default: return fmt.Errorf("invalid request type %v", req.Type) } @@ -597,63 +723,63 @@ func CallRPC(ctx context.Context, client tikvpb.TikvClient, req *Request) (*Resp var err error switch req.Type { case CmdGet: - resp.Get, err = client.KvGet(ctx, req.Get) + resp.Get, err = client.KvGet(ctx, req.Get()) case CmdScan: - resp.Scan, err = client.KvScan(ctx, req.Scan) + resp.Scan, err = client.KvScan(ctx, req.Scan()) case CmdPrewrite: - resp.Prewrite, err = client.KvPrewrite(ctx, req.Prewrite) + resp.Prewrite, err = client.KvPrewrite(ctx, req.Prewrite()) case CmdPessimisticLock: - resp.PessimisticLock, err = client.KvPessimisticLock(ctx, req.PessimisticLock) + resp.PessimisticLock, err = client.KvPessimisticLock(ctx, req.PessimisticLock()) case CmdPessimisticRollback: - resp.PessimisticRollback, err = client.KVPessimisticRollback(ctx, req.PessimisticRollback) + resp.PessimisticRollback, err = client.KVPessimisticRollback(ctx, req.PessimisticRollback()) case CmdCommit: - resp.Commit, err = client.KvCommit(ctx, req.Commit) + resp.Commit, err = client.KvCommit(ctx, req.Commit()) case CmdCleanup: - resp.Cleanup, err = client.KvCleanup(ctx, req.Cleanup) + resp.Cleanup, err = client.KvCleanup(ctx, req.Cleanup()) case CmdBatchGet: - resp.BatchGet, err = client.KvBatchGet(ctx, req.BatchGet) + resp.BatchGet, err = client.KvBatchGet(ctx, req.BatchGet()) case CmdBatchRollback: - resp.BatchRollback, err = client.KvBatchRollback(ctx, req.BatchRollback) + resp.BatchRollback, err = client.KvBatchRollback(ctx, req.BatchRollback()) case CmdScanLock: - resp.ScanLock, err = client.KvScanLock(ctx, req.ScanLock) + resp.ScanLock, err = client.KvScanLock(ctx, req.ScanLock()) case CmdResolveLock: - resp.ResolveLock, err = client.KvResolveLock(ctx, req.ResolveLock) + resp.ResolveLock, err = client.KvResolveLock(ctx, req.ResolveLock()) case CmdGC: - resp.GC, err = client.KvGC(ctx, req.GC) + resp.GC, err = client.KvGC(ctx, req.GC()) case CmdDeleteRange: - resp.DeleteRange, err = client.KvDeleteRange(ctx, req.DeleteRange) + resp.DeleteRange, err = client.KvDeleteRange(ctx, req.DeleteRange()) case CmdRawGet: - resp.RawGet, err = client.RawGet(ctx, req.RawGet) + resp.RawGet, err = client.RawGet(ctx, req.RawGet()) case CmdRawBatchGet: - resp.RawBatchGet, err = client.RawBatchGet(ctx, req.RawBatchGet) + resp.RawBatchGet, err = client.RawBatchGet(ctx, req.RawBatchGet()) case CmdRawPut: - resp.RawPut, err = client.RawPut(ctx, req.RawPut) + resp.RawPut, err = client.RawPut(ctx, req.RawPut()) case CmdRawBatchPut: - resp.RawBatchPut, err = client.RawBatchPut(ctx, req.RawBatchPut) + resp.RawBatchPut, err = client.RawBatchPut(ctx, req.RawBatchPut()) case CmdRawDelete: - resp.RawDelete, err = client.RawDelete(ctx, req.RawDelete) + resp.RawDelete, err = client.RawDelete(ctx, req.RawDelete()) case CmdRawBatchDelete: - resp.RawBatchDelete, err = client.RawBatchDelete(ctx, req.RawBatchDelete) + resp.RawBatchDelete, err = client.RawBatchDelete(ctx, req.RawBatchDelete()) case CmdRawDeleteRange: - resp.RawDeleteRange, err = client.RawDeleteRange(ctx, req.RawDeleteRange) + resp.RawDeleteRange, err = client.RawDeleteRange(ctx, req.RawDeleteRange()) case CmdRawScan: - resp.RawScan, err = client.RawScan(ctx, req.RawScan) + resp.RawScan, err = client.RawScan(ctx, req.RawScan()) case CmdUnsafeDestroyRange: - resp.UnsafeDestroyRange, err = client.UnsafeDestroyRange(ctx, req.UnsafeDestroyRange) + resp.UnsafeDestroyRange, err = client.UnsafeDestroyRange(ctx, req.UnsafeDestroyRange()) case CmdCop: - resp.Cop, err = client.Coprocessor(ctx, req.Cop) + resp.Cop, err = client.Coprocessor(ctx, req.Cop()) case CmdCopStream: var streamClient tikvpb.Tikv_CoprocessorStreamClient - streamClient, err = client.CoprocessorStream(ctx, req.Cop) + streamClient, err = client.CoprocessorStream(ctx, req.Cop()) resp.CopStream = &CopStreamResponse{ Tikv_CoprocessorStreamClient: streamClient, } case CmdMvccGetByKey: - resp.MvccGetByKey, err = client.MvccGetByKey(ctx, req.MvccGetByKey) + resp.MvccGetByKey, err = client.MvccGetByKey(ctx, req.MvccGetByKey()) case CmdMvccGetByStartTs: - resp.MvccGetByStartTS, err = client.MvccGetByStartTs(ctx, req.MvccGetByStartTs) + resp.MvccGetByStartTS, err = client.MvccGetByStartTs(ctx, req.MvccGetByStartTs()) case CmdSplitRegion: - resp.SplitRegion, err = client.SplitRegion(ctx, req.SplitRegion) + resp.SplitRegion, err = client.SplitRegion(ctx, req.SplitRegion()) case CmdEmpty: resp.Empty, err = &tikvpb.BatchCommandsEmptyResponse{}, nil default: @@ -672,7 +798,7 @@ func CallDebugRPC(ctx context.Context, client debugpb.DebugClient, req *Request) var err error switch req.Type { case CmdDebugGetRegionProperties: - resp.DebugGetRegionProperties, err = client.GetRegionProperties(ctx, req.DebugGetRegionProperties) + resp.DebugGetRegionProperties, err = client.GetRegionProperties(ctx, req.DebugGetRegionProperties()) default: return nil, errors.Errorf("invalid request type: %v", req.Type) } From 3f1d234e2036a3163f7774ff106fa6397d585287 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Mon, 15 Jul 2019 16:27:31 +0800 Subject: [PATCH 032/196] planner: disable projection elimination for select plan of update (#10962) --- planner/core/cbo_test.go | 15 +++++++++++++++ planner/core/logical_plan_builder.go | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 33b06eefa4881..cdce14d6e8f73 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -1090,3 +1090,18 @@ func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { " └─TableScan_19 6.00 cop table:t, keep order:false", )) } + +func (s *testAnalyzeSuite) TestUpdateProjEliminate(c *C) { + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b int)") + tk.MustExec("explain update t t1, (select distinct b from t) t2 set t1.b = t2.b") +} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index fb19ca7f6169f..d053c88bff08e 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2576,7 +2576,9 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { updt := Update{OrderedList: orderedList}.Init(b.ctx) updt.SetSchema(p.Schema()) - updt.SelectPlan, err = DoOptimize(b.optFlag, p) + // We cannot apply projection elimination when building the subplan, because + // columns in orderedList cannot be resolved. + updt.SelectPlan, err = DoOptimize(b.optFlag&^flagEliminateProjection, p) if err != nil { return nil, err } From 368119b8df10d65a7575a90de8d77def660ac5ca Mon Sep 17 00:00:00 2001 From: kennytm Date: Tue, 16 Jul 2019 07:39:17 +0800 Subject: [PATCH 033/196] =?UTF-8?q?executor:=20compute=20ADMIN=20CHECKSUM?= =?UTF-8?q?=20for=20partitioned=20tables=20correc=E2=80=A6=20(#11089)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/checksum.go | 44 ++++++++++++++++++++++++++++----------- executor/executor_test.go | 11 ++++++++++ 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/executor/checksum.go b/executor/checksum.go index 9ca8692088825..c84579fe85ee8 100644 --- a/executor/checksum.go +++ b/executor/checksum.go @@ -187,27 +187,47 @@ func newChecksumContext(db *model.DBInfo, table *model.TableInfo, startTs uint64 } func (c *checksumContext) BuildRequests(ctx sessionctx.Context) ([]*kv.Request, error) { - reqs := make([]*kv.Request, 0, len(c.TableInfo.Indices)+1) - req, err := c.buildTableRequest(ctx) - if err != nil { + var partDefs []model.PartitionDefinition + if part := c.TableInfo.Partition; part != nil { + partDefs = part.Definitions + } + + reqs := make([]*kv.Request, 0, (len(c.TableInfo.Indices)+1)*(len(partDefs)+1)) + if err := c.appendRequest(ctx, c.TableInfo.ID, &reqs); err != nil { return nil, err } - reqs = append(reqs, req) + + for _, partDef := range partDefs { + if err := c.appendRequest(ctx, partDef.ID, &reqs); err != nil { + return nil, err + } + } + + return reqs, nil +} + +func (c *checksumContext) appendRequest(ctx sessionctx.Context, tableID int64, reqs *[]*kv.Request) error { + req, err := c.buildTableRequest(ctx, tableID) + if err != nil { + return err + } + + *reqs = append(*reqs, req) for _, indexInfo := range c.TableInfo.Indices { if indexInfo.State != model.StatePublic { continue } - req, err = c.buildIndexRequest(ctx, indexInfo) + req, err = c.buildIndexRequest(ctx, tableID, indexInfo) if err != nil { - return nil, err + return err } - reqs = append(reqs, req) + *reqs = append(*reqs, req) } - return reqs, nil + return nil } -func (c *checksumContext) buildTableRequest(ctx sessionctx.Context) (*kv.Request, error) { +func (c *checksumContext) buildTableRequest(ctx sessionctx.Context, tableID int64) (*kv.Request, error) { checksum := &tipb.ChecksumRequest{ StartTs: c.StartTs, ScanOn: tipb.ChecksumScanOn_Table, @@ -217,13 +237,13 @@ func (c *checksumContext) buildTableRequest(ctx sessionctx.Context) (*kv.Request ranges := ranger.FullIntRange(false) var builder distsql.RequestBuilder - return builder.SetTableRanges(c.TableInfo.ID, ranges, nil). + return builder.SetTableRanges(tableID, ranges, nil). SetChecksumRequest(checksum). SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency). Build() } -func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, indexInfo *model.IndexInfo) (*kv.Request, error) { +func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, tableID int64, indexInfo *model.IndexInfo) (*kv.Request, error) { checksum := &tipb.ChecksumRequest{ StartTs: c.StartTs, ScanOn: tipb.ChecksumScanOn_Index, @@ -233,7 +253,7 @@ func (c *checksumContext) buildIndexRequest(ctx sessionctx.Context, indexInfo *m ranges := ranger.FullRange() var builder distsql.RequestBuilder - return builder.SetIndexRanges(ctx.GetSessionVars().StmtCtx, c.TableInfo.ID, indexInfo.ID, ranges). + return builder.SetIndexRanges(ctx.GetSessionVars().StmtCtx, tableID, indexInfo.ID, ranges). SetChecksumRequest(checksum). SetConcurrency(ctx.GetSessionVars().DistSQLScanConcurrency). Build() diff --git a/executor/executor_test.go b/executor/executor_test.go index e860a6e200d52..b68bebd4bfefb 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -412,6 +412,17 @@ func (s *testSuite) TestAdmin(c *C) { c.Assert(historyJobs, DeepEquals, historyJobs2) } +func (s *testSuite) TestAdminChecksumOfPartitionedTable(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("USE test;") + tk.MustExec("DROP TABLE IF EXISTS admin_checksum_partition_test;") + tk.MustExec("CREATE TABLE admin_checksum_partition_test (a INT) PARTITION BY HASH(a) PARTITIONS 4;") + tk.MustExec("INSERT INTO admin_checksum_partition_test VALUES (1), (2);") + + r := tk.MustQuery("ADMIN CHECKSUM TABLE admin_checksum_partition_test;") + r.Check(testkit.Rows("test admin_checksum_partition_test 1 5 5")) +} + func (s *testSuite) fillData(tk *testkit.TestKit, table string) { tk.MustExec("use test") tk.MustExec(fmt.Sprintf("create table %s(id int not null default 1, name varchar(255), PRIMARY KEY(id));", table)) From 7403ce331e2b170cab92088d76ba9d732f4a4e84 Mon Sep 17 00:00:00 2001 From: baishen Date: Tue, 16 Jul 2019 11:15:54 +0800 Subject: [PATCH 034/196] planner: fix NAME_CONST function compatibility (#11241) --- expression/integration_test.go | 4 ++++ planner/core/preprocess.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index d1f7272f8eb68..0f73a64807965 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4130,6 +4130,10 @@ func (s *testIntegrationSuite) TestFuncNameConst(c *C) { r.Check(testkit.Rows("2")) r = tk.MustQuery("SELECT concat('hello', name_const('test_string', 'world')) FROM t;") r.Check(testkit.Rows("helloworld")) + r = tk.MustQuery("SELECT NAME_CONST('come', -1);") + r.Check(testkit.Rows("-1")) + r = tk.MustQuery("SELECT NAME_CONST('come', -1.0);") + r.Check(testkit.Rows("-1.0")) err := tk.ExecToErr(`select name_const(a,b) from t;`) c.Assert(err.Error(), Equals, "[planner:1210]Incorrect arguments to NAME_CONST") err = tk.ExecToErr(`select name_const(a,"hello") from t;`) diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 26d93eabaf9ab..770ba6f683e5a 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -178,7 +178,12 @@ func (p *preprocessor) Leave(in ast.Node) (out ast.Node, ok bool) { p.err = expression.ErrIncorrectParameterCount.GenWithStackByArgs(x.FnName.L) } else { _, isValueExpr1 := x.Args[0].(*driver.ValueExpr) - _, isValueExpr2 := x.Args[1].(*driver.ValueExpr) + isValueExpr2 := false + switch x.Args[1].(type) { + case *driver.ValueExpr, *ast.UnaryOperationExpr: + isValueExpr2 = true + } + if !isValueExpr1 || !isValueExpr2 { p.err = ErrWrongArguments.GenWithStackByArgs("NAME_CONST") } From eae30ebbcb8464d4644598db459337998ab1d43b Mon Sep 17 00:00:00 2001 From: lysu Date: Tue, 16 Jul 2019 13:21:00 +0800 Subject: [PATCH 035/196] *: reduce tikvrpc resp struct size (#11056) --- executor/analyze.go | 4 +- server/http_handler.go | 2 +- store/helper/helper.go | 2 +- store/mockstore/mocktikv/rpc.go | 113 ++++++----- store/tikv/2pc.go | 14 +- store/tikv/2pc_test.go | 4 +- store/tikv/client.go | 2 +- store/tikv/coprocessor.go | 12 +- store/tikv/delete_range.go | 4 +- store/tikv/gcworker/gc_worker.go | 12 +- store/tikv/gcworker/gc_worker_test.go | 7 +- store/tikv/lock_resolver.go | 12 +- store/tikv/lock_test.go | 5 +- store/tikv/rawkv.go | 48 ++--- store/tikv/region_request.go | 12 +- store/tikv/region_request_test.go | 12 +- store/tikv/scan.go | 4 +- store/tikv/snapshot.go | 8 +- store/tikv/split_region.go | 7 +- store/tikv/store_test.go | 4 +- store/tikv/tikvrpc/tikvrpc.go | 273 +++++++++----------------- 21 files changed, 238 insertions(+), 323 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index cb2840e3d05b1..c159b82313206 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -635,11 +635,11 @@ func (e *AnalyzeFastExec) getSampRegionsRowCount(bo *tikv.Backoffer, needRebuild if *err != nil { return } - if resp.DebugGetRegionProperties == nil || len(resp.DebugGetRegionProperties.Props) == 0 { + if resp.Resp == nil || len(resp.Resp.(*debugpb.GetRegionPropertiesResponse).Props) == 0 { *needRebuild = true return } - for _, prop := range resp.DebugGetRegionProperties.Props { + for _, prop := range resp.Resp.(*debugpb.GetRegionPropertiesResponse).Props { if prop.Name == "mvcc.num_rows" { var cnt uint64 cnt, *err = strconv.ParseUint(prop.Value, 10, 64) diff --git a/server/http_handler.go b/server/http_handler.go index deaf0b7c316cb..2b65bc030be70 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -167,7 +167,7 @@ func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey []by zap.Error(err)) return nil, errors.Trace(err) } - data := kvResp.MvccGetByStartTS + data := kvResp.Resp.(*kvrpcpb.MvccGetByStartTsResponse) if err := data.GetRegionError(); err != nil { logutil.BgLogger().Warn("get MVCC by startTS failed", zap.Uint64("txnStartTS", startTS), diff --git a/store/helper/helper.go b/store/helper/helper.go index 43cbcf0177e0c..ce202f9bb2ef1 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -62,7 +62,7 @@ func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyRe zap.Error(err)) return nil, errors.Trace(err) } - return kvResp.MvccGetByKey, nil + return kvResp.Resp.(*kvrpcpb.MvccGetByKeyResponse), nil } // StoreHotRegionInfos records all hog region stores. diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index fb9f104b7a3c0..e4455fe589b1c 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -668,44 +668,43 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } reqCtx := &req.Context resp := &tikvrpc.Response{} - resp.Type = req.Type switch req.Type { case tikvrpc.CmdGet: r := req.Get() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.Get = &kvrpcpb.GetResponse{RegionError: err} + resp.Resp = &kvrpcpb.GetResponse{RegionError: err} return resp, nil } - resp.Get = handler.handleKvGet(r) + resp.Resp = handler.handleKvGet(r) case tikvrpc.CmdScan: r := req.Scan() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.Scan = &kvrpcpb.ScanResponse{RegionError: err} + resp.Resp = &kvrpcpb.ScanResponse{RegionError: err} return resp, nil } - resp.Scan = handler.handleKvScan(r) + resp.Resp = handler.handleKvScan(r) case tikvrpc.CmdPrewrite: r := req.Prewrite() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.Prewrite = &kvrpcpb.PrewriteResponse{RegionError: err} + resp.Resp = &kvrpcpb.PrewriteResponse{RegionError: err} return resp, nil } - resp.Prewrite = handler.handleKvPrewrite(r) + resp.Resp = handler.handleKvPrewrite(r) case tikvrpc.CmdPessimisticLock: r := req.PessimisticLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.PessimisticLock = &kvrpcpb.PessimisticLockResponse{RegionError: err} + resp.Resp = &kvrpcpb.PessimisticLockResponse{RegionError: err} return resp, nil } - resp.PessimisticLock = handler.handleKvPessimisticLock(r) + resp.Resp = handler.handleKvPessimisticLock(r) case tikvrpc.CmdPessimisticRollback: r := req.PessimisticRollback() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.PessimisticRollback = &kvrpcpb.PessimisticRollbackResponse{RegionError: err} + resp.Resp = &kvrpcpb.PessimisticRollbackResponse{RegionError: err} return resp, nil } - resp.PessimisticRollback = handler.handleKvPessimisticRollback(r) + resp.Resp = handler.handleKvPessimisticRollback(r) case tikvrpc.CmdCommit: failpoint.Inject("rpcCommitResult", func(val failpoint.Value) { switch val.(string) { @@ -713,23 +712,21 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R failpoint.Return(nil, errors.New("timeout")) case "notLeader": failpoint.Return(&tikvrpc.Response{ - Type: tikvrpc.CmdCommit, - Commit: &kvrpcpb.CommitResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + Resp: &kvrpcpb.CommitResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, }, nil) case "keyError": failpoint.Return(&tikvrpc.Response{ - Type: tikvrpc.CmdCommit, - Commit: &kvrpcpb.CommitResponse{Error: &kvrpcpb.KeyError{}}, + Resp: &kvrpcpb.CommitResponse{Error: &kvrpcpb.KeyError{}}, }, nil) } }) r := req.Commit() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.Commit = &kvrpcpb.CommitResponse{RegionError: err} + resp.Resp = &kvrpcpb.CommitResponse{RegionError: err} return resp, nil } - resp.Commit = handler.handleKvCommit(r) + resp.Resp = handler.handleKvCommit(r) failpoint.Inject("rpcCommitTimeout", func(val failpoint.Value) { if val.(bool) { failpoint.Return(nil, undeterminedErr) @@ -738,113 +735,113 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R case tikvrpc.CmdCleanup: r := req.Cleanup() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.Cleanup = &kvrpcpb.CleanupResponse{RegionError: err} + resp.Resp = &kvrpcpb.CleanupResponse{RegionError: err} return resp, nil } - resp.Cleanup = handler.handleKvCleanup(r) + resp.Resp = handler.handleKvCleanup(r) case tikvrpc.CmdBatchGet: r := req.BatchGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.BatchGet = &kvrpcpb.BatchGetResponse{RegionError: err} + resp.Resp = &kvrpcpb.BatchGetResponse{RegionError: err} return resp, nil } - resp.BatchGet = handler.handleKvBatchGet(r) + resp.Resp = handler.handleKvBatchGet(r) case tikvrpc.CmdBatchRollback: r := req.BatchRollback() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.BatchRollback = &kvrpcpb.BatchRollbackResponse{RegionError: err} + resp.Resp = &kvrpcpb.BatchRollbackResponse{RegionError: err} return resp, nil } - resp.BatchRollback = handler.handleKvBatchRollback(r) + resp.Resp = handler.handleKvBatchRollback(r) case tikvrpc.CmdScanLock: r := req.ScanLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.ScanLock = &kvrpcpb.ScanLockResponse{RegionError: err} + resp.Resp = &kvrpcpb.ScanLockResponse{RegionError: err} return resp, nil } - resp.ScanLock = handler.handleKvScanLock(r) + resp.Resp = handler.handleKvScanLock(r) case tikvrpc.CmdResolveLock: r := req.ResolveLock() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.ResolveLock = &kvrpcpb.ResolveLockResponse{RegionError: err} + resp.Resp = &kvrpcpb.ResolveLockResponse{RegionError: err} return resp, nil } - resp.ResolveLock = handler.handleKvResolveLock(r) + resp.Resp = handler.handleKvResolveLock(r) case tikvrpc.CmdGC: r := req.GC() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.GC = &kvrpcpb.GCResponse{RegionError: err} + resp.Resp = &kvrpcpb.GCResponse{RegionError: err} return resp, nil } - resp.GC = &kvrpcpb.GCResponse{} + resp.Resp = &kvrpcpb.GCResponse{} case tikvrpc.CmdDeleteRange: r := req.DeleteRange() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.DeleteRange = &kvrpcpb.DeleteRangeResponse{RegionError: err} + resp.Resp = &kvrpcpb.DeleteRangeResponse{RegionError: err} return resp, nil } - resp.DeleteRange = handler.handleKvDeleteRange(r) + resp.Resp = handler.handleKvDeleteRange(r) case tikvrpc.CmdRawGet: r := req.RawGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawGet = &kvrpcpb.RawGetResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawGetResponse{RegionError: err} return resp, nil } - resp.RawGet = handler.handleKvRawGet(r) + resp.Resp = handler.handleKvRawGet(r) case tikvrpc.CmdRawBatchGet: r := req.RawBatchGet() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawBatchGet = &kvrpcpb.RawBatchGetResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawBatchGetResponse{RegionError: err} return resp, nil } - resp.RawBatchGet = handler.handleKvRawBatchGet(r) + resp.Resp = handler.handleKvRawBatchGet(r) case tikvrpc.CmdRawPut: r := req.RawPut() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawPut = &kvrpcpb.RawPutResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawPutResponse{RegionError: err} return resp, nil } - resp.RawPut = handler.handleKvRawPut(r) + resp.Resp = handler.handleKvRawPut(r) case tikvrpc.CmdRawBatchPut: r := req.RawBatchPut() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawBatchPut = &kvrpcpb.RawBatchPutResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawBatchPutResponse{RegionError: err} return resp, nil } - resp.RawBatchPut = handler.handleKvRawBatchPut(r) + resp.Resp = handler.handleKvRawBatchPut(r) case tikvrpc.CmdRawDelete: r := req.RawDelete() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawDelete = &kvrpcpb.RawDeleteResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawDeleteResponse{RegionError: err} return resp, nil } - resp.RawDelete = handler.handleKvRawDelete(r) + resp.Resp = handler.handleKvRawDelete(r) case tikvrpc.CmdRawBatchDelete: r := req.RawBatchDelete() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawBatchDelete = &kvrpcpb.RawBatchDeleteResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawBatchDeleteResponse{RegionError: err} } - resp.RawBatchDelete = handler.handleKvRawBatchDelete(r) + resp.Resp = handler.handleKvRawBatchDelete(r) case tikvrpc.CmdRawDeleteRange: r := req.RawDeleteRange() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawDeleteRange = &kvrpcpb.RawDeleteRangeResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawDeleteRangeResponse{RegionError: err} return resp, nil } - resp.RawDeleteRange = handler.handleKvRawDeleteRange(r) + resp.Resp = handler.handleKvRawDeleteRange(r) case tikvrpc.CmdRawScan: r := req.RawScan() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.RawScan = &kvrpcpb.RawScanResponse{RegionError: err} + resp.Resp = &kvrpcpb.RawScanResponse{RegionError: err} return resp, nil } - resp.RawScan = handler.handleKvRawScan(r) + resp.Resp = handler.handleKvRawScan(r) case tikvrpc.CmdUnsafeDestroyRange: panic("unimplemented") case tikvrpc.CmdCop: r := req.Cop() if err := handler.checkRequestContext(reqCtx); err != nil { - resp.Cop = &coprocessor.Response{RegionError: err} + resp.Resp = &coprocessor.Response{RegionError: err} return resp, nil } handler.rawStartKey = MvccKey(handler.startKey).Raw() @@ -860,11 +857,11 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R default: panic(fmt.Sprintf("unknown coprocessor request type: %v", r.GetTp())) } - resp.Cop = res + resp.Resp = res case tikvrpc.CmdCopStream: r := req.Cop() if err := handler.checkRequestContext(reqCtx); err != nil { - resp.CopStream = &tikvrpc.CopStreamResponse{ + resp.Resp = &tikvrpc.CopStreamResponse{ Tikv_CoprocessorStreamClient: &mockCopStreamErrClient{Error: err}, Response: &coprocessor.Response{ RegionError: err, @@ -893,34 +890,34 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R return nil, errors.Trace(err) } streamResp.Response = first - resp.CopStream = streamResp + resp.Resp = streamResp case tikvrpc.CmdMvccGetByKey: r := req.MvccGetByKey() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.MvccGetByKey = &kvrpcpb.MvccGetByKeyResponse{RegionError: err} + resp.Resp = &kvrpcpb.MvccGetByKeyResponse{RegionError: err} return resp, nil } - resp.MvccGetByKey = handler.handleMvccGetByKey(r) + resp.Resp = handler.handleMvccGetByKey(r) case tikvrpc.CmdMvccGetByStartTs: r := req.MvccGetByStartTs() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.MvccGetByStartTS = &kvrpcpb.MvccGetByStartTsResponse{RegionError: err} + resp.Resp = &kvrpcpb.MvccGetByStartTsResponse{RegionError: err} return resp, nil } - resp.MvccGetByStartTS = handler.handleMvccGetByStartTS(r) + resp.Resp = handler.handleMvccGetByStartTS(r) case tikvrpc.CmdSplitRegion: r := req.SplitRegion() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { - resp.SplitRegion = &kvrpcpb.SplitRegionResponse{RegionError: err} + resp.Resp = &kvrpcpb.SplitRegionResponse{RegionError: err} return resp, nil } - resp.SplitRegion = handler.handleSplitRegion(r) + resp.Resp = handler.handleSplitRegion(r) // DebugGetRegionProperties is for fast analyze in mock tikv. case tikvrpc.CmdDebugGetRegionProperties: r := req.DebugGetRegionProperties() region, _ := c.Cluster.GetRegion(r.RegionId) scanResp := handler.handleKvScan(&kvrpcpb.ScanRequest{StartKey: MvccKey(region.StartKey).Raw(), EndKey: MvccKey(region.EndKey).Raw(), Version: math.MaxUint64, Limit: math.MaxUint32}) - resp.DebugGetRegionProperties = &debugpb.GetRegionPropertiesResponse{ + resp.Resp = &debugpb.GetRegionPropertiesResponse{ Props: []*debugpb.Property{{ Name: "mvcc.num_rows", Value: strconv.Itoa(len(scanResp.Pairs)), diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 5db1f24307e06..1e11c3f891185 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -504,10 +504,10 @@ func (c *twoPhaseCommitter) prewriteSingleBatch(bo *Backoffer, batch batchKeys) err = c.prewriteKeys(bo, batch.keys) return errors.Trace(err) } - prewriteResp := resp.Prewrite - if prewriteResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + prewriteResp := resp.Resp.(*pb.PrewriteResponse) keyErrs := prewriteResp.GetErrors() if len(keyErrs) == 0 { return nil @@ -591,10 +591,10 @@ func (c *twoPhaseCommitter) pessimisticLockSingleBatch(bo *Backoffer, batch batc err = c.pessimisticLockKeys(bo, batch.keys) return errors.Trace(err) } - lockResp := resp.PessimisticLock - if lockResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + lockResp := resp.Resp.(*pb.PessimisticLockResponse) keyErrs := lockResp.GetErrors() if len(keyErrs) == 0 { return nil @@ -728,10 +728,10 @@ func (c *twoPhaseCommitter) commitSingleBatch(bo *Backoffer, batch batchKeys) er err = c.commitKeys(bo, batch.keys) return errors.Trace(err) } - commitResp := resp.Commit - if commitResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + commitResp := resp.Resp.(*pb.CommitResponse) // Here we can make sure tikv has processed the commit primary key request. So // we can clean undetermined error. if isPrimary { @@ -785,7 +785,7 @@ func (c *twoPhaseCommitter) cleanupSingleBatch(bo *Backoffer, batch batchKeys) e err = c.cleanupKeys(bo, batch.keys) return errors.Trace(err) } - if keyErr := resp.BatchRollback.GetError(); keyErr != nil { + if keyErr := resp.Resp.(*pb.BatchRollbackResponse).GetError(); keyErr != nil { err = errors.Errorf("conn%d 2PC cleanup failed: %s", c.connID, keyErr) logutil.BgLogger().Debug("2PC failed cleanup key", zap.Error(err), diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index 52883f24247cf..63426bd9fb0e8 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -245,8 +245,8 @@ func (s *testCommitterSuite) isKeyLocked(c *C, key []byte) bool { c.Assert(err, IsNil) resp, err := s.store.SendReq(bo, req, loc.Region, readTimeoutShort) c.Assert(err, IsNil) - c.Assert(resp.Get, NotNil) - keyErr := resp.Get.GetError() + c.Assert(resp.Resp, NotNil) + keyErr := (resp.Resp.(*kvrpcpb.GetResponse)).GetError() return keyErr.GetLocked() != nil } diff --git a/store/tikv/client.go b/store/tikv/client.go index 4944830a3bd46..3278a2ebb9fd8 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -314,7 +314,7 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } // Put the lease object to the timeout channel, so it would be checked periodically. - copStream := resp.CopStream + copStream := resp.Resp.(*tikvrpc.CopStreamResponse) copStream.Timeout = timeout copStream.Lease.Cancel = cancel connArray.streamTimeout <- &copStream.Lease diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index b3e96859a5f7b..83f9395c4f96b 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -637,11 +637,11 @@ func (worker *copIteratorWorker) handleTaskOnce(bo *Backoffer, task *copTask, ch metrics.TiKVCoprocessorHistogram.Observe(costTime.Seconds()) if task.cmdType == tikvrpc.CmdCopStream { - return worker.handleCopStreamResult(bo, rpcCtx, resp.CopStream, task, ch) + return worker.handleCopStreamResult(bo, rpcCtx, resp.Resp.(*tikvrpc.CopStreamResponse), task, ch) } // Handles the response for non-streaming copTask. - return worker.handleCopResponse(bo, rpcCtx, &copResponse{pbResp: resp.Cop}, task, ch, nil) + return worker.handleCopResponse(bo, rpcCtx, &copResponse{pbResp: resp.Resp.(*coprocessor.Response)}, task, ch, nil) } const ( @@ -657,11 +657,11 @@ func (worker *copIteratorWorker) logTimeCopTask(costTime time.Duration, task *co logStr += fmt.Sprintf(" backoff_ms:%d backoff_types:%s", bo.totalSleep, backoffTypes) } var detail *kvrpcpb.ExecDetails - if resp.Cop != nil { - detail = resp.Cop.ExecDetails - } else if resp.CopStream != nil && resp.CopStream.Response != nil { + if resp.Resp != nil { + detail = resp.Resp.(*coprocessor.Response).ExecDetails + } else if resp.Resp != nil && resp.Resp.(*tikvrpc.CopStreamResponse).Response != nil { // streaming request returns io.EOF, so the first resp.CopStream.Response maybe nil. - detail = resp.CopStream.ExecDetails + detail = (resp.Resp.(*tikvrpc.CopStreamResponse)).ExecDetails } if detail != nil && detail.HandleTime != nil { diff --git a/store/tikv/delete_range.go b/store/tikv/delete_range.go index aa719c8977c81..81a8eaab7bf75 100644 --- a/store/tikv/delete_range.go +++ b/store/tikv/delete_range.go @@ -125,10 +125,10 @@ func (t *DeleteRangeTask) sendReqOnRange(ctx context.Context, r kv.KeyRange) (Ra } continue } - deleteRangeResp := resp.DeleteRange - if deleteRangeResp == nil { + if resp.Resp == nil { return stat, errors.Trace(ErrBodyMissing) } + deleteRangeResp := resp.Resp.(*kvrpcpb.DeleteRangeResponse) if err := deleteRangeResp.GetError(); err != "" { return stat, errors.Errorf("unexpected delete range err: %v", err) } diff --git a/store/tikv/gcworker/gc_worker.go b/store/tikv/gcworker/gc_worker.go index 522b9226ced39..fe4794abf9f5d 100644 --- a/store/tikv/gcworker/gc_worker.go +++ b/store/tikv/gcworker/gc_worker.go @@ -675,10 +675,10 @@ func (w *GCWorker) doUnsafeDestroyRangeRequest(ctx context.Context, startKey []b resp, err1 := w.store.GetTiKVClient().SendRequest(ctx, address, req, tikv.UnsafeDestroyRangeTimeout) if err1 == nil { - if resp == nil || resp.UnsafeDestroyRange == nil { + if resp == nil || resp.Resp == nil { err1 = errors.Errorf("unsafe destroy range returns nil response from store %v", storeID) } else { - errStr := resp.UnsafeDestroyRange.Error + errStr := (resp.Resp.(*kvrpcpb.UnsafeDestroyRangeResponse)).Error if len(errStr) > 0 { err1 = errors.Errorf("unsafe destroy range failed on store %v: %s", storeID, errStr) } @@ -854,10 +854,10 @@ func (w *GCWorker) resolveLocksForRange(ctx context.Context, safePoint uint64, s } continue } - locksResp := resp.ScanLock - if locksResp == nil { + if resp.Resp == nil { return stat, errors.Trace(tikv.ErrBodyMissing) } + locksResp := resp.Resp.(*kvrpcpb.ScanLockResponse) if locksResp.GetError() != nil { return stat, errors.Errorf("unexpected scanlock error: %s", locksResp) } @@ -996,10 +996,10 @@ func (w *GCWorker) doGCForRegion(bo *tikv.Backoffer, safePoint uint64, region ti return regionErr, nil } - gcResp := resp.GC - if gcResp == nil { + if resp.Resp == nil { return nil, errors.Trace(tikv.ErrBodyMissing) } + gcResp := resp.Resp.(*kvrpcpb.GCResponse) if gcResp.GetError() != nil { return nil, errors.Errorf("unexpected gc error: %s", gcResp.GetError()) } diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index 781b6be23b689..b56d5ebaac0ef 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -399,16 +399,15 @@ func (s *testGCWorkerSuite) testDeleteRangesFailureImpl(c *C, failType int) { s.client.unsafeDestroyRangeHandler = func(addr string, req *tikvrpc.Request) (*tikvrpc.Response, error) { sendReqCh <- SentReq{req, addr} resp := &tikvrpc.Response{ - Type: tikvrpc.CmdUnsafeDestroyRange, - UnsafeDestroyRange: &kvrpcpb.UnsafeDestroyRangeResponse{}, + Resp: &kvrpcpb.UnsafeDestroyRangeResponse{}, } if bytes.Equal(req.UnsafeDestroyRange().GetStartKey(), failKey) && addr == failStore.GetAddress() { if failType == failRPCErr { return nil, errors.New("error") } else if failType == failNilResp { - resp.UnsafeDestroyRange = nil + resp.Resp = nil } else if failType == failErrResp { - resp.UnsafeDestroyRange.Error = "error" + (resp.Resp.(*kvrpcpb.UnsafeDestroyRangeResponse)).Error = "error" } else { panic("unreachable") } diff --git a/store/tikv/lock_resolver.go b/store/tikv/lock_resolver.go index d0fc33122a1dc..f20116c26af55 100644 --- a/store/tikv/lock_resolver.go +++ b/store/tikv/lock_resolver.go @@ -240,10 +240,10 @@ func (lr *LockResolver) BatchResolveLocks(bo *Backoffer, locks []*Lock, loc Regi return false, nil } - cmdResp := resp.ResolveLock - if cmdResp == nil { + if resp.Resp == nil { return false, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.ResolveLockResponse) if keyErr := cmdResp.GetError(); keyErr != nil { return false, errors.Errorf("unexpected resolve err: %s", keyErr) } @@ -360,10 +360,10 @@ func (lr *LockResolver) getTxnStatus(bo *Backoffer, txnID uint64, primary []byte } continue } - cmdResp := resp.Cleanup - if cmdResp == nil { + if resp.Resp == nil { return status, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.CleanupResponse) if keyErr := cmdResp.GetError(); keyErr != nil { err = errors.Errorf("unexpected cleanup err: %s, tid: %v", keyErr, txnID) logutil.BgLogger().Error("getTxnStatus error", zap.Error(err)) @@ -419,10 +419,10 @@ func (lr *LockResolver) resolveLock(bo *Backoffer, l *Lock, status TxnStatus, cl } continue } - cmdResp := resp.ResolveLock - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.ResolveLockResponse) if keyErr := cmdResp.GetError(); keyErr != nil { err = errors.Errorf("unexpected resolve err: %s, lock: %v", keyErr, l) logutil.BgLogger().Error("resolveLock error", zap.Error(err)) diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index ea0aaf6aff78f..f2379cd142f87 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -221,9 +221,8 @@ func (s *testLockSuite) mustGetLock(c *C, key []byte) *Lock { c.Assert(err, IsNil) resp, err := s.store.SendReq(bo, req, loc.Region, readTimeoutShort) c.Assert(err, IsNil) - cmdGetResp := resp.Get - c.Assert(cmdGetResp, NotNil) - keyErr := cmdGetResp.GetError() + c.Assert(resp.Resp, NotNil) + keyErr := resp.Resp.(*kvrpcpb.GetResponse).GetError() c.Assert(keyErr, NotNil) lock, err := extractLockFromKeyErr(keyErr) c.Assert(err, IsNil) diff --git a/store/tikv/rawkv.go b/store/tikv/rawkv.go index 9a43eb0437d68..c510dddba9d9f 100644 --- a/store/tikv/rawkv.go +++ b/store/tikv/rawkv.go @@ -109,10 +109,10 @@ func (c *RawKVClient) Get(key []byte) ([]byte, error) { if err != nil { return nil, errors.Trace(err) } - cmdResp := resp.RawGet - if cmdResp == nil { + if resp.Resp == nil { return nil, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawGetResponse) if cmdResp.GetError() != "" { return nil, errors.New(cmdResp.GetError()) } @@ -135,10 +135,10 @@ func (c *RawKVClient) BatchGet(keys [][]byte) ([][]byte, error) { return nil, errors.Trace(err) } - cmdResp := resp.RawBatchGet - if cmdResp == nil { + if resp.Resp == nil { return nil, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawBatchGetResponse) keyToValue := make(map[string][]byte, len(keys)) for _, pair := range cmdResp.Pairs { @@ -171,10 +171,10 @@ func (c *RawKVClient) Put(key, value []byte) error { if err != nil { return errors.Trace(err) } - cmdResp := resp.RawPut - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawPutResponse) if cmdResp.GetError() != "" { return errors.New(cmdResp.GetError()) } @@ -213,10 +213,10 @@ func (c *RawKVClient) Delete(key []byte) error { if err != nil { return errors.Trace(err) } - cmdResp := resp.RawDelete - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawDeleteResponse) if cmdResp.GetError() != "" { return errors.New(cmdResp.GetError()) } @@ -235,10 +235,10 @@ func (c *RawKVClient) BatchDelete(keys [][]byte) error { if err != nil { return errors.Trace(err) } - cmdResp := resp.RawBatchDelete - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawBatchDeleteResponse) if cmdResp.GetError() != "" { return errors.New(cmdResp.GetError()) } @@ -265,10 +265,10 @@ func (c *RawKVClient) DeleteRange(startKey []byte, endKey []byte) error { if err != nil { return errors.Trace(err) } - cmdResp := resp.RawDeleteRange - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawDeleteRangeResponse) if cmdResp.GetError() != "" { return errors.New(cmdResp.GetError()) } @@ -301,10 +301,10 @@ func (c *RawKVClient) Scan(startKey, endKey []byte, limit int) (keys [][]byte, v if err != nil { return nil, nil, errors.Trace(err) } - cmdResp := resp.RawScan - if cmdResp == nil { + if resp.Resp == nil { return nil, nil, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawScanResponse) for _, pair := range cmdResp.Kvs { keys = append(keys, pair.Key) values = append(values, pair.Value) @@ -345,10 +345,10 @@ func (c *RawKVClient) ReverseScan(startKey, endKey []byte, limit int) (keys [][] if err != nil { return nil, nil, errors.Trace(err) } - cmdResp := resp.RawScan - if cmdResp == nil { + if resp.Resp == nil { return nil, nil, errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawScanResponse) for _, pair := range cmdResp.Kvs { keys = append(keys, pair.Key) values = append(values, pair.Value) @@ -419,9 +419,9 @@ func (c *RawKVClient) sendBatchReq(bo *Backoffer, keys [][]byte, cmdType tikvrpc var resp *tikvrpc.Response switch cmdType { case tikvrpc.CmdRawBatchGet: - resp = &tikvrpc.Response{Type: tikvrpc.CmdRawBatchGet, RawBatchGet: &kvrpcpb.RawBatchGetResponse{}} + resp = &tikvrpc.Response{Resp: &kvrpcpb.RawBatchGetResponse{}} case tikvrpc.CmdRawBatchDelete: - resp = &tikvrpc.Response{Type: tikvrpc.CmdRawBatchDelete, RawBatchDelete: &kvrpcpb.RawBatchDeleteResponse{}} + resp = &tikvrpc.Response{Resp: &kvrpcpb.RawBatchDeleteResponse{}} } for i := 0; i < len(batches); i++ { singleResp, ok := <-ches @@ -432,8 +432,8 @@ func (c *RawKVClient) sendBatchReq(bo *Backoffer, keys [][]byte, cmdType tikvrpc firstError = singleResp.err } } else if cmdType == tikvrpc.CmdRawBatchGet { - cmdResp := singleResp.resp.RawBatchGet - resp.RawBatchGet.Pairs = append(resp.RawBatchGet.Pairs, cmdResp.Pairs...) + cmdResp := singleResp.resp.Resp.(*kvrpcpb.RawBatchGetResponse) + resp.Resp.(*kvrpcpb.RawBatchGetResponse).Pairs = append(resp.Resp.(*kvrpcpb.RawBatchGetResponse).Pairs, cmdResp.Pairs...) } } } @@ -483,11 +483,11 @@ func (c *RawKVClient) doBatchReq(bo *Backoffer, batch batch, cmdType tikvrpc.Cmd case tikvrpc.CmdRawBatchGet: batchResp.resp = resp case tikvrpc.CmdRawBatchDelete: - cmdResp := resp.RawBatchDelete - if cmdResp == nil { + if resp.Resp == nil { batchResp.err = errors.Trace(ErrBodyMissing) return batchResp } + cmdResp := resp.Resp.(*kvrpcpb.RawBatchDeleteResponse) if cmdResp.GetError() != "" { batchResp.err = errors.New(cmdResp.GetError()) return batchResp @@ -642,10 +642,10 @@ func (c *RawKVClient) doBatchPut(bo *Backoffer, batch batch) error { return c.sendBatchPut(bo, batch.keys, batch.values) } - cmdResp := resp.RawBatchPut - if cmdResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdResp := resp.Resp.(*kvrpcpb.RawBatchPutResponse) if cmdResp.GetError() != "" { return errors.New(cmdResp.GetError()) } diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index 3fcd149707f2b..45c0d53667e44 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -15,6 +15,9 @@ package tikv import ( "context" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" "sync/atomic" "time" @@ -26,9 +29,6 @@ import ( "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/util/logutil" - "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" ) // ShuttingDown is a flag to indicate tidb-server is exiting (Ctrl+C signal @@ -82,15 +82,13 @@ func (s *RegionRequestSender) SendReqCtx(bo *Backoffer, req *tikvrpc.Request, re case "GCNotLeader": if req.Type == tikvrpc.CmdGC { failpoint.Return(&tikvrpc.Response{ - Type: tikvrpc.CmdGC, - GC: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{NotLeader: &errorpb.NotLeader{}}}, }, nil, nil) } case "GCServerIsBusy": if req.Type == tikvrpc.CmdGC { failpoint.Return(&tikvrpc.Response{ - Type: tikvrpc.CmdGC, - GC: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, + Resp: &kvrpcpb.GCResponse{RegionError: &errorpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}}, }, nil, nil) } } diff --git a/store/tikv/region_request_test.go b/store/tikv/region_request_test.go index 5ee508e817d08..9d80af4c67efa 100644 --- a/store/tikv/region_request_test.go +++ b/store/tikv/region_request_test.go @@ -69,7 +69,7 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithStoreRestart(c *C) { c.Assert(region, NotNil) resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) // stop store. s.cluster.StopStore(s.store) @@ -86,7 +86,7 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithStoreRestart(c *C) { c.Assert(region, NotNil) resp, err = s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) } func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOne(c *C) { @@ -99,7 +99,7 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOn c.Assert(region, NotNil) resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) // add new unknown region store2 := s.cluster.AllocID() @@ -134,7 +134,7 @@ func (s *testRegionRequestSuite) TestSendReqCtx(c *C) { c.Assert(region, NotNil) resp, ctx, err := s.regionRequestSender.SendReqCtx(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) c.Assert(ctx, NotNil) } @@ -148,7 +148,7 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCancelled(c *C) { c.Assert(region, NotNil) resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) // set store to cancel state. s.cluster.CancelStore(s.store) @@ -165,7 +165,7 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCancelled(c *C) { c.Assert(region, NotNil) resp, err = s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) - c.Assert(resp.RawPut, NotNil) + c.Assert(resp.Resp, NotNil) } func (s *testRegionRequestSuite) TestNoReloadRegionWhenCtxCanceled(c *C) { diff --git a/store/tikv/scan.go b/store/tikv/scan.go index c50313ae91be5..5804c5bf6d3aa 100644 --- a/store/tikv/scan.go +++ b/store/tikv/scan.go @@ -218,10 +218,10 @@ func (s *Scanner) getData(bo *Backoffer) error { } continue } - cmdScanResp := resp.Scan - if cmdScanResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + cmdScanResp := resp.Resp.(*pb.ScanResponse) err = s.snapshot.store.CheckVisibility(s.startTS()) if err != nil { diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index f4fbe811ec605..258856fb6eb3b 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -177,10 +177,10 @@ func (s *tikvSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, coll err = s.batchGetKeysByRegions(bo, pending, collectF) return errors.Trace(err) } - batchGetResp := resp.BatchGet - if batchGetResp == nil { + if resp.Resp == nil { return errors.Trace(ErrBodyMissing) } + batchGetResp := resp.Resp.(*pb.BatchGetResponse) var ( lockedKeys [][]byte locks []*Lock @@ -260,10 +260,10 @@ func (s *tikvSnapshot) get(bo *Backoffer, k kv.Key) ([]byte, error) { } continue } - cmdGetResp := resp.Get - if cmdGetResp == nil { + if resp.Resp == nil { return nil, errors.Trace(ErrBodyMissing) } + cmdGetResp := resp.Resp.(*pb.GetResponse) val := cmdGetResp.GetValue() if keyErr := cmdGetResp.GetError(); keyErr != nil { lock, err := extractLockFromKeyErr(keyErr) diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 89986bad22b87..a232573575682 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -63,11 +63,12 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key, scatter bool) (regionID uint64, } continue } + splitRegion := res.Resp.(*kvrpcpb.SplitRegionResponse) logutil.BgLogger().Info("split region complete", zap.Binary("at", splitKey), - zap.Stringer("new region left", res.SplitRegion.GetLeft()), - zap.Stringer("new region right", res.SplitRegion.GetRight())) - left := res.SplitRegion.GetLeft() + zap.Stringer("new region left", splitRegion.GetLeft()), + zap.Stringer("new region right", splitRegion.GetRight())) + left := splitRegion.GetLeft() if left == nil { return 0, nil } diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index 40fb48a2b004b..5fff7dd47ccc5 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -220,8 +220,8 @@ type checkRequestClient struct { func (c *checkRequestClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { resp, err := c.Client.SendRequest(ctx, addr, req, timeout) if c.priority != req.Priority { - if resp.Get != nil { - resp.Get.Error = &pb.KeyError{ + if resp.Resp != nil { + (resp.Resp.(*pb.GetResponse)).Error = &pb.KeyError{ Abort: "request check error", } } diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index 2f9101a480135..8d2e813e6d9d6 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -358,90 +358,58 @@ func (req *Request) IsDebugReq() bool { // Response wraps all kv/coprocessor responses. type Response struct { - Type CmdType - Get *kvrpcpb.GetResponse - Scan *kvrpcpb.ScanResponse - Prewrite *kvrpcpb.PrewriteResponse - Commit *kvrpcpb.CommitResponse - Cleanup *kvrpcpb.CleanupResponse - BatchGet *kvrpcpb.BatchGetResponse - BatchRollback *kvrpcpb.BatchRollbackResponse - ScanLock *kvrpcpb.ScanLockResponse - ResolveLock *kvrpcpb.ResolveLockResponse - GC *kvrpcpb.GCResponse - DeleteRange *kvrpcpb.DeleteRangeResponse - RawGet *kvrpcpb.RawGetResponse - RawBatchGet *kvrpcpb.RawBatchGetResponse - RawPut *kvrpcpb.RawPutResponse - RawBatchPut *kvrpcpb.RawBatchPutResponse - RawDelete *kvrpcpb.RawDeleteResponse - RawBatchDelete *kvrpcpb.RawBatchDeleteResponse - RawDeleteRange *kvrpcpb.RawDeleteRangeResponse - RawScan *kvrpcpb.RawScanResponse - UnsafeDestroyRange *kvrpcpb.UnsafeDestroyRangeResponse - Cop *coprocessor.Response - CopStream *CopStreamResponse - MvccGetByKey *kvrpcpb.MvccGetByKeyResponse - MvccGetByStartTS *kvrpcpb.MvccGetByStartTsResponse - SplitRegion *kvrpcpb.SplitRegionResponse - - PessimisticLock *kvrpcpb.PessimisticLockResponse - PessimisticRollback *kvrpcpb.PessimisticRollbackResponse - - DebugGetRegionProperties *debugpb.GetRegionPropertiesResponse - - Empty *tikvpb.BatchCommandsEmptyResponse + Resp interface{} } // FromBatchCommandsResponse converts a BatchCommands response to Response. func FromBatchCommandsResponse(res *tikvpb.BatchCommandsResponse_Response) *Response { switch res := res.GetCmd().(type) { case *tikvpb.BatchCommandsResponse_Response_Get: - return &Response{Type: CmdGet, Get: res.Get} + return &Response{Resp: res.Get} case *tikvpb.BatchCommandsResponse_Response_Scan: - return &Response{Type: CmdScan, Scan: res.Scan} + return &Response{Resp: res.Scan} case *tikvpb.BatchCommandsResponse_Response_Prewrite: - return &Response{Type: CmdPrewrite, Prewrite: res.Prewrite} + return &Response{Resp: res.Prewrite} case *tikvpb.BatchCommandsResponse_Response_Commit: - return &Response{Type: CmdCommit, Commit: res.Commit} + return &Response{Resp: res.Commit} case *tikvpb.BatchCommandsResponse_Response_Cleanup: - return &Response{Type: CmdCleanup, Cleanup: res.Cleanup} + return &Response{Resp: res.Cleanup} case *tikvpb.BatchCommandsResponse_Response_BatchGet: - return &Response{Type: CmdBatchGet, BatchGet: res.BatchGet} + return &Response{Resp: res.BatchGet} case *tikvpb.BatchCommandsResponse_Response_BatchRollback: - return &Response{Type: CmdBatchRollback, BatchRollback: res.BatchRollback} + return &Response{Resp: res.BatchRollback} case *tikvpb.BatchCommandsResponse_Response_ScanLock: - return &Response{Type: CmdScanLock, ScanLock: res.ScanLock} + return &Response{Resp: res.ScanLock} case *tikvpb.BatchCommandsResponse_Response_ResolveLock: - return &Response{Type: CmdResolveLock, ResolveLock: res.ResolveLock} + return &Response{Resp: res.ResolveLock} case *tikvpb.BatchCommandsResponse_Response_GC: - return &Response{Type: CmdGC, GC: res.GC} + return &Response{Resp: res.GC} case *tikvpb.BatchCommandsResponse_Response_DeleteRange: - return &Response{Type: CmdDeleteRange, DeleteRange: res.DeleteRange} + return &Response{Resp: res.DeleteRange} case *tikvpb.BatchCommandsResponse_Response_RawGet: - return &Response{Type: CmdRawGet, RawGet: res.RawGet} + return &Response{Resp: res.RawGet} case *tikvpb.BatchCommandsResponse_Response_RawBatchGet: - return &Response{Type: CmdRawBatchGet, RawBatchGet: res.RawBatchGet} + return &Response{Resp: res.RawBatchGet} case *tikvpb.BatchCommandsResponse_Response_RawPut: - return &Response{Type: CmdRawPut, RawPut: res.RawPut} + return &Response{Resp: res.RawPut} case *tikvpb.BatchCommandsResponse_Response_RawBatchPut: - return &Response{Type: CmdRawBatchPut, RawBatchPut: res.RawBatchPut} + return &Response{Resp: res.RawBatchPut} case *tikvpb.BatchCommandsResponse_Response_RawDelete: - return &Response{Type: CmdRawDelete, RawDelete: res.RawDelete} + return &Response{Resp: res.RawDelete} case *tikvpb.BatchCommandsResponse_Response_RawBatchDelete: - return &Response{Type: CmdRawBatchDelete, RawBatchDelete: res.RawBatchDelete} + return &Response{Resp: res.RawBatchDelete} case *tikvpb.BatchCommandsResponse_Response_RawDeleteRange: - return &Response{Type: CmdRawDeleteRange, RawDeleteRange: res.RawDeleteRange} + return &Response{Resp: res.RawDeleteRange} case *tikvpb.BatchCommandsResponse_Response_RawScan: - return &Response{Type: CmdRawScan, RawScan: res.RawScan} + return &Response{Resp: res.RawScan} case *tikvpb.BatchCommandsResponse_Response_Coprocessor: - return &Response{Type: CmdCop, Cop: res.Coprocessor} + return &Response{Resp: res.Coprocessor} case *tikvpb.BatchCommandsResponse_Response_PessimisticLock: - return &Response{Type: CmdPessimisticLock, PessimisticLock: res.PessimisticLock} + return &Response{Resp: res.PessimisticLock} case *tikvpb.BatchCommandsResponse_Response_PessimisticRollback: - return &Response{Type: CmdPessimisticRollback, PessimisticRollback: res.PessimisticRollback} + return &Response{Resp: res.PessimisticRollback} case *tikvpb.BatchCommandsResponse_Response_Empty: - return &Response{Type: CmdEmpty, Empty: res.Empty} + return &Response{Resp: res.Empty} } return nil } @@ -529,189 +497,144 @@ func SetContext(req *Request, region *metapb.Region, peer *metapb.Peer) error { // GenRegionErrorResp returns corresponding Response with specified RegionError // according to the given req. func GenRegionErrorResp(req *Request, e *errorpb.Error) (*Response, error) { + var p interface{} resp := &Response{} - resp.Type = req.Type switch req.Type { case CmdGet: - resp.Get = &kvrpcpb.GetResponse{ + p = &kvrpcpb.GetResponse{ RegionError: e, } case CmdScan: - resp.Scan = &kvrpcpb.ScanResponse{ + p = &kvrpcpb.ScanResponse{ RegionError: e, } case CmdPrewrite: - resp.Prewrite = &kvrpcpb.PrewriteResponse{ + p = &kvrpcpb.PrewriteResponse{ RegionError: e, } case CmdPessimisticLock: - resp.PessimisticLock = &kvrpcpb.PessimisticLockResponse{ + p = &kvrpcpb.PessimisticLockResponse{ RegionError: e, } case CmdPessimisticRollback: - resp.PessimisticRollback = &kvrpcpb.PessimisticRollbackResponse{ + p = &kvrpcpb.PessimisticRollbackResponse{ RegionError: e, } case CmdCommit: - resp.Commit = &kvrpcpb.CommitResponse{ + p = &kvrpcpb.CommitResponse{ RegionError: e, } case CmdCleanup: - resp.Cleanup = &kvrpcpb.CleanupResponse{ + p = &kvrpcpb.CleanupResponse{ RegionError: e, } case CmdBatchGet: - resp.BatchGet = &kvrpcpb.BatchGetResponse{ + p = &kvrpcpb.BatchGetResponse{ RegionError: e, } case CmdBatchRollback: - resp.BatchRollback = &kvrpcpb.BatchRollbackResponse{ + p = &kvrpcpb.BatchRollbackResponse{ RegionError: e, } case CmdScanLock: - resp.ScanLock = &kvrpcpb.ScanLockResponse{ + p = &kvrpcpb.ScanLockResponse{ RegionError: e, } case CmdResolveLock: - resp.ResolveLock = &kvrpcpb.ResolveLockResponse{ + p = &kvrpcpb.ResolveLockResponse{ RegionError: e, } case CmdGC: - resp.GC = &kvrpcpb.GCResponse{ + p = &kvrpcpb.GCResponse{ RegionError: e, } case CmdDeleteRange: - resp.DeleteRange = &kvrpcpb.DeleteRangeResponse{ + p = &kvrpcpb.DeleteRangeResponse{ RegionError: e, } case CmdRawGet: - resp.RawGet = &kvrpcpb.RawGetResponse{ + p = &kvrpcpb.RawGetResponse{ RegionError: e, } case CmdRawBatchGet: - resp.RawBatchGet = &kvrpcpb.RawBatchGetResponse{ + p = &kvrpcpb.RawBatchGetResponse{ RegionError: e, } case CmdRawPut: - resp.RawPut = &kvrpcpb.RawPutResponse{ + p = &kvrpcpb.RawPutResponse{ RegionError: e, } case CmdRawBatchPut: - resp.RawBatchPut = &kvrpcpb.RawBatchPutResponse{ + p = &kvrpcpb.RawBatchPutResponse{ RegionError: e, } case CmdRawDelete: - resp.RawDelete = &kvrpcpb.RawDeleteResponse{ + p = &kvrpcpb.RawDeleteResponse{ RegionError: e, } case CmdRawBatchDelete: - resp.RawBatchDelete = &kvrpcpb.RawBatchDeleteResponse{ + p = &kvrpcpb.RawBatchDeleteResponse{ RegionError: e, } case CmdRawDeleteRange: - resp.RawDeleteRange = &kvrpcpb.RawDeleteRangeResponse{ + p = &kvrpcpb.RawDeleteRangeResponse{ RegionError: e, } case CmdRawScan: - resp.RawScan = &kvrpcpb.RawScanResponse{ + p = &kvrpcpb.RawScanResponse{ RegionError: e, } case CmdUnsafeDestroyRange: - resp.UnsafeDestroyRange = &kvrpcpb.UnsafeDestroyRangeResponse{ + p = &kvrpcpb.UnsafeDestroyRangeResponse{ RegionError: e, } case CmdCop: - resp.Cop = &coprocessor.Response{ + p = &coprocessor.Response{ RegionError: e, } case CmdCopStream: - resp.CopStream = &CopStreamResponse{ + p = &CopStreamResponse{ Response: &coprocessor.Response{ RegionError: e, }, } case CmdMvccGetByKey: - resp.MvccGetByKey = &kvrpcpb.MvccGetByKeyResponse{ + p = &kvrpcpb.MvccGetByKeyResponse{ RegionError: e, } case CmdMvccGetByStartTs: - resp.MvccGetByStartTS = &kvrpcpb.MvccGetByStartTsResponse{ + p = &kvrpcpb.MvccGetByStartTsResponse{ RegionError: e, } case CmdSplitRegion: - resp.SplitRegion = &kvrpcpb.SplitRegionResponse{ + p = &kvrpcpb.SplitRegionResponse{ RegionError: e, } case CmdEmpty: default: return nil, fmt.Errorf("invalid request type %v", req.Type) } + resp.Resp = p return resp, nil } +type getRegionError interface { + GetRegionError() *errorpb.Error +} + // GetRegionError returns the RegionError of the underlying concrete response. func (resp *Response) GetRegionError() (*errorpb.Error, error) { - var e *errorpb.Error - switch resp.Type { - case CmdGet: - e = resp.Get.GetRegionError() - case CmdScan: - e = resp.Scan.GetRegionError() - case CmdPessimisticLock: - e = resp.PessimisticLock.GetRegionError() - case CmdPessimisticRollback: - e = resp.PessimisticRollback.GetRegionError() - case CmdPrewrite: - e = resp.Prewrite.GetRegionError() - case CmdCommit: - e = resp.Commit.GetRegionError() - case CmdCleanup: - e = resp.Cleanup.GetRegionError() - case CmdBatchGet: - e = resp.BatchGet.GetRegionError() - case CmdBatchRollback: - e = resp.BatchRollback.GetRegionError() - case CmdScanLock: - e = resp.ScanLock.GetRegionError() - case CmdResolveLock: - e = resp.ResolveLock.GetRegionError() - case CmdGC: - e = resp.GC.GetRegionError() - case CmdDeleteRange: - e = resp.DeleteRange.GetRegionError() - case CmdRawGet: - e = resp.RawGet.GetRegionError() - case CmdRawBatchGet: - e = resp.RawBatchGet.GetRegionError() - case CmdRawPut: - e = resp.RawPut.GetRegionError() - case CmdRawBatchPut: - e = resp.RawBatchPut.GetRegionError() - case CmdRawDelete: - e = resp.RawDelete.GetRegionError() - case CmdRawBatchDelete: - e = resp.RawBatchDelete.GetRegionError() - case CmdRawDeleteRange: - e = resp.RawDeleteRange.GetRegionError() - case CmdRawScan: - e = resp.RawScan.GetRegionError() - case CmdUnsafeDestroyRange: - e = resp.UnsafeDestroyRange.GetRegionError() - case CmdCop: - e = resp.Cop.GetRegionError() - case CmdCopStream: - e = resp.CopStream.Response.GetRegionError() - case CmdMvccGetByKey: - e = resp.MvccGetByKey.GetRegionError() - case CmdMvccGetByStartTs: - e = resp.MvccGetByStartTS.GetRegionError() - case CmdSplitRegion: - e = resp.SplitRegion.GetRegionError() - case CmdEmpty: - default: - return nil, fmt.Errorf("invalid response type %v", resp.Type) + if resp.Resp == nil { + return nil, nil + } + err, ok := resp.Resp.(getRegionError) + if !ok { + if _, isEmpty := resp.Resp.(*tikvpb.BatchCommandsEmptyResponse); isEmpty { + return nil, nil + } + return nil, fmt.Errorf("invalid response type %v", resp) } - return e, nil + return err.GetRegionError(), nil } // CallRPC launches a rpc call. @@ -719,69 +642,68 @@ func (resp *Response) GetRegionError() (*errorpb.Error, error) { // cancel function will be sent to the channel, together with a lease checked by a background goroutine. func CallRPC(ctx context.Context, client tikvpb.TikvClient, req *Request) (*Response, error) { resp := &Response{} - resp.Type = req.Type var err error switch req.Type { case CmdGet: - resp.Get, err = client.KvGet(ctx, req.Get()) + resp.Resp, err = client.KvGet(ctx, req.Get()) case CmdScan: - resp.Scan, err = client.KvScan(ctx, req.Scan()) + resp.Resp, err = client.KvScan(ctx, req.Scan()) case CmdPrewrite: - resp.Prewrite, err = client.KvPrewrite(ctx, req.Prewrite()) + resp.Resp, err = client.KvPrewrite(ctx, req.Prewrite()) case CmdPessimisticLock: - resp.PessimisticLock, err = client.KvPessimisticLock(ctx, req.PessimisticLock()) + resp.Resp, err = client.KvPessimisticLock(ctx, req.PessimisticLock()) case CmdPessimisticRollback: - resp.PessimisticRollback, err = client.KVPessimisticRollback(ctx, req.PessimisticRollback()) + resp.Resp, err = client.KVPessimisticRollback(ctx, req.PessimisticRollback()) case CmdCommit: - resp.Commit, err = client.KvCommit(ctx, req.Commit()) + resp.Resp, err = client.KvCommit(ctx, req.Commit()) case CmdCleanup: - resp.Cleanup, err = client.KvCleanup(ctx, req.Cleanup()) + resp.Resp, err = client.KvCleanup(ctx, req.Cleanup()) case CmdBatchGet: - resp.BatchGet, err = client.KvBatchGet(ctx, req.BatchGet()) + resp.Resp, err = client.KvBatchGet(ctx, req.BatchGet()) case CmdBatchRollback: - resp.BatchRollback, err = client.KvBatchRollback(ctx, req.BatchRollback()) + resp.Resp, err = client.KvBatchRollback(ctx, req.BatchRollback()) case CmdScanLock: - resp.ScanLock, err = client.KvScanLock(ctx, req.ScanLock()) + resp.Resp, err = client.KvScanLock(ctx, req.ScanLock()) case CmdResolveLock: - resp.ResolveLock, err = client.KvResolveLock(ctx, req.ResolveLock()) + resp.Resp, err = client.KvResolveLock(ctx, req.ResolveLock()) case CmdGC: - resp.GC, err = client.KvGC(ctx, req.GC()) + resp.Resp, err = client.KvGC(ctx, req.GC()) case CmdDeleteRange: - resp.DeleteRange, err = client.KvDeleteRange(ctx, req.DeleteRange()) + resp.Resp, err = client.KvDeleteRange(ctx, req.DeleteRange()) case CmdRawGet: - resp.RawGet, err = client.RawGet(ctx, req.RawGet()) + resp.Resp, err = client.RawGet(ctx, req.RawGet()) case CmdRawBatchGet: - resp.RawBatchGet, err = client.RawBatchGet(ctx, req.RawBatchGet()) + resp.Resp, err = client.RawBatchGet(ctx, req.RawBatchGet()) case CmdRawPut: - resp.RawPut, err = client.RawPut(ctx, req.RawPut()) + resp.Resp, err = client.RawPut(ctx, req.RawPut()) case CmdRawBatchPut: - resp.RawBatchPut, err = client.RawBatchPut(ctx, req.RawBatchPut()) + resp.Resp, err = client.RawBatchPut(ctx, req.RawBatchPut()) case CmdRawDelete: - resp.RawDelete, err = client.RawDelete(ctx, req.RawDelete()) + resp.Resp, err = client.RawDelete(ctx, req.RawDelete()) case CmdRawBatchDelete: - resp.RawBatchDelete, err = client.RawBatchDelete(ctx, req.RawBatchDelete()) + resp.Resp, err = client.RawBatchDelete(ctx, req.RawBatchDelete()) case CmdRawDeleteRange: - resp.RawDeleteRange, err = client.RawDeleteRange(ctx, req.RawDeleteRange()) + resp.Resp, err = client.RawDeleteRange(ctx, req.RawDeleteRange()) case CmdRawScan: - resp.RawScan, err = client.RawScan(ctx, req.RawScan()) + resp.Resp, err = client.RawScan(ctx, req.RawScan()) case CmdUnsafeDestroyRange: - resp.UnsafeDestroyRange, err = client.UnsafeDestroyRange(ctx, req.UnsafeDestroyRange()) + resp.Resp, err = client.UnsafeDestroyRange(ctx, req.UnsafeDestroyRange()) case CmdCop: - resp.Cop, err = client.Coprocessor(ctx, req.Cop()) + resp.Resp, err = client.Coprocessor(ctx, req.Cop()) case CmdCopStream: var streamClient tikvpb.Tikv_CoprocessorStreamClient streamClient, err = client.CoprocessorStream(ctx, req.Cop()) - resp.CopStream = &CopStreamResponse{ + resp.Resp = &CopStreamResponse{ Tikv_CoprocessorStreamClient: streamClient, } case CmdMvccGetByKey: - resp.MvccGetByKey, err = client.MvccGetByKey(ctx, req.MvccGetByKey()) + resp.Resp, err = client.MvccGetByKey(ctx, req.MvccGetByKey()) case CmdMvccGetByStartTs: - resp.MvccGetByStartTS, err = client.MvccGetByStartTs(ctx, req.MvccGetByStartTs()) + resp.Resp, err = client.MvccGetByStartTs(ctx, req.MvccGetByStartTs()) case CmdSplitRegion: - resp.SplitRegion, err = client.SplitRegion(ctx, req.SplitRegion()) + resp.Resp, err = client.SplitRegion(ctx, req.SplitRegion()) case CmdEmpty: - resp.Empty, err = &tikvpb.BatchCommandsEmptyResponse{}, nil + resp.Resp, err = &tikvpb.BatchCommandsEmptyResponse{}, nil default: return nil, errors.Errorf("invalid request type: %v", req.Type) } @@ -793,12 +715,11 @@ func CallRPC(ctx context.Context, client tikvpb.TikvClient, req *Request) (*Resp // CallDebugRPC launches a debug rpc call. func CallDebugRPC(ctx context.Context, client debugpb.DebugClient, req *Request) (*Response, error) { - resp := &Response{Type: req.Type} - resp.Type = req.Type + resp := &Response{} var err error switch req.Type { case CmdDebugGetRegionProperties: - resp.DebugGetRegionProperties, err = client.GetRegionProperties(ctx, req.DebugGetRegionProperties()) + resp.Resp, err = client.GetRegionProperties(ctx, req.DebugGetRegionProperties()) default: return nil, errors.Errorf("invalid request type: %v", req.Type) } From 593fb7def59c615de46193cfb704483c31d8de8f Mon Sep 17 00:00:00 2001 From: amyangfei Date: Tue, 16 Jul 2019 15:56:51 +0800 Subject: [PATCH 036/196] expression: add max_allowed_packet check in concat/concat_ws (#11137) --- expression/builtin_string.go | 55 +++++++++++++++---- expression/builtin_string_test.go | 91 +++++++++++++++++++++++++++++++ util/mock/context.go | 3 + 3 files changed, 137 insertions(+), 12 deletions(-) diff --git a/expression/builtin_string.go b/expression/builtin_string.go index ed553476a74a1..38cd2c09faffc 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -272,17 +272,26 @@ func (c *concatFunctionClass) getFunction(ctx sessionctx.Context, args []Express if bf.tp.Flen >= mysql.MaxBlobWidth { bf.tp.Flen = mysql.MaxBlobWidth } - sig := &builtinConcatSig{bf} + + valStr, _ := ctx.GetSessionVars().GetSystemVar(variable.MaxAllowedPacket) + maxAllowedPacket, err := strconv.ParseUint(valStr, 10, 64) + if err != nil { + return nil, err + } + + sig := &builtinConcatSig{bf, maxAllowedPacket} return sig, nil } type builtinConcatSig struct { baseBuiltinFunc + maxAllowedPacket uint64 } func (b *builtinConcatSig) Clone() builtinFunc { newSig := &builtinConcatSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.maxAllowedPacket = b.maxAllowedPacket return newSig } @@ -295,6 +304,10 @@ func (b *builtinConcatSig) evalString(row chunk.Row) (d string, isNull bool, err if isNull || err != nil { return d, isNull, err } + if uint64(len(s)+len(d)) > b.maxAllowedPacket { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errWarnAllowedPacketOverflowed.GenWithStackByArgs("concat", b.maxAllowedPacket)) + return "", true, nil + } s = append(s, []byte(d)...) } return string(s), false, nil @@ -337,17 +350,25 @@ func (c *concatWSFunctionClass) getFunction(ctx sessionctx.Context, args []Expre bf.tp.Flen = mysql.MaxBlobWidth } - sig := &builtinConcatWSSig{bf} + valStr, _ := ctx.GetSessionVars().GetSystemVar(variable.MaxAllowedPacket) + maxAllowedPacket, err := strconv.ParseUint(valStr, 10, 64) + if err != nil { + return nil, err + } + + sig := &builtinConcatWSSig{bf, maxAllowedPacket} return sig, nil } type builtinConcatWSSig struct { baseBuiltinFunc + maxAllowedPacket uint64 } func (b *builtinConcatWSSig) Clone() builtinFunc { newSig := &builtinConcatWSSig{} newSig.cloneFrom(&b.baseBuiltinFunc) + newSig.maxAllowedPacket = b.maxAllowedPacket return newSig } @@ -357,25 +378,35 @@ func (b *builtinConcatWSSig) evalString(row chunk.Row) (string, bool, error) { args := b.getArgs() strs := make([]string, 0, len(args)) var sep string - for i, arg := range args { - val, isNull, err := arg.EvalString(b.ctx, row) + var targetLength int + + N := len(args) + if N > 0 { + val, isNull, err := args[0].EvalString(b.ctx, row) + if err != nil || isNull { + // If the separator is NULL, the result is NULL. + return val, isNull, err + } + sep = val + } + for i := 1; i < N; i++ { + val, isNull, err := args[i].EvalString(b.ctx, row) if err != nil { return val, isNull, err } - if isNull { - // If the separator is NULL, the result is NULL. - if i == 0 { - return val, isNull, nil - } // CONCAT_WS() does not skip empty strings. However, // it does skip any NULL values after the separator argument. continue } - if i == 0 { - sep = val - continue + targetLength += len(val) + if i > 1 { + targetLength += len(sep) + } + if uint64(targetLength) > b.maxAllowedPacket { + b.ctx.GetSessionVars().StmtCtx.AppendWarning(errWarnAllowedPacketOverflowed.GenWithStackByArgs("concat_ws", b.maxAllowedPacket)) + return "", true, nil } strs = append(strs, val) } diff --git a/expression/builtin_string_test.go b/expression/builtin_string_test.go index 5c9b6debb62e4..b07ebd7d11f13 100644 --- a/expression/builtin_string_test.go +++ b/expression/builtin_string_test.go @@ -171,6 +171,50 @@ func (s *testEvaluatorSuite) TestConcat(c *C) { } } +func (s *testEvaluatorSuite) TestConcatSig(c *C) { + colTypes := []*types.FieldType{ + {Tp: mysql.TypeVarchar}, + {Tp: mysql.TypeVarchar}, + } + resultType := &types.FieldType{Tp: mysql.TypeVarchar, Flen: 1000} + args := []Expression{ + &Column{Index: 0, RetType: colTypes[0]}, + &Column{Index: 1, RetType: colTypes[1]}, + } + base := baseBuiltinFunc{args: args, ctx: s.ctx, tp: resultType} + concat := &builtinConcatSig{base, 5} + + cases := []struct { + args []interface{} + warnings int + res string + }{ + {[]interface{}{"a", "b"}, 0, "ab"}, + {[]interface{}{"aaa", "bbb"}, 1, ""}, + {[]interface{}{"中", "a"}, 0, "中a"}, + {[]interface{}{"中文", "a"}, 2, ""}, + } + + for _, t := range cases { + input := chunk.NewChunkWithCapacity(colTypes, 10) + input.AppendString(0, t.args[0].(string)) + input.AppendString(1, t.args[1].(string)) + + res, isNull, err := concat.evalString(input.GetRow(0)) + c.Assert(res, Equals, t.res) + c.Assert(err, IsNil) + if t.warnings == 0 { + c.Assert(isNull, IsFalse) + } else { + c.Assert(isNull, IsTrue) + warnings := s.ctx.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(warnings, HasLen, t.warnings) + lastWarn := warnings[len(warnings)-1] + c.Assert(terror.ErrorEqual(errWarnAllowedPacketOverflowed, lastWarn.Err), IsTrue) + } + } +} + func (s *testEvaluatorSuite) TestConcatWS(c *C) { defer testleak.AfterTest(c)() cases := []struct { @@ -246,6 +290,53 @@ func (s *testEvaluatorSuite) TestConcatWS(c *C) { c.Assert(err, IsNil) } +func (s *testEvaluatorSuite) TestConcatWSSig(c *C) { + colTypes := []*types.FieldType{ + {Tp: mysql.TypeVarchar}, + {Tp: mysql.TypeVarchar}, + {Tp: mysql.TypeVarchar}, + } + resultType := &types.FieldType{Tp: mysql.TypeVarchar, Flen: 1000} + args := []Expression{ + &Column{Index: 0, RetType: colTypes[0]}, + &Column{Index: 1, RetType: colTypes[1]}, + &Column{Index: 2, RetType: colTypes[2]}, + } + base := baseBuiltinFunc{args: args, ctx: s.ctx, tp: resultType} + concat := &builtinConcatWSSig{base, 6} + + cases := []struct { + args []interface{} + warnings int + res string + }{ + {[]interface{}{",", "a", "b"}, 0, "a,b"}, + {[]interface{}{",", "aaa", "bbb"}, 1, ""}, + {[]interface{}{",", "中", "a"}, 0, "中,a"}, + {[]interface{}{",", "中文", "a"}, 2, ""}, + } + + for _, t := range cases { + input := chunk.NewChunkWithCapacity(colTypes, 10) + input.AppendString(0, t.args[0].(string)) + input.AppendString(1, t.args[1].(string)) + input.AppendString(2, t.args[2].(string)) + + res, isNull, err := concat.evalString(input.GetRow(0)) + c.Assert(res, Equals, t.res) + c.Assert(err, IsNil) + if t.warnings == 0 { + c.Assert(isNull, IsFalse) + } else { + c.Assert(isNull, IsTrue) + warnings := s.ctx.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(warnings, HasLen, t.warnings) + lastWarn := warnings[len(warnings)-1] + c.Assert(terror.ErrorEqual(errWarnAllowedPacketOverflowed, lastWarn.Err), IsTrue) + } + } +} + func (s *testEvaluatorSuite) TestLeft(c *C) { defer testleak.AfterTest(c)() stmtCtx := s.ctx.GetSessionVars().StmtCtx diff --git a/util/mock/context.go b/util/mock/context.go index befd00241b08f..31ddddb5fe209 100644 --- a/util/mock/context.go +++ b/util/mock/context.go @@ -264,6 +264,9 @@ func NewContext() *Context { sctx.sessionVars.MaxChunkSize = 32 sctx.sessionVars.StmtCtx.TimeZone = time.UTC sctx.sessionVars.GlobalVarsAccessor = variable.NewMockGlobalAccessor() + if err := sctx.GetSessionVars().SetSystemVar(variable.MaxAllowedPacket, "67108864"); err != nil { + panic(err) + } return sctx } From 3ec46b0286e2a9dd095cfd77498c9bc5b5235b80 Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Tue, 16 Jul 2019 16:02:44 +0800 Subject: [PATCH 037/196] types: fix overflow check of types/convert.go::floatStrToIntStr (#11114) Signed-off-by: H-ZeX --- types/convert.go | 17 +++++++++-------- types/convert_test.go | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/types/convert.go b/types/convert.go index 5e16ad43ac2d1..58913cb2252eb 100644 --- a/types/convert.go +++ b/types/convert.go @@ -394,6 +394,9 @@ func roundIntStr(numNextDot byte, intStr string) string { // strconv.ParseInt, we can't parse float first then convert it to string because precision will // be lost. For example, the string value "18446744073709551615" which is the max number of unsigned // int will cause some precision to lose. intStr[0] may be a positive and negative sign like '+' or '-'. +// +// This func will find serious overflow such as the len of intStr > 20 (without prefix `+/-`) +// however, it will not check whether the intStr overflow BIGINT. func floatStrToIntStr(sc *stmtctx.StatementContext, validFloat string, oriStr string) (intStr string, _ error) { var dotIdx = -1 var eIdx = -1 @@ -443,12 +446,15 @@ func floatStrToIntStr(sc *stmtctx.StatementContext, validFloat string, oriStr st if err != nil { return validFloat, errors.Trace(err) } - if exp > 0 && int64(intCnt) > (math.MaxInt64-int64(exp)) { - // (exp + incCnt) overflows MaxInt64. + intCnt += exp + if exp >= 0 && (intCnt > 21 || intCnt < 0) { + // MaxInt64 has 19 decimal digits. + // MaxUint64 has 20 decimal digits. + // And the intCnt may contain the len of `+/-`, + // so I use 21 here as the early detection. sc.AppendWarning(ErrOverflow.GenWithStackByArgs("BIGINT", oriStr)) return validFloat[:eIdx], nil } - intCnt += exp if intCnt <= 0 { intStr = "0" if intCnt == 0 && len(digits) > 0 { @@ -474,11 +480,6 @@ func floatStrToIntStr(sc *stmtctx.StatementContext, validFloat string, oriStr st } else { // convert scientific notation decimal number extraZeroCount := intCnt - len(digits) - if extraZeroCount > 20 { - // Append overflow warning and return to avoid allocating too much memory. - sc.AppendWarning(ErrOverflow.GenWithStackByArgs("BIGINT", oriStr)) - return validFloat[:eIdx], nil - } intStr = string(digits) + strings.Repeat("0", extraZeroCount) } return intStr, nil diff --git a/types/convert_test.go b/types/convert_test.go index c591650952056..47f13e99306ad 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -708,6 +708,7 @@ func (s *testTypeConvertSuite) TestGetValidFloat(c *C) { {".5e0", "1"}, {"+.5e0", "+1"}, {"-.5e0", "-1"}, + {".5", "1"}, {"123.456789e5", "12345679"}, {"123.456784e5", "12345678"}, } From 2640d6d69ad7038555dc40ebceb501be9898e835 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 16 Jul 2019 17:03:23 +0800 Subject: [PATCH 038/196] config: refine comment (#11272) --- config/config.toml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.toml.example b/config/config.toml.example index 6ff6fd9b0741a..15f079e1de12d 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -57,7 +57,7 @@ treat-old-version-utf8-as-utf8mb4 = true # enable-table-lock is used to control table lock feature. Default is false, indicate the table lock feature is disabled. enable-table-lock = false -# delay-clean-table-lock is used to control whether delayed-release the table lock in the abnormal situation. (Milliseconds) +# delay-clean-table-lock is used to control the time (Milliseconds) of delay before unlock the table in the abnormal situation. delay-clean-table-lock = 0 [log] From f772318035685ceace8f6b140ef9583b9d93c0e6 Mon Sep 17 00:00:00 2001 From: MyonKeminta <9948422+MyonKeminta@users.noreply.github.com> Date: Tue, 16 Jul 2019 17:47:32 +0800 Subject: [PATCH 039/196] store/tikv: Make GC's tests more strict (#11084) Signed-off-by: MyonKeminta --- store/mockstore/mocktikv/mock_tikv_test.go | 48 ++++ store/mockstore/mocktikv/mvcc.go | 1 + store/mockstore/mocktikv/mvcc_leveldb.go | 66 ++++++ store/mockstore/mocktikv/pd.go | 11 +- store/mockstore/mocktikv/rpc.go | 14 +- store/tikv/gcworker/gc_worker_test.go | 259 ++++++++++++++++++--- 6 files changed, 359 insertions(+), 40 deletions(-) diff --git a/store/mockstore/mocktikv/mock_tikv_test.go b/store/mockstore/mocktikv/mock_tikv_test.go index 250f81ab81809..d36700f8efd75 100644 --- a/store/mockstore/mocktikv/mock_tikv_test.go +++ b/store/mockstore/mocktikv/mock_tikv_test.go @@ -201,6 +201,10 @@ func (s *testMockTiKVSuite) mustBatchResolveLock(c *C, txnInfos map[uint64]uint6 c.Assert(s.store.BatchResolveLock(nil, nil, txnInfos), IsNil) } +func (s *testMockTiKVSuite) mustGC(c *C, safePoint uint64) { + c.Assert(s.store.GC(nil, nil, safePoint), IsNil) +} + func (s *testMockTiKVSuite) mustDeleteRange(c *C, startKey, endKey string) { err := s.store.DeleteRange([]byte(startKey), []byte(endKey)) c.Assert(err, IsNil) @@ -488,6 +492,50 @@ func (s *testMockTiKVSuite) TestBatchResolveLock(c *C) { s.mustScanLock(c, 30, nil) } +func (s *testMockTiKVSuite) TestGC(c *C) { + var safePoint uint64 = 100 + + // Prepare data + s.mustPutOK(c, "k1", "v1", 1, 2) + s.mustPutOK(c, "k1", "v2", 11, 12) + + s.mustPutOK(c, "k2", "v1", 1, 2) + s.mustPutOK(c, "k2", "v2", 11, 12) + s.mustPutOK(c, "k2", "v3", 101, 102) + + s.mustPutOK(c, "k3", "v1", 1, 2) + s.mustPutOK(c, "k3", "v2", 11, 12) + s.mustDeleteOK(c, "k3", 101, 102) + + s.mustPutOK(c, "k4", "v1", 1, 2) + s.mustDeleteOK(c, "k4", 11, 12) + + // Check prepared data + s.mustGetOK(c, "k1", 5, "v1") + s.mustGetOK(c, "k1", 15, "v2") + s.mustGetOK(c, "k2", 5, "v1") + s.mustGetOK(c, "k2", 15, "v2") + s.mustGetOK(c, "k2", 105, "v3") + s.mustGetOK(c, "k3", 5, "v1") + s.mustGetOK(c, "k3", 15, "v2") + s.mustGetNone(c, "k3", 105) + s.mustGetOK(c, "k4", 5, "v1") + s.mustGetNone(c, "k4", 105) + + s.mustGC(c, safePoint) + + s.mustGetNone(c, "k1", 5) + s.mustGetOK(c, "k1", 15, "v2") + s.mustGetNone(c, "k2", 5) + s.mustGetOK(c, "k2", 15, "v2") + s.mustGetOK(c, "k2", 105, "v3") + s.mustGetNone(c, "k3", 5) + s.mustGetOK(c, "k3", 15, "v2") + s.mustGetNone(c, "k3", 105) + s.mustGetNone(c, "k4", 5) + s.mustGetNone(c, "k4", 105) +} + func (s *testMockTiKVSuite) TestRollbackAndWriteConflict(c *C) { s.mustPutOK(c, "test", "test", 1, 3) req := &kvrpcpb.PrewriteRequest{ diff --git a/store/mockstore/mocktikv/mvcc.go b/store/mockstore/mocktikv/mvcc.go index e5825de671494..091373a1030b2 100644 --- a/store/mockstore/mocktikv/mvcc.go +++ b/store/mockstore/mocktikv/mvcc.go @@ -257,6 +257,7 @@ type MVCCStore interface { ScanLock(startKey, endKey []byte, maxTS uint64) ([]*kvrpcpb.LockInfo, error) ResolveLock(startKey, endKey []byte, startTS, commitTS uint64) error BatchResolveLock(startKey, endKey []byte, txnInfos map[uint64]uint64) error + GC(startKey, endKey []byte, safePoint uint64) error DeleteRange(startKey, endKey []byte) error Close() error } diff --git a/store/mockstore/mocktikv/mvcc_leveldb.go b/store/mockstore/mocktikv/mvcc_leveldb.go index d4413de96ae53..ad059f1a4cbd4 100644 --- a/store/mockstore/mocktikv/mvcc_leveldb.go +++ b/store/mockstore/mocktikv/mvcc_leveldb.go @@ -1047,6 +1047,72 @@ func (mvcc *MVCCLevelDB) BatchResolveLock(startKey, endKey []byte, txnInfos map[ return mvcc.db.Write(batch, nil) } +// GC implements the MVCCStore interface +func (mvcc *MVCCLevelDB) GC(startKey, endKey []byte, safePoint uint64) error { + mvcc.mu.Lock() + defer mvcc.mu.Unlock() + + iter, currKey, err := newScanIterator(mvcc.db, startKey, endKey) + defer iter.Release() + if err != nil { + return errors.Trace(err) + } + + // Mock TiKV usually doesn't need to process large amount of data. So write it in a single batch. + batch := &leveldb.Batch{} + + for iter.Valid() { + lockDec := lockDecoder{expectKey: currKey} + ok, err := lockDec.Decode(iter) + if err != nil { + return errors.Trace(err) + } + if ok && lockDec.lock.startTS <= safePoint { + return errors.Errorf( + "key %+q has lock with startTs %v which is under safePoint %v", + currKey, + lockDec.lock.startTS, + safePoint) + } + + keepNext := true + dec := valueDecoder{expectKey: currKey} + + for iter.Valid() { + ok, err := dec.Decode(iter) + if err != nil { + return errors.Trace(err) + } + + if !ok { + // Go to the next key + currKey, _, err = mvccDecode(iter.Key()) + if err != nil { + return errors.Trace(err) + } + break + } + + if dec.value.commitTS > safePoint { + continue + } + + if dec.value.valueType == typePut || dec.value.valueType == typeDelete { + // Keep the latest version if it's `typePut` + if !keepNext || dec.value.valueType == typeDelete { + batch.Delete(mvccEncode(currKey, dec.value.commitTS)) + } + keepNext = false + } else { + // Delete all other types + batch.Delete(mvccEncode(currKey, dec.value.commitTS)) + } + } + } + + return mvcc.db.Write(batch, nil) +} + // DeleteRange implements the MVCCStore interface. func (mvcc *MVCCLevelDB) DeleteRange(startKey, endKey []byte) error { return mvcc.doRawDeleteRange(codec.EncodeBytes(nil, startKey), codec.EncodeBytes(nil, endKey)) diff --git a/store/mockstore/mocktikv/pd.go b/store/mockstore/mocktikv/pd.go index b259357a51281..e1c0dda9633de 100644 --- a/store/mockstore/mocktikv/pd.go +++ b/store/mockstore/mocktikv/pd.go @@ -32,6 +32,9 @@ var tsMu = struct { type pdClient struct { cluster *Cluster + // SafePoint set by `UpdateGCSafePoint`. Not to be confused with SafePointKV. + gcSafePoint uint64 + gcSafePointMu sync.Mutex } // NewPDClient creates a mock pd.Client that uses local timestamp and meta data @@ -108,7 +111,13 @@ func (c *pdClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) } func (c *pdClient) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint64, error) { - return 0, nil + c.gcSafePointMu.Lock() + defer c.gcSafePointMu.Unlock() + + if safePoint > c.gcSafePoint { + c.gcSafePoint = safePoint + } + return c.gcSafePoint, nil } func (c *pdClient) Close() { diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index e4455fe589b1c..99866195b17a9 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -443,6 +443,18 @@ func (h *rpcHandler) handleKvResolveLock(req *kvrpcpb.ResolveLockRequest) *kvrpc return &kvrpcpb.ResolveLockResponse{} } +func (h *rpcHandler) handleKvGC(req *kvrpcpb.GCRequest) *kvrpcpb.GCResponse { + startKey := MvccKey(h.startKey).Raw() + endKey := MvccKey(h.endKey).Raw() + err := h.mvccStore.GC(startKey, endKey, req.GetSafePoint()) + if err != nil { + return &kvrpcpb.GCResponse{ + Error: convertToKeyError(err), + } + } + return &kvrpcpb.GCResponse{} +} + func (h *rpcHandler) handleKvDeleteRange(req *kvrpcpb.DeleteRangeRequest) *kvrpcpb.DeleteRangeResponse { if !h.checkKeyInRegion(req.StartKey) { panic("KvDeleteRange: key not in region") @@ -773,7 +785,7 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R resp.Resp = &kvrpcpb.GCResponse{RegionError: err} return resp, nil } - resp.Resp = &kvrpcpb.GCResponse{} + resp.Resp = handler.handleKvGC(r) case tikvrpc.CmdDeleteRange: r := req.DeleteRange() if err := handler.checkRequest(reqCtx, r.Size()); err != nil { diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index b56d5ebaac0ef..b73f263d63830 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -17,6 +17,8 @@ import ( "bytes" "context" "errors" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/store/tikv/oracle" "math" "sort" "strconv" @@ -28,7 +30,7 @@ import ( "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/kvproto/pkg/metapb" - pd "github.com/pingcap/pd/client" + "github.com/pingcap/pd/client" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/util" "github.com/pingcap/tidb/domain" @@ -98,6 +100,83 @@ func (s *testGCWorkerSuite) timeEqual(c *C, t1, t2 time.Time, epsilon time.Durat c.Assert(math.Abs(float64(t1.Sub(t2))), Less, float64(epsilon)) } +func (s *testGCWorkerSuite) mustPut(c *C, key, value string) { + txn, err := s.store.Begin() + c.Assert(err, IsNil) + err = txn.Set([]byte(key), []byte(value)) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) +} + +func (s *testGCWorkerSuite) mustGet(c *C, key string, ts uint64) string { + snap, err := s.store.GetSnapshot(kv.Version{Ver: ts}) + c.Assert(err, IsNil) + value, err := snap.Get([]byte(key)) + c.Assert(err, IsNil) + return string(value) +} + +func (s *testGCWorkerSuite) mustGetNone(c *C, key string, ts uint64) { + snap, err := s.store.GetSnapshot(kv.Version{Ver: ts}) + c.Assert(err, IsNil) + _, err = snap.Get([]byte(key)) + c.Assert(err, Equals, kv.ErrNotExist) +} + +func (s *testGCWorkerSuite) mustAllocTs(c *C) uint64 { + ts, err := s.oracle.GetTimestamp(context.Background()) + c.Assert(err, IsNil) + return ts +} + +func (s *testGCWorkerSuite) mustGetSafePointFromPd(c *C) uint64 { + // UpdateGCSafePoint returns the newest safePoint after the updating, which can be used to check whether the + // safePoint is successfully uploaded. + safePoint, err := s.pdClient.UpdateGCSafePoint(context.Background(), 0) + c.Assert(err, IsNil) + return safePoint +} + +// gcProbe represents a key that contains multiple versions, one of which should be collected. Execution of GC with +// greater ts will be detected, but it may not work properly if there are newer versions of the key. +// This is not used to check the correctness of GC algorithm, but only for checking whether GC has been executed on the +// specified key. Create this using `s.createGCProbe`. +type gcProbe struct { + key string + // The ts that can see the version that should be deleted. + v1Ts uint64 + // The ts that can see the version that should be kept. + v2Ts uint64 +} + +// createGCProbe creates gcProbe on specified key. +func (s *testGCWorkerSuite) createGCProbe(c *C, key string) *gcProbe { + s.mustPut(c, key, "v1") + ts1 := s.mustAllocTs(c) + s.mustPut(c, key, "v2") + ts2 := s.mustAllocTs(c) + p := &gcProbe{ + key: key, + v1Ts: ts1, + v2Ts: ts2, + } + s.checkNotCollected(c, p) + return p +} + +// checkCollected asserts the gcProbe has been correctly collected. +func (s *testGCWorkerSuite) checkCollected(c *C, p *gcProbe) { + s.mustGetNone(c, p.key, p.v1Ts) + c.Assert(s.mustGet(c, p.key, p.v2Ts), Equals, "v2") +} + +// checkNotCollected asserts the gcProbe has not been collected. +func (s *testGCWorkerSuite) checkNotCollected(c *C, p *gcProbe) { + c.Assert(s.mustGet(c, p.key, p.v1Ts), Equals, "v1") + c.Assert(s.mustGet(c, p.key, p.v2Ts), Equals, "v2") +} + func (s *testGCWorkerSuite) TestGetOracleTime(c *C) { t1, err := s.gcWorker.getOracleTime() c.Assert(err, IsNil) @@ -241,24 +320,27 @@ func (s *testGCWorkerSuite) TestDoGCForOneRegion(c *C) { loc, err := s.store.GetRegionCache().LocateKey(bo, []byte("")) c.Assert(err, IsNil) var regionErr *errorpb.Error - regionErr, err = s.gcWorker.doGCForRegion(bo, 20, loc.Region) + + p := s.createGCProbe(c, "k1") + regionErr, err = s.gcWorker.doGCForRegion(bo, s.mustAllocTs(c), loc.Region) c.Assert(regionErr, IsNil) c.Assert(err, IsNil) + s.checkCollected(c, p) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult", `return("timeout")`), IsNil) - regionErr, err = s.gcWorker.doGCForRegion(bo, 20, loc.Region) + regionErr, err = s.gcWorker.doGCForRegion(bo, s.mustAllocTs(c), loc.Region) c.Assert(regionErr, IsNil) c.Assert(err, NotNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult"), IsNil) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult", `return("GCNotLeader")`), IsNil) - regionErr, err = s.gcWorker.doGCForRegion(bo, 20, loc.Region) + regionErr, err = s.gcWorker.doGCForRegion(bo, s.mustAllocTs(c), loc.Region) c.Assert(regionErr.GetNotLeader(), NotNil) c.Assert(err, IsNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult"), IsNil) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult", `return("GCServerIsBusy")`), IsNil) - regionErr, err = s.gcWorker.doGCForRegion(bo, 20, loc.Region) + regionErr, err = s.gcWorker.doGCForRegion(bo, s.mustAllocTs(c), loc.Region) c.Assert(regionErr.GetServerIsBusy(), NotNil) c.Assert(err, IsNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/tikvStoreSendReqResult"), IsNil) @@ -292,26 +374,20 @@ func (s *testGCWorkerSuite) TestDoGC(c *C) { gcSafePointCacheInterval = 1 - err = s.gcWorker.saveValueToSysTable(gcConcurrencyKey, strconv.Itoa(gcDefaultConcurrency)) - c.Assert(err, IsNil) - concurrency, err := s.gcWorker.loadGCConcurrencyWithDefault() - c.Assert(err, IsNil) - err = s.gcWorker.doGC(ctx, 20, concurrency) + p := s.createGCProbe(c, "k1") + err = s.gcWorker.doGC(ctx, s.mustAllocTs(c), gcDefaultConcurrency) c.Assert(err, IsNil) + s.checkCollected(c, p) - err = s.gcWorker.saveValueToSysTable(gcConcurrencyKey, strconv.Itoa(gcMinConcurrency)) - c.Assert(err, IsNil) - concurrency, err = s.gcWorker.loadGCConcurrencyWithDefault() - c.Assert(err, IsNil) - err = s.gcWorker.doGC(ctx, 20, concurrency) + p = s.createGCProbe(c, "k1") + err = s.gcWorker.doGC(ctx, s.mustAllocTs(c), gcMinConcurrency) c.Assert(err, IsNil) + s.checkCollected(c, p) - err = s.gcWorker.saveValueToSysTable(gcConcurrencyKey, strconv.Itoa(gcMaxConcurrency)) - c.Assert(err, IsNil) - concurrency, err = s.gcWorker.loadGCConcurrencyWithDefault() - c.Assert(err, IsNil) - err = s.gcWorker.doGC(ctx, 20, concurrency) + p = s.createGCProbe(c, "k1") + err = s.gcWorker.doGC(ctx, s.mustAllocTs(c), gcMaxConcurrency) c.Assert(err, IsNil) + s.checkCollected(c, p) } func (s *testGCWorkerSuite) TestCheckGCMode(c *C) { @@ -521,33 +597,140 @@ func (c *testGCWorkerClient) SendRequest(ctx context.Context, addr string, req * return c.Client.SendRequest(ctx, addr, req, timeout) } -func (s *testGCWorkerSuite) TestRunGCJob(c *C) { +func (s *testGCWorkerSuite) TestLeaderTick(c *C) { gcSafePointCacheInterval = 0 - err := RunGCJob(context.Background(), s.store, 0, "mock", 1) + + veryLong := gcDefaultLifeTime * 10 + // Avoid failing at interval check. `lastFinish` is checked by os time. + s.gcWorker.lastFinish = time.Now().Add(-veryLong) + // Use central mode to do this test. + err := s.gcWorker.saveValueToSysTable(gcModeKey, gcModeCentral) c.Assert(err, IsNil) - gcWorker, err := NewGCWorker(s.store, s.pdClient) + p := s.createGCProbe(c, "k1") + s.oracle.AddOffset(gcDefaultLifeTime * 2) + + // Skip if GC is running. + s.gcWorker.gcIsRunning = true + err = s.gcWorker.leaderTick(context.Background()) + c.Assert(err, IsNil) + s.checkNotCollected(c, p) + s.gcWorker.gcIsRunning = false + // Reset GC last run time + err = s.gcWorker.saveTime(gcLastRunTimeKey, oracle.GetTimeFromTS(s.mustAllocTs(c)).Add(-veryLong)) + c.Assert(err, IsNil) + + // Skip if prepare failed (disabling GC will make prepare returns ok = false). + err = s.gcWorker.saveValueToSysTable(gcEnableKey, booleanFalse) + c.Assert(err, IsNil) + err = s.gcWorker.leaderTick(context.Background()) + c.Assert(err, IsNil) + s.checkNotCollected(c, p) + err = s.gcWorker.saveValueToSysTable(gcEnableKey, booleanTrue) + c.Assert(err, IsNil) + // Reset GC last run time + err = s.gcWorker.saveTime(gcLastRunTimeKey, oracle.GetTimeFromTS(s.mustAllocTs(c)).Add(-veryLong)) + c.Assert(err, IsNil) + + // Skip if gcWaitTime not exceeded. + s.gcWorker.lastFinish = time.Now() + err = s.gcWorker.leaderTick(context.Background()) + c.Assert(err, IsNil) + s.checkNotCollected(c, p) + s.gcWorker.lastFinish = time.Now().Add(-veryLong) + // Reset GC last run time + err = s.gcWorker.saveTime(gcLastRunTimeKey, oracle.GetTimeFromTS(s.mustAllocTs(c)).Add(-veryLong)) + c.Assert(err, IsNil) + + // Continue GC if all those checks passed. + err = s.gcWorker.leaderTick(context.Background()) + c.Assert(err, IsNil) + // Wait for GC finish + select { + case err = <-s.gcWorker.done: + break + case <-time.After(time.Second * 10): + err = errors.New("receive from s.gcWorker.done timeout") + } + c.Assert(err, IsNil) + s.checkCollected(c, p) +} + +func (s *testGCWorkerSuite) TestRunGCJob(c *C) { + gcSafePointCacheInterval = 0 + + // Avoid blocking runGCJob function + s.gcWorker.done = make(chan error, 1) + + // Test distributed mode + useDistributedGC, err := s.gcWorker.checkUseDistributedGC() c.Assert(err, IsNil) - gcWorker.Start() - useDistributedGC, err := gcWorker.(*GCWorker).checkUseDistributedGC() c.Assert(useDistributedGC, IsTrue) + safePoint := s.mustAllocTs(c) + s.gcWorker.runGCJob(context.Background(), safePoint, 1) + err = <-s.gcWorker.done c.Assert(err, IsNil) - safePoint := uint64(time.Now().Unix()) - gcWorker.(*GCWorker).runGCJob(context.Background(), safePoint, 1) - getSafePoint, err := loadSafePoint(gcWorker.(*GCWorker).store.GetSafePointKV()) + + pdSafePoint := s.mustGetSafePointFromPd(c) + c.Assert(pdSafePoint, Equals, safePoint) + + etcdSafePoint := s.loadEtcdSafePoint(c) c.Assert(err, IsNil) - c.Assert(getSafePoint, Equals, safePoint) - gcWorker.Close() + c.Assert(etcdSafePoint, Equals, safePoint) + + // Test distributed mode with safePoint regressing (although this is impossible) + s.gcWorker.runGCJob(context.Background(), safePoint-1, 1) + err = <-s.gcWorker.done + c.Assert(err, NotNil) + + // Test central mode + err = s.gcWorker.saveValueToSysTable(gcModeKey, gcModeCentral) + c.Assert(err, IsNil) + useDistributedGC, err = s.gcWorker.checkUseDistributedGC() + c.Assert(err, IsNil) + c.Assert(useDistributedGC, IsFalse) + + p := s.createGCProbe(c, "k1") + safePoint = s.mustAllocTs(c) + s.gcWorker.runGCJob(context.Background(), safePoint, 1) + s.checkCollected(c, p) + err = <-s.gcWorker.done + c.Assert(err, IsNil) + + etcdSafePoint = s.loadEtcdSafePoint(c) + c.Assert(err, IsNil) + c.Assert(etcdSafePoint, Equals, safePoint) } -func loadSafePoint(kv tikv.SafePointKV) (uint64, error) { - val, err := kv.Get(tikv.GcSavedSafePoint) - if err != nil { - return 0, err - } - return strconv.ParseUint(val, 10, 64) +func (s *testGCWorkerSuite) TestRunGCJobAPI(c *C) { + gcSafePointCacheInterval = 0 + + p := s.createGCProbe(c, "k1") + safePoint := s.mustAllocTs(c) + err := RunGCJob(context.Background(), s.store, safePoint, "mock", 1) + c.Assert(err, IsNil) + s.checkCollected(c, p) + etcdSafePoint := s.loadEtcdSafePoint(c) + c.Assert(err, IsNil) + c.Assert(etcdSafePoint, Equals, safePoint) } -func (s *testGCWorkerSuite) TestRunDistGCJob(c *C) { - err := RunDistributedGCJob(context.Background(), s.store, s.pdClient, 0, "mock", 1) +func (s *testGCWorkerSuite) TestRunDistGCJobAPI(c *C) { + gcSafePointCacheInterval = 0 + + safePoint := s.mustAllocTs(c) + err := RunDistributedGCJob(context.Background(), s.store, s.pdClient, safePoint, "mock", 1) + c.Assert(err, IsNil) + pdSafePoint := s.mustGetSafePointFromPd(c) + c.Assert(pdSafePoint, Equals, safePoint) + etcdSafePoint := s.loadEtcdSafePoint(c) + c.Assert(err, IsNil) + c.Assert(etcdSafePoint, Equals, safePoint) +} + +func (s *testGCWorkerSuite) loadEtcdSafePoint(c *C) uint64 { + val, err := s.gcWorker.store.GetSafePointKV().Get(tikv.GcSavedSafePoint) + c.Assert(err, IsNil) + res, err := strconv.ParseUint(val, 10, 64) c.Assert(err, IsNil) + return res } From fcc15b1256e4e33132a80e39b841a6e062c353a0 Mon Sep 17 00:00:00 2001 From: DQYuan <932087612@qq.com> Date: Tue, 16 Jul 2019 19:50:04 +0800 Subject: [PATCH 040/196] types: fix delete error when convert string to float or int (#10861) --- executor/executor_test.go | 1 + types/convert.go | 4 ++++ types/convert_test.go | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/executor/executor_test.go b/executor/executor_test.go index b68bebd4bfefb..7bd75859cdaf7 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -451,6 +451,7 @@ func checkCases(tests []testCase, ld *executor.LoadDataInfo, ctx.GetSessionVars().StmtCtx.DupKeyAsWarning = true ctx.GetSessionVars().StmtCtx.BadNullAsWarning = true ctx.GetSessionVars().StmtCtx.InLoadDataStmt = true + ctx.GetSessionVars().StmtCtx.InDeleteStmt = false data, reachLimit, err1 := ld.InsertData(context.Background(), tt.data1, tt.data2) c.Assert(err1, IsNil) c.Assert(reachLimit, IsFalse) diff --git a/types/convert.go b/types/convert.go index 58913cb2252eb..7ddfa5e1dd2b1 100644 --- a/types/convert.go +++ b/types/convert.go @@ -581,6 +581,10 @@ func ConvertJSONToDecimal(sc *stmtctx.StatementContext, j json.BinaryJSON) (*MyD // getValidFloatPrefix gets prefix of string which can be successfully parsed as float. func getValidFloatPrefix(sc *stmtctx.StatementContext, s string) (valid string, err error) { + if sc.InDeleteStmt && s == "" { + return "0", nil + } + var ( sawDot bool sawDigit bool diff --git a/types/convert_test.go b/types/convert_test.go index 47f13e99306ad..52e1b58832683 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -461,6 +461,29 @@ func (s *testTypeConvertSuite) TestStrToNum(c *C) { testStrToFloat(c, "1e649", math.MaxFloat64, false, nil) testStrToFloat(c, "-1e649", -math.MaxFloat64, true, ErrTruncatedWrongVal) testStrToFloat(c, "-1e649", -math.MaxFloat64, false, nil) + + // for issue #10806 + testDeleteEmptyStringError(c) +} + +func testDeleteEmptyStringError(c *C) { + sc := new(stmtctx.StatementContext) + sc.InDeleteStmt = true + + str := "" + expect := 0 + + val, err := StrToInt(sc, str) + c.Assert(err, IsNil) + c.Assert(val, Equals, int64(expect)) + + val1, err := StrToUint(sc, str) + c.Assert(err, IsNil) + c.Assert(val1, Equals, uint64(expect)) + + val2, err := StrToFloat(sc, str) + c.Assert(err, IsNil) + c.Assert(val2, Equals, float64(expect)) } func (s *testTypeConvertSuite) TestFieldTypeToStr(c *C) { From 23d4c97e2213e7f63f66f6c2094f728c9321200a Mon Sep 17 00:00:00 2001 From: GameOver <41495709+wty4427300@users.noreply.github.com> Date: Tue, 16 Jul 2019 20:28:37 -0500 Subject: [PATCH 041/196] Update README.md (#11267) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1bb34299bc0ae..d4f38ad22cdc3 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ [![GitHub release date](https://img.shields.io/github/release-date/pingcap/tidb.svg)](https://github.com/pingcap/tidb/releases) [![CircleCI Status](https://circleci.com/gh/pingcap/tidb.svg?style=shield)](https://circleci.com/gh/pingcap/tidb) [![Coverage Status](https://codecov.io/gh/pingcap/tidb/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/tidb) +[![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://godoc.org/github.com/pingcap/tidb) - [**Stack Overflow**](https://stackoverflow.com/questions/tagged/tidb) - Community [**Slack Channel**](https://pingcap.com/tidbslack/) From 77b6858bf08c2c20dd16b9f7f8d291b17c13b91e Mon Sep 17 00:00:00 2001 From: lysu Date: Wed, 17 Jul 2019 13:01:22 +0800 Subject: [PATCH 042/196] *: refine text protocol multiple query response (#11263) --- server/conn.go | 34 +++++++++++--- session/session.go | 59 ++++++++++++++++++++++++- util/sqlexec/restricted_sql_executor.go | 14 ++++++ 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/server/conn.go b/server/conn.go index 391cd779f4da7..11516087fc266 100644 --- a/server/conn.go +++ b/server/conn.go @@ -65,6 +65,7 @@ import ( "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" + "github.com/pingcap/tidb/util/sqlexec" "go.uber.org/zap" ) @@ -945,6 +946,10 @@ func (cc *clientConn) flush() error { func (cc *clientConn) writeOK() error { msg := cc.ctx.LastMessage() + return cc.writeOkWith(msg, cc.ctx.AffectedRows(), cc.ctx.LastInsertID(), cc.ctx.Status(), cc.ctx.WarningCount()) +} + +func (cc *clientConn) writeOkWith(msg string, affectedRows, lastInsertID uint64, status, warnCnt uint16) error { enclen := 0 if len(msg) > 0 { enclen = lengthEncodedIntSize(uint64(len(msg))) + len(msg) @@ -952,11 +957,11 @@ func (cc *clientConn) writeOK() error { data := cc.alloc.AllocWithLen(4, 32+enclen) data = append(data, mysql.OKHeader) - data = dumpLengthEncodedInt(data, cc.ctx.AffectedRows()) - data = dumpLengthEncodedInt(data, cc.ctx.LastInsertID()) + data = dumpLengthEncodedInt(data, affectedRows) + data = dumpLengthEncodedInt(data, lastInsertID) if cc.capability&mysql.ClientProtocol41 > 0 { - data = dumpUint16(data, cc.ctx.Status()) - data = dumpUint16(data, cc.ctx.WarningCount()) + data = dumpUint16(data, status) + data = dumpUint16(data, warnCnt) } if enclen > 0 { // although MySQL manual says the info message is string(https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html), @@ -1403,12 +1408,27 @@ func (cc *clientConn) writeChunksWithFetchSize(ctx context.Context, rs ResultSet } func (cc *clientConn) writeMultiResultset(ctx context.Context, rss []ResultSet, binary bool) error { - for _, rs := range rss { - if err := cc.writeResultset(ctx, rs, binary, mysql.ServerMoreResultsExists, 0); err != nil { + for i, rs := range rss { + lastRs := i == len(rss)-1 + if r, ok := rs.(*tidbResultSet).recordSet.(sqlexec.MultiQueryNoDelayResult); ok { + status := r.Status() + if !lastRs { + status |= mysql.ServerMoreResultsExists + } + if err := cc.writeOkWith(r.LastMessage(), r.AffectedRows(), r.LastInsertID(), status, r.WarnCount()); err != nil { + return err + } + continue + } + status := uint16(0) + if !lastRs { + status |= mysql.ServerMoreResultsExists + } + if err := cc.writeResultset(ctx, rs, binary, status, 0); err != nil { return err } } - return cc.writeOK() + return nil } func (cc *clientConn) setConn(conn net.Conn) { diff --git a/session/session.go b/session/session.go index 37b5fab958b94..07290a295b01f 100644 --- a/session/session.go +++ b/session/session.go @@ -991,7 +991,7 @@ func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecu s.processInfo.Store(&pi) } -func (s *session) executeStatement(ctx context.Context, connID uint64, stmtNode ast.StmtNode, stmt sqlexec.Statement, recordSets []sqlexec.RecordSet) ([]sqlexec.RecordSet, error) { +func (s *session) executeStatement(ctx context.Context, connID uint64, stmtNode ast.StmtNode, stmt sqlexec.Statement, recordSets []sqlexec.RecordSet, inMulitQuery bool) ([]sqlexec.RecordSet, error) { s.SetValue(sessionctx.QueryString, stmt.OriginText()) if _, ok := stmtNode.(ast.DDLNode); ok { s.SetValue(sessionctx.LastExecuteDDL, true) @@ -1016,6 +1016,16 @@ func (s *session) executeStatement(ctx context.Context, connID uint64, stmtNode sessionExecuteRunDurationGeneral.Observe(time.Since(startTime).Seconds()) } + if inMulitQuery && recordSet == nil { + recordSet = &multiQueryNoDelayRecordSet{ + affectedRows: s.AffectedRows(), + lastMessage: s.LastMessage(), + warnCount: s.sessionVars.StmtCtx.WarningCount(), + lastInsertID: s.sessionVars.StmtCtx.LastInsertID, + status: s.sessionVars.Status, + } + } + if recordSet != nil { recordSets = append(recordSets, recordSet) } @@ -1062,6 +1072,7 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec var tempStmtNodes []ast.StmtNode compiler := executor.Compiler{Ctx: s} + multiQuery := len(stmtNodes) > 1 for idx, stmtNode := range stmtNodes { s.PrepareTxnCtx(ctx) @@ -1098,7 +1109,7 @@ func (s *session) execute(ctx context.Context, sql string) (recordSets []sqlexec s.currentPlan = stmt.Plan // Step3: Execute the physical plan. - if recordSets, err = s.executeStatement(ctx, connID, stmtNode, stmt, recordSets); err != nil { + if recordSets, err = s.executeStatement(ctx, connID, stmtNode, stmt, recordSets, multiQuery); err != nil { return nil, err } } @@ -1952,3 +1963,47 @@ func (s *session) recordTransactionCounter(err error) { } } } + +type multiQueryNoDelayRecordSet struct { + affectedRows uint64 + lastMessage string + status uint16 + warnCount uint16 + lastInsertID uint64 +} + +func (c *multiQueryNoDelayRecordSet) Fields() []*ast.ResultField { + panic("unsupported method") +} + +func (c *multiQueryNoDelayRecordSet) Next(ctx context.Context, chk *chunk.Chunk) error { + panic("unsupported method") +} + +func (c *multiQueryNoDelayRecordSet) NewChunk() *chunk.Chunk { + panic("unsupported method") +} + +func (c *multiQueryNoDelayRecordSet) Close() error { + return nil +} + +func (c *multiQueryNoDelayRecordSet) AffectedRows() uint64 { + return c.affectedRows +} + +func (c *multiQueryNoDelayRecordSet) LastMessage() string { + return c.lastMessage +} + +func (c *multiQueryNoDelayRecordSet) WarnCount() uint16 { + return c.warnCount +} + +func (c *multiQueryNoDelayRecordSet) Status() uint16 { + return c.status +} + +func (c *multiQueryNoDelayRecordSet) LastInsertID() uint64 { + return c.lastInsertID +} diff --git a/util/sqlexec/restricted_sql_executor.go b/util/sqlexec/restricted_sql_executor.go index 5c5c0d31711f0..ec42336087752 100644 --- a/util/sqlexec/restricted_sql_executor.go +++ b/util/sqlexec/restricted_sql_executor.go @@ -96,3 +96,17 @@ type RecordSet interface { // restart the iteration. Close() error } + +// MultiQueryNoDelayResult is an interface for one no-delay result for one statement in multi-queries. +type MultiQueryNoDelayResult interface { + // AffectedRows return affected row for one statement in multi-queries. + AffectedRows() uint64 + // LastMessage return last message for one statement in multi-queries. + LastMessage() string + // WarnCount return warn count for one statement in multi-queries. + WarnCount() uint16 + // Status return status when executing one statement in multi-queries. + Status() uint16 + // LastInsertID return last insert id for one statement in multi-queries. + LastInsertID() uint64 +} From dd06ebb31518095c39054db9c9df806a742bbee3 Mon Sep 17 00:00:00 2001 From: wshwsh12 <793703860@qq.com> Date: Wed, 17 Jul 2019 14:39:55 +0800 Subject: [PATCH 043/196] fix microseconds behaviour in DATE_ADD() (#11280) --- expression/integration_test.go | 12 ++++++++++++ types/fsp.go | 5 ++++- types/fsp_test.go | 4 ++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 0f73a64807965..a04a4b4d50e61 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4464,3 +4464,15 @@ func (s *testIntegrationSuite) TestIssue10675(c *C) { testkit.Rows("1")) tk.MustQuery(`select * from t where a > 184467440737095516167.1;`).Check(testkit.Rows()) } + +func (s *testIntegrationSuite) TestIssue11257(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 SECOND_MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.800000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 MINUTE_MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.800000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 HOUR_MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.800000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 DAY_MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.800000")) +} diff --git a/types/fsp.go b/types/fsp.go index fe5a656cd87bb..c9709822c6453 100644 --- a/types/fsp.go +++ b/types/fsp.go @@ -86,9 +86,12 @@ func ParseFrac(s string, fsp int) (v int, overflow bool, err error) { return } -// alignFrac is used to generate alignment frac, like `100` -> `100000` +// alignFrac is used to generate alignment frac, like `100` -> `100000` ,`-100` -> `-100000` func alignFrac(s string, fsp int) string { sl := len(s) + if sl > 0 && s[0] == '-' { + sl = sl - 1 + } if sl < fsp { return s + strings.Repeat("0", fsp-sl) } diff --git a/types/fsp_test.go b/types/fsp_test.go index 8802e87d5b3e4..b8f29cd4077d7 100644 --- a/types/fsp_test.go +++ b/types/fsp_test.go @@ -115,4 +115,8 @@ func (s *FspTest) TestAlignFrac(c *C) { c.Assert(obtained, Equals, "100000") obtained = alignFrac("10000000000", 6) c.Assert(obtained, Equals, "10000000000") + obtained = alignFrac("-100", 6) + c.Assert(obtained, Equals, "-100000") + obtained = alignFrac("-10000000000", 6) + c.Assert(obtained, Equals, "-10000000000") } From db454602801fea40b098fb5942576c3cd89dfebc Mon Sep 17 00:00:00 2001 From: MyonKeminta <9948422+MyonKeminta@users.noreply.github.com> Date: Wed, 17 Jul 2019 17:31:10 +0800 Subject: [PATCH 044/196] store/tikv: Avoid sending to channel everywhere in runGCJob (#11032) Signed-off-by: MyonKeminta --- store/tikv/gcworker/gc_worker.go | 31 ++++++----------- store/tikv/gcworker/gc_worker_test.go | 49 ++++++++++++++++++++------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/store/tikv/gcworker/gc_worker.go b/store/tikv/gcworker/gc_worker.go index fe4794abf9f5d..382c0c474ca5f 100644 --- a/store/tikv/gcworker/gc_worker.go +++ b/store/tikv/gcworker/gc_worker.go @@ -233,13 +233,11 @@ func (w *GCWorker) leaderTick(ctx context.Context) error { if err != nil { metrics.GCJobFailureCounter.WithLabelValues("prepare").Inc() } - w.gcIsRunning = false return errors.Trace(err) } // When the worker is just started, or an old GC job has just finished, // wait a while before starting a new job. if time.Since(w.lastFinish) < gcWaitTime { - w.gcIsRunning = false logutil.Logger(ctx).Info("[gc worker] another gc job has just finished, skipped.", zap.String("leaderTick on ", w.uuid)) return nil @@ -258,7 +256,9 @@ func (w *GCWorker) leaderTick(ctx context.Context) error { zap.String("uuid", w.uuid), zap.Uint64("safePoint", safePoint), zap.Int("concurrency", concurrency)) - go w.runGCJob(ctx, safePoint, concurrency) + go func() { + w.done <- w.runGCJob(ctx, safePoint, concurrency) + }() return nil } @@ -466,7 +466,7 @@ func (w *GCWorker) calculateNewSafePoint(now time.Time) (*time.Time, error) { return &safePoint, nil } -func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency int) { +func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency int) error { metrics.GCWorkerCounter.WithLabelValues("run_job").Inc() err := w.resolveLocks(ctx, safePoint, concurrency) if err != nil { @@ -474,8 +474,7 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i zap.String("uuid", w.uuid), zap.Error(err)) metrics.GCJobFailureCounter.WithLabelValues("resolve_lock").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } // Save safe point to pd. err = w.saveSafePoint(w.store.GetSafePointKV(), tikv.GcSavedSafePoint, safePoint) @@ -483,10 +482,8 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i logutil.Logger(ctx).Error("[gc worker] failed to save safe point to PD", zap.String("uuid", w.uuid), zap.Error(err)) - w.gcIsRunning = false metrics.GCJobFailureCounter.WithLabelValues("save_safe_point").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } // Sleep to wait for all other tidb instances update their safepoint cache. time.Sleep(gcSafePointCacheInterval) @@ -497,8 +494,7 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i zap.String("uuid", w.uuid), zap.Error(err)) metrics.GCJobFailureCounter.WithLabelValues("delete_range").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } err = w.redoDeleteRanges(ctx, safePoint, concurrency) if err != nil { @@ -506,8 +502,7 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i zap.String("uuid", w.uuid), zap.Error(err)) metrics.GCJobFailureCounter.WithLabelValues("redo_delete_range").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } useDistributedGC, err := w.checkUseDistributedGC() @@ -525,10 +520,8 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i logutil.Logger(ctx).Error("[gc worker] failed to upload safe point to PD", zap.String("uuid", w.uuid), zap.Error(err)) - w.gcIsRunning = false metrics.GCJobFailureCounter.WithLabelValues("upload_safe_point").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } } else { err = w.doGC(ctx, safePoint, concurrency) @@ -536,14 +529,12 @@ func (w *GCWorker) runGCJob(ctx context.Context, safePoint uint64, concurrency i logutil.Logger(ctx).Error("[gc worker] do GC returns an error", zap.String("uuid", w.uuid), zap.Error(err)) - w.gcIsRunning = false metrics.GCJobFailureCounter.WithLabelValues("gc").Inc() - w.done <- errors.Trace(err) - return + return errors.Trace(err) } } - w.done <- nil + return nil } // deleteRanges processes all delete range records whose ts < safePoint in table `gc_delete_range` diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index b73f263d63830..4f762242dcc94 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -16,7 +16,6 @@ package gcworker import ( "bytes" "context" - "errors" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/tikv/oracle" "math" @@ -26,6 +25,7 @@ import ( "time" . "github.com/pingcap/check" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" @@ -647,39 +647,64 @@ func (s *testGCWorkerSuite) TestLeaderTick(c *C) { // Wait for GC finish select { case err = <-s.gcWorker.done: + s.gcWorker.gcIsRunning = false break case <-time.After(time.Second * 10): err = errors.New("receive from s.gcWorker.done timeout") } c.Assert(err, IsNil) s.checkCollected(c, p) + + // Test again to ensure the synchronization between goroutines is correct. + err = s.gcWorker.saveTime(gcLastRunTimeKey, oracle.GetTimeFromTS(s.mustAllocTs(c)).Add(-veryLong)) + c.Assert(err, IsNil) + s.gcWorker.lastFinish = time.Now().Add(-veryLong) + p = s.createGCProbe(c, "k1") + s.oracle.AddOffset(gcDefaultLifeTime * 2) + + err = s.gcWorker.leaderTick(context.Background()) + c.Assert(err, IsNil) + // Wait for GC finish + select { + case err = <-s.gcWorker.done: + s.gcWorker.gcIsRunning = false + break + case <-time.After(time.Second * 10): + err = errors.New("receive from s.gcWorker.done timeout") + } + c.Assert(err, IsNil) + s.checkCollected(c, p) + + // No more signals in the channel + select { + case err = <-s.gcWorker.done: + err = errors.Errorf("received signal s.gcWorker.done which shouldn't exist: %v", err) + break + case <-time.After(time.Second): + break + } + c.Assert(err, IsNil) } func (s *testGCWorkerSuite) TestRunGCJob(c *C) { gcSafePointCacheInterval = 0 - // Avoid blocking runGCJob function - s.gcWorker.done = make(chan error, 1) - // Test distributed mode useDistributedGC, err := s.gcWorker.checkUseDistributedGC() c.Assert(err, IsNil) c.Assert(useDistributedGC, IsTrue) safePoint := s.mustAllocTs(c) - s.gcWorker.runGCJob(context.Background(), safePoint, 1) - err = <-s.gcWorker.done + err = s.gcWorker.runGCJob(context.Background(), safePoint, 1) c.Assert(err, IsNil) pdSafePoint := s.mustGetSafePointFromPd(c) c.Assert(pdSafePoint, Equals, safePoint) etcdSafePoint := s.loadEtcdSafePoint(c) - c.Assert(err, IsNil) c.Assert(etcdSafePoint, Equals, safePoint) // Test distributed mode with safePoint regressing (although this is impossible) - s.gcWorker.runGCJob(context.Background(), safePoint-1, 1) - err = <-s.gcWorker.done + err = s.gcWorker.runGCJob(context.Background(), safePoint-1, 1) c.Assert(err, NotNil) // Test central mode @@ -691,13 +716,11 @@ func (s *testGCWorkerSuite) TestRunGCJob(c *C) { p := s.createGCProbe(c, "k1") safePoint = s.mustAllocTs(c) - s.gcWorker.runGCJob(context.Background(), safePoint, 1) - s.checkCollected(c, p) - err = <-s.gcWorker.done + err = s.gcWorker.runGCJob(context.Background(), safePoint, 1) c.Assert(err, IsNil) + s.checkCollected(c, p) etcdSafePoint = s.loadEtcdSafePoint(c) - c.Assert(err, IsNil) c.Assert(etcdSafePoint, Equals, safePoint) } From 651632e7928998ddb23e38c2e48cd5a2474e5f24 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 17 Jul 2019 20:49:36 +0800 Subject: [PATCH 045/196] fix scatter range (#11281) Signed-off-by: Ryan Leung --- server/http_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index 2b65bc030be70..c3507fed709e0 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -823,8 +823,8 @@ func (h tableHandler) addScatterSchedule(startKey, endKey []byte, name string) e } input := map[string]string{ "name": "scatter-range", - "start_key": string(startKey), - "end_key": string(endKey), + "start_key": url.QueryEscape(string(startKey)), + "end_key": url.QueryEscape(string(endKey)), "range_name": name, } v, err := json.Marshal(input) From 1768bf401b7c13a12339e382ea509d1a7dccde12 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Thu, 18 Jul 2019 16:20:17 +0800 Subject: [PATCH 046/196] metrics: add missing tikv_txn_cmd_counter (#11292) Signed-off-by: Shuaipeng Yu --- metrics/metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/metrics/metrics.go b/metrics/metrics.go index a21fd394c4d63..ba6a3ce73736d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -115,6 +115,7 @@ func RegisterMetrics() { prometheus.MustRegister(TiKVSecondaryLockCleanupFailureCounter) prometheus.MustRegister(TiKVSendReqHistogram) prometheus.MustRegister(TiKVSnapshotCounter) + prometheus.MustRegister(TiKVTxnCmdCounter) prometheus.MustRegister(TiKVTxnCmdHistogram) prometheus.MustRegister(TiKVTxnCounter) prometheus.MustRegister(TiKVTxnRegionsNumHistogram) From b6a634638f32c9a6ae1295de548b54c26582b6c1 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 18 Jul 2019 19:16:22 +0800 Subject: [PATCH 047/196] executor: speedup unit test (#10930) --- executor/executor_test.go | 121 ++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 56 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 7bd75859cdaf7..3efde355754f3 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -87,7 +87,8 @@ func TestT(t *testing.T) { testleak.AfterTestT(t)() } -var _ = Suite(&testSuite{}) +var _ = Suite(&testSuite{&baseTestSuite{}}) +var _ = Suite(&testSuiteP1{&baseTestSuite{}}) var _ = Suite(&testSuite1{}) var _ = Suite(&testSuite2{}) var _ = Suite(&testSuite3{}) @@ -98,9 +99,11 @@ var _ = Suite(&testOOMSuite{}) var _ = Suite(&testPointGetSuite{}) var _ = Suite(&testRecoverTable{}) var _ = Suite(&testFlushSuite{}) -var _ = Suite(&testShowStatsSuite{}) -type testSuite struct { +type testSuite struct{ *baseTestSuite } +type testSuiteP1 struct{ *baseTestSuite } + +type baseTestSuite struct { cluster *mocktikv.Cluster mvccStore mocktikv.MVCCStore store kv.Storage @@ -111,7 +114,7 @@ type testSuite struct { var mockTikv = flag.Bool("mockTikv", true, "use mock tikv store in executor test") -func (s *testSuite) SetUpSuite(c *C) { +func (s *baseTestSuite) SetUpSuite(c *C) { s.Parser = parser.New() flag.Lookup("mockTikv") useMockTikv := *mockTikv @@ -134,7 +137,7 @@ func (s *testSuite) SetUpSuite(c *C) { s.domain = d } -func (s *testSuite) TearDownSuite(c *C) { +func (s *baseTestSuite) TearDownSuite(c *C) { s.domain.Close() s.store.Close() } @@ -145,7 +148,7 @@ func enablePessimisticTxn(enable bool) { config.StoreGlobalConfig(newConf) } -func (s *testSuite) TestPessimisticSelectForUpdate(c *C) { +func (s *testSuiteP1) TestPessimisticSelectForUpdate(c *C) { defer func() { enablePessimisticTxn(false) }() enablePessimisticTxn(true) tk := testkit.NewTestKit(c, s.store) @@ -170,7 +173,7 @@ func (s *testSuite) TearDownTest(c *C) { } } -func (s *testSuite) TestBind(c *C) { +func (s *testSuiteP1) TestBind(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists testbind") @@ -185,7 +188,7 @@ func (s *testSuite) TestBind(c *C) { tk.MustExec("drop session binding for select * from testbind") } -func (s *testSuite) TestChange(c *C) { +func (s *testSuiteP1) TestChange(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -195,18 +198,20 @@ func (s *testSuite) TestChange(c *C) { c.Assert(tk.ExecToErr("alter table t change c d varchar(100)"), NotNil) } -func (s *testSuite) TestLoadStats(c *C) { +func (s *testSuiteP1) TestLoadStats(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") c.Assert(tk.ExecToErr("load stats"), NotNil) c.Assert(tk.ExecToErr("load stats ./xxx.json"), NotNil) } -func (s *testSuite) TestShow(c *C) { +func (s *testSuiteP1) TestShow(c *C) { tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") + tk.MustExec("create database test_show;") + tk.MustExec("use test_show") tk.MustQuery("show engines") + tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int primary key)") c.Assert(len(tk.MustQuery("show index in t").Rows()), Equals, 1) c.Assert(len(tk.MustQuery("show index from t").Rows()), Equals, 1) @@ -218,7 +223,7 @@ func (s *testSuite) TestShow(c *C) { "latin1 Latin1 latin1_bin 1", "binary binary binary 1")) c.Assert(len(tk.MustQuery("show master status").Rows()), Equals, 1) - tk.MustQuery("show create database test").Check(testkit.Rows("test CREATE DATABASE `test` /*!40100 DEFAULT CHARACTER SET utf8mb4 */")) + tk.MustQuery("show create database test_show").Check(testkit.Rows("test_show CREATE DATABASE `test_show` /*!40100 DEFAULT CHARACTER SET utf8mb4 */")) tk.MustQuery("show privileges").Check(testkit.Rows("Alter Tables To alter the table", "Alter Tables To alter the table", "Alter routine Functions,Procedures To alter or drop stored functions/procedures", @@ -254,7 +259,7 @@ func (s *testSuite) TestShow(c *C) { c.Assert(len(tk.MustQuery("show table status").Rows()), Equals, 1) } -func (s *testSuite) TestAdmin(c *C) { +func (s *testSuiteP1) TestAdmin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists admin_test") @@ -423,7 +428,7 @@ func (s *testSuite) TestAdminChecksumOfPartitionedTable(c *C) { r.Check(testkit.Rows("test admin_checksum_partition_test 1 5 5")) } -func (s *testSuite) fillData(tk *testkit.TestKit, table string) { +func (s *baseTestSuite) fillData(tk *testkit.TestKit, table string) { tk.MustExec("use test") tk.MustExec(fmt.Sprintf("create table %s(id int not null default 1, name varchar(255), PRIMARY KEY(id));", table)) @@ -478,7 +483,7 @@ func checkCases(tests []testCase, ld *executor.LoadDataInfo, } } -func (s *testSuite) TestSelectWithoutFrom(c *C) { +func (s *testSuiteP1) TestSelectWithoutFrom(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -493,7 +498,7 @@ func (s *testSuite) TestSelectWithoutFrom(c *C) { } // TestSelectBackslashN Issue 3685. -func (s *testSuite) TestSelectBackslashN(c *C) { +func (s *testSuiteP1) TestSelectBackslashN(c *C) { tk := testkit.NewTestKit(c, s.store) sql := `select \N;` @@ -584,7 +589,7 @@ func (s *testSuite) TestSelectBackslashN(c *C) { } // TestSelectNull Issue #4053. -func (s *testSuite) TestSelectNull(c *C) { +func (s *testSuiteP1) TestSelectNull(c *C) { tk := testkit.NewTestKit(c, s.store) sql := `select nUll;` @@ -617,7 +622,7 @@ func (s *testSuite) TestSelectNull(c *C) { } // TestSelectStringLiteral Issue #3686. -func (s *testSuite) TestSelectStringLiteral(c *C) { +func (s *testSuiteP1) TestSelectStringLiteral(c *C) { tk := testkit.NewTestKit(c, s.store) sql := `select 'abc';` @@ -772,7 +777,7 @@ func (s *testSuite) TestSelectStringLiteral(c *C) { c.Check(fields[0].Column.Name.O, Equals, "ss") } -func (s *testSuite) TestSelectLimit(c *C) { +func (s *testSuiteP1) TestSelectLimit(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") s.fillData(tk, "select_limit") @@ -801,7 +806,7 @@ func (s *testSuite) TestSelectLimit(c *C) { c.Assert(err, NotNil) } -func (s *testSuite) TestSelectOrderBy(c *C) { +func (s *testSuiteP1) TestSelectOrderBy(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") s.fillData(tk, "select_order_test") @@ -898,7 +903,7 @@ func (s *testSuite) TestSelectOrderBy(c *C) { tk.MustQuery("select a from t use index(b) order by b").Check(testkit.Rows("9", "8", "7", "6", "5", "4", "3", "2", "1", "0")) } -func (s *testSuite) TestOrderBy(c *C) { +func (s *testSuiteP1) TestOrderBy(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t") tk.MustExec("create table t (c1 int, c2 int, c3 varchar(20))") @@ -921,7 +926,7 @@ func (s *testSuite) TestOrderBy(c *C) { tk.MustQuery("select c1, c2 from t order by binary c3").Check(testkit.Rows("1 2", "2 1")) } -func (s *testSuite) TestSelectErrorRow(c *C) { +func (s *testSuiteP1) TestSelectErrorRow(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -951,7 +956,7 @@ func (s *testSuite) TestSelectErrorRow(c *C) { } // TestIssue2612 is related with https://github.com/pingcap/tidb/issues/2612 -func (s *testSuite) TestIssue2612(c *C) { +func (s *testSuiteP1) TestIssue2612(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`drop table if exists t`) @@ -969,7 +974,7 @@ func (s *testSuite) TestIssue2612(c *C) { } // TestIssue345 is related with https://github.com/pingcap/tidb/issues/345 -func (s *testSuite) TestIssue345(c *C) { +func (s *testSuiteP1) TestIssue345(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`drop table if exists t1, t2`) @@ -1000,7 +1005,7 @@ func (s *testSuite) TestIssue345(c *C) { c.Assert(err, NotNil) } -func (s *testSuite) TestIssue5055(c *C) { +func (s *testSuiteP1) TestIssue5055(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`drop table if exists t1, t2`) @@ -1197,6 +1202,7 @@ func (s *testSuite) TestUnion(c *C) { tk.MustExec(`insert into t1 select * from t1;`) tk.MustExec(`insert into t2 values(1, 1);`) tk.MustExec(`set @@tidb_init_chunk_size=2;`) + tk.MustExec(`set @@sql_mode="";`) tk.MustQuery(`select count(*) from (select t1.a, t1.b from t1 left join t2 on t1.a=t2.a union all select t1.a, t1.a from t1 left join t2 on t1.a=t2.a) tmp;`).Check(testkit.Rows("128")) tk.MustQuery(`select tmp.a, count(*) from (select t1.a, t1.b from t1 left join t2 on t1.a=t2.a union all select t1.a, t1.a from t1 left join t2 on t1.a=t2.a) tmp;`).Check(testkit.Rows("1 128")) @@ -1251,7 +1257,7 @@ func (s *testSuite) TestUnion(c *C) { tk.MustQuery("select count(distinct a), sum(distinct a), avg(distinct a) from (select a from t union all select b from t) tmp;").Check(testkit.Rows("1 1.000 1.0000000")) } -func (s *testSuite) TestNeighbouringProj(c *C) { +func (s *testSuiteP1) TestNeighbouringProj(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1269,7 +1275,7 @@ func (s *testSuite) TestNeighbouringProj(c *C) { rs.Check(testkit.Rows("1 1 1", "1 2 2", "1 3 3")) } -func (s *testSuite) TestIn(c *C) { +func (s *testSuiteP1) TestIn(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`drop table if exists t`) @@ -1285,7 +1291,7 @@ func (s *testSuite) TestIn(c *C) { tk.MustQuery(queryStr).Check(testkit.Rows("7")) } -func (s *testSuite) TestTablePKisHandleScan(c *C) { +func (s *testSuiteP1) TestTablePKisHandleScan(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1347,7 +1353,7 @@ func (s *testSuite) TestTablePKisHandleScan(c *C) { } } -func (s *testSuite) TestIndexScan(c *C) { +func (s *testSuiteP1) TestIndexScan(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1430,7 +1436,7 @@ func (s *testSuite) TestIndexScan(c *C) { result.Check(testkit.Rows()) } -func (s *testSuite) TestIndexReverseOrder(c *C) { +func (s *testSuiteP1) TestIndexReverseOrder(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1448,7 +1454,7 @@ func (s *testSuite) TestIndexReverseOrder(c *C) { result.Check(testkit.Rows("0 2", "0 1", "0 0", "1 2", "1 1", "1 0", "2 2", "2 1", "2 0")) } -func (s *testSuite) TestTableReverseOrder(c *C) { +func (s *testSuiteP1) TestTableReverseOrder(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1460,7 +1466,7 @@ func (s *testSuite) TestTableReverseOrder(c *C) { result.Check(testkit.Rows("7", "6", "2", "1")) } -func (s *testSuite) TestDefaultNull(c *C) { +func (s *testSuiteP1) TestDefaultNull(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1476,7 +1482,7 @@ func (s *testSuite) TestDefaultNull(c *C) { tk.MustQuery("select * from t").Check(testkit.Rows("1 1 ")) } -func (s *testSuite) TestUnsignedPKColumn(c *C) { +func (s *testSuiteP1) TestUnsignedPKColumn(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1489,7 +1495,7 @@ func (s *testSuite) TestUnsignedPKColumn(c *C) { result.Check(testkit.Rows("1 1 2")) } -func (s *testSuite) TestJSON(c *C) { +func (s *testSuiteP1) TestJSON(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -1577,7 +1583,7 @@ func (s *testSuite) TestJSON(c *C) { "1234567890123456789012345678901234567890123456789012345.12")) } -func (s *testSuite) TestMultiUpdate(c *C) { +func (s *testSuiteP1) TestMultiUpdate(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`CREATE TABLE test_mu (a int primary key, b int, c int)`) @@ -1606,7 +1612,7 @@ func (s *testSuite) TestMultiUpdate(c *C) { result.Check(testkit.Rows(`1 7 2`, `4 8 8`, `7 8 8`)) } -func (s *testSuite) TestGeneratedColumnWrite(c *C) { +func (s *testSuiteP1) TestGeneratedColumnWrite(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") _, err := tk.Exec(`CREATE TABLE test_gc_write (a int primary key auto_increment, b int, c int as (a+8) virtual)`) @@ -1660,7 +1666,7 @@ func (s *testSuite) TestGeneratedColumnWrite(c *C) { // TestGeneratedColumnRead tests select generated columns from table. // They should be calculated from their generation expressions. -func (s *testSuite) TestGeneratedColumnRead(c *C) { +func (s *testSuiteP1) TestGeneratedColumnRead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`CREATE TABLE test_gc_read(a int primary key, b int, c int as (a+b), d int as (a*b) stored, e int as (c*2))`) @@ -1822,7 +1828,7 @@ func (s *testSuite) TestGeneratedColumnRead(c *C) { } } -func (s *testSuite) TestToPBExpr(c *C) { +func (s *testSuiteP1) TestToPBExpr(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1870,7 +1876,7 @@ func (s *testSuite) TestToPBExpr(c *C) { result.Check(testkit.Rows("1", "2")) } -func (s *testSuite) TestDatumXAPI(c *C) { +func (s *testSuiteP1) TestDatumXAPI(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1895,7 +1901,7 @@ func (s *testSuite) TestDatumXAPI(c *C) { result.Check(testkit.Rows("11:11:12.000 11:11:12", "11:11:13.000 11:11:13")) } -func (s *testSuite) TestSQLMode(c *C) { +func (s *testSuiteP1) TestSQLMode(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -1933,6 +1939,7 @@ func (s *testSuite) TestSQLMode(c *C) { s.domain.GetGlobalVarsCache().Disable() tk2 := testkit.NewTestKit(c, s.store) tk2.MustExec("use test") + tk2.MustExec("drop table if exists t2") tk2.MustExec("create table t2 (a varchar(3))") tk2.MustExec("insert t2 values ('abcd')") tk2.MustQuery("select * from t2").Check(testkit.Rows("abc")) @@ -1944,7 +1951,7 @@ func (s *testSuite) TestSQLMode(c *C) { tk.MustExec("set @@global.sql_mode = 'STRICT_TRANS_TABLES'") } -func (s *testSuite) TestTableDual(c *C) { +func (s *testSuiteP1) TestTableDual(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") result := tk.MustQuery("Select 1") @@ -1956,11 +1963,12 @@ func (s *testSuite) TestTableDual(c *C) { result = tk.MustQuery("Select 1 from dual where 1") result.Check(testkit.Rows("1")) + tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int primary key)") tk.MustQuery("select t1.* from t t1, t t2 where t1.a=t2.a and 1=0").Check(testkit.Rows()) } -func (s *testSuite) TestTableScan(c *C) { +func (s *testSuiteP1) TestTableScan(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use information_schema") result := tk.MustQuery("select * from schemata") @@ -1979,7 +1987,7 @@ func (s *testSuite) TestTableScan(c *C) { result.Check(testkit.Rows("1")) } -func (s *testSuite) TestAdapterStatement(c *C) { +func (s *testSuiteP1) TestAdapterStatement(c *C) { se, err := session.CreateSession4Test(s.store) c.Check(err, IsNil) se.GetSessionVars().TxnCtx.InfoSchema = domain.GetDomain(se).InfoSchema() @@ -1997,7 +2005,7 @@ func (s *testSuite) TestAdapterStatement(c *C) { c.Check(stmt.OriginText(), Equals, "create table test.t (a int)") } -func (s *testSuite) TestIsPointGet(c *C) { +func (s *testSuiteP1) TestIsPointGet(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use mysql") ctx := tk.Se.(sessionctx.Context) @@ -2022,7 +2030,7 @@ func (s *testSuite) TestIsPointGet(c *C) { } } -func (s *testSuite) TestPointGetRepeatableRead(c *C) { +func (s *testSuiteP1) TestPointGetRepeatableRead(c *C) { tk1 := testkit.NewTestKit(c, s.store) tk1.MustExec("use test") tk1.MustExec(`create table point_get (a int, b int, c int, @@ -2058,7 +2066,7 @@ func (s *testSuite) TestPointGetRepeatableRead(c *C) { c.Assert(failpoint.Disable(step2), IsNil) } -func (s *testSuite) TestSplitRegionTimeout(c *C) { +func (s *testSuite4) TestSplitRegionTimeout(c *C) { c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockSplitRegionTimeout", `return(true)`), IsNil) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -2078,7 +2086,7 @@ func (s *testSuite) TestSplitRegionTimeout(c *C) { c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/mockScatterRegionTimeout"), IsNil) } -func (s *testSuite) TestRow(c *C) { +func (s *testSuiteP1) TestRow(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -2127,7 +2135,7 @@ func (s *testSuite) TestRow(c *C) { result.Check(testkit.Rows("1")) } -func (s *testSuite) TestColumnName(c *C) { +func (s *testSuiteP1) TestColumnName(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -2208,7 +2216,7 @@ func (s *testSuite) TestColumnName(c *C) { rs.Close() } -func (s *testSuite) TestSelectVar(c *C) { +func (s *testSuiteP1) TestSelectVar(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -2223,7 +2231,7 @@ func (s *testSuite) TestSelectVar(c *C) { tk.MustExec("select SQL_BUFFER_RESULT d from t group by d") } -func (s *testSuite) TestHistoryRead(c *C) { +func (s *testSuiteP1) TestHistoryRead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists history_read") @@ -2287,7 +2295,7 @@ func (s *testSuite) TestHistoryRead(c *C) { tk.MustQuery("select * from history_read order by a").Check(testkit.Rows("2 ", "4 ", "8 8", "9 9")) } -func (s *testSuite) TestLowResolutionTSORead(c *C) { +func (s *testSuiteP1) TestLowResolutionTSORead(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("set @@autocommit=1") tk.MustExec("use test") @@ -3894,7 +3902,7 @@ func (s *testSuite4) TearDownTest(c *C) { } } -func (s *testSuite) TestStrToDateBuiltin(c *C) { +func (s *testSuiteP1) TestStrToDateBuiltin(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustQuery(`select str_to_date('18/10/22','%y/%m/%d') from dual`).Check(testkit.Rows("2018-10-22")) tk.MustQuery(`select str_to_date('a18/10/22','%y/%m/%d') from dual`).Check(testkit.Rows("")) @@ -3934,7 +3942,7 @@ func (s *testSuite) TestStrToDateBuiltin(c *C) { tk.MustQuery(`select str_to_date('18_10_22','%y_%m_%d') from dual`).Check(testkit.Rows("2018-10-22")) } -func (s *testSuite) TestReadPartitionedTable(c *C) { +func (s *testSuiteP1) TestReadPartitionedTable(c *C) { // Test three reader on partitioned table. tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -3951,7 +3959,7 @@ func (s *testSuite) TestReadPartitionedTable(c *C) { tk.MustQuery("select a from pt where b = 3").Check(testkit.Rows("3")) } -func (s *testSuite) TestSplitRegion(c *C) { +func (s *testSuiteP1) TestSplitRegion(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -4151,9 +4159,10 @@ func testGetTableByName(c *C, ctx sessionctx.Context, db, table string) table.Ta return tbl } -func (s *testSuite) TestIssue10435(c *C) { +func (s *testSuiteP1) TestIssue10435(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") + tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(i int, j int, k int)") tk.MustExec("insert into t1 VALUES (1,1,1),(2,2,2),(3,3,3),(4,4,4)") tk.MustExec("INSERT INTO t1 SELECT 10*i,j,5*j FROM t1 UNION SELECT 20*i,j,5*j FROM t1 UNION SELECT 30*i,j,5*j FROM t1") @@ -4164,7 +4173,7 @@ func (s *testSuite) TestIssue10435(c *C) { ) } -func (s *testSuite) TestUnsignedFeedback(c *C) { +func (s *testSuiteP1) TestUnsignedFeedback(c *C) { tk := testkit.NewTestKit(c, s.store) oriProbability := statistics.FeedbackProbability.Load() statistics.FeedbackProbability.Store(1.0) From bedd1b07b02932a3f73e241dbf63494dca20e600 Mon Sep 17 00:00:00 2001 From: lysu Date: Thu, 18 Jul 2019 19:21:01 +0800 Subject: [PATCH 048/196] plugin: add "exec start-time" to GeneralEvent (#11293) --- executor/adapter.go | 3 ++- plugin/audit.go | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/executor/adapter.go b/executor/adapter.go index ee7bc38441d1a..a08c26ab11abf 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -608,7 +608,8 @@ func (a *ExecStmt) logAudit() { audit := plugin.DeclareAuditManifest(p.Manifest) if audit.OnGeneralEvent != nil { cmd := mysql.Command2Str[byte(atomic.LoadUint32(&a.Ctx.GetSessionVars().CommandValue))] - audit.OnGeneralEvent(context.Background(), sessVars, plugin.Log, cmd) + ctx := context.WithValue(context.Background(), plugin.ExecStartTimeCtxKey, a.StartTime) + audit.OnGeneralEvent(ctx, sessVars, plugin.Log, cmd) } return nil }) diff --git a/plugin/audit.go b/plugin/audit.go index 603b7e0f8982f..f1471562fc657 100644 --- a/plugin/audit.go +++ b/plugin/audit.go @@ -84,3 +84,8 @@ type AuditManifest struct { // OnParseEvent will be called around parse logic. OnParseEvent func(ctx context.Context, sctx *variable.SessionVars, event ParseEvent) error } + +const ( + // ExecStartTimeCtxKey indicates stmt start execution time. + ExecStartTimeCtxKey = "ExecStartTime" +) From 3217bff3d04358c85168b5fbbab6a41e6c85e82b Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 18 Jul 2019 19:47:56 +0800 Subject: [PATCH 049/196] =?UTF-8?q?expression:=20let=20`testEvaluatorSuite?= =?UTF-8?q?`=20run=20serially=20to=20avoid=20aff=E2=80=A6=20(#11310)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- expression/evaluator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/evaluator_test.go b/expression/evaluator_test.go index db1bbd54d02b5..ef86d53f0b242 100644 --- a/expression/evaluator_test.go +++ b/expression/evaluator_test.go @@ -32,7 +32,7 @@ import ( "github.com/pingcap/tidb/util/testutil" ) -var _ = Suite(&testEvaluatorSuite{}) +var _ = SerialSuites(&testEvaluatorSuite{}) func TestT(t *testing.T) { CustomVerboseFlag = true From 5aef053c163524aa167e6bb4ddabd0671a75e712 Mon Sep 17 00:00:00 2001 From: wshwsh12 <793703860@qq.com> Date: Thu, 18 Jul 2019 23:18:41 +0800 Subject: [PATCH 050/196] expression: fix getIntervalFromDecimal in DATE_ADD() (#11297) --- expression/builtin_time.go | 52 ++++++++----- expression/integration_test.go | 134 ++++++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 21 deletions(-) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 75cfa8acacad1..50a8c4ac75129 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -2662,26 +2662,38 @@ func (du *baseDateArithmitical) getIntervalFromDecimal(ctx sessionctx.Context, a } switch strings.ToUpper(unit) { - case "HOUR_MINUTE", "MINUTE_SECOND": - interval = strings.Replace(interval, ".", ":", -1) - case "YEAR_MONTH": - interval = strings.Replace(interval, ".", "-", -1) - case "DAY_HOUR": - interval = strings.Replace(interval, ".", " ", -1) - case "DAY_MINUTE": - interval = "0 " + strings.Replace(interval, ".", ":", -1) - case "DAY_SECOND": - interval = "0 00:" + strings.Replace(interval, ".", ":", -1) - case "DAY_MICROSECOND": - interval = "0 00:00:" + interval - case "HOUR_MICROSECOND": - interval = "00:00:" + interval - case "HOUR_SECOND": - interval = "00:" + strings.Replace(interval, ".", ":", -1) - case "MINUTE_MICROSECOND": - interval = "00:" + interval - case "SECOND_MICROSECOND": - /* keep interval as original decimal */ + case "HOUR_MINUTE", "MINUTE_SECOND", "YEAR_MONTH", "DAY_HOUR", "DAY_MINUTE", + "DAY_SECOND", "DAY_MICROSECOND", "HOUR_MICROSECOND", "HOUR_SECOND", "MINUTE_MICROSECOND", "SECOND_MICROSECOND": + neg := false + if interval != "" && interval[0] == '-' { + neg = true + interval = interval[1:] + } + switch strings.ToUpper(unit) { + case "HOUR_MINUTE", "MINUTE_SECOND": + interval = strings.Replace(interval, ".", ":", -1) + case "YEAR_MONTH": + interval = strings.Replace(interval, ".", "-", -1) + case "DAY_HOUR": + interval = strings.Replace(interval, ".", " ", -1) + case "DAY_MINUTE": + interval = "0 " + strings.Replace(interval, ".", ":", -1) + case "DAY_SECOND": + interval = "0 00:" + strings.Replace(interval, ".", ":", -1) + case "DAY_MICROSECOND": + interval = "0 00:00:" + interval + case "HOUR_MICROSECOND": + interval = "00:00:" + interval + case "HOUR_SECOND": + interval = "00:" + strings.Replace(interval, ".", ":", -1) + case "MINUTE_MICROSECOND": + interval = "00:" + interval + case "SECOND_MICROSECOND": + /* keep interval as original decimal */ + } + if neg { + interval = "-" + interval + } case "SECOND": // Decimal's EvalString is like %f format. interval, isNull, err = args[1].EvalString(ctx, row) diff --git a/expression/integration_test.go b/expression/integration_test.go index a04a4b4d50e61..a42ed09e60c98 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4465,8 +4465,9 @@ func (s *testIntegrationSuite) TestIssue10675(c *C) { tk.MustQuery(`select * from t where a > 184467440737095516167.1;`).Check(testkit.Rows()) } -func (s *testIntegrationSuite) TestIssue11257(c *C) { +func (s *testIntegrationSuite) TestDatetimeMicrosecond(c *C) { tk := testkit.NewTestKit(c, s.store) + // For int tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 SECOND_MICROSECOND);`).Check( testkit.Rows("2007-03-28 22:08:27.800000")) tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 MINUTE_MICROSECOND);`).Check( @@ -4475,4 +4476,135 @@ func (s *testIntegrationSuite) TestIssue11257(c *C) { testkit.Rows("2007-03-28 22:08:27.800000")) tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2 DAY_MICROSECOND);`).Check( testkit.Rows("2007-03-28 22:08:27.800000")) + + // For Decimal + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR_MINUTE);`).Check( + testkit.Rows("2007-03-29 00:10:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MINUTE_SECOND);`).Check( + testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 YEAR_MONTH);`).Check( + testkit.Rows("2009-05-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_HOUR);`).Check( + testkit.Rows("2007-03-31 00:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_MINUTE);`).Check( + testkit.Rows("2007-03-29 00:10:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_SECOND);`).Check( + testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR_SECOND);`).Check( + testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 SECOND);`).Check( + testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 YEAR);`).Check( + testkit.Rows("2009-03-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 QUARTER);`).Check( + testkit.Rows("2007-09-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MONTH);`).Check( + testkit.Rows("2007-05-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 WEEK);`).Check( + testkit.Rows("2007-04-11 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY);`).Check( + testkit.Rows("2007-03-30 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR);`).Check( + testkit.Rows("2007-03-29 00:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MINUTE);`).Check( + testkit.Rows("2007-03-28 22:10:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:28.000002")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MINUTE_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 YEAR_MONTH);`).Check( + testkit.Rows("2005-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_HOUR);`).Check( + testkit.Rows("2007-03-26 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 SECOND);`).Check( + // testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 YEAR);`).Check( + testkit.Rows("2005-03-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 QUARTER);`).Check( + testkit.Rows("2006-09-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MONTH);`).Check( + testkit.Rows("2007-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 WEEK);`).Check( + testkit.Rows("2007-03-14 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY);`).Check( + testkit.Rows("2007-03-26 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR);`).Check( + testkit.Rows("2007-03-28 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MINUTE);`).Check( + testkit.Rows("2007-03-28 22:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.999998")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" HOUR_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" MINUTE_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" YEAR_MONTH);`).Check( + testkit.Rows("2005-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" DAY_HOUR);`).Check( + testkit.Rows("2007-03-26 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" DAY_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" DAY_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" HOUR_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" SECOND);`).Check( + // testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" YEAR);`).Check( + testkit.Rows("2005-03-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" QUARTER);`).Check( + testkit.Rows("2006-09-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" MONTH);`).Check( + testkit.Rows("2007-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" WEEK);`).Check( + testkit.Rows("2007-03-14 22:08:28")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" DAY);`).Check( + // testkit.Rows("2007-03-26 22:08:28")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" HOUR);`).Check( + // testkit.Rows("2007-03-28 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" MINUTE);`).Check( + testkit.Rows("2007-03-28 22:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.2" MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.999998")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" HOUR_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" MINUTE_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" YEAR_MONTH);`).Check( + testkit.Rows("2005-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" DAY_HOUR);`).Check( + testkit.Rows("2007-03-26 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" DAY_MINUTE);`).Check( + testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" DAY_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" HOUR_SECOND);`).Check( + testkit.Rows("2007-03-28 22:06:26")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" SECOND);`).Check( + // testkit.Rows("2007-03-28 22:08:26")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" YEAR);`).Check( + testkit.Rows("2005-03-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" QUARTER);`).Check( + testkit.Rows("2006-09-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" MONTH);`).Check( + testkit.Rows("2007-01-28 22:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" WEEK);`).Check( + testkit.Rows("2007-03-14 22:08:28")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" DAY);`).Check( + // testkit.Rows("2007-03-26 22:08:28")) + // tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" HOUR);`).Check( + // testkit.Rows("2007-03-28 20:08:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" MINUTE);`).Check( + testkit.Rows("2007-03-28 22:06:28")) + tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" MICROSECOND);`).Check( + testkit.Rows("2007-03-28 22:08:27.999998")) + } From ddb6132e82f79a91518d75c91617f3d60d0de06b Mon Sep 17 00:00:00 2001 From: wjHuang Date: Fri, 19 Jul 2019 12:37:25 +0800 Subject: [PATCH 051/196] expression, types: fix Mod(%), Multiple(*), Minus(-) operators meets inconsistent 0 results. (#11251) --- expression/integration_test.go | 7 +++++-- go.sum | 2 ++ types/mydecimal.go | 14 +++++++++++--- types/mydecimal_test.go | 13 +++++++++---- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index a42ed09e60c98..f7caf84ceaf60 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2839,7 +2839,10 @@ func (s *testIntegrationSuite) TestArithmeticBuiltin(c *C) { c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "[types:1690]BIGINT UNSIGNED value is out of range in '(18446744073709551615 - -1)'") c.Assert(rs.Close(), IsNil) + tk.MustQuery(`select cast(-3 as unsigned) - cast(-1 as signed);`).Check(testkit.Rows("18446744073709551614")) + tk.MustQuery("select 1.11 - 1.11;").Check(testkit.Rows("0.00")) + // for multiply tk.MustQuery("select 1234567890 * 1234567890").Check(testkit.Rows("1524157875019052100")) rs, err = tk.Exec("select 1234567890 * 12345671890") c.Assert(err, IsNil) @@ -2866,8 +2869,7 @@ func (s *testIntegrationSuite) TestArithmeticBuiltin(c *C) { _, err = session.GetRows4Test(ctx, tk.Se, rs) c.Assert(terror.ErrorEqual(err, types.ErrOverflow), IsTrue) c.Assert(rs.Close(), IsNil) - result = tk.MustQuery(`select cast(-3 as unsigned) - cast(-1 as signed);`) - result.Check(testkit.Rows("18446744073709551614")) + tk.MustQuery("select 0.0 * -1;").Check(testkit.Rows("0.0")) tk.MustExec("DROP TABLE IF EXISTS t;") tk.MustExec("CREATE TABLE t(a DECIMAL(4, 2), b DECIMAL(5, 3));") @@ -2937,6 +2939,7 @@ func (s *testIntegrationSuite) TestArithmeticBuiltin(c *C) { tk.MustExec("INSERT IGNORE INTO t VALUE(12 MOD 0);") tk.MustQuery("show warnings;").Check(testkit.Rows("Warning 1365 Division by 0")) tk.MustQuery("select v from t;").Check(testkit.Rows("")) + tk.MustQuery("select 0.000 % 0.11234500000000000000;").Check(testkit.Rows("0.00000000000000000000")) _, err = tk.Exec("INSERT INTO t VALUE(12 MOD 0);") c.Assert(terror.ErrorEqual(err, expression.ErrDivisionByZero), IsTrue) diff --git a/go.sum b/go.sum index 2bf2b6d4c859c..726d1d63d51be 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mo github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -191,6 +192,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFd github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= diff --git a/types/mydecimal.go b/types/mydecimal.go index 06ca50c4204fb..996323fa8706f 100644 --- a/types/mydecimal.go +++ b/types/mydecimal.go @@ -107,6 +107,14 @@ var ( zeroMyDecimal = MyDecimal{} ) +// get the zero of MyDecimal with the specified result fraction digits +func zeroMyDecimalWithFrac(frac int8) MyDecimal { + zero := MyDecimal{} + zero.digitsFrac = frac + zero.resultFrac = frac + return zero +} + // add adds a and b and carry, returns the sum and new carry. func add(a, b, carry int32) (int32, int32) { sum := a + b + carry @@ -1556,7 +1564,7 @@ func doSub(from1, from2, to *MyDecimal) (cmp int, err error) { if to == nil { return 0, nil } - *to = zeroMyDecimal + *to = zeroMyDecimalWithFrac(to.resultFrac) return 0, nil } } @@ -1911,7 +1919,7 @@ func DecimalMul(from1, from2, to *MyDecimal) error { idx++ /* We got decimal zero */ if idx == end { - *to = zeroMyDecimal + *to = zeroMyDecimalWithFrac(to.resultFrac) break } } @@ -2010,7 +2018,7 @@ func doDivMod(from1, from2, to, mod *MyDecimal, fracIncr int) error { } if prec1 <= 0 { /* short-circuit everything: from1 == 0 */ - *to = zeroMyDecimal + *to = zeroMyDecimalWithFrac(to.resultFrac) return nil } prec1 -= countLeadingZeroes((prec1-1)%digitsPerWord, from1.wordBuf[idx1]) diff --git a/types/mydecimal_test.go b/types/mydecimal_test.go index e799692231c6a..551105987b3d0 100644 --- a/types/mydecimal_test.go +++ b/types/mydecimal_test.go @@ -694,6 +694,7 @@ func (s *testMyDecimalSuite) TestAdd(c *C) { {"-123.45", "12345", "12221.55", nil}, {"5", "-6.0", "-1.0", nil}, {"2" + strings.Repeat("1", 71), strings.Repeat("8", 81), "8888888890" + strings.Repeat("9", 71), nil}, + {"-1234.1234", "1234.1234", "0.0000", nil}, } for _, tt := range tests { a := NewDecFromStringForTest(tt.a) @@ -718,7 +719,7 @@ func (s *testMyDecimalSuite) TestSub(c *C) { {"1234500009876.5", ".00012345000098765", "1234500009876.49987654999901235", nil}, {"9999900000000.5", ".555", "9999899999999.945", nil}, {"1111.5551", "1111.555", "0.0001", nil}, - {".555", ".555", "0", nil}, + {".555", ".555", "0.000", nil}, {"10000000", "1", "9999999", nil}, {"1000001000", ".1", "1000000999.9", nil}, {"1000000000", ".1", "999999999.9", nil}, @@ -728,6 +729,7 @@ func (s *testMyDecimalSuite) TestSub(c *C) { {"-123.45", "-12345", "12221.55", nil}, {"-12345", "123.45", "-12468.45", nil}, {"12345", "-123.45", "12468.45", nil}, + {"12.12", "12.12", "0.00", nil}, } for _, tt := range tests { var a, b, sum MyDecimal @@ -759,6 +761,7 @@ func (s *testMyDecimalSuite) TestMul(c *C) { {"1" + strings.Repeat("0", 60), "1" + strings.Repeat("0", 60), "0", ErrOverflow}, {"0.5999991229316", "0.918755041726043", "0.5512522192246113614062276588", nil}, {"0.5999991229317", "0.918755041726042", "0.5512522192247026369112773314", nil}, + {"0.000", "-1", "0.000", nil}, } for _, tt := range tests { var a, b, product MyDecimal @@ -786,7 +789,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) { {"0", "0", "", ErrDivByZero}, {"-12193185.1853376", "98765.4321", "-123.456000000000000000", nil}, {"121931851853376", "987654321", "123456.000000000", nil}, - {"0", "987", "0", nil}, + {"0", "987", "0.00000", nil}, {"1", "3", "0.333333333", nil}, {"1.000000000000", "3", "0.333333333333333333", nil}, {"1", "1", "1.000000000", nil}, @@ -799,7 +802,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) { var a, b, to MyDecimal a.FromString([]byte(tt.a)) b.FromString([]byte(tt.b)) - err := doDivMod(&a, &b, &to, nil, 5) + err := DecimalDiv(&a, &b, &to, 5) c.Check(err, Equals, tt.err) if tt.err == ErrDivByZero { continue @@ -816,12 +819,13 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) { {"99999999999999999999999999999999999999", "3", "0", nil}, {"51", "0.003430", "0.002760", nil}, {"0.0000000001", "1.0", "0.0000000001", nil}, + {"0.000", "0.1", "0.000", nil}, } for _, tt := range tests { var a, b, to MyDecimal a.FromString([]byte(tt.a)) b.FromString([]byte(tt.b)) - ec := doDivMod(&a, &b, nil, &to, 0) + ec := DecimalMod(&a, &b, &to) c.Check(ec, Equals, tt.err) if tt.err == ErrDivByZero { continue @@ -836,6 +840,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) { {"1", "1.000", "1.0000", nil}, {"2", "3", "0.6667", nil}, {"51", "0.003430", "14868.8047", nil}, + {"0.000", "0.1", "0.0000000", nil}, } for _, tt := range tests { var a, b, to MyDecimal From 4b928a52c0bcbdea3b3e3e1779ac4c7ef6f7e1a0 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 19 Jul 2019 17:58:58 +0800 Subject: [PATCH 052/196] executor: update test since DELETE FROM ... AS ... syntax is now supported (#11184) --- executor/write_test.go | 7 ++++--- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/executor/write_test.go b/executor/write_test.go index 69874534a57bd..3a60e95c97bd0 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -1764,8 +1764,9 @@ func (s *testSuite4) TestQualifiedDelete(c *C) { r := tk.MustQuery("select * from t1") c.Assert(r.Rows(), HasLen, 0) - _, err := tk.Exec("delete from t1 as a where a.c1 = 1") - c.Assert(err, NotNil) + tk.MustExec("insert into t1 values (1, 3)") + tk.MustExec("delete from t1 as a where a.c1 = 1") + tk.CheckExecResult(1, 0) tk.MustExec("insert into t1 values (1, 1), (2, 2)") tk.MustExec("insert into t2 values (2, 1), (3,1)") @@ -1776,7 +1777,7 @@ func (s *testSuite4) TestQualifiedDelete(c *C) { tk.MustExec("delete a, b from t1 as a join t2 as b where a.c2 = b.c1") tk.CheckExecResult(2, 0) - _, err = tk.Exec("delete t1, t2 from t1 as a join t2 as b where a.c2 = b.c1") + _, err := tk.Exec("delete t1, t2 from t1 as a join t2 as b where a.c2 = b.c1") c.Assert(err, NotNil) } diff --git a/go.mod b/go.mod index 0fd738a958d1f..5950f1dc2f5c0 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d + github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 726d1d63d51be..10bd3f3646065 100644 --- a/go.sum +++ b/go.sum @@ -164,8 +164,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d h1:vOZjn1ami1LIjtIj0i5QunGh/sHawbhiBCb1qPx373w= -github.com/pingcap/parser v0.0.0-20190710072914-6cd203114f2d/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903 h1:mRZH1M//ZhlpJ9ByB6TyEFErVO5vsfeWyA8a0SklkF0= +github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From d977edf8a39ccd1972d1fe1b17855b63b934a09d Mon Sep 17 00:00:00 2001 From: zhaoyanxing Date: Fri, 19 Jul 2019 19:49:41 +0800 Subject: [PATCH 053/196] =?UTF-8?q?Function=20SUBTIME=20ADDTIME=20should?= =?UTF-8?q?=20return=20NULL=20with=20a=20warning=20if=E2=80=A6=20(#11262)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- expression/builtin_time.go | 64 +++++++++++++++++++++++++++++---- expression/builtin_time_test.go | 56 +++++++++++++++++++++++++++++ expression/integration_test.go | 35 +++++++++++++++++- 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 50a8c4ac75129..9b302873eb852 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -4800,6 +4800,10 @@ func (b *builtinAddDatetimeAndStringSig) evalTime(row chunk.Row) (types.Time, bo sc := b.ctx.GetSessionVars().StmtCtx arg1, err := types.ParseDuration(sc, s, types.GetFsp(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return types.ZeroDatetime, true, nil + } return types.ZeroDatetime, true, err } result, err := arg0.Add(sc, arg1) @@ -4874,8 +4878,13 @@ func (b *builtinAddDurationAndStringSig) evalDuration(row chunk.Row) (types.Dura if !isDuration(s) { return types.ZeroDuration, true, nil } - arg1, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, s, types.GetFsp(s)) + sc := b.ctx.GetSessionVars().StmtCtx + arg1, err := types.ParseDuration(sc, s, types.GetFsp(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return types.ZeroDuration, true, nil + } return types.ZeroDuration, true, err } result, err := arg0.Add(arg1) @@ -4930,6 +4939,10 @@ func (b *builtinAddStringAndDurationSig) evalString(row chunk.Row) (result strin if isDuration(arg0) { result, err = strDurationAddDuration(sc, arg0, arg1) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } return result, false, nil @@ -4970,11 +4983,19 @@ func (b *builtinAddStringAndStringSig) evalString(row chunk.Row) (result string, sc := b.ctx.GetSessionVars().StmtCtx arg1, err = types.ParseDuration(sc, arg1Str, getFsp4TimeAddSub(arg1Str)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } if isDuration(arg0) { result, err = strDurationAddDuration(sc, arg0, arg1) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } return result, false, nil @@ -5032,8 +5053,13 @@ func (b *builtinAddDateAndStringSig) evalString(row chunk.Row) (string, bool, er if !isDuration(s) { return "", true, nil } - arg1, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, s, getFsp4TimeAddSub(s)) + sc := b.ctx.GetSessionVars().StmtCtx + arg1, err := types.ParseDuration(sc, s, getFsp4TimeAddSub(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } result, err := arg0.Add(arg1) @@ -5705,6 +5731,10 @@ func (b *builtinSubDatetimeAndStringSig) evalTime(row chunk.Row) (types.Time, bo sc := b.ctx.GetSessionVars().StmtCtx arg1, err := types.ParseDuration(sc, s, types.GetFsp(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return types.ZeroDatetime, true, nil + } return types.ZeroDatetime, true, err } arg1time, err := arg1.ConvertToTime(sc, mysql.TypeDatetime) @@ -5761,6 +5791,10 @@ func (b *builtinSubStringAndDurationSig) evalString(row chunk.Row) (result strin if isDuration(arg0) { result, err = strDurationSubDuration(sc, arg0, arg1) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } return result, false, nil @@ -5798,14 +5832,22 @@ func (b *builtinSubStringAndStringSig) evalString(row chunk.Row) (result string, if isNull || err != nil { return "", isNull, err } - arg1, err = types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, s, getFsp4TimeAddSub(s)) + sc := b.ctx.GetSessionVars().StmtCtx + arg1, err = types.ParseDuration(sc, s, getFsp4TimeAddSub(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } - sc := b.ctx.GetSessionVars().StmtCtx if isDuration(arg0) { result, err = strDurationSubDuration(sc, arg0, arg1) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } return result, false, nil @@ -5882,8 +5924,13 @@ func (b *builtinSubDurationAndStringSig) evalDuration(row chunk.Row) (types.Dura if !isDuration(s) { return types.ZeroDuration, true, nil } - arg1, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, s, types.GetFsp(s)) + sc := b.ctx.GetSessionVars().StmtCtx + arg1, err := types.ParseDuration(sc, s, types.GetFsp(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return types.ZeroDuration, true, nil + } return types.ZeroDuration, true, err } result, err := arg0.Sub(arg1) @@ -5955,8 +6002,13 @@ func (b *builtinSubDateAndStringSig) evalString(row chunk.Row) (string, bool, er if !isDuration(s) { return "", true, nil } - arg1, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, s, getFsp4TimeAddSub(s)) + sc := b.ctx.GetSessionVars().StmtCtx + arg1, err := types.ParseDuration(sc, s, getFsp4TimeAddSub(s)) if err != nil { + if terror.ErrorEqual(err, types.ErrTruncatedWrongVal) { + sc.AppendWarning(err) + return "", true, nil + } return "", true, err } result, err := arg0.Sub(arg1) diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index 4c570ffa15c6c..eebb80b524d51 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" @@ -937,6 +938,33 @@ func (s *testEvaluatorSuite) TestAddTimeSig(c *C) { c.Assert(result, Equals, t.expect) } + tblWarning := []struct { + Input interface{} + InputDuration interface{} + warning *terror.Error + }{ + {"0", "-32073", types.ErrTruncatedWrongVal}, + {"-32073", "0", types.ErrTruncatedWrongVal}, + {types.ZeroDuration, "-32073", types.ErrTruncatedWrongVal}, + {"-32073", types.ZeroDuration, types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeTimestamp), "-32073", types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeDate), "-32073", types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeDatetime), "-32073", types.ErrTruncatedWrongVal}, + } + for i, t := range tblWarning { + tmpInput := types.NewDatum(t.Input) + tmpInputDuration := types.NewDatum(t.InputDuration) + f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{tmpInput, tmpInputDuration})) + c.Assert(err, IsNil) + d, err := evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + result, _ := d.ToString() + c.Assert(result, Equals, "") + c.Assert(d.IsNull(), Equals, true) + warnings := s.ctx.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(len(warnings), Equals, i+1) + c.Assert(terror.ErrorEqual(t.warning, warnings[i].Err), IsTrue, Commentf("err %v", warnings[i].Err)) + } } func (s *testEvaluatorSuite) TestSubTimeSig(c *C) { @@ -1002,6 +1030,34 @@ func (s *testEvaluatorSuite) TestSubTimeSig(c *C) { result, _ := d.ToString() c.Assert(result, Equals, t.expect) } + + tblWarning := []struct { + Input interface{} + InputDuration interface{} + warning *terror.Error + }{ + {"0", "-32073", types.ErrTruncatedWrongVal}, + {"-32073", "0", types.ErrTruncatedWrongVal}, + {types.ZeroDuration, "-32073", types.ErrTruncatedWrongVal}, + {"-32073", types.ZeroDuration, types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeTimestamp), "-32073", types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeDate), "-32073", types.ErrTruncatedWrongVal}, + {types.CurrentTime(mysql.TypeDatetime), "-32073", types.ErrTruncatedWrongVal}, + } + for i, t := range tblWarning { + tmpInput := types.NewDatum(t.Input) + tmpInputDuration := types.NewDatum(t.InputDuration) + f, err := fc.getFunction(s.ctx, s.datumsToConstants([]types.Datum{tmpInput, tmpInputDuration})) + c.Assert(err, IsNil) + d, err := evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + result, _ := d.ToString() + c.Assert(result, Equals, "") + c.Assert(d.IsNull(), Equals, true) + warnings := s.ctx.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(len(warnings), Equals, i+1) + c.Assert(terror.ErrorEqual(t.warning, warnings[i].Err), IsTrue, Commentf("err %v", warnings[i].Err)) + } } func (s *testEvaluatorSuite) TestSysDate(c *C) { diff --git a/expression/integration_test.go b/expression/integration_test.go index f7caf84ceaf60..3e551f0199e2f 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -1442,7 +1442,6 @@ func (s *testIntegrationSuite) TestTimeBuiltin(c *C) { result.Check(testkit.Rows(" ")) result = tk.MustQuery("select addtime('2017-01-01', 1), addtime('2017-01-01 01:01:01', 1), addtime(cast('2017-01-01' as date), 1)") result.Check(testkit.Rows("2017-01-01 00:00:01 2017-01-01 01:01:02 00:00:01")) - result = tk.MustQuery("select subtime(a, e), subtime(b, e), subtime(c, e), subtime(d, e) from t") result.Check(testkit.Rows(" ")) result = tk.MustQuery("select subtime('2017-01-01 01:01:01', 0b1), subtime('2017-01-01', b'1'), subtime('01:01:01', 0b1011)") @@ -1450,6 +1449,40 @@ func (s *testIntegrationSuite) TestTimeBuiltin(c *C) { result = tk.MustQuery("select subtime('2017-01-01', 1), subtime('2017-01-01 01:01:01', 1), subtime(cast('2017-01-01' as date), 1)") result.Check(testkit.Rows("2016-12-31 23:59:59 2017-01-01 01:01:00 -00:00:01")) + result = tk.MustQuery("select addtime(-32073, 0), addtime(0, -32073);") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + result = tk.MustQuery("select addtime(-32073, c), addtime(c, -32073) from t;") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + result = tk.MustQuery("select addtime(a, -32073), addtime(b, -32073), addtime(d, -32073) from t;") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + + result = tk.MustQuery("select subtime(-32073, 0), subtime(0, -32073);") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + result = tk.MustQuery("select subtime(-32073, c), subtime(c, -32073) from t;") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + result = tk.MustQuery("select subtime(a, -32073), subtime(b, -32073), subtime(d, -32073) from t;") + result.Check(testkit.Rows(" ")) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'", + "Warning|1292|Truncated incorrect time value: '-32073'")) + // fixed issue #3986 tk.MustExec("SET SQL_MODE='NO_ENGINE_SUBSTITUTION';") tk.MustExec("SET TIME_ZONE='+03:00';") From 85de5df03fd47f80b90c36a63b7d213aeda4468b Mon Sep 17 00:00:00 2001 From: Lynn Date: Sat, 20 Jul 2019 09:09:35 +0800 Subject: [PATCH 054/196] ddl: fix the parallel problem of "set default value" and other DDL (#11341) --- ddl/column.go | 9 ++++++--- ddl/db_change_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ddl/column.go b/ddl/column.go index 2b652672b0e23..8f8c354606863 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -324,7 +324,7 @@ func onSetDefaultValue(t *meta.Meta, job *model.Job) (ver int64, _ error) { return ver, errors.Trace(err) } - return updateColumn(t, job, newCol, &newCol.Name) + return updateColumnDefaultValue(t, job, newCol, &newCol.Name) } func (w *worker) onModifyColumn(t *meta.Meta, job *model.Job) (ver int64, _ error) { @@ -489,7 +489,7 @@ func checkForNullValue(ctx sessionctx.Context, isDataTruncated bool, schema, tab return nil } -func updateColumn(t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldColName *model.CIStr) (ver int64, _ error) { +func updateColumnDefaultValue(t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldColName *model.CIStr) (ver int64, _ error) { tblInfo, err := getTableInfoAndCancelFaultJob(t, job, job.SchemaID) if err != nil { return ver, errors.Trace(err) @@ -499,7 +499,10 @@ func updateColumn(t *meta.Meta, job *model.Job, newCol *model.ColumnInfo, oldCol job.State = model.JobStateCancelled return ver, infoschema.ErrColumnNotExists.GenWithStackByArgs(newCol.Name, tblInfo.Name) } - *oldCol = *newCol + // The newCol's offset may be the value of the old schema version, so we can't use newCol directly. + oldCol.DefaultValue = newCol.DefaultValue + oldCol.DefaultValueBit = newCol.DefaultValueBit + oldCol.Flag = newCol.Flag ver, err = updateVersionAndTableInfo(t, job, tblInfo, true) if err != nil { diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index 541620f47cbcb..ee89526611a23 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -682,6 +682,30 @@ func (s *testStateChangeSuite) TestParallelAlterModifyColumn(c *C) { // s.testControlParallelExecSQL(c, sql1, sql2, f) // } +func (s *testStateChangeSuite) TestParallelAddColumAndSetDefaultValue(c *C) { + _, err := s.se.Execute(context.Background(), "use test_db_state") + c.Assert(err, IsNil) + _, err = s.se.Execute(context.Background(), `create table tx ( + c1 varchar(64), + c2 enum('N','Y') not null default 'N', + primary key idx2 (c2, c1))`) + c.Assert(err, IsNil) + _, err = s.se.Execute(context.Background(), "insert into tx values('a', 'N')") + c.Assert(err, IsNil) + defer s.se.Execute(context.Background(), "drop table tx") + + sql1 := "alter table tx add column cx int after c1" + sql2 := "alter table tx alter c2 set default 'N'" + + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2, IsNil) + _, err := s.se.Execute(context.Background(), "delete from tx where c1='a'") + c.Assert(err, IsNil) + } + s.testControlParallelExecSQL(c, sql1, sql2, f) +} + func (s *testStateChangeSuite) TestParallelChangeColumnName(c *C) { sql1 := "ALTER TABLE t CHANGE a aa int;" sql2 := "ALTER TABLE t CHANGE b aa int;" From 1c8038cceb065ab010539c29687146a435f35fa0 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 22 Jul 2019 11:50:30 +0800 Subject: [PATCH 055/196] store/tikv: refine streaming client re-create log and use a smarter backoff strategy (#11307) --- store/tikv/client_batch.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index f959deafa55bf..467fab1827d41 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -16,6 +16,7 @@ package tikv import ( "context" + "math" "sync" "sync/atomic" "time" @@ -23,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/tikvpb" + "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/store/tikv/tikvrpc" @@ -209,7 +211,7 @@ func (c *batchCommandsClient) failPendingRequests(err error) { }) } -func (c *batchCommandsClient) reCreateStreamingClient(err error) bool { +func (c *batchCommandsClient) reCreateStreamingClient(err error) error { // Hold the lock to forbid batchSendLoop using the old client. c.clientLock.Lock() defer c.clientLock.Unlock() @@ -224,14 +226,14 @@ func (c *batchCommandsClient) reCreateStreamingClient(err error) bool { zap.String("target", c.target), ) c.client = streamClient - return true + return nil } logutil.BgLogger().Error( "batchRecvLoop re-create streaming fail", zap.String("target", c.target), zap.Error(err), ) - return false + return err } func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { @@ -249,23 +251,27 @@ func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { for { resp, err := c.recv() if err != nil { + logutil.BgLogger().Error( + "batchRecvLoop error when receive", + zap.String("target", c.target), + zap.Error(err), + ) + + b := NewBackoffer(context.Background(), math.MaxInt32) now := time.Now() for { // try to re-create the streaming in the loop. if c.isStopped() { return } - logutil.BgLogger().Error( - "batchRecvLoop error when receive", - zap.String("target", c.target), - zap.Error(err), - ) - if c.reCreateStreamingClient(err) { + err1 := c.reCreateStreamingClient(err) + if err1 == nil { break } - - // TODO: Use a more smart backoff strategy. - time.Sleep(time.Second) + err2 := b.Backoff(boTiKVRPC, err1) + // As timeout is set to math.MaxUint32, err2 should always be nil. + // This line is added to make the 'make errcheck' pass. + terror.Log(err2) } metrics.TiKVBatchClientUnavailable.Observe(time.Since(now).Seconds()) continue From d927f9ae18f49b732bf691c7172bf18e33401aa2 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 22 Jul 2019 12:40:55 +0800 Subject: [PATCH 056/196] executor: expose `chunk.Column` to prepare for vectorized expression evaluation (#11351) --- util/chunk/chunk.go | 60 +++++++++++++++++++-------------------- util/chunk/chunk_test.go | 2 +- util/chunk/chunk_util.go | 4 +-- util/chunk/codec.go | 12 ++++---- util/chunk/codec_test.go | 12 ++++---- util/chunk/column.go | 60 ++++++++++++++++++++++++--------------- util/chunk/column_test.go | 4 +-- util/chunk/compare.go | 4 +-- util/chunk/mutrow.go | 30 ++++++++++---------- util/chunk/pool.go | 16 +++++------ 10 files changed, 109 insertions(+), 95 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index df331d049bfb7..66e88d94fddea 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -28,8 +28,8 @@ import ( // Values are appended in compact format and can be directly accessed without decoding. // When the chunk is done processing, we can reuse the allocated memory by resetting it. type Chunk struct { - columns []*column - // numVirtualRows indicates the number of virtual rows, which have zero column. + columns []*Column + // numVirtualRows indicates the number of virtual rows, which have zero Column. // It is used only when this Chunk doesn't hold any data, i.e. "len(columns)==0". numVirtualRows int // capacity indicates the max number of rows this chunk can hold. @@ -56,7 +56,7 @@ func NewChunkWithCapacity(fields []*types.FieldType, cap int) *Chunk { // maxChunkSize: the max limit for the number of rows. func New(fields []*types.FieldType, cap, maxChunkSize int) *Chunk { chk := &Chunk{ - columns: make([]*column, 0, len(fields)), + columns: make([]*Column, 0, len(fields)), capacity: mathutil.Min(cap, maxChunkSize), // set the default value of requiredRows to maxChunkSize to let chk.IsFull() behave // like how we judge whether a chunk is full now, then the statement @@ -97,8 +97,8 @@ func Renew(chk *Chunk, maxChunkSize int) *Chunk { // renewColumns creates the columns of a Chunk. The capacity of the newly // created columns is equal to cap. -func renewColumns(oldCol []*column, cap int) []*column { - columns := make([]*column, 0, len(oldCol)) +func renewColumns(oldCol []*Column, cap int) []*Column { + columns := make([]*Column, 0, len(oldCol)) for _, col := range oldCol { if col.isFixed() { columns = append(columns, newFixedLenColumn(len(col.elemBuf), cap)) @@ -110,7 +110,7 @@ func renewColumns(oldCol []*column, cap int) []*column { } // MemoryUsage returns the total memory usage of a Chunk in B. -// We ignore the size of column.length and column.nullCount +// We ignore the size of Column.length and Column.nullCount // since they have little effect of the total memory usage. func (c *Chunk) MemoryUsage() (sum int64) { for _, col := range c.columns { @@ -120,17 +120,17 @@ func (c *Chunk) MemoryUsage() (sum int64) { return } -// newFixedLenColumn creates a fixed length column with elemLen and initial data capacity. -func newFixedLenColumn(elemLen, cap int) *column { - return &column{ +// newFixedLenColumn creates a fixed length Column with elemLen and initial data capacity. +func newFixedLenColumn(elemLen, cap int) *Column { + return &Column{ elemBuf: make([]byte, elemLen), data: make([]byte, 0, cap*elemLen), nullBitmap: make([]byte, 0, cap>>3), } } -// newVarLenColumn creates a variable length column with initial data capacity. -func newVarLenColumn(cap int, old *column) *column { +// newVarLenColumn creates a variable length Column with initial data capacity. +func newVarLenColumn(cap int, old *Column) *Column { estimatedElemLen := 8 // For varLenColumn (e.g. varchar), the accurate length of an element is unknown. // Therefore, in the first executor.Next we use an experience value -- 8 (so it may make runtime.growslice) @@ -138,7 +138,7 @@ func newVarLenColumn(cap int, old *column) *column { if old != nil && old.length != 0 { estimatedElemLen = (len(old.data) + len(old.data)/8) / old.length } - return &column{ + return &Column{ offsets: make([]int64, 1, cap+1), data: make([]byte, 0, cap*estimatedElemLen), nullBitmap: make([]byte, 0, cap>>3), @@ -164,7 +164,7 @@ func (c *Chunk) IsFull() bool { return c.NumRows() >= c.requiredRows } -// MakeRef makes column in "dstColIdx" reference to column in "srcColIdx". +// MakeRef makes Column in "dstColIdx" reference to Column in "srcColIdx". func (c *Chunk) MakeRef(srcColIdx, dstColIdx int) { c.columns[dstColIdx] = c.columns[srcColIdx] } @@ -174,11 +174,11 @@ func (c *Chunk) MakeRefTo(dstColIdx int, src *Chunk, srcColIdx int) { c.columns[dstColIdx] = src.columns[srcColIdx] } -// SwapColumn swaps column "c.columns[colIdx]" with column -// "other.columns[otherIdx]". If there exists columns refer to the column to be +// SwapColumn swaps Column "c.columns[colIdx]" with Column +// "other.columns[otherIdx]". If there exists columns refer to the Column to be // swapped, we need to re-build the reference. func (c *Chunk) SwapColumn(colIdx int, other *Chunk, otherIdx int) { - // Find the leftmost column of the reference which is the actual column to + // Find the leftmost Column of the reference which is the actual Column to // be swapped. for i := 0; i < colIdx; i++ { if c.columns[i] == c.columns[colIdx] { @@ -191,7 +191,7 @@ func (c *Chunk) SwapColumn(colIdx int, other *Chunk, otherIdx int) { } } - // Find the columns which refer to the actual column to be swapped. + // Find the columns which refer to the actual Column to be swapped. refColsIdx := make([]int, 0, len(c.columns)-colIdx) for i := colIdx; i < len(c.columns); i++ { if c.columns[i] == c.columns[colIdx] { @@ -224,7 +224,7 @@ func (c *Chunk) SwapColumns(other *Chunk) { } // SetNumVirtualRows sets the virtual row number for a Chunk. -// It should only be used when there exists no column in the Chunk. +// It should only be used when there exists no Column in the Chunk. func (c *Chunk) SetNumVirtualRows(numVirtualRows int) { c.numVirtualRows = numVirtualRows } @@ -236,14 +236,14 @@ func (c *Chunk) Reset() { return } for _, col := range c.columns { - col.reset() + col.Reset() } c.numVirtualRows = 0 } // CopyConstruct creates a new chunk and copies this chunk's data into it. func (c *Chunk) CopyConstruct() *Chunk { - newChk := &Chunk{numVirtualRows: c.numVirtualRows, capacity: c.capacity, columns: make([]*column, len(c.columns))} + newChk := &Chunk{numVirtualRows: c.numVirtualRows, capacity: c.capacity, columns: make([]*Column, len(c.columns))} for i := range c.columns { newChk.columns[i] = c.columns[i].copyConstruct() } @@ -461,17 +461,17 @@ func (c *Chunk) TruncateTo(numRows int) { // AppendNull appends a null value to the chunk. func (c *Chunk) AppendNull(colIdx int) { - c.columns[colIdx].appendNull() + c.columns[colIdx].AppendNull() } // AppendInt64 appends a int64 value to the chunk. func (c *Chunk) AppendInt64(colIdx int, i int64) { - c.columns[colIdx].appendInt64(i) + c.columns[colIdx].AppendInt64(i) } // AppendUint64 appends a uint64 value to the chunk. func (c *Chunk) AppendUint64(colIdx int, u uint64) { - c.columns[colIdx].appendUint64(u) + c.columns[colIdx].AppendUint64(u) } // AppendFloat32 appends a float32 value to the chunk. @@ -481,33 +481,33 @@ func (c *Chunk) AppendFloat32(colIdx int, f float32) { // AppendFloat64 appends a float64 value to the chunk. func (c *Chunk) AppendFloat64(colIdx int, f float64) { - c.columns[colIdx].appendFloat64(f) + c.columns[colIdx].AppendFloat64(f) } // AppendString appends a string value to the chunk. func (c *Chunk) AppendString(colIdx int, str string) { - c.columns[colIdx].appendString(str) + c.columns[colIdx].AppendString(str) } // AppendBytes appends a bytes value to the chunk. func (c *Chunk) AppendBytes(colIdx int, b []byte) { - c.columns[colIdx].appendBytes(b) + c.columns[colIdx].AppendBytes(b) } // AppendTime appends a Time value to the chunk. // TODO: change the time structure so it can be directly written to memory. func (c *Chunk) AppendTime(colIdx int, t types.Time) { - c.columns[colIdx].appendTime(t) + c.columns[colIdx].AppendTime(t) } // AppendDuration appends a Duration value to the chunk. func (c *Chunk) AppendDuration(colIdx int, dur types.Duration) { - c.columns[colIdx].appendDuration(dur) + c.columns[colIdx].AppendDuration(dur) } // AppendMyDecimal appends a MyDecimal value to the chunk. func (c *Chunk) AppendMyDecimal(colIdx int, dec *types.MyDecimal) { - c.columns[colIdx].appendMyDecimal(dec) + c.columns[colIdx].AppendMyDecimal(dec) } // AppendEnum appends an Enum value to the chunk. @@ -522,7 +522,7 @@ func (c *Chunk) AppendSet(colIdx int, set types.Set) { // AppendJSON appends a JSON value to the chunk. func (c *Chunk) AppendJSON(colIdx int, j json.BinaryJSON) { - c.columns[colIdx].appendJSON(j) + c.columns[colIdx].AppendJSON(j) } // AppendDatum appends a datum into the chunk. diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 704ad3fe93fdd..809530ac5e2ed 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -300,7 +300,7 @@ func (s *testChunkSuite) TestChunkSizeControl(c *check.C) { } // newChunk creates a new chunk and initialize columns with element length. -// 0 adds an varlen column, positive len add a fixed length column, negative len adds a interface column. +// 0 adds an varlen Column, positive len add a fixed length Column, negative len adds a interface Column. func newChunk(elemLen ...int) *Chunk { chk := &Chunk{} for _, l := range elemLen { diff --git a/util/chunk/chunk_util.go b/util/chunk/chunk_util.go index ea2da90585a7e..9bc6dddb73fda 100644 --- a/util/chunk/chunk_util.go +++ b/util/chunk/chunk_util.go @@ -34,7 +34,7 @@ func CopySelectedJoinRows(src *Chunk, innerColOffset, outerColOffset int, select // return the number of rows which is selected. func copySelectedInnerRows(innerColOffset, outerColOffset int, src *Chunk, selected []bool, dst *Chunk) int { oldLen := dst.columns[innerColOffset].length - var srcCols []*column + var srcCols []*Column if innerColOffset == 0 { srcCols = src.columns[:outerColOffset] } else { @@ -78,7 +78,7 @@ func copyOuterRows(innerColOffset, outerColOffset int, src *Chunk, numRows int, return } row := src.GetRow(0) - var srcCols []*column + var srcCols []*Column if innerColOffset == 0 { srcCols = src.columns[outerColOffset:] } else { diff --git a/util/chunk/codec.go b/util/chunk/codec.go index 4458dbe76e1bc..601052cdcd84f 100644 --- a/util/chunk/codec.go +++ b/util/chunk/codec.go @@ -27,7 +27,7 @@ import ( // 1. encode a Chunk to a byte slice. // 2. decode a Chunk from a byte slice. type Codec struct { - // colTypes is used to check whether a column is fixed sized and what the + // colTypes is used to check whether a Column is fixed sized and what the // fixed size for every element. // NOTE: It's only used for decoding. colTypes []*types.FieldType @@ -47,7 +47,7 @@ func (c *Codec) Encode(chk *Chunk) []byte { return buffer } -func (c *Codec) encodeColumn(buffer []byte, col *column) []byte { +func (c *Codec) encodeColumn(buffer []byte, col *Column) []byte { var lenBuffer [4]byte // encode length. binary.LittleEndian.PutUint32(lenBuffer[:], uint32(col.length)) @@ -90,7 +90,7 @@ func (c *Codec) i64SliceToBytes(i64s []int64) (b []byte) { func (c *Codec) Decode(buffer []byte) (*Chunk, []byte) { chk := &Chunk{} for ordinal := 0; len(buffer) > 0; ordinal++ { - col := &column{} + col := &Column{} buffer = c.decodeColumn(buffer, col, ordinal) chk.columns = append(chk.columns, col) } @@ -105,7 +105,7 @@ func (c *Codec) DecodeToChunk(buffer []byte, chk *Chunk) (remained []byte) { return buffer } -func (c *Codec) decodeColumn(buffer []byte, col *column, ordinal int) (remained []byte) { +func (c *Codec) decodeColumn(buffer []byte, col *Column, ordinal int) (remained []byte) { // decode length. col.length = int(binary.LittleEndian.Uint32(buffer)) buffer = buffer[4:] @@ -142,7 +142,7 @@ func (c *Codec) decodeColumn(buffer []byte, col *column, ordinal int) (remained var allNotNullBitmap [128]byte -func (c *Codec) setAllNotNull(col *column) { +func (c *Codec) setAllNotNull(col *Column) { numNullBitmapBytes := (col.length + 7) / 8 col.nullBitmap = col.nullBitmap[:0] for i := 0; i < numNullBitmapBytes; { @@ -163,7 +163,7 @@ func (c *Codec) bytesToI64Slice(b []byte) (i64s []int64) { return i64s } -// varElemLen indicates this column is a variable length column. +// varElemLen indicates this Column is a variable length Column. const varElemLen = -1 func getFixedLen(colType *types.FieldType) int { diff --git a/util/chunk/codec_test.go b/util/chunk/codec_test.go index 3ce2a48847df5..bf23a9849b834 100644 --- a/util/chunk/codec_test.go +++ b/util/chunk/codec_test.go @@ -81,9 +81,9 @@ func BenchmarkEncodeChunk(b *testing.B) { numCols := 4 numRows := 1024 - chk := &Chunk{columns: make([]*column, numCols)} + chk := &Chunk{columns: make([]*Column, numCols)} for i := 0; i < numCols; i++ { - chk.columns[i] = &column{ + chk.columns[i] = &Column{ length: numRows, nullCount: 14, nullBitmap: make([]byte, numRows/8+1), @@ -104,9 +104,9 @@ func BenchmarkDecode(b *testing.B) { numRows := 1024 colTypes := make([]*types.FieldType, numCols) - chk := &Chunk{columns: make([]*column, numCols)} + chk := &Chunk{columns: make([]*Column, numCols)} for i := 0; i < numCols; i++ { - chk.columns[i] = &column{ + chk.columns[i] = &Column{ length: numRows, nullCount: 14, nullBitmap: make([]byte, numRows/8+1), @@ -131,10 +131,10 @@ func BenchmarkDecodeToChunk(b *testing.B) { colTypes := make([]*types.FieldType, numCols) chk := &Chunk{ - columns: make([]*column, numCols), + columns: make([]*Column, numCols), } for i := 0; i < numCols; i++ { - chk.columns[i] = &column{ + chk.columns[i] = &Column{ length: numRows, nullCount: 14, nullBitmap: make([]byte, numRows/8+1), diff --git a/util/chunk/column.go b/util/chunk/column.go index 39cac8de0fb63..d1ff51f803200 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -20,16 +20,18 @@ import ( "github.com/pingcap/tidb/types/json" ) -func (c *column) appendDuration(dur types.Duration) { - c.appendInt64(int64(dur.Duration)) +// AppendDuration appends a duration value into this Column. +func (c *Column) AppendDuration(dur types.Duration) { + c.AppendInt64(int64(dur.Duration)) } -func (c *column) appendMyDecimal(dec *types.MyDecimal) { +// AppendMyDecimal appends a MyDecimal value into this Column. +func (c *Column) AppendMyDecimal(dec *types.MyDecimal) { *(*types.MyDecimal)(unsafe.Pointer(&c.elemBuf[0])) = *dec c.finishAppendFixed() } -func (c *column) appendNameValue(name string, val uint64) { +func (c *Column) appendNameValue(name string, val uint64) { var buf [8]byte *(*uint64)(unsafe.Pointer(&buf[0])) = val c.data = append(c.data, buf[:]...) @@ -37,13 +39,16 @@ func (c *column) appendNameValue(name string, val uint64) { c.finishAppendVar() } -func (c *column) appendJSON(j json.BinaryJSON) { +// AppendJSON appends a BinaryJSON value into this Column. +func (c *Column) AppendJSON(j json.BinaryJSON) { c.data = append(c.data, j.TypeCode) c.data = append(c.data, j.Value...) c.finishAppendVar() } -type column struct { +// Column stores one column of data in Apache Arrow format. +// See https://arrow.apache.org/docs/memory_layout.html +type Column struct { length int nullCount int nullBitmap []byte @@ -52,11 +57,12 @@ type column struct { elemBuf []byte } -func (c *column) isFixed() bool { +func (c *Column) isFixed() bool { return c.elemBuf != nil } -func (c *column) reset() { +// Reset resets this Column. +func (c *Column) Reset() { c.length = 0 c.nullCount = 0 c.nullBitmap = c.nullBitmap[:0] @@ -67,13 +73,13 @@ func (c *column) reset() { c.data = c.data[:0] } -func (c *column) isNull(rowIdx int) bool { +func (c *Column) isNull(rowIdx int) bool { nullByte := c.nullBitmap[rowIdx/8] return nullByte&(1<<(uint(rowIdx)&7)) == 0 } -func (c *column) copyConstruct() *column { - newCol := &column{length: c.length, nullCount: c.nullCount} +func (c *Column) copyConstruct() *Column { + newCol := &Column{length: c.length, nullCount: c.nullCount} newCol.nullBitmap = append(newCol.nullBitmap, c.nullBitmap...) newCol.offsets = append(newCol.offsets, c.offsets...) newCol.data = append(newCol.data, c.data...) @@ -81,7 +87,7 @@ func (c *column) copyConstruct() *column { return newCol } -func (c *column) appendNullBitmap(notNull bool) { +func (c *Column) appendNullBitmap(notNull bool) { idx := c.length >> 3 if idx >= len(c.nullBitmap) { c.nullBitmap = append(c.nullBitmap, 0) @@ -97,7 +103,7 @@ func (c *column) appendNullBitmap(notNull bool) { // appendMultiSameNullBitmap appends multiple same bit value to `nullBitMap`. // notNull means not null. // num means the number of bits that should be appended. -func (c *column) appendMultiSameNullBitmap(notNull bool, num int) { +func (c *Column) appendMultiSameNullBitmap(notNull bool, num int) { numNewBytes := ((c.length + num + 7) >> 3) - len(c.nullBitmap) b := byte(0) if notNull { @@ -120,7 +126,8 @@ func (c *column) appendMultiSameNullBitmap(notNull bool, num int) { c.nullBitmap[len(c.nullBitmap)-1] &= bitMask } -func (c *column) appendNull() { +// AppendNull appends a null value into this Column. +func (c *Column) AppendNull() { c.appendNullBitmap(false) if c.isFixed() { c.data = append(c.data, c.elemBuf...) @@ -130,49 +137,56 @@ func (c *column) appendNull() { c.length++ } -func (c *column) finishAppendFixed() { +func (c *Column) finishAppendFixed() { c.data = append(c.data, c.elemBuf...) c.appendNullBitmap(true) c.length++ } -func (c *column) appendInt64(i int64) { +// AppendInt64 appends an int64 value into this Column. +func (c *Column) AppendInt64(i int64) { *(*int64)(unsafe.Pointer(&c.elemBuf[0])) = i c.finishAppendFixed() } -func (c *column) appendUint64(u uint64) { +// AppendUint64 appends a uint64 value into this Column. +func (c *Column) AppendUint64(u uint64) { *(*uint64)(unsafe.Pointer(&c.elemBuf[0])) = u c.finishAppendFixed() } -func (c *column) appendFloat32(f float32) { +// appendFloat32 appends a float32 value into this Column. +func (c *Column) appendFloat32(f float32) { *(*float32)(unsafe.Pointer(&c.elemBuf[0])) = f c.finishAppendFixed() } -func (c *column) appendFloat64(f float64) { +// AppendFloat64 appends a float64 value into this Column. +func (c *Column) AppendFloat64(f float64) { *(*float64)(unsafe.Pointer(&c.elemBuf[0])) = f c.finishAppendFixed() } -func (c *column) finishAppendVar() { +func (c *Column) finishAppendVar() { c.appendNullBitmap(true) c.offsets = append(c.offsets, int64(len(c.data))) c.length++ } -func (c *column) appendString(str string) { +// AppendString appends a string value into this Column. +func (c *Column) AppendString(str string) { c.data = append(c.data, str...) c.finishAppendVar() } -func (c *column) appendBytes(b []byte) { +// AppendBytes appends a byte slice into this Column. +func (c *Column) AppendBytes(b []byte) { c.data = append(c.data, b...) c.finishAppendVar() } -func (c *column) appendTime(t types.Time) { +// AppendTime appends a time value into this Column. +func (c *Column) AppendTime(t types.Time) { writeTime(c.elemBuf, t) c.finishAppendFixed() } diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 4b209569fd4eb..ff577fea30ac4 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -17,7 +17,7 @@ import ( "github.com/pingcap/check" ) -func equalColumn(c1, c2 *column) bool { +func equalColumn(c1, c2 *Column) bool { if c1.length != c2.length || c1.nullCount != c2.nullCount { return false @@ -54,7 +54,7 @@ func equalColumn(c1, c2 *column) bool { func (s *testChunkSuite) TestColumnCopy(c *check.C) { col := newFixedLenColumn(8, 10) for i := 0; i < 10; i++ { - col.appendInt64(int64(i)) + col.AppendInt64(int64(i)) } c1 := col.copyConstruct() diff --git a/util/chunk/compare.go b/util/chunk/compare.go index a8adbfc4174a9..2ecaf196ee0c9 100644 --- a/util/chunk/compare.go +++ b/util/chunk/compare.go @@ -209,7 +209,7 @@ func Compare(row Row, colIdx int, ad *types.Datum) int { } } -// LowerBound searches on the non-decreasing column colIdx, +// LowerBound searches on the non-decreasing Column colIdx, // returns the smallest index i such that the value at row i is not less than `d`. func (c *Chunk) LowerBound(colIdx int, d *types.Datum) (index int, match bool) { index = sort.Search(c.NumRows(), func(i int) bool { @@ -222,7 +222,7 @@ func (c *Chunk) LowerBound(colIdx int, d *types.Datum) (index int, match bool) { return } -// UpperBound searches on the non-decreasing column colIdx, +// UpperBound searches on the non-decreasing Column colIdx, // returns the smallest index i such that the value at row i is larger than `d`. func (c *Chunk) UpperBound(colIdx int, d *types.Datum) int { return sort.Search(c.NumRows(), func(i int) bool { diff --git a/util/chunk/mutrow.go b/util/chunk/mutrow.go index 15b5f29999db5..aa36326559930 100644 --- a/util/chunk/mutrow.go +++ b/util/chunk/mutrow.go @@ -40,7 +40,7 @@ func (mr MutRow) Len() int { // MutRowFromValues creates a MutRow from a interface slice. func MutRowFromValues(vals ...interface{}) MutRow { - c := &Chunk{columns: make([]*column, 0, len(vals))} + c := &Chunk{columns: make([]*Column, 0, len(vals))} for _, val := range vals { col := makeMutRowColumn(val) c.columns = append(c.columns, col) @@ -50,7 +50,7 @@ func MutRowFromValues(vals ...interface{}) MutRow { // MutRowFromDatums creates a MutRow from a datum slice. func MutRowFromDatums(datums []types.Datum) MutRow { - c := &Chunk{columns: make([]*column, 0, len(datums))} + c := &Chunk{columns: make([]*Column, 0, len(datums))} for _, d := range datums { col := makeMutRowColumn(d.GetValue()) c.columns = append(c.columns, col) @@ -58,9 +58,9 @@ func MutRowFromDatums(datums []types.Datum) MutRow { return MutRow{c: c, idx: 0} } -// MutRowFromTypes creates a MutRow from a FieldType slice, each column is initialized to zero value. +// MutRowFromTypes creates a MutRow from a FieldType slice, each Column is initialized to zero value. func MutRowFromTypes(types []*types.FieldType) MutRow { - c := &Chunk{columns: make([]*column, 0, len(types))} + c := &Chunk{columns: make([]*Column, 0, len(types))} for _, tp := range types { col := makeMutRowColumn(zeroValForType(tp)) c.columns = append(c.columns, col) @@ -106,7 +106,7 @@ func zeroValForType(tp *types.FieldType) interface{} { } } -func makeMutRowColumn(in interface{}) *column { +func makeMutRowColumn(in interface{}) *Column { switch x := in.(type) { case nil: col := makeMutRowUint64Column(uint64(0)) @@ -162,9 +162,9 @@ func makeMutRowColumn(in interface{}) *column { } } -func newMutRowFixedLenColumn(elemSize int) *column { +func newMutRowFixedLenColumn(elemSize int) *Column { buf := make([]byte, elemSize+1) - col := &column{ + col := &Column{ length: 1, elemBuf: buf[:elemSize], data: buf[:elemSize], @@ -174,9 +174,9 @@ func newMutRowFixedLenColumn(elemSize int) *column { return col } -func newMutRowVarLenColumn(valSize int) *column { +func newMutRowVarLenColumn(valSize int) *Column { buf := make([]byte, valSize+1) - col := &column{ + col := &Column{ length: 1, offsets: []int64{0, int64(valSize)}, data: buf[:valSize], @@ -186,13 +186,13 @@ func newMutRowVarLenColumn(valSize int) *column { return col } -func makeMutRowUint64Column(val uint64) *column { +func makeMutRowUint64Column(val uint64) *Column { col := newMutRowFixedLenColumn(8) *(*uint64)(unsafe.Pointer(&col.data[0])) = val return col } -func makeMutRowBytesColumn(bin []byte) *column { +func makeMutRowBytesColumn(bin []byte) *Column { col := newMutRowVarLenColumn(len(bin)) copy(col.data, bin) col.nullBitmap[0] = 1 @@ -305,7 +305,7 @@ func (mr MutRow) SetDatum(colIdx int, d types.Datum) { col.nullBitmap[0] = 1 } -func setMutRowBytes(col *column, bin []byte) { +func setMutRowBytes(col *Column, bin []byte) { if len(col.data) >= len(bin) { col.data = col.data[:len(bin)] } else { @@ -317,7 +317,7 @@ func setMutRowBytes(col *column, bin []byte) { col.offsets[1] = int64(len(bin)) } -func setMutRowNameValue(col *column, name string, val uint64) { +func setMutRowNameValue(col *Column, name string, val uint64) { dataLen := len(name) + 8 if len(col.data) >= dataLen { col.data = col.data[:dataLen] @@ -331,12 +331,12 @@ func setMutRowNameValue(col *column, name string, val uint64) { col.offsets[1] = int64(dataLen) } -func setMutRowJSON(col *column, j json.BinaryJSON) { +func setMutRowJSON(col *Column, j json.BinaryJSON) { dataLen := len(j.Value) + 1 if len(col.data) >= dataLen { col.data = col.data[:dataLen] } else { - // In MutRow, there always exists 1 data in every column, + // In MutRow, there always exists 1 data in every Column, // we should allocate one more byte for null bitmap. buf := make([]byte, dataLen+1) col.data = buf[:dataLen] diff --git a/util/chunk/pool.go b/util/chunk/pool.go index 69a95735c2f3a..a378fbe4ccea2 100644 --- a/util/chunk/pool.go +++ b/util/chunk/pool.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/tidb/types" ) -// Pool is the column pool. +// Pool is the Column pool. // NOTE: Pool is non-copyable. type Pool struct { initCap int @@ -47,19 +47,19 @@ func NewPool(initCap int) *Pool { func (p *Pool) GetChunk(fields []*types.FieldType) *Chunk { chk := new(Chunk) chk.capacity = p.initCap - chk.columns = make([]*column, len(fields)) + chk.columns = make([]*Column, len(fields)) for i, f := range fields { switch elemLen := getFixedLen(f); elemLen { case varElemLen: - chk.columns[i] = p.varLenColPool.Get().(*column) + chk.columns[i] = p.varLenColPool.Get().(*Column) case 4: - chk.columns[i] = p.fixLenColPool4.Get().(*column) + chk.columns[i] = p.fixLenColPool4.Get().(*Column) case 8: - chk.columns[i] = p.fixLenColPool8.Get().(*column) + chk.columns[i] = p.fixLenColPool8.Get().(*Column) case 16: - chk.columns[i] = p.fixLenColPool16.Get().(*column) + chk.columns[i] = p.fixLenColPool16.Get().(*Column) case 40: - chk.columns[i] = p.fixLenColPool40.Get().(*column) + chk.columns[i] = p.fixLenColPool40.Get().(*Column) } } return chk @@ -81,5 +81,5 @@ func (p *Pool) PutChunk(fields []*types.FieldType, chk *Chunk) { p.fixLenColPool40.Put(chk.columns[i]) } } - chk.columns = nil // release the column references. + chk.columns = nil // release the Column references. } From c00d8a95cfa1d3519d830c71fd40aa62da4ded5f Mon Sep 17 00:00:00 2001 From: Lingyu Song Date: Mon, 22 Jul 2019 13:08:03 +0800 Subject: [PATCH 057/196] executor, privilege: fix some two bug of RBAC (#11273) --- executor/simple.go | 19 +++++++++++++++++-- executor/simple_test.go | 15 +++++++++++++++ expression/builtin_info.go | 11 +++++++++-- privilege/privileges/cache.go | 10 ++++++++-- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/executor/simple.go b/executor/simple.go index 81181033a7136..d67d270d7cb79 100644 --- a/executor/simple.go +++ b/executor/simple.go @@ -471,13 +471,20 @@ func (e *SimpleExec) executeRevokeRole(s *ast.RevokeRoleStmt) error { } return ErrCannotUser.GenWithStackByArgs("REVOKE ROLE", role.String()) } + sql = fmt.Sprintf(`DELETE IGNORE FROM %s.%s WHERE DEFAULT_ROLE_HOST='%s' and DEFAULT_ROLE_USER='%s' and HOST='%s' and USER='%s'`, mysql.SystemDB, mysql.DefaultRoleTable, role.Hostname, role.Username, user.Hostname, user.Username) + if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), sql); err != nil { + if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "rollback"); err != nil { + return errors.Trace(err) + } + return ErrCannotUser.GenWithStackByArgs("REVOKE ROLE", role.String()) + } } } if _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.Background(), "commit"); err != nil { return err } - err := domain.GetDomain(e.ctx).PrivilegeHandle().Update(e.ctx.(sessionctx.Context)) - return errors.Trace(err) + domain.GetDomain(e.ctx).NotifyUpdatePrivilege(e.ctx) + return nil } func (e *SimpleExec) executeCommit(s *ast.CommitStmt) { @@ -597,6 +604,14 @@ func (e *SimpleExec) executeAlterUser(s *ast.AlterUserStmt) error { func (e *SimpleExec) executeGrantRole(s *ast.GrantRoleStmt) error { failedUsers := make([]string, 0, len(s.Users)) + sessionVars := e.ctx.GetSessionVars() + for i, user := range s.Users { + if user.CurrentUser { + s.Users[i].Username = sessionVars.User.AuthUsername + s.Users[i].Hostname = sessionVars.User.AuthHostname + } + } + for _, role := range s.Roles { exists, err := userExists(e.ctx, role.Username, role.Hostname) if err != nil { diff --git a/executor/simple_test.go b/executor/simple_test.go index 0de9f78d714a0..d9233db0e8806 100644 --- a/executor/simple_test.go +++ b/executor/simple_test.go @@ -136,6 +136,15 @@ func (s *testSuite3) TestRole(c *C) { grantRoleSQL = `GRANT 'r_1'@'localhost' TO 'r_3'@'localhost', 'r_4'@'localhost';` _, err = tk.Exec(grantRoleSQL) c.Check(err, NotNil) + + // Test grant role for current_user(); + sessionVars := tk.Se.GetSessionVars() + originUser := sessionVars.User + sessionVars.User = &auth.UserIdentity{Username: "root", Hostname: "localhost", AuthUsername: "root", AuthHostname: "%"} + tk.MustExec("grant 'r_1'@'localhost' to current_user();") + tk.MustExec("revoke 'r_1'@'localhost' from 'root'@'%';") + sessionVars.User = originUser + result = tk.MustQuery(`SELECT FROM_USER FROM mysql.role_edges WHERE TO_USER="r_3" and TO_HOST="localhost"`) result.Check(nil) @@ -152,14 +161,20 @@ func (s *testSuite3) TestRole(c *C) { tk.MustExec("insert into mysql.role_edges (FROM_HOST,FROM_USER,TO_HOST,TO_USER) values ('localhost','test','%','root')") tk.MustExec("insert into mysql.role_edges (FROM_HOST,FROM_USER,TO_HOST,TO_USER) values ('%','r_1','%','root')") tk.MustExec("insert into mysql.role_edges (FROM_HOST,FROM_USER,TO_HOST,TO_USER) values ('%','r_2','%','root')") + tk.MustExec("flush privileges") + tk.MustExec("SET DEFAULT ROLE r_1, r_2 TO root") _, err = tk.Exec("revoke test@localhost, r_1 from root;") c.Check(err, IsNil) _, err = tk.Exec("revoke `r_2`@`%` from root, u_2;") c.Check(err, NotNil) _, err = tk.Exec("revoke `r_2`@`%` from root;") c.Check(err, IsNil) + _, err = tk.Exec("revoke `r_1`@`%` from root;") + c.Check(err, IsNil) result = tk.MustQuery(`SELECT * FROM mysql.default_roles WHERE DEFAULT_ROLE_USER="test" and DEFAULT_ROLE_HOST="localhost"`) result.Check(nil) + result = tk.MustQuery(`SELECT * FROM mysql.default_roles WHERE USER="root" and HOST="%"`) + result.Check(nil) dropRoleSQL = `DROP ROLE 'test'@'localhost', r_1, r_2;` tk.MustExec(dropRoleSQL) diff --git a/expression/builtin_info.go b/expression/builtin_info.go index 0653afef87847..2e6ab91c6b5c5 100644 --- a/expression/builtin_info.go +++ b/expression/builtin_info.go @@ -18,6 +18,8 @@ package expression import ( + "sort" + "github.com/pingcap/errors" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx" @@ -192,8 +194,13 @@ func (b *builtinCurrentRoleSig) evalString(row chunk.Row) (string, bool, error) return "", false, nil } res := "" - for i, r := range data.ActiveRoles { - res += r.String() + sortedRes := make([]string, 0, 10) + for _, r := range data.ActiveRoles { + sortedRes = append(sortedRes, r.String()) + } + sort.Strings(sortedRes) + for i, r := range sortedRes { + res += r if i != len(data.ActiveRoles)-1 { res += "," } diff --git a/privilege/privileges/cache.go b/privilege/privileges/cache.go index d574a8759dfbc..699bfe616dacc 100644 --- a/privilege/privileges/cache.go +++ b/privilege/privileges/cache.go @@ -848,13 +848,19 @@ func (p *MySQLPrivilege) showGrants(user, host string, roles []*auth.RoleIdentit edgeTable, ok := p.RoleGraph[graphKey] g = "" if ok { + sortedRes := make([]string, 0, 10) for k := range edgeTable.roleList { role := strings.Split(k, "@") roleName, roleHost := role[0], role[1] - if g != "" { + tmp := fmt.Sprintf("'%s'@'%s'", roleName, roleHost) + sortedRes = append(sortedRes, tmp) + } + sort.Strings(sortedRes) + for i, r := range sortedRes { + g += r + if i != len(sortedRes)-1 { g += ", " } - g += fmt.Sprintf("'%s'@'%s'", roleName, roleHost) } s := fmt.Sprintf(`GRANT %s TO '%s'@'%s'`, g, user, host) gs = append(gs, s) From 5f7f43a5718c414f1658af2e761bf05238d41668 Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Mon, 22 Jul 2019 13:27:46 +0800 Subject: [PATCH 058/196] planner: fix bugs related to TIDB_INLJ hint (#11253) * fix logic of extractTableName and tryToGetIndexJoin * add unit test * fix test case * add comment * rewrite comments * rewrite comment * rewrite comments & clear code --- planner/core/exhaust_physical_plans.go | 10 ++++-- planner/core/logical_plan_builder.go | 9 +++++ planner/core/physical_plan_test.go | 49 ++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 8f856003f80c3..5ff411d6e5e7f 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1029,15 +1029,19 @@ func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJ } if leftJoins != nil && lhsCardinality < rhsCardinality { - return leftJoins, hasIndexJoinHint + return leftJoins, leftOuter } if rightJoins != nil && rhsCardinality < lhsCardinality { - return rightJoins, hasIndexJoinHint + return rightJoins, rightOuter } + canForceLeft := leftJoins != nil && leftOuter + canForceRight := rightJoins != nil && rightOuter + forced = canForceLeft || canForceRight + joins := append(leftJoins, rightJoins...) - return joins, hasIndexJoinHint && len(joins) != 0 + return joins, forced } return nil, false diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index d053c88bff08e..7285b51f9a22d 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -310,8 +310,17 @@ func (p *LogicalJoin) extractOnCondition(conditions []expression.Expression, der return } +// extractTableAlias returns table alias of the LogicalPlan's columns. +// It will return nil when there are multiple table alias, because the alias is only used to check if +// the logicalPlan match some optimizer hints, and hints are not expected to take effect in this case. func extractTableAlias(p LogicalPlan) *model.CIStr { if p.Schema().Len() > 0 && p.Schema().Columns[0].TblName.L != "" { + tblName := p.Schema().Columns[0].TblName.L + for _, column := range p.Schema().Columns { + if column.TblName.L != tblName { + return nil + } + } return &(p.Schema().Columns[0].TblName) } return nil diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 12354f17d9eba..459aa9cc5d113 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1526,3 +1526,52 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { } } } + +func (s *testPlanSuite) TestIndexJoinHint(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + warning string + }{ + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t2.a, t3.a from t t1, t t2, t t3 where t1.a = t2.a and t2.a = t3.a;", + best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t1.a)}(test.t3.a,test.t2.a)->Projection", + warning: "", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.b, t2.a from t t1, t t2 where t1.b = t2.a;", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.b,test.t2.a)", + warning: "[planner:1815]Optimizer Hint /*+ TIDB_INLJ(t1) */ is inapplicable", + }, + } + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + if test.warning == "" { + c.Assert(len(warnings), Equals, 0) + } else { + c.Assert(len(warnings), Equals, 1) + c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning) + c.Assert(warnings[0].Err.Error(), Equals, test.warning) + } + } +} From f03a0212699ea3a457508781ffd91834d4dd1e21 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 22 Jul 2019 17:10:27 +0800 Subject: [PATCH 059/196] store/tikv: reduce the lock contend between sending message and re-creating streaming client (#11301) --- store/tikv/client.go | 15 ++-- store/tikv/client_batch.go | 164 +++++++++++++++++++++++-------------- store/tikv/client_test.go | 4 +- 3 files changed, 113 insertions(+), 70 deletions(-) diff --git a/store/tikv/client.go b/store/tikv/client.go index 3278a2ebb9fd8..4f3049236ee4f 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -152,16 +152,15 @@ func (a *connArray) Init(addr string, security config.Security, idleNotify *uint return errors.Trace(err) } batchClient := &batchCommandsClient{ - target: a.target, - conn: conn, - client: streamClient, - batched: sync.Map{}, - idAlloc: 0, - tikvTransportLayerLoad: &a.tikvTransportLayerLoad, - closed: 0, + target: a.target, + conn: conn, + client: streamClient, + batched: sync.Map{}, + idAlloc: 0, + closed: 0, } a.batchCommandsClients = append(a.batchCommandsClients, batchClient) - go batchClient.batchRecvLoop(cfg.TiKVClient) + go batchClient.batchRecvLoop(cfg.TiKVClient, &a.tikvTransportLayerLoad) } } go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout) diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index 467fab1827d41..bb4da4a8b71e3 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -152,20 +152,50 @@ func fetchMorePendingRequests( } } +type tryLock struct { + sync.RWMutex + reCreating bool +} + +func (l *tryLock) tryLockForSend() bool { + l.RLock() + if l.reCreating { + l.RUnlock() + return false + } + return true +} + +func (l *tryLock) unlockForSend() { + l.RUnlock() +} + +func (l *tryLock) lockForRecreate() { + l.Lock() + l.reCreating = true + l.Unlock() + +} + +func (l *tryLock) unlockForRecreate() { + l.Lock() + l.reCreating = false + l.Unlock() +} + type batchCommandsClient struct { // The target host. target string - conn *grpc.ClientConn - client tikvpb.Tikv_BatchCommandsClient - batched sync.Map - idAlloc uint64 - tikvTransportLayerLoad *uint64 + conn *grpc.ClientConn + client tikvpb.Tikv_BatchCommandsClient + batched sync.Map + idAlloc uint64 // closed indicates the batch client is closed explicitly or not. closed int32 - // clientLock protects client when re-create the streaming. - clientLock sync.Mutex + // tryLock protects client when re-create the streaming. + tryLock } func (c *batchCommandsClient) isStopped() bool { @@ -173,10 +203,6 @@ func (c *batchCommandsClient) isStopped() bool { } func (c *batchCommandsClient) send(request *tikvpb.BatchCommandsRequest, entries []*batchCommandsEntry) { - // Use the lock to protect the stream client won't be replaced by RecvLoop, - // and new added request won't be removed by `failPendingRequests`. - c.clientLock.Lock() - defer c.clientLock.Unlock() for i, requestID := range request.RequestIds { c.batched.Store(requestID, entries[i]) } @@ -211,10 +237,7 @@ func (c *batchCommandsClient) failPendingRequests(err error) { }) } -func (c *batchCommandsClient) reCreateStreamingClient(err error) error { - // Hold the lock to forbid batchSendLoop using the old client. - c.clientLock.Lock() - defer c.clientLock.Unlock() +func (c *batchCommandsClient) reCreateStreamingClientOnce(err error) error { c.failPendingRequests(err) // fail all pending requests. // Re-establish a application layer stream. TCP layer is handled by gRPC. @@ -226,6 +249,7 @@ func (c *batchCommandsClient) reCreateStreamingClient(err error) error { zap.String("target", c.target), ) c.client = streamClient + return nil } logutil.BgLogger().Error( @@ -236,7 +260,7 @@ func (c *batchCommandsClient) reCreateStreamingClient(err error) error { return err } -func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { +func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient, tikvTransportLayerLoad *uint64) { defer func() { if r := recover(); r != nil { metrics.PanicCounter.WithLabelValues(metrics.LabelBatchRecvLoop).Inc() @@ -244,7 +268,7 @@ func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { zap.Reflect("r", r), zap.Stack("stack")) logutil.BgLogger().Info("restart batchRecvLoop") - go c.batchRecvLoop(cfg) + go c.batchRecvLoop(cfg, tikvTransportLayerLoad) } }() @@ -257,21 +281,9 @@ func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { zap.Error(err), ) - b := NewBackoffer(context.Background(), math.MaxInt32) now := time.Now() - for { // try to re-create the streaming in the loop. - if c.isStopped() { - return - } - - err1 := c.reCreateStreamingClient(err) - if err1 == nil { - break - } - err2 := b.Backoff(boTiKVRPC, err1) - // As timeout is set to math.MaxUint32, err2 should always be nil. - // This line is added to make the 'make errcheck' pass. - terror.Log(err2) + if stopped := c.reCreateStreamingClient(err); stopped { + return } metrics.TiKVBatchClientUnavailable.Observe(time.Since(now).Seconds()) continue @@ -294,14 +306,37 @@ func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient) { c.batched.Delete(requestID) } - tikvTransportLayerLoad := resp.GetTransportLayerLoad() - if tikvTransportLayerLoad > 0.0 && cfg.MaxBatchWaitTime > 0 { + transportLayerLoad := resp.GetTransportLayerLoad() + if transportLayerLoad > 0.0 && cfg.MaxBatchWaitTime > 0 { // We need to consider TiKV load only if batch-wait strategy is enabled. - atomic.StoreUint64(c.tikvTransportLayerLoad, tikvTransportLayerLoad) + atomic.StoreUint64(tikvTransportLayerLoad, transportLayerLoad) } } } +func (c *batchCommandsClient) reCreateStreamingClient(err error) (stopped bool) { + // Forbids the batchSendLoop using the old client. + c.lockForRecreate() + defer c.unlockForRecreate() + + b := NewBackoffer(context.Background(), math.MaxInt32) + for { // try to re-create the streaming in the loop. + if c.isStopped() { + return true + } + err1 := c.reCreateStreamingClientOnce(err) + if err1 == nil { + break + } + + err2 := b.Backoff(boTiKVRPC, err1) + // As timeout is set to math.MaxUint32, err2 should always be nil. + // This line is added to make the 'make errcheck' pass. + terror.Log(err2) + } + return false +} + type batchCommandsEntry struct { req *tikvpb.BatchCommandsRequest_Request res chan *tikvpb.BatchCommandsResponse_Response @@ -335,10 +370,6 @@ func (a *batchConn) batchSendLoop(cfg config.TiKVClient) { var bestBatchWaitSize = cfg.BatchWaitSize for { - // Choose a connection by round-robbin. - next := atomic.AddUint32(&a.index, 1) % uint32(len(a.batchCommandsClients)) - batchCommandsClient := a.batchCommandsClients[next] - entries = entries[:0] requests = requests[:0] requestIDs = requestIDs[:0] @@ -347,9 +378,8 @@ func (a *batchConn) batchSendLoop(cfg config.TiKVClient) { a.fetchAllPendingRequests(int(cfg.MaxBatchSize), &entries, &requests) if len(entries) < int(cfg.MaxBatchSize) && cfg.MaxBatchWaitTime > 0 { - tikvTransportLayerLoad := atomic.LoadUint64(batchCommandsClient.tikvTransportLayerLoad) // If the target TiKV is overload, wait a while to collect more requests. - if uint(tikvTransportLayerLoad) >= cfg.OverloadThreshold { + if atomic.LoadUint64(&a.tikvTransportLayerLoad) >= uint64(cfg.OverloadThreshold) { fetchMorePendingRequests( a.batchCommandsCh, int(cfg.MaxBatchSize), int(bestBatchWaitSize), cfg.MaxBatchWaitTime, &entries, &requests, @@ -367,23 +397,40 @@ func (a *batchConn) batchSendLoop(cfg config.TiKVClient) { bestBatchWaitSize += 1 } - length = removeCanceledRequests(&entries, &requests) - if length == 0 { + entries, requests = removeCanceledRequests(entries, requests) + if len(entries) == 0 { continue // All requests are canceled. } - maxBatchID := atomic.AddUint64(&batchCommandsClient.idAlloc, uint64(length)) - for i := 0; i < length; i++ { - requestID := uint64(i) + maxBatchID - uint64(length) - requestIDs = append(requestIDs, requestID) - } - req := &tikvpb.BatchCommandsRequest{ - Requests: requests, - RequestIds: requestIDs, + a.getClientAndSend(entries, requests, requestIDs) + } +} + +func (a *batchConn) getClientAndSend(entries []*batchCommandsEntry, requests []*tikvpb.BatchCommandsRequest_Request, requestIDs []uint64) { + // Choose a connection by round-robbin. + var cli *batchCommandsClient + for { + a.index = (a.index + 1) % uint32(len(a.batchCommandsClients)) + cli = a.batchCommandsClients[a.index] + // The lock protects the batchCommandsClient from been closed while it's inuse. + if cli.tryLockForSend() { + break } + } + defer cli.unlockForSend() - batchCommandsClient.send(req, entries) + maxBatchID := atomic.AddUint64(&cli.idAlloc, uint64(len(requests))) + for i := 0; i < len(requests); i++ { + requestID := uint64(i) + maxBatchID - uint64(len(requests)) + requestIDs = append(requestIDs, requestID) } + req := &tikvpb.BatchCommandsRequest{ + Requests: requests, + RequestIds: requestIDs, + } + + cli.send(req, entries) + return } func (a *batchConn) Close() { @@ -399,20 +446,17 @@ func (a *batchConn) Close() { } // removeCanceledRequests removes canceled requests before sending. -func removeCanceledRequests( - entries *[]*batchCommandsEntry, - requests *[]*tikvpb.BatchCommandsRequest_Request) int { - validEntries := (*entries)[:0] - validRequets := (*requests)[:0] - for _, e := range *entries { +func removeCanceledRequests(entries []*batchCommandsEntry, + requests []*tikvpb.BatchCommandsRequest_Request) ([]*batchCommandsEntry, []*tikvpb.BatchCommandsRequest_Request) { + validEntries := entries[:0] + validRequets := requests[:0] + for _, e := range entries { if !e.isCanceled() { validEntries = append(validEntries, e) validRequets = append(validRequets, e.req) } } - *entries = validEntries - *requests = validRequets - return len(*entries) + return validEntries, validRequets } func sendBatchRequest( diff --git a/store/tikv/client_test.go b/store/tikv/client_test.go index 5f685534e40ba..4871a78afa557 100644 --- a/store/tikv/client_test.go +++ b/store/tikv/client_test.go @@ -76,8 +76,8 @@ func (s *testClientSuite) TestRemoveCanceledRequests(c *C) { for i := range entries { requests[i] = entries[i].req } - length := removeCanceledRequests(&entries, &requests) - c.Assert(length, Equals, 2) + entries, requests = removeCanceledRequests(entries, requests) + c.Assert(len(entries), Equals, 2) for _, e := range entries { c.Assert(e.isCanceled(), IsFalse) } From 268be86807040a8767565e0885124d403f062277 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 22 Jul 2019 19:06:58 +0800 Subject: [PATCH 060/196] executor: add vectorized data access methods to `Column` (#11368) --- util/chunk/chunk.go | 22 ++-- util/chunk/chunk_util.go | 6 +- util/chunk/column.go | 125 ++++++++++++++++++- util/chunk/column_test.go | 254 ++++++++++++++++++++++++++++++++++++++ util/chunk/mutrow.go | 4 +- util/chunk/row.go | 2 +- 6 files changed, 393 insertions(+), 20 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 66e88d94fddea..60bc217ac74e5 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -66,12 +66,7 @@ func New(fields []*types.FieldType, cap, maxChunkSize int) *Chunk { } for _, f := range fields { - elemLen := getFixedLen(f) - if elemLen == varElemLen { - chk.columns = append(chk.columns, newVarLenColumn(chk.capacity, nil)) - } else { - chk.columns = append(chk.columns, newFixedLenColumn(elemLen, chk.capacity)) - } + chk.columns = append(chk.columns, NewColumn(f, chk.capacity)) } return chk @@ -310,7 +305,7 @@ func (c *Chunk) AppendRow(row Row) { func (c *Chunk) AppendPartialRow(colIdx int, row Row) { for i, rowCol := range row.c.columns { chkCol := c.columns[colIdx+i] - chkCol.appendNullBitmap(!rowCol.isNull(row.idx)) + chkCol.appendNullBitmap(!rowCol.IsNull(row.idx)) if rowCol.isFixed() { elemLen := len(rowCol.elemBuf) offset := row.idx * elemLen @@ -338,7 +333,7 @@ func (c *Chunk) PreAlloc(row Row) (rowIdx uint32) { rowIdx = uint32(c.NumRows()) for i, srcCol := range row.c.columns { dstCol := c.columns[i] - dstCol.appendNullBitmap(!srcCol.isNull(row.idx)) + dstCol.appendNullBitmap(!srcCol.IsNull(row.idx)) elemLen := len(srcCol.elemBuf) if !srcCol.isFixed() { elemLen = int(srcCol.offsets[row.idx+1] - srcCol.offsets[row.idx]) @@ -421,7 +416,7 @@ func (c *Chunk) Append(other *Chunk, begin, end int) { } } for i := begin; i < end; i++ { - dst.appendNullBitmap(!src.isNull(i)) + dst.appendNullBitmap(!src.IsNull(i)) dst.length++ } } @@ -439,7 +434,7 @@ func (c *Chunk) TruncateTo(numRows int) { col.offsets = col.offsets[:numRows+1] } for i := numRows; i < col.length; i++ { - if col.isNull(i) { + if col.IsNull(i) { col.nullCount-- } } @@ -476,7 +471,7 @@ func (c *Chunk) AppendUint64(colIdx int, u uint64) { // AppendFloat32 appends a float32 value to the chunk. func (c *Chunk) AppendFloat32(colIdx int, f float32) { - c.columns[colIdx].appendFloat32(f) + c.columns[colIdx].AppendFloat32(f) } // AppendFloat64 appends a float64 value to the chunk. @@ -555,6 +550,11 @@ func (c *Chunk) AppendDatum(colIdx int, d *types.Datum) { } } +// Column returns the specific column. +func (c *Chunk) Column(colIdx int) *Column { + return c.columns[colIdx] +} + func writeTime(buf []byte, t types.Time) { binary.BigEndian.PutUint16(buf, uint16(t.Time.Year())) buf[2] = uint8(t.Time.Month()) diff --git a/util/chunk/chunk_util.go b/util/chunk/chunk_util.go index 9bc6dddb73fda..be15dafe44a87 100644 --- a/util/chunk/chunk_util.go +++ b/util/chunk/chunk_util.go @@ -47,7 +47,7 @@ func copySelectedInnerRows(innerColOffset, outerColOffset int, src *Chunk, selec if !selected[i] { continue } - dstCol.appendNullBitmap(!srcCol.isNull(i)) + dstCol.appendNullBitmap(!srcCol.IsNull(i)) dstCol.length++ elemLen := len(srcCol.elemBuf) @@ -59,7 +59,7 @@ func copySelectedInnerRows(innerColOffset, outerColOffset int, src *Chunk, selec if !selected[i] { continue } - dstCol.appendNullBitmap(!srcCol.isNull(i)) + dstCol.appendNullBitmap(!srcCol.IsNull(i)) dstCol.length++ start, end := srcCol.offsets[i], srcCol.offsets[i+1] @@ -86,7 +86,7 @@ func copyOuterRows(innerColOffset, outerColOffset int, src *Chunk, numRows int, } for i, srcCol := range srcCols { dstCol := dst.columns[outerColOffset+i] - dstCol.appendMultiSameNullBitmap(!srcCol.isNull(row.idx), numRows) + dstCol.appendMultiSameNullBitmap(!srcCol.IsNull(row.idx), numRows) dstCol.length += numRows if srcCol.isFixed() { elemLen := len(srcCol.elemBuf) diff --git a/util/chunk/column.go b/util/chunk/column.go index d1ff51f803200..d06b1f2e92669 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -14,10 +14,13 @@ package chunk import ( + "reflect" + "time" "unsafe" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/hack" ) // AppendDuration appends a duration value into this Column. @@ -46,6 +49,11 @@ func (c *Column) AppendJSON(j json.BinaryJSON) { c.finishAppendVar() } +// AppendSet appends a Set value into this Column. +func (c *Column) AppendSet(set types.Set) { + c.appendNameValue(set.Name, set.Value) +} + // Column stores one column of data in Apache Arrow format. // See https://arrow.apache.org/docs/memory_layout.html type Column struct { @@ -57,6 +65,15 @@ type Column struct { elemBuf []byte } +// NewColumn creates a new column with the specific length and capacity. +func NewColumn(ft *types.FieldType, cap int) *Column { + typeSize := getFixedLen(ft) + if typeSize == varElemLen { + return newVarLenColumn(cap, nil) + } + return newFixedLenColumn(typeSize, cap) +} + func (c *Column) isFixed() bool { return c.elemBuf != nil } @@ -73,7 +90,8 @@ func (c *Column) Reset() { c.data = c.data[:0] } -func (c *Column) isNull(rowIdx int) bool { +// IsNull returns if this row is null. +func (c *Column) IsNull(rowIdx int) bool { nullByte := c.nullBitmap[rowIdx/8] return nullByte&(1<<(uint(rowIdx)&7)) == 0 } @@ -155,8 +173,8 @@ func (c *Column) AppendUint64(u uint64) { c.finishAppendFixed() } -// appendFloat32 appends a float32 value into this Column. -func (c *Column) appendFloat32(f float32) { +// AppendFloat32 appends a float32 value into this Column. +func (c *Column) AppendFloat32(f float32) { *(*float32)(unsafe.Pointer(&c.elemBuf[0])) = f c.finishAppendFixed() } @@ -190,3 +208,104 @@ func (c *Column) AppendTime(t types.Time) { writeTime(c.elemBuf, t) c.finishAppendFixed() } + +// AppendEnum appends a Enum value into this Column. +func (c *Column) AppendEnum(enum types.Enum) { + c.appendNameValue(enum.Name, enum.Value) +} + +const ( + sizeInt64 = int(unsafe.Sizeof(int64(0))) + sizeUint64 = int(unsafe.Sizeof(uint64(0))) + sizeFloat32 = int(unsafe.Sizeof(float32(0))) + sizeFloat64 = int(unsafe.Sizeof(float64(0))) + sizeMyDecimal = int(unsafe.Sizeof(types.MyDecimal{})) +) + +func (c *Column) castSliceHeader(header *reflect.SliceHeader, typeSize int) { + header.Data = uintptr(unsafe.Pointer(&c.data[0])) + header.Len = c.length + header.Cap = cap(c.data) / typeSize +} + +// Int64s returns an int64 slice stored in this Column. +func (c *Column) Int64s() []int64 { + var res []int64 + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeInt64) + return res +} + +// Uint64s returns a uint64 slice stored in this Column. +func (c *Column) Uint64s() []uint64 { + var res []uint64 + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeUint64) + return res +} + +// Float32s returns a float32 slice stored in this Column. +func (c *Column) Float32s() []float32 { + var res []float32 + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeFloat32) + return res +} + +// Float64s returns a float64 slice stored in this Column. +func (c *Column) Float64s() []float64 { + var res []float64 + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeFloat64) + return res +} + +// Decimals returns a MyDecimal slice stored in this Column. +func (c *Column) Decimals() []types.MyDecimal { + var res []types.MyDecimal + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeMyDecimal) + return res +} + +// GetString returns the string in the specific row. +func (c *Column) GetString(rowID int) string { + return string(hack.String(c.data[c.offsets[rowID]:c.offsets[rowID+1]])) +} + +// GetJSON returns the JSON in the specific row. +func (c *Column) GetJSON(rowID int) json.BinaryJSON { + start := c.offsets[rowID] + return json.BinaryJSON{TypeCode: c.data[start], Value: c.data[start+1 : c.offsets[rowID+1]]} +} + +// GetBytes returns the byte slice in the specific row. +func (c *Column) GetBytes(rowID int) []byte { + return c.data[c.offsets[rowID]:c.offsets[rowID+1]] +} + +// GetEnum returns the Enum in the specific row. +func (c *Column) GetEnum(rowID int) types.Enum { + name, val := c.getNameValue(rowID) + return types.Enum{Name: name, Value: val} +} + +// GetSet returns the Set in the specific row. +func (c *Column) GetSet(rowID int) types.Set { + name, val := c.getNameValue(rowID) + return types.Set{Name: name, Value: val} +} + +// GetTime returns the Time in the specific row. +func (c *Column) GetTime(rowID int) types.Time { + return readTime(c.data[rowID*16:]) +} + +// GetDuration returns the Duration in the specific row. +func (c *Column) GetDuration(rowID int, fillFsp int) types.Duration { + dur := *(*int64)(unsafe.Pointer(&c.data[rowID*8])) + return types.Duration{Duration: time.Duration(dur), Fsp: fillFsp} +} + +func (c *Column) getNameValue(rowID int) (string, uint64) { + start, end := c.offsets[rowID], c.offsets[rowID+1] + if start == end { + return "", 0 + } + return string(hack.String(c.data[start+8 : end])), *(*uint64)(unsafe.Pointer(&c.data[start])) +} diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index ff577fea30ac4..210e270155b84 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -14,7 +14,13 @@ package chunk import ( + "fmt" + "time" + "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/types/json" ) func equalColumn(c1, c2 *Column) bool { @@ -67,3 +73,251 @@ func (s *testChunkSuite) TestLargeStringColumnOffset(c *check.C) { col.offsets[0] = 6 << 30 c.Check(col.offsets[0], check.Equals, int64(6<<30)) // test no overflow. } + +func (s *testChunkSuite) TestI64Column(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendInt64(int64(i)) + } + + i64s := col.Int64s() + for i := 0; i < 1024; i++ { + c.Assert(i64s[i], check.Equals, int64(i)) + i64s[i]++ + } + + it := NewIterator4Chunk(chk) + var i int64 + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetInt64(0), check.Equals, int64(i+1)) + i++ + } +} + +func (s *testChunkSuite) TestF64Column(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDouble)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendFloat64(float64(i)) + } + + f64s := col.Float64s() + for i := 0; i < 1024; i++ { + c.Assert(f64s[i], check.Equals, float64(i)) + f64s[i] /= 2 + } + + it := NewIterator4Chunk(chk) + var i int64 + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetFloat64(0), check.Equals, float64(i)/2) + i++ + } +} + +func (s *testChunkSuite) TestF32Column(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeFloat)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendFloat32(float32(i)) + } + + f32s := col.Float32s() + for i := 0; i < 1024; i++ { + c.Assert(f32s[i], check.Equals, float32(i)) + f32s[i] /= 2 + } + + it := NewIterator4Chunk(chk) + var i int64 + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetFloat32(0), check.Equals, float32(i)/2) + i++ + } +} + +func (s *testChunkSuite) TestMyDecimal(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeNewDecimal)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + d := new(types.MyDecimal) + if err := d.FromFloat64(float64(i) * 1.1); err != nil { + c.Fatal(err) + } + col.AppendMyDecimal(d) + } + + ds := col.Decimals() + for i := 0; i < 1024; i++ { + d := new(types.MyDecimal) + if err := d.FromFloat64(float64(i) * 1.1); err != nil { + c.Fatal(err) + } + c.Assert(d.Compare(&ds[i]), check.Equals, 0) + + if err := types.DecimalAdd(&ds[i], d, &ds[i]); err != nil { + c.Fatal(err) + } + } + + it := NewIterator4Chunk(chk) + var i int64 + for row := it.Begin(); row != it.End(); row = it.Next() { + d := new(types.MyDecimal) + if err := d.FromFloat64(float64(i) * 1.1 * 2); err != nil { + c.Fatal(err) + } + + delta := new(types.MyDecimal) + if err := types.DecimalSub(d, row.GetMyDecimal(0), delta); err != nil { + c.Fatal(err) + } + + fDelta, err := delta.ToFloat64() + if err != nil { + c.Fatal(err) + } + if fDelta > 0.0001 || fDelta < -0.0001 { + c.Fatal() + } + + i++ + } +} + +func (s *testChunkSuite) TestStringColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeVarString)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendString(fmt.Sprintf("%v", i*i)) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetString(0), check.Equals, fmt.Sprintf("%v", i*i)) + c.Assert(col.GetString(i), check.Equals, fmt.Sprintf("%v", i*i)) + i++ + } +} + +func (s *testChunkSuite) TestSetColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeSet)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendSet(types.Set{Name: fmt.Sprintf("%v", i), Value: uint64(i)}) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + s1 := col.GetSet(i) + s2 := row.GetSet(0) + c.Assert(s1.Name, check.Equals, s2.Name) + c.Assert(s1.Value, check.Equals, s2.Value) + c.Assert(s1.Name, check.Equals, fmt.Sprintf("%v", i)) + c.Assert(s1.Value, check.Equals, uint64(i)) + i++ + } +} + +func (s *testChunkSuite) TestJSONColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeJSON)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + j := new(json.BinaryJSON) + if err := j.UnmarshalJSON([]byte(fmt.Sprintf(`{"%v":%v}`, i, i))); err != nil { + c.Fatal(err) + } + col.AppendJSON(*j) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + j1 := col.GetJSON(i) + j2 := row.GetJSON(0) + c.Assert(j1.String(), check.Equals, j2.String()) + i++ + } +} + +func (s *testChunkSuite) TestTimeColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDatetime)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendTime(types.CurrentTime(mysql.TypeDatetime)) + time.Sleep(time.Millisecond / 10) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + j1 := col.GetTime(i) + j2 := row.GetTime(0) + c.Assert(j1.Compare(j2), check.Equals, 0) + i++ + } +} + +func (s *testChunkSuite) TestDurationColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDuration)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendDuration(types.Duration{Duration: time.Second * time.Duration(i)}) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + j1 := col.GetDuration(i, 0) + j2 := row.GetDuration(0, 0) + c.Assert(j1.Compare(j2), check.Equals, 0) + i++ + } +} + +func (s *testChunkSuite) TestEnumColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeEnum)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendEnum(types.Enum{Name: fmt.Sprintf("%v", i), Value: uint64(i)}) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + s1 := col.GetEnum(i) + s2 := row.GetEnum(0) + c.Assert(s1.Name, check.Equals, s2.Name) + c.Assert(s1.Value, check.Equals, s2.Value) + c.Assert(s1.Name, check.Equals, fmt.Sprintf("%v", i)) + c.Assert(s1.Value, check.Equals, uint64(i)) + i++ + } +} + +func (s *testChunkSuite) TestNullsColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + if i%2 == 0 { + col.AppendNull() + continue + } + col.AppendInt64(int64(i)) + } + + it := NewIterator4Chunk(chk) + var i int + for row := it.Begin(); row != it.End(); row = it.Next() { + if i%2 == 0 { + c.Assert(row.IsNull(0), check.Equals, true) + c.Assert(col.IsNull(i), check.Equals, true) + } else { + c.Assert(row.GetInt64(0), check.Equals, int64(i)) + } + i++ + } +} diff --git a/util/chunk/mutrow.go b/util/chunk/mutrow.go index aa36326559930..b0bddc18f3029 100644 --- a/util/chunk/mutrow.go +++ b/util/chunk/mutrow.go @@ -203,7 +203,7 @@ func makeMutRowBytesColumn(bin []byte) *Column { func (mr MutRow) SetRow(row Row) { for colIdx, rCol := range row.c.columns { mrCol := mr.c.columns[colIdx] - if rCol.isNull(row.idx) { + if rCol.IsNull(row.idx) { mrCol.nullBitmap[0] = 0 continue } @@ -351,7 +351,7 @@ func setMutRowJSON(col *Column, j json.BinaryJSON) { func (mr MutRow) ShallowCopyPartialRow(colIdx int, row Row) { for i, srcCol := range row.c.columns { dstCol := mr.c.columns[colIdx+i] - if !srcCol.isNull(row.idx) { + if !srcCol.IsNull(row.idx) { // MutRow only contains one row, so we can directly set the whole byte. dstCol.nullBitmap[0] = 1 } else { diff --git a/util/chunk/row.go b/util/chunk/row.go index 0d282558fc0e1..b4c22d6b02210 100644 --- a/util/chunk/row.go +++ b/util/chunk/row.go @@ -220,5 +220,5 @@ func (r Row) GetDatum(colIdx int, tp *types.FieldType) types.Datum { // IsNull returns if the datum in the chunk.Row is null. func (r Row) IsNull(colIdx int) bool { - return r.c.columns[colIdx].isNull(r.idx) + return r.c.columns[colIdx].IsNull(r.idx) } From b3f71868c159ba33a8b88956a828d06d43a5eb6e Mon Sep 17 00:00:00 2001 From: LanceLi Date: Mon, 22 Jul 2019 19:19:54 +0800 Subject: [PATCH 061/196] =?UTF-8?q?executor:=20fix=20autoid=20doesn't=20ha?= =?UTF-8?q?ndle=20float,=20double=20type=20and=20tin=E2=80=A6=20(#11110)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/insert_common.go | 30 ++++- executor/insert_test.go | 246 ++++++++++++++++++++++++++++++++++++++ executor/update_test.go | 125 +++++++++++++++++++ executor/write.go | 6 +- 4 files changed, 400 insertions(+), 7 deletions(-) diff --git a/executor/insert_common.go b/executor/insert_common.go index 861630ea03c11..f2ef0ed89a68c 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -15,6 +15,7 @@ package executor import ( "context" + "math" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" @@ -488,12 +489,10 @@ func (e *InsertValues) adjustAutoIncrementDatum(ctx context.Context, d types.Dat d.SetNull() } if !d.IsNull() { - sc := e.ctx.GetSessionVars().StmtCtx - datum, err1 := d.ConvertTo(sc, &c.FieldType) - if e.filterErr(err1) != nil { - return types.Datum{}, err1 + recordID, err = getAutoRecordID(d, &c.FieldType, true) + if err != nil { + return types.Datum{}, err } - recordID = datum.GetInt64() } // Use the value if it's not null and not 0. if recordID != 0 { @@ -503,7 +502,6 @@ func (e *InsertValues) adjustAutoIncrementDatum(ctx context.Context, d types.Dat } e.ctx.GetSessionVars().StmtCtx.InsertID = uint64(recordID) retryInfo.AddAutoIncrementID(recordID) - d.SetAutoID(recordID, c.Flag) return d, nil } @@ -531,6 +529,26 @@ func (e *InsertValues) adjustAutoIncrementDatum(ctx context.Context, d types.Dat return casted, nil } +func getAutoRecordID(d types.Datum, target *types.FieldType, isInsert bool) (int64, error) { + var recordID int64 + + switch target.Tp { + case mysql.TypeFloat, mysql.TypeDouble: + f := d.GetFloat64() + if isInsert { + recordID = int64(math.Round(f)) + } else { + recordID = int64(f) + } + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong: + recordID = d.GetInt64() + default: + return 0, errors.Errorf("unexpected field type [%v]", target.Tp) + } + + return recordID, nil +} + func (e *InsertValues) handleWarning(err error) { sc := e.ctx.GetSessionVars().StmtCtx sc.AppendWarning(err) diff --git a/executor/insert_test.go b/executor/insert_test.go index e26834572f12b..c5d63e86d919c 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -287,6 +287,251 @@ func (s *testSuite3) TestAllowInvalidDates(c *C) { runWithMode("ALLOW_INVALID_DATES") } +func (s *testSuite3) TestInsertWithAutoidSchema(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec(`create table t1(id int primary key auto_increment, n int);`) + tk.MustExec(`create table t2(id int unsigned primary key auto_increment, n int);`) + tk.MustExec(`create table t3(id tinyint primary key auto_increment, n int);`) + tk.MustExec(`create table t4(id int primary key, n float auto_increment, key I_n(n));`) + tk.MustExec(`create table t5(id int primary key, n float unsigned auto_increment, key I_n(n));`) + tk.MustExec(`create table t6(id int primary key, n double auto_increment, key I_n(n));`) + tk.MustExec(`create table t7(id int primary key, n double unsigned auto_increment, key I_n(n));`) + + tests := []struct { + insert string + query string + result [][]interface{} + }{ + { + `insert into t1(id, n) values(1, 1)`, + `select * from t1 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t1(n) values(2)`, + `select * from t1 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t1(n) values(3)`, + `select * from t1 where id = 3`, + testkit.Rows(`3 3`), + }, + { + `insert into t1(id, n) values(-1, 4)`, + `select * from t1 where id = -1`, + testkit.Rows(`-1 4`), + }, + { + `insert into t1(n) values(5)`, + `select * from t1 where id = 4`, + testkit.Rows(`4 5`), + }, + { + `insert into t1(id, n) values('5', 6)`, + `select * from t1 where id = 5`, + testkit.Rows(`5 6`), + }, + { + `insert into t1(n) values(7)`, + `select * from t1 where id = 6`, + testkit.Rows(`6 7`), + }, + { + `insert into t1(id, n) values(7.4, 8)`, + `select * from t1 where id = 7`, + testkit.Rows(`7 8`), + }, + { + `insert into t1(id, n) values(7.5, 9)`, + `select * from t1 where id = 8`, + testkit.Rows(`8 9`), + }, + { + `insert into t1(n) values(9)`, + `select * from t1 where id = 9`, + testkit.Rows(`9 9`), + }, + { + `insert into t2(id, n) values(1, 1)`, + `select * from t2 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t2(n) values(2)`, + `select * from t2 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t2(n) values(3)`, + `select * from t2 where id = 3`, + testkit.Rows(`3 3`), + }, + { + `insert into t3(id, n) values(1, 1)`, + `select * from t3 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t3(n) values(2)`, + `select * from t3 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t3(n) values(3)`, + `select * from t3 where id = 3`, + testkit.Rows(`3 3`), + }, + { + `insert into t3(id, n) values(-1, 4)`, + `select * from t3 where id = -1`, + testkit.Rows(`-1 4`), + }, + { + `insert into t3(n) values(5)`, + `select * from t3 where id = 4`, + testkit.Rows(`4 5`), + }, + { + `insert into t4(id, n) values(1, 1)`, + `select * from t4 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t4(id) values(2)`, + `select * from t4 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t4(id, n) values(3, -1)`, + `select * from t4 where id = 3`, + testkit.Rows(`3 -1`), + }, + { + `insert into t4(id) values(4)`, + `select * from t4 where id = 4`, + testkit.Rows(`4 3`), + }, + { + `insert into t4(id, n) values(5, 5.5)`, + `select * from t4 where id = 5`, + testkit.Rows(`5 5.5`), + }, + { + `insert into t4(id) values(6)`, + `select * from t4 where id = 6`, + testkit.Rows(`6 7`), + }, + { + `insert into t4(id, n) values(7, '7.7')`, + `select * from t4 where id = 7`, + testkit.Rows(`7 7.7`), + }, + { + `insert into t4(id) values(8)`, + `select * from t4 where id = 8`, + testkit.Rows(`8 9`), + }, + { + `insert into t4(id, n) values(9, 10.4)`, + `select * from t4 where id = 9`, + testkit.Rows(`9 10.4`), + }, + { + `insert into t4(id) values(10)`, + `select * from t4 where id = 10`, + testkit.Rows(`10 11`), + }, + { + `insert into t5(id, n) values(1, 1)`, + `select * from t5 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t5(id) values(2)`, + `select * from t5 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t5(id) values(3)`, + `select * from t5 where id = 3`, + testkit.Rows(`3 3`), + }, + { + `insert into t6(id, n) values(1, 1)`, + `select * from t6 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t6(id) values(2)`, + `select * from t6 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t6(id, n) values(3, -1)`, + `select * from t6 where id = 3`, + testkit.Rows(`3 -1`), + }, + { + `insert into t6(id) values(4)`, + `select * from t6 where id = 4`, + testkit.Rows(`4 3`), + }, + { + `insert into t6(id, n) values(5, 5.5)`, + `select * from t6 where id = 5`, + testkit.Rows(`5 5.5`), + }, + { + `insert into t6(id) values(6)`, + `select * from t6 where id = 6`, + testkit.Rows(`6 7`), + }, + { + `insert into t6(id, n) values(7, '7.7')`, + `select * from t4 where id = 7`, + testkit.Rows(`7 7.7`), + }, + { + `insert into t6(id) values(8)`, + `select * from t4 where id = 8`, + testkit.Rows(`8 9`), + }, + { + `insert into t6(id, n) values(9, 10.4)`, + `select * from t6 where id = 9`, + testkit.Rows(`9 10.4`), + }, + { + `insert into t6(id) values(10)`, + `select * from t6 where id = 10`, + testkit.Rows(`10 11`), + }, + { + `insert into t7(id, n) values(1, 1)`, + `select * from t7 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `insert into t7(id) values(2)`, + `select * from t7 where id = 2`, + testkit.Rows(`2 2`), + }, + { + `insert into t7(id) values(3)`, + `select * from t7 where id = 3`, + testkit.Rows(`3 3`), + }, + } + + for _, tt := range tests { + tk.MustExec(tt.insert) + tk.MustQuery(tt.query).Check(tt.result) + } + +} + func (s *testSuite3) TestPartitionInsertOnDuplicate(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec(`use test`) @@ -299,4 +544,5 @@ func (s *testSuite3) TestPartitionInsertOnDuplicate(c *C) { tk.MustExec(`insert into t2 set a=1,b=1;`) tk.MustExec(`insert into t2 set a=1,b=1 on duplicate key update a=1,b=1`) tk.MustQuery(`select * from t2`).Check(testkit.Rows("1 1")) + } diff --git a/executor/update_test.go b/executor/update_test.go index e91a95dbba96c..410ba65fc4246 100644 --- a/executor/update_test.go +++ b/executor/update_test.go @@ -87,3 +87,128 @@ func (s *testUpdateSuite) TestUpdateGenColInTxn(c *C) { tk.MustQuery(`select * from t;`).Check(testkit.Rows( `1 2`)) } + +func (s *testUpdateSuite) TestUpdateWithAutoidSchema(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test`) + tk.MustExec(`create table t1(id int primary key auto_increment, n int);`) + tk.MustExec(`create table t2(id int primary key, n float auto_increment, key I_n(n));`) + tk.MustExec(`create table t3(id int primary key, n double auto_increment, key I_n(n));`) + + tests := []struct { + exec string + query string + result [][]interface{} + }{ + { + `insert into t1 set n = 1`, + `select * from t1 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `update t1 set id = id+1`, + `select * from t1 where id = 2`, + testkit.Rows(`2 1`), + }, + { + `insert into t1 set n = 2`, + `select * from t1 where id = 3`, + testkit.Rows(`3 2`), + }, + { + `update t1 set id = id + '1.1' where id = 3`, + `select * from t1 where id = 4`, + testkit.Rows(`4 2`), + }, + { + `insert into t1 set n = 3`, + `select * from t1 where id = 5`, + testkit.Rows(`5 3`), + }, + { + `update t1 set id = id + '0.5' where id = 5`, + `select * from t1 where id = 6`, + testkit.Rows(`6 3`), + }, + { + `insert into t1 set n = 4`, + `select * from t1 where id = 7`, + testkit.Rows(`7 4`), + }, + { + `insert into t2 set id = 1`, + `select * from t2 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `update t2 set n = n+1`, + `select * from t2 where id = 1`, + testkit.Rows(`1 2`), + }, + { + `insert into t2 set id = 2`, + `select * from t2 where id = 2`, + testkit.Rows(`2 3`), + }, + { + `update t2 set n = n + '2.2'`, + `select * from t2 where id = 2`, + testkit.Rows(`2 5.2`), + }, + { + `insert into t2 set id = 3`, + `select * from t2 where id = 3`, + testkit.Rows(`3 6`), + }, + { + `update t2 set n = n + '0.5' where id = 3`, + `select * from t2 where id = 3`, + testkit.Rows(`3 6.5`), + }, + { + `insert into t2 set id = 4`, + `select * from t2 where id = 4`, + testkit.Rows(`4 7`), + }, + { + `insert into t3 set id = 1`, + `select * from t3 where id = 1`, + testkit.Rows(`1 1`), + }, + { + `update t3 set n = n+1`, + `select * from t3 where id = 1`, + testkit.Rows(`1 2`), + }, + { + `insert into t3 set id = 2`, + `select * from t3 where id = 2`, + testkit.Rows(`2 3`), + }, + { + `update t3 set n = n + '3.3'`, + `select * from t3 where id = 2`, + testkit.Rows(`2 6.3`), + }, + { + `insert into t3 set id = 3`, + `select * from t3 where id = 3`, + testkit.Rows(`3 7`), + }, + { + `update t3 set n = n + '0.5' where id = 3`, + `select * from t3 where id = 3`, + testkit.Rows(`3 7.5`), + }, + { + `insert into t3 set id = 4`, + `select * from t3 where id = 4`, + testkit.Rows(`4 8`), + }, + } + + for _, tt := range tests { + tk.MustExec(tt.exec) + tk.MustQuery(tt.query).Check(tt.result) + } +} diff --git a/executor/write.go b/executor/write.go index c5304d321a967..62574621b19de 100644 --- a/executor/write.go +++ b/executor/write.go @@ -87,7 +87,11 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu modified[i] = true // Rebase auto increment id if the field is changed. if mysql.HasAutoIncrementFlag(col.Flag) { - if err = t.RebaseAutoID(ctx, newData[i].GetInt64(), true); err != nil { + recordID, err := getAutoRecordID(newData[i], &col.FieldType, false) + if err != nil { + return false, false, 0, err + } + if err = t.RebaseAutoID(ctx, recordID, true); err != nil { return false, false, 0, err } } From 3a1ba358a4b8d58bb5b190fbbe596e0d3e709ee7 Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Tue, 23 Jul 2019 13:13:05 +0800 Subject: [PATCH 062/196] =?UTF-8?q?planner:=20fix=20bugs=20and=20make=20it?= =?UTF-8?q?=20more=20effective=20in=20outer=20join=20eli=E2=80=A6=20(#1116?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/explaintest/r/explain_complex.result | 59 +++++++++ cmd/explaintest/t/explain_complex.test | 43 +++++++ planner/core/logical_plan_test.go | 16 ++- planner/core/plan.go | 8 ++ planner/core/rule_join_elimination.go | 153 +++++++++++++---------- 5 files changed, 215 insertions(+), 64 deletions(-) diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result index e659323792973..44425e7a554f7 100644 --- a/cmd/explaintest/r/explain_complex.result +++ b/cmd/explaintest/r/explain_complex.result @@ -200,3 +200,62 @@ HashAgg_34 72000.00 root group by:col_1, funcs:sum(col_0) │ └─TableScan_58 10000.00 cop table:tbl_008, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_62 10000.00 root data:TableScan_61 └─TableScan_61 10000.00 cop table:tbl_009, range:[-inf,+inf], keep order:false, stats:pseudo +CREATE TABLE org_department ( +id int(11) NOT NULL AUTO_INCREMENT, +ctx int(11) DEFAULT '0' COMMENT 'organization id', +name varchar(128) DEFAULT NULL, +left_value int(11) DEFAULT NULL, +right_value int(11) DEFAULT NULL, +depth int(11) DEFAULT NULL, +leader_id bigint(20) DEFAULT NULL, +status int(11) DEFAULT '1000', +created_on datetime DEFAULT NULL, +updated_on datetime DEFAULT NULL, +PRIMARY KEY (id), +UNIQUE KEY org_department_id_uindex (id), +KEY org_department_leader_id_index (leader_id), +KEY org_department_ctx_index (ctx) +); +CREATE TABLE org_position ( +id int(11) NOT NULL AUTO_INCREMENT, +ctx int(11) DEFAULT NULL, +name varchar(128) DEFAULT NULL, +left_value int(11) DEFAULT NULL, +right_value int(11) DEFAULT NULL, +depth int(11) DEFAULT NULL, +department_id int(11) DEFAULT NULL, +status int(2) DEFAULT NULL, +created_on datetime DEFAULT NULL, +updated_on datetime DEFAULT NULL, +PRIMARY KEY (id), +UNIQUE KEY org_position_id_uindex (id), +KEY org_position_department_id_index (department_id) +) ENGINE=InnoDB AUTO_INCREMENT=22 DEFAULT CHARSET=utf8; +CREATE TABLE org_employee_position ( +hotel_id int(11) DEFAULT NULL, +user_id bigint(20) DEFAULT NULL, +position_id int(11) DEFAULT NULL, +status int(11) DEFAULT NULL, +created_on datetime DEFAULT NULL, +updated_on datetime DEFAULT NULL, +UNIQUE KEY org_employee_position_pk (hotel_id,user_id,position_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +explain SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.leader_id, d.status, d.created_on, d.updated_on FROM org_department AS d LEFT JOIN org_position AS p ON p.department_id = d.id AND p.status = 1000 LEFT JOIN org_employee_position AS ep ON ep.position_id = p.id AND ep.status = 1000 WHERE (d.ctx = 1 AND (ep.user_id = 62 OR d.id = 20 OR d.id = 20) AND d.status = 1000) GROUP BY d.id ORDER BY d.left_value; +id count task operator info +Sort_10 1.00 root test.d.left_value:asc +└─HashAgg_15 1.00 root group by:test.d.id, funcs:firstrow(test.d.id), firstrow(test.d.ctx), firstrow(test.d.name), firstrow(test.d.left_value), firstrow(test.d.right_value), firstrow(test.d.depth), firstrow(test.d.leader_id), firstrow(test.d.status), firstrow(test.d.created_on), firstrow(test.d.updated_on) + └─Selection_22 0.01 root or(eq(test.ep.user_id, 62), or(eq(test.d.id, 20), eq(test.d.id, 20))) + └─HashLeftJoin_23 0.02 root left outer join, inner:TableReader_58, equal:[eq(test.p.id, test.ep.position_id)] + ├─IndexJoin_32 0.01 root left outer join, inner:IndexLookUp_31, outer key:test.d.id, inner key:test.p.department_id + │ ├─IndexLookUp_48 0.01 root + │ │ ├─IndexScan_45 10.00 cop table:d, index:ctx, range:[1,1], keep order:false, stats:pseudo + │ │ └─Selection_47 0.01 cop eq(test.d.status, 1000) + │ │ └─TableScan_46 10.00 cop table:org_department, keep order:false, stats:pseudo + │ └─IndexLookUp_31 0.01 root + │ ├─Selection_29 9.99 cop not(isnull(test.p.department_id)) + │ │ └─IndexScan_27 10.00 cop table:p, index:department_id, range: decided by [eq(test.p.department_id, test.d.id)], keep order:false, stats:pseudo + │ └─Selection_30 0.01 cop eq(test.p.status, 1000) + │ └─TableScan_28 9.99 cop table:org_position, keep order:false, stats:pseudo + └─TableReader_58 9.99 root data:Selection_57 + └─Selection_57 9.99 cop eq(test.ep.status, 1000), not(isnull(test.ep.position_id)) + └─TableScan_56 10000.00 cop table:ep, range:[-inf,+inf], keep order:false, stats:pseudo diff --git a/cmd/explaintest/t/explain_complex.test b/cmd/explaintest/t/explain_complex.test index e39412ba4a114..aae669cc9274f 100644 --- a/cmd/explaintest/t/explain_complex.test +++ b/cmd/explaintest/t/explain_complex.test @@ -131,3 +131,46 @@ CREATE TABLE `tbl_008` (`a` int, `b` int); CREATE TABLE `tbl_009` (`a` int, `b` int); explain select sum(a) from (select * from tbl_001 union all select * from tbl_002 union all select * from tbl_003 union all select * from tbl_004 union all select * from tbl_005 union all select * from tbl_006 union all select * from tbl_007 union all select * from tbl_008 union all select * from tbl_009) x group by b; + +CREATE TABLE org_department ( + id int(11) NOT NULL AUTO_INCREMENT, + ctx int(11) DEFAULT '0' COMMENT 'organization id', + name varchar(128) DEFAULT NULL, + left_value int(11) DEFAULT NULL, + right_value int(11) DEFAULT NULL, + depth int(11) DEFAULT NULL, + leader_id bigint(20) DEFAULT NULL, + status int(11) DEFAULT '1000', + created_on datetime DEFAULT NULL, + updated_on datetime DEFAULT NULL, + PRIMARY KEY (id), + UNIQUE KEY org_department_id_uindex (id), + KEY org_department_leader_id_index (leader_id), + KEY org_department_ctx_index (ctx) +); +CREATE TABLE org_position ( + id int(11) NOT NULL AUTO_INCREMENT, + ctx int(11) DEFAULT NULL, + name varchar(128) DEFAULT NULL, + left_value int(11) DEFAULT NULL, + right_value int(11) DEFAULT NULL, + depth int(11) DEFAULT NULL, + department_id int(11) DEFAULT NULL, + status int(2) DEFAULT NULL, + created_on datetime DEFAULT NULL, + updated_on datetime DEFAULT NULL, + PRIMARY KEY (id), + UNIQUE KEY org_position_id_uindex (id), + KEY org_position_department_id_index (department_id) +) ENGINE=InnoDB AUTO_INCREMENT=22 DEFAULT CHARSET=utf8; + CREATE TABLE org_employee_position ( + hotel_id int(11) DEFAULT NULL, + user_id bigint(20) DEFAULT NULL, + position_id int(11) DEFAULT NULL, + status int(11) DEFAULT NULL, + created_on datetime DEFAULT NULL, + updated_on datetime DEFAULT NULL, + UNIQUE KEY org_employee_position_pk (hotel_id,user_id,position_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +explain SELECT d.id, d.ctx, d.name, d.left_value, d.right_value, d.depth, d.leader_id, d.status, d.created_on, d.updated_on FROM org_department AS d LEFT JOIN org_position AS p ON p.department_id = d.id AND p.status = 1000 LEFT JOIN org_employee_position AS ep ON ep.position_id = p.id AND ep.status = 1000 WHERE (d.ctx = 1 AND (ep.user_id = 62 OR d.id = 20 OR d.id = 20) AND d.status = 1000) GROUP BY d.id ORDER BY d.left_value; diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 450db19a241de..b9faf05e017fa 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2053,7 +2053,21 @@ func (s *testPlanSuite) TestOuterJoinEliminator(c *C) { // For complex join query { sql: "select max(t3.b) from (t t1 left join t t2 on t1.a = t2.a) right join t t3 on t1.b = t3.b", - best: "DataScan(t3)->TopN([test.t3.b true],0,1)->Aggr(max(test.t3.b))->Projection", + best: "Join{Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->DataScan(t3)->TopN([test.t3.b true],0,1)}(test.t1.b,test.t3.b)->TopN([test.t3.b true],0,1)->Aggr(max(test.t3.b))->Projection", + }, + { + sql: "select t1.a ta, t1.b tb from t t1 left join t t2 on t1.a = t2.a", + best: "DataScan(t1)->Projection", + }, + { + // Because the `order by` uses t2.a, the `join` can't be eliminated. + sql: "select t1.a, t1.b from t t1 left join t t2 on t1.a = t2.a order by t2.a", + best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->Sort->Projection", + }, + // For issue 11167 + { + sql: "select a.a from t a natural left join t b natural left join t c", + best: "DataScan(a)->Projection", }, } diff --git a/planner/core/plan.go b/planner/core/plan.go index 12fd4b402e67e..f2d90ddf648d6 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -113,6 +113,9 @@ type LogicalPlan interface { // SetChildren sets the children for the plan. SetChildren(...LogicalPlan) + + // SetChild sets the ith child for the plan. + SetChild(i int, child LogicalPlan) } // PhysicalPlan is a tree of the physical operators. @@ -299,6 +302,11 @@ func (p *basePhysicalPlan) SetChildren(children ...PhysicalPlan) { p.children = children } +// SetChild implements LogicalPlan SetChild interface. +func (p *baseLogicalPlan) SetChild(i int, child LogicalPlan) { + p.children[i] = child +} + // SetChild implements PhysicalPlan SetChild interface. func (p *basePhysicalPlan) SetChild(i int, child PhysicalPlan) { p.children[i] = child diff --git a/planner/core/rule_join_elimination.go b/planner/core/rule_join_elimination.go index 983da726cdd01..b392169dc626c 100644 --- a/planner/core/rule_join_elimination.go +++ b/planner/core/rule_join_elimination.go @@ -16,6 +16,7 @@ package core import ( "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/util/set" ) type outerJoinEliminator struct { @@ -28,7 +29,7 @@ type outerJoinEliminator struct { // 2. outer join elimination with duplicate agnostic aggregate functions: For example left outer join. // If the parent only use the columns from left table with 'distinct' label. The left outer join can // be eliminated. -func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentSchema *expression.Schema) (LogicalPlan, error) { +func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentCols []*expression.Column) (LogicalPlan, bool, error) { var innerChildIdx int switch p.JoinType { case LeftOuterJoin: @@ -36,32 +37,42 @@ func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols [] case RightOuterJoin: innerChildIdx = 0 default: - return p, nil + return p, false, nil } outerPlan := p.children[1^innerChildIdx] innerPlan := p.children[innerChildIdx] + outerUniqueIDs := set.NewInt64Set() + for _, outerCol := range outerPlan.Schema().Columns { + outerUniqueIDs.Insert(outerCol.UniqueID) + } + matched := o.isColsAllFromOuterTable(parentCols, outerUniqueIDs) + if !matched { + return p, false, nil + } // outer join elimination with duplicate agnostic aggregate functions - matched, err := o.isAggColsAllFromOuterTable(outerPlan, aggCols) - if err != nil || matched { - return outerPlan, err + matched = o.isColsAllFromOuterTable(aggCols, outerUniqueIDs) + if matched { + return outerPlan, true, nil } // outer join elimination without duplicate agnostic aggregate functions - matched, err = o.isParentColsAllFromOuterTable(outerPlan, parentSchema) - if err != nil || !matched { - return p, err - } innerJoinKeys := o.extractInnerJoinKeys(p, innerChildIdx) contain, err := o.isInnerJoinKeysContainUniqueKey(innerPlan, innerJoinKeys) - if err != nil || contain { - return outerPlan, err + if err != nil { + return p, false, err + } + if contain { + return outerPlan, true, nil } contain, err = o.isInnerJoinKeysContainIndex(innerPlan, innerJoinKeys) - if err != nil || contain { - return outerPlan, err + if err != nil { + return p, false, err + } + if contain { + return outerPlan, true, nil } - return p, nil + return p, false, nil } // extract join keys as a schema for inner child of a outer join @@ -73,33 +84,20 @@ func (o *outerJoinEliminator) extractInnerJoinKeys(join *LogicalJoin, innerChild return expression.NewSchema(joinKeys...) } -func (o *outerJoinEliminator) isAggColsAllFromOuterTable(outerPlan LogicalPlan, aggCols []*expression.Column) (bool, error) { - if len(aggCols) == 0 { - return false, nil - } - for _, col := range aggCols { - columnName := &ast.ColumnName{Schema: col.DBName, Table: col.TblName, Name: col.ColName} - c, err := outerPlan.Schema().FindColumn(columnName) - if err != nil || c == nil { - return false, err - } - } - return true, nil -} - -// check whether schema cols of join's parent plan are all from outer join table -func (o *outerJoinEliminator) isParentColsAllFromOuterTable(outerPlan LogicalPlan, parentSchema *expression.Schema) (bool, error) { - if parentSchema == nil { - return false, nil - } - for _, col := range parentSchema.Columns { - columnName := &ast.ColumnName{Schema: col.DBName, Table: col.TblName, Name: col.ColName} - c, err := outerPlan.Schema().FindColumn(columnName) - if err != nil || c == nil { - return false, err +// check whether the cols all from outer plan +func (o *outerJoinEliminator) isColsAllFromOuterTable(cols []*expression.Column, outerUniqueIDs set.Int64Set) bool { + // There are two cases "return false" here: + // 1. If cols represents aggCols, then "len(cols) == 0" means not all aggregate functions are duplicate agnostic before. + // 2. If cols represents parentCols, then "len(cols) == 0" means no parent logical plan of this join plan. + if len(cols) == 0 { + return false + } + for _, col := range cols { + if !outerUniqueIDs.Exist(col.UniqueID) { + return false } } - return true, nil + return true } // check whether one of unique keys sets is contained by inner join keys @@ -157,52 +155,81 @@ func (o *outerJoinEliminator) isInnerJoinKeysContainIndex(innerPlan LogicalPlan, return false, nil } -// Check whether a LogicalPlan is a LogicalAggregation and its all aggregate functions is duplicate agnostic. -// Also, check all the args are expression.Column. -func (o *outerJoinEliminator) isDuplicateAgnosticAgg(p LogicalPlan) (_ bool, cols []*expression.Column) { +// getDupAgnosticAggCols checks whether a LogicalPlan is LogicalAggregation. +// It extracts all the columns from the duplicate agnostic aggregate functions. +// The returned column set is nil if not all the aggregate functions are duplicate agnostic. +// Only the following functions are considered to be duplicate agnostic: +// 1. MAX(arg) +// 2. MIN(arg) +// 3. FIRST_ROW(arg) +// 4. Other agg functions with DISTINCT flag, like SUM(DISTINCT arg) +func (o *outerJoinEliminator) getDupAgnosticAggCols( + p LogicalPlan, + oldAggCols []*expression.Column, // Reuse the original buffer. +) (isAgg bool, newAggCols []*expression.Column) { agg, ok := p.(*LogicalAggregation) if !ok { return false, nil } - cols = agg.groupByCols + newAggCols = oldAggCols[:0] for _, aggDesc := range agg.AggFuncs { if !aggDesc.HasDistinct && aggDesc.Name != ast.AggFuncFirstRow && aggDesc.Name != ast.AggFuncMax && aggDesc.Name != ast.AggFuncMin { - return false, nil + // If not all aggregate functions are duplicate agnostic, + // we should clean the aggCols, so `return true, newAggCols[:0]`. + return true, newAggCols[:0] } for _, expr := range aggDesc.Args { - if col, ok := expr.(*expression.Column); ok { - cols = append(cols, col) - } else { - return false, nil - } + newAggCols = append(newAggCols, expression.ExtractColumns(expr)...) } } - return true, cols + return true, newAggCols } -func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Column, parentSchema *expression.Schema) (LogicalPlan, error) { - // check the duplicate agnostic aggregate functions - if ok, newCols := o.isDuplicateAgnosticAgg(p); ok { +func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Column, parentCols []*expression.Column) (LogicalPlan, error) { + var err error + var isEliminated bool + for join, isJoin := p.(*LogicalJoin); isJoin; join, isJoin = p.(*LogicalJoin) { + p, isEliminated, err = o.tryToEliminateOuterJoin(join, aggCols, parentCols) + if err != nil { + return p, err + } + if !isEliminated { + break + } + } + + switch x := p.(type) { + case *LogicalProjection: + parentCols = parentCols[:0] + for _, expr := range x.Exprs { + parentCols = append(parentCols, expression.ExtractColumns(expr)...) + } + case *LogicalAggregation: + parentCols = append(parentCols[:0], x.groupByCols...) + for _, aggDesc := range x.AggFuncs { + for _, expr := range aggDesc.Args { + parentCols = append(parentCols, expression.ExtractColumns(expr)...) + } + } + default: + parentCols = append(parentCols[:0], p.Schema().Columns...) + } + + if ok, newCols := o.getDupAgnosticAggCols(p, aggCols); ok { aggCols = newCols } - newChildren := make([]LogicalPlan, 0, len(p.Children())) - for _, child := range p.Children() { - newChild, err := o.doOptimize(child, aggCols, p.Schema()) + for i, child := range p.Children() { + newChild, err := o.doOptimize(child, aggCols, parentCols) if err != nil { return nil, err } - newChildren = append(newChildren, newChild) - } - p.SetChildren(newChildren...) - join, isJoin := p.(*LogicalJoin) - if !isJoin { - return p, nil + p.SetChild(i, newChild) } - return o.tryToEliminateOuterJoin(join, aggCols, parentSchema) + return p, nil } func (o *outerJoinEliminator) optimize(p LogicalPlan) (LogicalPlan, error) { From d47b65570d1a370a1a6432d88bc95b8155d7f398 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Tue, 23 Jul 2019 13:56:11 +0800 Subject: [PATCH 063/196] ddl: don't rely on `expression.Column.ColName` (#11255) --- ddl/column.go | 5 ++-- ddl/partition.go | 68 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/ddl/column.go b/ddl/column.go index 8f8c354606863..9308254a80748 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/ddl/util" - "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/sessionctx" @@ -596,9 +595,9 @@ func generateOriginDefaultValue(col *model.ColumnInfo) (interface{}, error) { return odValue, nil } -func findColumnInIndexCols(c *expression.Column, cols []*ast.IndexColName) bool { +func findColumnInIndexCols(c *model.ColumnInfo, cols []*ast.IndexColName) bool { for _, c1 := range cols { - if c.ColName.L == c1.Column.Name.L { + if c.Name.L == c1.Column.Name.L { return true } } diff --git a/ddl/partition.go b/ddl/partition.go index 8b73dc040e94d..c45669c64cfb0 100644 --- a/ddl/partition.go +++ b/ddl/partition.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/pingcap/errors" + "github.com/pingcap/parser" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -179,13 +180,13 @@ func checkPartitionNameUnique(tbInfo *model.TableInfo, pi *model.PartitionInfo) // See https://github.com/mysql/mysql-server/blob/5.7/sql/item_func.h#L387 func hasTimestampField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.ExprNode) (bool, error) { - partCols, err := partitionColumns(ctx, tblInfo, expr) + partCols, err := checkPartitionColumns(tblInfo, expr) if err != nil { return false, err } for _, c := range partCols { - if c.GetType().Tp == mysql.TypeTimestamp { + if c.FieldType.Tp == mysql.TypeTimestamp { return true, nil } } @@ -195,13 +196,13 @@ func hasTimestampField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr as // See https://github.com/mysql/mysql-server/blob/5.7/sql/item_func.h#L399 func hasDateField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.ExprNode) (bool, error) { - partCols, err := partitionColumns(ctx, tblInfo, expr) + partCols, err := checkPartitionColumns(tblInfo, expr) if err != nil { return false, err } for _, c := range partCols { - if c.GetType().Tp == mysql.TypeDate || c.GetType().Tp == mysql.TypeDatetime { + if c.FieldType.Tp == mysql.TypeDate || c.FieldType.Tp == mysql.TypeDatetime { return true, nil } } @@ -211,13 +212,13 @@ func hasDateField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.Exp // See https://github.com/mysql/mysql-server/blob/5.7/sql/item_func.h#L412 func hasTimeField(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.ExprNode) (bool, error) { - partCols, err := partitionColumns(ctx, tblInfo, expr) + partCols, err := checkPartitionColumns(tblInfo, expr) if err != nil { return false, err } for _, c := range partCols { - if c.GetType().Tp == mysql.TypeDatetime || c.GetType().Tp == mysql.TypeDuration { + if c.FieldType.Tp == mysql.TypeDatetime || c.FieldType.Tp == mysql.TypeDuration { return true, nil } } @@ -283,7 +284,7 @@ func checkPartitionFuncValid(ctx sessionctx.Context, tblInfo *model.TableInfo, e } // check constant. - _, err := partitionColumns(ctx, tblInfo, expr) + _, err := checkPartitionColumns(tblInfo, expr) return err } @@ -302,12 +303,12 @@ func checkPartitionFunc(isTimezoneDependent bool, err error) error { return nil } -func partitionColumns(ctx sessionctx.Context, tblInfo *model.TableInfo, expr ast.ExprNode) ([]*expression.Column, error) { +func checkPartitionColumns(tblInfo *model.TableInfo, expr ast.ExprNode) ([]*model.ColumnInfo, error) { buf := new(bytes.Buffer) expr.Format(buf) - partCols, err := extractPartitionColumns(ctx, buf.String(), tblInfo) + partCols, err := extractPartitionColumns(buf.String(), tblInfo) if err != nil { - return nil, errors.Trace(err) + return nil, err } if len(partCols) == 0 { @@ -597,7 +598,7 @@ func checkRangePartitioningKeysConstraints(sctx sessionctx.Context, s *ast.Creat // Parse partitioning key, extract the column names in the partitioning key to slice. buf := new(bytes.Buffer) s.Partition.Expr.Format(buf) - partCols, err := extractPartitionColumns(sctx, buf.String(), tblInfo) + partCols, err := extractPartitionColumns(buf.String(), tblInfo) if err != nil { return err } @@ -621,7 +622,7 @@ func checkRangePartitioningKeysConstraints(sctx sessionctx.Context, s *ast.Creat func checkPartitionKeysConstraint(sctx sessionctx.Context, partExpr string, idxColNames []*ast.IndexColName, tblInfo *model.TableInfo) error { // Parse partitioning key, extract the column names in the partitioning key to slice. - partCols, err := extractPartitionColumns(sctx, partExpr, tblInfo) + partCols, err := extractPartitionColumns(partExpr, tblInfo) if err != nil { return err } @@ -634,16 +635,49 @@ func checkPartitionKeysConstraint(sctx sessionctx.Context, partExpr string, idxC return nil } -func extractPartitionColumns(sctx sessionctx.Context, partExpr string, tblInfo *model.TableInfo) ([]*expression.Column, error) { - e, err := expression.ParseSimpleExprWithTableInfo(sctx, partExpr, tblInfo) +type columnNameExtractor struct { + extractedColumns []*model.ColumnInfo + tblInfo *model.TableInfo + err error +} + +func (cne *columnNameExtractor) Enter(node ast.Node) (ast.Node, bool) { + return node, false +} + +func (cne *columnNameExtractor) Leave(node ast.Node) (ast.Node, bool) { + if c, ok := node.(*ast.ColumnNameExpr); ok { + for _, info := range cne.tblInfo.Columns { + if info.Name.L == c.Name.Name.L { + cne.extractedColumns = append(cne.extractedColumns, info) + return node, true + } + } + cne.err = ErrBadField.GenWithStackByArgs(c.Name.Name.O, "expression") + return nil, false + } + return node, true +} + +func extractPartitionColumns(partExpr string, tblInfo *model.TableInfo) ([]*model.ColumnInfo, error) { + partExpr = "select " + partExpr + stmts, _, err := parser.New().Parse(partExpr, "", "") if err != nil { - return nil, errors.Trace(err) + return nil, err + } + extractor := &columnNameExtractor{ + tblInfo: tblInfo, + extractedColumns: make([]*model.ColumnInfo, 0), + } + stmts[0].Accept(extractor) + if extractor.err != nil { + return nil, extractor.err } - return expression.ExtractColumns(e), nil + return extractor.extractedColumns, nil } // checkUniqueKeyIncludePartKey checks that the partitioning key is included in the constraint. -func checkUniqueKeyIncludePartKey(partCols []*expression.Column, idxCols []*ast.IndexColName) bool { +func checkUniqueKeyIncludePartKey(partCols []*model.ColumnInfo, idxCols []*ast.IndexColName) bool { for _, partCol := range partCols { if !findColumnInIndexCols(partCol, idxCols) { return false From c1a5e79b285d7708f7b4dc27b976945770db4f79 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Tue, 23 Jul 2019 14:17:49 +0800 Subject: [PATCH 064/196] planner: correct the generation of the field name (#11324) --- executor/executor_test.go | 7 +++++++ planner/core/logical_plan_builder.go | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/executor/executor_test.go b/executor/executor_test.go index 3efde355754f3..3eda72e7bbe07 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2214,6 +2214,13 @@ func (s *testSuiteP1) TestColumnName(c *C) { c.Assert(fields[1].ColumnAsName.L, Equals, "num") tk.MustExec("set @@tidb_enable_window_function = 0") rs.Close() + + rs, err = tk.Exec("select if(1,c,c) from t;") + c.Check(err, IsNil) + fields = rs.Fields() + c.Assert(fields[0].Column.Name.L, Equals, "if(1,c,c)") + // It's a compatibility issue. Should be empty instead. + c.Assert(fields[0].ColumnAsName.L, Equals, "if(1,c,c)") } func (s *testSuiteP1) TestSelectVar(c *C) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 7285b51f9a22d..99a68742f3d63 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -600,12 +600,10 @@ func (b *PlanBuilder) buildSelection(p LogicalPlan, where ast.ExprNode, AggMappe } // buildProjectionFieldNameFromColumns builds the field name, table name and database name when field expression is a column reference. -func (b *PlanBuilder) buildProjectionFieldNameFromColumns(field *ast.SelectField, c *expression.Column) (colName, origColName, tblName, origTblName, dbName model.CIStr) { - if astCol, ok := getInnerFromParenthesesAndUnaryPlus(field.Expr).(*ast.ColumnNameExpr); ok { - origColName, tblName, dbName = astCol.Name.Name, astCol.Name.Table, astCol.Name.Schema - } - if field.AsName.L != "" { - colName = field.AsName +func (b *PlanBuilder) buildProjectionFieldNameFromColumns(origField *ast.SelectField, colNameField *ast.ColumnNameExpr, c *expression.Column) (colName, origColName, tblName, origTblName, dbName model.CIStr) { + origColName, tblName, dbName = colNameField.Name.Name, colNameField.Name.Table, colNameField.Name.Schema + if origField.AsName.L != "" { + colName = origField.AsName } else { colName = origColName } @@ -686,9 +684,13 @@ func (b *PlanBuilder) buildProjectionFieldNameFromExpressions(field *ast.SelectF // buildProjectionField builds the field object according to SelectField in projection. func (b *PlanBuilder) buildProjectionField(id, position int, field *ast.SelectField, expr expression.Expression) (*expression.Column, error) { var origTblName, tblName, origColName, colName, dbName model.CIStr - if c, ok := expr.(*expression.Column); ok && !c.IsReferenced { + innerNode := getInnerFromParenthesesAndUnaryPlus(field.Expr) + col, isCol := expr.(*expression.Column) + // Correlated column won't affect the final output names. So we can put it in any of the three logic block. + // Don't put it into the first block just for simplifying the codes. + if colNameField, ok := innerNode.(*ast.ColumnNameExpr); ok && isCol { // Field is a column reference. - colName, origColName, tblName, origTblName, dbName = b.buildProjectionFieldNameFromColumns(field, c) + colName, origColName, tblName, origTblName, dbName = b.buildProjectionFieldNameFromColumns(field, colNameField, col) } else if field.AsName.L != "" { // Field has alias. colName = field.AsName From eb9e749a408ede5171f33e171b0f375954b19631 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 23 Jul 2019 14:26:04 +0800 Subject: [PATCH 065/196] ddl: add check for max index key length when alter modify/change column (#11220) --- ddl/db_integration_test.go | 15 ++++ ddl/ddl_api.go | 43 ++++++++++- ddl/index.go | 153 +++++++++++++++++++++++-------------- 3 files changed, 151 insertions(+), 60 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 367549acc13cb..84698837156af 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -1453,6 +1453,21 @@ func (s *testIntegrationSuite3) TestAlterColumn(c *C) { c.Assert(createSQL, Equals, expected) _, err = s.tk.Exec("alter table mc modify column a bigint auto_increment") // Adds auto_increment should throw error c.Assert(err, NotNil) + + s.tk.MustExec("drop table if exists t") + // TODO: fix me, below sql should execute successfully. Currently, the result of calculate key length is wrong. + //s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3071),index(a),index(a,b),index (c,d));") + s.tk.MustExec("create table t1 (a varchar(10),b varchar(100),c tinyint,d varchar(3068),index(a),index(a,b),index (c,d));") + _, err = s.tk.Exec("alter table t1 modify column a varchar(3000);") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") + // check modify column with rename column. + _, err = s.tk.Exec("alter table t1 change column a x varchar(3000);") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") + _, err = s.tk.Exec("alter table t1 modify column c bigint;") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[ddl:1071]Specified key was too long; max key length is 3072 bytes") } func (s *testIntegrationSuite) assertWarningExec(c *C, sql string, expectedWarn *terror.Error) { diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 5a930f6aafcf6..cac7a60239285 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2642,7 +2642,11 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or } if err = checkColumnFieldLength(newCol); err != nil { - return nil, errors.Trace(err) + return nil, err + } + + if err = checkColumnWithIndexConstraint(t.Meta(), col.ColumnInfo, newCol.ColumnInfo); err != nil { + return nil, err } // As same with MySQL, we don't support modifying the stored status for generated columns. @@ -2660,6 +2664,43 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or return job, nil } +// checkColumnWithIndexConstraint is used to check the related index constraint of the modified column. +// Index has a max-prefix-length constraint. eg: a varchar(100), index idx(a), modifying column a to a varchar(4000) +// will cause index idx to break the max-prefix-length constraint. +func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol *model.ColumnInfo) error { + var columns []*model.ColumnInfo + for _, indexInfo := range tbInfo.Indices { + containColumn := false + for _, col := range indexInfo.Columns { + if col.Name.L == originalCol.Name.L { + containColumn = true + break + } + } + if containColumn == false { + continue + } + if columns == nil { + columns = make([]*model.ColumnInfo, 0, len(tbInfo.Columns)) + columns = append(columns, tbInfo.Columns...) + // replace old column with new column. + for i, col := range columns { + if col.Name.L != originalCol.Name.L { + continue + } + columns[i] = newCol.Clone() + columns[i].Name = originalCol.Name + break + } + } + err := checkIndexPrefixLength(columns, indexInfo.Columns) + if err != nil { + return err + } + } + return nil +} + // ChangeColumn renames an existing column and modifies the column's definition, // currently we only support limited kind of changes // that do not need to change or check data on the table. diff --git a/ddl/index.go b/ddl/index.go index b09e0e1c89fa9..c6ffa9e3a1bb1 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -54,73 +54,21 @@ func buildIndexColumns(columns []*model.ColumnInfo, idxColNames []*ast.IndexColN // The sum of length of all index columns. sumLength := 0 - for _, ic := range idxColNames { - col := model.FindColumnInfo(columns, ic.Column.Name.O) + col := model.FindColumnInfo(columns, ic.Column.Name.L) if col == nil { return nil, errKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Column.Name) } - if col.Flen == 0 && (types.IsTypeChar(col.FieldType.Tp) || types.IsTypeVarchar(col.FieldType.Tp)) { - return nil, errors.Trace(errWrongKeyColumn.GenWithStackByArgs(ic.Column.Name)) - } - - // JSON column cannot index. - if col.FieldType.Tp == mysql.TypeJSON { - return nil, errors.Trace(errJSONUsedAsKey.GenWithStackByArgs(col.Name.O)) - } - - // Length must be specified for BLOB and TEXT column indexes. - if types.IsTypeBlob(col.FieldType.Tp) && ic.Length == types.UnspecifiedLength { - return nil, errors.Trace(errBlobKeyWithoutLength) - } - - // Length can only be specified for specifiable types. - if ic.Length != types.UnspecifiedLength && !types.IsTypePrefixable(col.FieldType.Tp) { - return nil, errors.Trace(errIncorrectPrefixKey) - } - - // Key length must be shorter or equal to the column length. - if ic.Length != types.UnspecifiedLength && - types.IsTypeChar(col.FieldType.Tp) && col.Flen < ic.Length { - return nil, errors.Trace(errIncorrectPrefixKey) - } - - // Specified length must be shorter than the max length for prefix. - if ic.Length > maxPrefixLength { - return nil, errors.Trace(errTooLongKey) + if err := checkIndexColumn(col, ic); err != nil { + return nil, err } - // Take care of the sum of length of all index columns. - if ic.Length != types.UnspecifiedLength { - sumLength += ic.Length - } else { - // Specified data types. - if col.Flen != types.UnspecifiedLength { - // Special case for the bit type. - if col.FieldType.Tp == mysql.TypeBit { - sumLength += (col.Flen + 7) >> 3 - } else { - sumLength += col.Flen - } - } else { - if length, ok := mysql.DefaultLengthOfMysqlTypes[col.FieldType.Tp]; ok { - sumLength += length - } else { - return nil, errUnknownTypeLength.GenWithStackByArgs(col.FieldType.Tp) - } - - // Special case for time fraction. - if types.IsTypeFractionable(col.FieldType.Tp) && - col.FieldType.Decimal != types.UnspecifiedLength { - if length, ok := mysql.DefaultLengthOfTimeFraction[col.FieldType.Decimal]; ok { - sumLength += length - } else { - return nil, errUnknownFractionLength.GenWithStackByArgs(col.FieldType.Tp, col.FieldType.Decimal) - } - } - } + indexColumnLength, err := getIndexColumnLength(col, ic.Length) + if err != nil { + return nil, err } + sumLength += indexColumnLength // The sum of all lengths must be shorter than the max length for prefix. if sumLength > maxPrefixLength { @@ -137,6 +85,93 @@ func buildIndexColumns(columns []*model.ColumnInfo, idxColNames []*ast.IndexColN return idxColumns, nil } +func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn) error { + // The sum of length of all index columns. + sumLength := 0 + for _, ic := range idxColumns { + col := model.FindColumnInfo(columns, ic.Name.L) + if col == nil { + return errKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Name) + } + + indexColumnLength, err := getIndexColumnLength(col, ic.Length) + if err != nil { + return err + } + sumLength += indexColumnLength + // The sum of all lengths must be shorter than the max length for prefix. + if sumLength > maxPrefixLength { + return errors.Trace(errTooLongKey) + } + } + return nil +} + +func checkIndexColumn(col *model.ColumnInfo, ic *ast.IndexColName) error { + if col.Flen == 0 && (types.IsTypeChar(col.FieldType.Tp) || types.IsTypeVarchar(col.FieldType.Tp)) { + return errors.Trace(errWrongKeyColumn.GenWithStackByArgs(ic.Column.Name)) + } + + // JSON column cannot index. + if col.FieldType.Tp == mysql.TypeJSON { + return errors.Trace(errJSONUsedAsKey.GenWithStackByArgs(col.Name.O)) + } + + // Length must be specified for BLOB and TEXT column indexes. + if types.IsTypeBlob(col.FieldType.Tp) && ic.Length == types.UnspecifiedLength { + return errors.Trace(errBlobKeyWithoutLength) + } + + // Length can only be specified for specifiable types. + if ic.Length != types.UnspecifiedLength && !types.IsTypePrefixable(col.FieldType.Tp) { + return errors.Trace(errIncorrectPrefixKey) + } + + // Key length must be shorter or equal to the column length. + if ic.Length != types.UnspecifiedLength && + types.IsTypeChar(col.FieldType.Tp) && col.Flen < ic.Length { + return errors.Trace(errIncorrectPrefixKey) + } + + // Specified length must be shorter than the max length for prefix. + if ic.Length > maxPrefixLength { + return errors.Trace(errTooLongKey) + } + return nil +} + +func getIndexColumnLength(col *model.ColumnInfo, colLen int) (int, error) { + // Take care of the sum of length of all index columns. + if colLen != types.UnspecifiedLength { + return colLen, nil + } + // Specified data types. + if col.Flen != types.UnspecifiedLength { + // Special case for the bit type. + if col.FieldType.Tp == mysql.TypeBit { + return (col.Flen + 7) >> 3, nil + } + return col.Flen, nil + + } + + length, ok := mysql.DefaultLengthOfMysqlTypes[col.FieldType.Tp] + if !ok { + return length, errUnknownTypeLength.GenWithStackByArgs(col.FieldType.Tp) + } + + // Special case for time fraction. + if types.IsTypeFractionable(col.FieldType.Tp) && + col.FieldType.Decimal != types.UnspecifiedLength { + decimalLength, ok := mysql.DefaultLengthOfTimeFraction[col.FieldType.Decimal] + if !ok { + return length, errUnknownFractionLength.GenWithStackByArgs(col.FieldType.Tp, col.FieldType.Decimal) + } + length += decimalLength + } + return length, nil +} + func buildIndexInfo(tblInfo *model.TableInfo, indexName model.CIStr, idxColNames []*ast.IndexColName, state model.SchemaState) (*model.IndexInfo, error) { idxColumns, err := buildIndexColumns(tblInfo.Columns, idxColNames) if err != nil { From 37bb4ba95f80add13036b276a418904fef90399f Mon Sep 17 00:00:00 2001 From: lysu Date: Tue, 23 Jul 2019 14:33:48 +0800 Subject: [PATCH 066/196] store/helper: remove timeout for fetch diagnose from info pd (#11352) --- store/helper/helper.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/store/helper/helper.go b/store/helper/helper.go index ce202f9bb2ef1..4be8a343cfec8 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -413,9 +413,7 @@ func (h *Helper) GetRegionsInfo() (*RegionsInfo, error) { if err != nil { return nil, errors.Trace(err) } - timeout, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) - resp, err := http.DefaultClient.Do(req.WithContext(timeout)) - defer cancelFunc() + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, errors.Trace(err) } @@ -492,9 +490,7 @@ func (h *Helper) GetStoresStat() (*StoresStat, error) { if err != nil { return nil, errors.Trace(err) } - timeout, cancelFunc := context.WithTimeout(context.Background(), 50*time.Millisecond) - resp, err := http.DefaultClient.Do(req.WithContext(timeout)) - defer cancelFunc() + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, errors.Trace(err) } From e1c6dfa6387421e5843ec51a739c1b62767d6416 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 23 Jul 2019 14:40:24 +0800 Subject: [PATCH 067/196] executor: refactor some methods of `Row` and `Chunk` (#11380) --- util/chunk/chunk.go | 11 +++++----- util/chunk/chunk_test.go | 8 ++++---- util/chunk/column.go | 25 +++++++++++++++++++++++ util/chunk/column_test.go | 5 ++++- util/chunk/list.go | 10 +++++----- util/chunk/list_test.go | 6 +++--- util/chunk/row.go | 42 +++++++++++---------------------------- 7 files changed, 59 insertions(+), 48 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 60bc217ac74e5..4b78ebdcd4de2 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -319,8 +319,8 @@ func (c *Chunk) AppendPartialRow(colIdx int, row Row) { } } -// PreAlloc pre-allocates the memory space in a Chunk to store the Row. -// NOTE: +// preAlloc pre-allocates the memory space in a Chunk to store the Row. +// NOTE: only used in test. // 1. The Chunk must be empty or holds no useful data. // 2. The schema of the Row must be the same with the Chunk. // 3. This API is paired with the `Insert()` function, which inserts all the @@ -329,7 +329,7 @@ func (c *Chunk) AppendPartialRow(colIdx int, row Row) { // when the Insert() function is called parallelly, the data race on a byte // can not be avoided although the manipulated bits are different inside a // byte. -func (c *Chunk) PreAlloc(row Row) (rowIdx uint32) { +func (c *Chunk) preAlloc(row Row) (rowIdx uint32) { rowIdx = uint32(c.NumRows()) for i, srcCol := range row.c.columns { dstCol := c.columns[i] @@ -379,10 +379,11 @@ func (c *Chunk) PreAlloc(row Row) (rowIdx uint32) { return } -// Insert inserts `row` on the position specified by `rowIdx`. +// insert inserts `row` on the position specified by `rowIdx`. +// NOTE: only used in test. // Note: Insert will cover the origin data, it should be called after // PreAlloc. -func (c *Chunk) Insert(rowIdx int, row Row) { +func (c *Chunk) insert(rowIdx int, row Row) { for i, srcCol := range row.c.columns { if row.IsNull(i) { continue diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 809530ac5e2ed..769b4f00b804f 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -646,7 +646,7 @@ func (s *testChunkSuite) TestPreAlloc4RowAndInsert(c *check.C) { // Test Chunk.PreAlloc. for i := 0; i < srcChk.NumRows(); i++ { c.Assert(destChk.NumRows(), check.Equals, i) - destChk.PreAlloc(srcChk.GetRow(i)) + destChk.preAlloc(srcChk.GetRow(i)) } for i, srcCol := range srcChk.columns { destCol := destChk.columns[i] @@ -673,7 +673,7 @@ func (s *testChunkSuite) TestPreAlloc4RowAndInsert(c *check.C) { // Test Chunk.Insert. for i := srcChk.NumRows() - 1; i >= 0; i-- { - destChk.Insert(i, srcChk.GetRow(i)) + destChk.insert(i, srcChk.GetRow(i)) } for i, srcCol := range srcChk.columns { destCol := destChk.columns[i] @@ -697,14 +697,14 @@ func (s *testChunkSuite) TestPreAlloc4RowAndInsert(c *check.C) { startWg, endWg := &sync.WaitGroup{}, &sync.WaitGroup{} startWg.Add(1) for i := 0; i < srcChk.NumRows(); i++ { - destChk.PreAlloc(srcChk.GetRow(i)) + destChk.preAlloc(srcChk.GetRow(i)) endWg.Add(1) go func(rowIdx int) { defer func() { endWg.Done() }() startWg.Wait() - destChk.Insert(rowIdx, srcChk.GetRow(rowIdx)) + destChk.insert(rowIdx, srcChk.GetRow(rowIdx)) }(i) } startWg.Done() diff --git a/util/chunk/column.go b/util/chunk/column.go index d06b1f2e92669..a10a699260db4 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -263,6 +263,31 @@ func (c *Column) Decimals() []types.MyDecimal { return res } +// GetInt64 returns the int64 in the specific row. +func (c *Column) GetInt64(rowID int) int64 { + return *(*int64)(unsafe.Pointer(&c.data[rowID*8])) +} + +// GetUint64 returns the uint64 in the specific row. +func (c *Column) GetUint64(rowID int) uint64 { + return *(*uint64)(unsafe.Pointer(&c.data[rowID*8])) +} + +// GetFloat32 returns the float32 in the specific row. +func (c *Column) GetFloat32(rowID int) float32 { + return *(*float32)(unsafe.Pointer(&c.data[rowID*4])) +} + +// GetFloat64 returns the float64 in the specific row. +func (c *Column) GetFloat64(rowID int) float64 { + return *(*float64)(unsafe.Pointer(&c.data[rowID*8])) +} + +// GetDecimal returns the decimal in the specific row. +func (c *Column) GetDecimal(rowID int) *types.MyDecimal { + return (*types.MyDecimal)(unsafe.Pointer(&c.data[rowID*types.MyDecimalStructSize])) +} + // GetString returns the string in the specific row. func (c *Column) GetString(rowID int) string { return string(hack.String(c.data[c.offsets[rowID]:c.offsets[rowID+1]])) diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 210e270155b84..e3748ad7583b3 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -88,9 +88,10 @@ func (s *testChunkSuite) TestI64Column(c *check.C) { } it := NewIterator4Chunk(chk) - var i int64 + var i int for row := it.Begin(); row != it.End(); row = it.Next() { c.Assert(row.GetInt64(0), check.Equals, int64(i+1)) + c.Assert(col.GetInt64(i), check.Equals, int64(i+1)) i++ } } @@ -112,6 +113,7 @@ func (s *testChunkSuite) TestF64Column(c *check.C) { var i int64 for row := it.Begin(); row != it.End(); row = it.Next() { c.Assert(row.GetFloat64(0), check.Equals, float64(i)/2) + c.Assert(col.GetFloat64(int(i)), check.Equals, float64(i)/2) i++ } } @@ -133,6 +135,7 @@ func (s *testChunkSuite) TestF32Column(c *check.C) { var i int64 for row := it.Begin(); row != it.End(); row = it.Next() { c.Assert(row.GetFloat32(0), check.Equals, float32(i)/2) + c.Assert(col.GetFloat32(int(i)), check.Equals, float32(i)/2) i++ } } diff --git a/util/chunk/list.go b/util/chunk/list.go index cf3ca1c5ab2dc..46f226310f506 100644 --- a/util/chunk/list.go +++ b/util/chunk/list.go @@ -145,13 +145,13 @@ func (l *List) Reset() { l.consumedIdx = -1 } -// PreAlloc4Row pre-allocates the storage memory for a Row. -// NOTE: +// preAlloc4Row pre-allocates the storage memory for a Row. +// NOTE: only used in test // 1. The List must be empty or holds no useful data. // 2. The schema of the Row must be the same with the List. // 3. This API is paired with the `Insert()` function, which inserts all the // rows data into the List after the pre-allocation. -func (l *List) PreAlloc4Row(row Row) (ptr RowPtr) { +func (l *List) preAlloc4Row(row Row) (ptr RowPtr) { chkIdx := len(l.chunks) - 1 if chkIdx == -1 || l.chunks[chkIdx].NumRows() >= l.chunks[chkIdx].Capacity() { newChk := l.allocChunk() @@ -163,7 +163,7 @@ func (l *List) PreAlloc4Row(row Row) (ptr RowPtr) { chkIdx++ } chk := l.chunks[chkIdx] - rowIdx := chk.PreAlloc(row) + rowIdx := chk.preAlloc(row) l.length++ return RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)} } @@ -172,7 +172,7 @@ func (l *List) PreAlloc4Row(row Row) (ptr RowPtr) { // Note: Insert will cover the origin data, it should be called after // PreAlloc. func (l *List) Insert(ptr RowPtr, row Row) { - l.chunks[ptr.ChkIdx].Insert(int(ptr.RowIdx), row) + l.chunks[ptr.ChkIdx].insert(int(ptr.RowIdx), row) } // ListWalkFunc is used to walk the list. diff --git a/util/chunk/list_test.go b/util/chunk/list_test.go index 014470a58bb3b..e1102d9c7f9a0 100644 --- a/util/chunk/list_test.go +++ b/util/chunk/list_test.go @@ -136,7 +136,7 @@ func (s *testChunkSuite) TestListPrePreAlloc4RowAndInsert(c *check.C) { destRowPtr := make([]RowPtr, srcChk.NumRows()) for i := 0; i < srcChk.NumRows(); i++ { srcList.AppendRow(srcChk.GetRow(i)) - destRowPtr[i] = destList.PreAlloc4Row(srcChk.GetRow(i)) + destRowPtr[i] = destList.preAlloc4Row(srcChk.GetRow(i)) } c.Assert(srcList.NumChunks(), check.Equals, 4) @@ -197,7 +197,7 @@ func BenchmarkPreAllocList(b *testing.B) { list.Reset() // 32768 indicates the number of int64 rows to fill 256KB L2 cache. for j := 0; j < 32768; j++ { - list.PreAlloc4Row(row) + list.preAlloc4Row(row) } } } @@ -214,7 +214,7 @@ func BenchmarkPreAllocChunk(b *testing.B) { for i := 0; i < b.N; i++ { finalChk.Reset() for j := 0; j < 32768; j++ { - finalChk.PreAlloc(row) + finalChk.preAlloc(row) } } } diff --git a/util/chunk/row.go b/util/chunk/row.go index b4c22d6b02210..7dc55ba636a7a 100644 --- a/util/chunk/row.go +++ b/util/chunk/row.go @@ -14,7 +14,6 @@ package chunk import ( - "time" "unsafe" "github.com/pingcap/parser/mysql" @@ -46,55 +45,43 @@ func (r Row) Len() int { // GetInt64 returns the int64 value with the colIdx. func (r Row) GetInt64(colIdx int) int64 { - col := r.c.columns[colIdx] - return *(*int64)(unsafe.Pointer(&col.data[r.idx*8])) + return r.c.columns[colIdx].GetInt64(r.idx) } // GetUint64 returns the uint64 value with the colIdx. func (r Row) GetUint64(colIdx int) uint64 { - col := r.c.columns[colIdx] - return *(*uint64)(unsafe.Pointer(&col.data[r.idx*8])) + return r.c.columns[colIdx].GetUint64(r.idx) } // GetFloat32 returns the float32 value with the colIdx. func (r Row) GetFloat32(colIdx int) float32 { - col := r.c.columns[colIdx] - return *(*float32)(unsafe.Pointer(&col.data[r.idx*4])) + return r.c.columns[colIdx].GetFloat32(r.idx) } // GetFloat64 returns the float64 value with the colIdx. func (r Row) GetFloat64(colIdx int) float64 { - col := r.c.columns[colIdx] - return *(*float64)(unsafe.Pointer(&col.data[r.idx*8])) + return r.c.columns[colIdx].GetFloat64(r.idx) } // GetString returns the string value with the colIdx. func (r Row) GetString(colIdx int) string { - col := r.c.columns[colIdx] - start, end := col.offsets[r.idx], col.offsets[r.idx+1] - str := string(hack.String(col.data[start:end])) - return str + return r.c.columns[colIdx].GetString(r.idx) } // GetBytes returns the bytes value with the colIdx. func (r Row) GetBytes(colIdx int) []byte { - col := r.c.columns[colIdx] - start, end := col.offsets[r.idx], col.offsets[r.idx+1] - return col.data[start:end] + return r.c.columns[colIdx].GetBytes(r.idx) } // GetTime returns the Time value with the colIdx. // TODO: use Time structure directly. func (r Row) GetTime(colIdx int) types.Time { - col := r.c.columns[colIdx] - return readTime(col.data[r.idx*16:]) + return r.c.columns[colIdx].GetTime(r.idx) } // GetDuration returns the Duration value with the colIdx. func (r Row) GetDuration(colIdx int, fillFsp int) types.Duration { - col := r.c.columns[colIdx] - dur := *(*int64)(unsafe.Pointer(&col.data[r.idx*8])) - return types.Duration{Duration: time.Duration(dur), Fsp: fillFsp} + return r.c.columns[colIdx].GetDuration(r.idx, fillFsp) } func (r Row) getNameValue(colIdx int) (string, uint64) { @@ -110,27 +97,22 @@ func (r Row) getNameValue(colIdx int) (string, uint64) { // GetEnum returns the Enum value with the colIdx. func (r Row) GetEnum(colIdx int) types.Enum { - name, val := r.getNameValue(colIdx) - return types.Enum{Name: name, Value: val} + return r.c.columns[colIdx].GetEnum(r.idx) } // GetSet returns the Set value with the colIdx. func (r Row) GetSet(colIdx int) types.Set { - name, val := r.getNameValue(colIdx) - return types.Set{Name: name, Value: val} + return r.c.columns[colIdx].GetSet(r.idx) } // GetMyDecimal returns the MyDecimal value with the colIdx. func (r Row) GetMyDecimal(colIdx int) *types.MyDecimal { - col := r.c.columns[colIdx] - return (*types.MyDecimal)(unsafe.Pointer(&col.data[r.idx*types.MyDecimalStructSize])) + return r.c.columns[colIdx].GetDecimal(r.idx) } // GetJSON returns the JSON value with the colIdx. func (r Row) GetJSON(colIdx int) json.BinaryJSON { - col := r.c.columns[colIdx] - start, end := col.offsets[r.idx], col.offsets[r.idx+1] - return json.BinaryJSON{TypeCode: col.data[start], Value: col.data[start+1 : end]} + return r.c.columns[colIdx].GetJSON(r.idx) } // GetDatumRow converts chunk.Row to types.DatumRow. From 435a912d294b7e71d57092903c1c050e56a86137 Mon Sep 17 00:00:00 2001 From: Tanner Date: Tue, 23 Jul 2019 15:13:24 +0800 Subject: [PATCH 068/196] ddl: disallow dropping index on auto_increment column (#11360) --- ddl/db_integration_test.go | 18 ++++++++++++++++++ ddl/ddl_api.go | 10 +++++++++- meta/autoid/errors.go | 2 ++ planner/core/preprocess.go | 3 ++- planner/core/preprocess_test.go | 6 +++--- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 84698837156af..1059930f69bc7 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -414,6 +414,8 @@ func (s *testIntegrationSuite5) TestMySQLErrorCode(c *C) { assertErrorCode(c, tk, sql, tmysql.ErrPrimaryCantHaveNull) sql = "create table t2 (id int null, age int, primary key(id));" assertErrorCode(c, tk, sql, tmysql.ErrPrimaryCantHaveNull) + sql = "create table t2 (id int auto_increment);" + assertErrorCode(c, tk, sql, tmysql.ErrWrongAutoKey) sql = "create table t2 (id int primary key , age int);" tk.MustExec(sql) @@ -1818,3 +1820,19 @@ func (s *testIntegrationSuite5) TestChangingDBCharset(c *C) { tk.MustExec("ALTER SCHEMA CHARACTER SET = 'utf8mb4' COLLATE = 'utf8mb4_general_ci'") verifyDBCharsetAndCollate("alterdb2", "utf8mb4", "utf8mb4_general_ci") } + +func (s *testIntegrationSuite4) TestDropAutoIncrementIndex(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test") + tk.MustExec("use test") + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (a int auto_increment, unique key (a))") + dropIndexSQL := "alter table t1 drop index a" + assertErrorCode(c, tk, dropIndexSQL, mysql.ErrWrongAutoKey) + + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (a int(11) not null auto_increment, b int(11), c bigint, unique key (a, b, c))") + dropIndexSQL = "alter table t1 drop index a" + assertErrorCode(c, tk, dropIndexSQL, mysql.ErrWrongAutoKey) +} diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index cac7a60239285..247ef1c172c9f 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -3304,7 +3304,8 @@ func (d *ddl) DropIndex(ctx sessionctx.Context, ti ast.Ident, indexName model.CI return errors.Trace(infoschema.ErrTableNotExists.GenWithStackByArgs(ti.Schema, ti.Name)) } - if indexInfo := t.Meta().FindIndexByName(indexName.L); indexInfo == nil { + indexInfo := t.Meta().FindIndexByName(indexName.L) + if indexInfo == nil { err = ErrCantDropFieldOrKey.GenWithStack("index %s doesn't exist", indexName) if ifExists { ctx.GetSessionVars().StmtCtx.AppendNote(err) @@ -3313,6 +3314,13 @@ func (d *ddl) DropIndex(ctx sessionctx.Context, ti ast.Ident, indexName model.CI return err } + cols := t.Cols() + for _, idxCol := range indexInfo.Columns { + if mysql.HasAutoIncrementFlag(cols[idxCol.Offset].Flag) { + return autoid.ErrWrongAutoKey + } + } + job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, diff --git a/meta/autoid/errors.go b/meta/autoid/errors.go index 44ef83650a202..e9b0b0fa6fae4 100644 --- a/meta/autoid/errors.go +++ b/meta/autoid/errors.go @@ -21,12 +21,14 @@ import ( // Error instances. var ( ErrAutoincReadFailed = terror.ClassAutoid.New(mysql.ErrAutoincReadFailed, mysql.MySQLErrName[mysql.ErrAutoincReadFailed]) + ErrWrongAutoKey = terror.ClassAutoid.New(mysql.ErrWrongAutoKey, mysql.MySQLErrName[mysql.ErrWrongAutoKey]) ) func init() { // Map error codes to mysql error codes. tableMySQLErrCodes := map[terror.ErrCode]uint16{ mysql.ErrAutoincReadFailed: mysql.ErrAutoincReadFailed, + mysql.ErrWrongAutoKey: mysql.ErrWrongAutoKey, } terror.ErrClassToMySQLCodes[terror.ClassAutoid] = tableMySQLErrCodes } diff --git a/planner/core/preprocess.go b/planner/core/preprocess.go index 770ba6f683e5a..d902ffba1f8e3 100644 --- a/planner/core/preprocess.go +++ b/planner/core/preprocess.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/parser_driver" @@ -295,7 +296,7 @@ func (p *preprocessor) checkAutoIncrement(stmt *ast.CreateTableStmt) { } } if (autoIncrementMustBeKey && !isKey) || count > 1 { - p.err = errors.New("Incorrect table definition; there can be only one auto column and it must be defined as a key") + p.err = autoid.ErrWrongAutoKey.GenWithStackByArgs() } switch autoIncrementCol.Tp.Tp { diff --git a/planner/core/preprocess_test.go b/planner/core/preprocess_test.go index fc652943c3821..01d04b9261208 100644 --- a/planner/core/preprocess_test.go +++ b/planner/core/preprocess_test.go @@ -53,11 +53,11 @@ func (s *testValidatorSuite) TestValidator(c *C) { {"create table t(id int auto_increment default null, primary key (id))", true, nil}, {"create table t(id int default null auto_increment, primary key (id))", true, nil}, {"create table t(id int not null auto_increment)", true, - errors.New("Incorrect table definition; there can be only one auto column and it must be defined as a key")}, + errors.New("[autoid:1075]Incorrect table definition; there can be only one auto column and it must be defined as a key")}, {"create table t(id int not null auto_increment, c int auto_increment, key (id, c))", true, - errors.New("Incorrect table definition; there can be only one auto column and it must be defined as a key")}, + errors.New("[autoid:1075]Incorrect table definition; there can be only one auto column and it must be defined as a key")}, {"create table t(id int not null auto_increment, c int, key (c, id))", true, - errors.New("Incorrect table definition; there can be only one auto column and it must be defined as a key")}, + errors.New("[autoid:1075]Incorrect table definition; there can be only one auto column and it must be defined as a key")}, {"create table t(id decimal auto_increment, key (id))", true, errors.New("Incorrect column specifier for column 'id'")}, {"create table t(id float auto_increment, key (id))", true, nil}, From 88db7ff17b5a3f4799dff0b4f4e2a233a7e16cd0 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 23 Jul 2019 15:52:29 +0800 Subject: [PATCH 069/196] =?UTF-8?q?infoschema:=20fix=20compatibility=20of?= =?UTF-8?q?=20auto=5Fincrement=20column=20value=20o=E2=80=A6=20(#10207)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ddl/ddl_api.go | 9 --------- ddl/generated_column.go | 3 ++- executor/show_test.go | 4 ++-- infoschema/infoschema.go | 10 ++++++++++ infoschema/tables.go | 35 +++++++++++++---------------------- infoschema/tables_test.go | 16 +++++++++++++++- 6 files changed, 42 insertions(+), 35 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 247ef1c172c9f..f24102e713d12 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1727,15 +1727,6 @@ func handleTableOptions(options []*ast.TableOption, tbInfo *model.TableInfo) err return nil } -func hasAutoIncrementColumn(tbInfo *model.TableInfo) (bool, string) { - for _, col := range tbInfo.Columns { - if mysql.HasAutoIncrementFlag(col.Flag) { - return true, col.Name.L - } - } - return false, "" -} - // isIgnorableSpec checks if the spec type is ignorable. // Some specs are parsed by ignored. This is for compatibility. func isIgnorableSpec(tp ast.AlterTableType) bool { diff --git a/ddl/generated_column.go b/ddl/generated_column.go index e9ee788ff033b..c0550a292efc3 100644 --- a/ddl/generated_column.go +++ b/ddl/generated_column.go @@ -18,6 +18,7 @@ import ( "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/table" ) @@ -238,7 +239,7 @@ func checkIndexOrStored(tbl table.Table, oldCol, newCol *table.Column) error { // checkAutoIncrementRef checks if an generated column depends on an auto-increment column and raises an error if so. // See https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html for details. func checkAutoIncrementRef(name string, dependencies map[string]struct{}, tbInfo *model.TableInfo) error { - exists, autoIncrementColumn := hasAutoIncrementColumn(tbInfo) + exists, autoIncrementColumn := infoschema.HasAutoIncrementColumn(tbInfo) if exists { if _, found := dependencies[autoIncrementColumn]; found { return ErrGeneratedColumnRefAutoInc.GenWithStackByArgs(name) diff --git a/executor/show_test.go b/executor/show_test.go index 675a2ccf1901e..c7306f63cd44d 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -267,7 +267,7 @@ func (s *testSuite2) TestShow2(c *C) { tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "192.168.0.1", AuthUsername: "root", AuthHostname: "%"}, nil, []byte("012345678901234567890")) r := tk.MustQuery("show table status from test like 't'") - r.Check(testkit.Rows(fmt.Sprintf("t InnoDB 10 Compact 0 0 0 0 0 0 0 %s utf8mb4_bin 注释", createTime))) + r.Check(testkit.Rows(fmt.Sprintf("t InnoDB 10 Compact 0 0 0 0 0 0 %s utf8mb4_bin 注释", createTime))) tk.MustQuery("show databases like 'test'").Check(testkit.Rows("test")) @@ -346,7 +346,7 @@ func (s *testSuite2) TestUnprivilegedShow(c *C) { c.Assert(err, IsNil) createTime := model.TSConvert2Time(tblInfo.Meta().UpdateTS).Format("2006-01-02 15:04:05") - tk.MustQuery("show table status from testshow").Check(testkit.Rows(fmt.Sprintf("t1 InnoDB 10 Compact 0 0 0 0 0 0 0 %s utf8mb4_bin ", createTime))) + tk.MustQuery("show table status from testshow").Check(testkit.Rows(fmt.Sprintf("t1 InnoDB 10 Compact 0 0 0 0 0 0 %s utf8mb4_bin ", createTime))) } diff --git a/infoschema/infoschema.go b/infoschema/infoschema.go index e2cfd3a7fe200..e9618d1544a29 100644 --- a/infoschema/infoschema.go +++ b/infoschema/infoschema.go @@ -411,3 +411,13 @@ func IsMemoryDB(dbName string) bool { } return false } + +// HasAutoIncrementColumn checks whether the table has auto_increment columns, if so, return true and the column name. +func HasAutoIncrementColumn(tbInfo *model.TableInfo) (bool, string) { + for _, col := range tbInfo.Columns { + if mysql.HasAutoIncrementFlag(col.Flag) { + return true, col.Name.L + } + } + return false, "" +} diff --git a/infoschema/tables.go b/infoschema/tables.go index 5282438b19549..3a447bb3264ae 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -1082,26 +1082,12 @@ func (c *statsCache) get(ctx sessionctx.Context) (map[int64]uint64, map[tableHis } func getAutoIncrementID(ctx sessionctx.Context, schema *model.DBInfo, tblInfo *model.TableInfo) (int64, error) { - hasAutoIncID := false - for _, col := range tblInfo.Cols() { - if mysql.HasAutoIncrementFlag(col.Flag) { - hasAutoIncID = true - break - } - } - autoIncID := tblInfo.AutoIncID - if hasAutoIncID { - is := ctx.GetSessionVars().TxnCtx.InfoSchema.(InfoSchema) - tbl, err := is.TableByName(schema.Name, tblInfo.Name) - if err != nil { - return 0, err - } - autoIncID, err = tbl.Allocator(ctx).NextGlobalAutoID(tblInfo.ID) - if err != nil { - return 0, err - } + is := ctx.GetSessionVars().TxnCtx.InfoSchema.(InfoSchema) + tbl, err := is.TableByName(schema.Name, tblInfo.Name) + if err != nil { + return 0, err } - return autoIncID, nil + return tbl.Allocator(ctx).Base() + 1, nil } func dataForViews(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.Datum, error) { @@ -1172,10 +1158,15 @@ func dataForTables(ctx sessionctx.Context, schemas []*model.DBInfo) ([][]types.D if table.GetPartitionInfo() != nil { createOptions = "partitioned" } - autoIncID, err := getAutoIncrementID(ctx, schema, table) - if err != nil { - return nil, err + var autoIncID interface{} + hasAutoIncID, _ := HasAutoIncrementColumn(table) + if hasAutoIncID { + autoIncID, err = getAutoIncrementID(ctx, schema, table) + if err != nil { + return nil, err + } } + rowCount := tableRowsMap[table.ID] dataLength, indexLength := getDataAndIndexLength(table, rowCount, colLengthMap) avgRowLength := uint64(0) diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index df0c95b2ab5a0..9e9220d015095 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -84,7 +84,21 @@ func (s *testTableSuite) TestInfoschemaFieldValue(c *C) { testkit.Rows("1")) tk.MustExec("insert into t(c, d) values(1, 1)") tk.MustQuery("select auto_increment from information_schema.tables where table_name='t'").Check( - testkit.Rows("30002")) + testkit.Rows("2")) + + tk.MustQuery("show create table t").Check( + testkit.Rows("" + + "t CREATE TABLE `t` (\n" + + " `c` int(11) NOT NULL AUTO_INCREMENT,\n" + + " `d` int(11) DEFAULT NULL,\n" + + " PRIMARY KEY (`c`)\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=30002")) + + // Test auto_increment for table without auto_increment column + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (d int)") + tk.MustQuery("select auto_increment from information_schema.tables where table_name='t'").Check( + testkit.Rows("")) tk.MustExec("create user xxx") tk.MustExec("flush privileges") From 388ce88743152411f046459cc8b3a3925f6d58d4 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 23 Jul 2019 16:18:48 +0800 Subject: [PATCH 070/196] ddl: refine ddl error when do ddl error cause by infoschema is changed (#11308) --- ddl/db_change_test.go | 13 +++++++++++++ executor/ddl.go | 18 +++++++++--------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index ee89526611a23..0c8e8e435903b 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -1058,3 +1058,16 @@ func (s *testStateChangeSuite) TestParallelAlterSchemaCharsetAndCollate(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustQuery(sql).Check(testkit.Rows("utf8mb4 utf8mb4_general_ci")) } + +// TestParallelTruncateTableAndAddColumn tests add column when truncate table. +func (s *testStateChangeSuite) TestParallelTruncateTableAndAddColumn(c *C) { + sql1 := "truncate table t" + sql2 := "alter table t add column c3 int" + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2, NotNil) + c.Assert(err2.Error(), Equals, "[domain:2]Information schema is changed. [try again later]") + + } + s.testControlParallelExecSQL(c, sql1, sql2, f) +} diff --git a/executor/ddl.go b/executor/ddl.go index fcbb14e1e9b3b..8d15d0cf1e6e1 100644 --- a/executor/ddl.go +++ b/executor/ddl.go @@ -49,14 +49,7 @@ type DDLExec struct { // toErr converts the error to the ErrInfoSchemaChanged when the schema is outdated. func (e *DDLExec) toErr(err error) error { - if e.ctx.GetSessionVars().StmtCtx.IsDDLJobInQueue { - return err - } - - // Before the DDL job is ready, it encouters an error that may be due to the outdated schema information. - // After the DDL job is ready, the ErrInfoSchemaChanged error won't happen because we are getting the schema directly from storage. - // So we needn't to consider this condition. - // Here we distinguish the ErrInfoSchemaChanged error from other errors. + // The err may be cause by schema changed, here we distinguish the ErrInfoSchemaChanged error from other errors. dom := domain.GetDomain(e.ctx) checker := domain.NewSchemaChecker(dom, e.is.SchemaMetaVersion(), nil) txn, err1 := e.ctx.Txn(true) @@ -118,7 +111,14 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { } if err != nil { - return e.toErr(err) + // If the owner return ErrTableNotExists error when running this DDL, it may be caused by schema changed, + // otherwise, ErrTableNotExists can be returned before putting this DDL job to the job queue. + if (e.ctx.GetSessionVars().StmtCtx.IsDDLJobInQueue && infoschema.ErrTableNotExists.Equal(err)) || + !e.ctx.GetSessionVars().StmtCtx.IsDDLJobInQueue { + return e.toErr(err) + } + return err + } dom := domain.GetDomain(e.ctx) From e0e5b50bf37088a2ba407ae45e78776e1b4b21f2 Mon Sep 17 00:00:00 2001 From: Liangliang Gu Date: Tue, 23 Jul 2019 16:45:47 +0800 Subject: [PATCH 071/196] test: port lock table ut from mysql (#10376) --- ddl/db_test.go | 132 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/ddl/db_test.go b/ddl/db_test.go index 0f4e890238aae..575b6017b4aca 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -2990,6 +2990,138 @@ func (s *testDBSuite4) TestAlterShardRowIDBits(c *C) { c.Assert(err.Error(), Equals, "[autoid:1467]Failed to read auto-increment value from storage engine") } +// port from mysql +// https://github.com/mysql/mysql-server/blob/124c7ab1d6f914637521fd4463a993aa73403513/mysql-test/t/lock.test +func (s *testDBSuite2) TestLock(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + tk := s.tk + tk.MustExec("use test") + + /* Testing of table locking */ + tk.MustExec("DROP TABLE IF EXISTS t1") + tk.MustExec("CREATE TABLE t1 ( `id` int(11) NOT NULL default '0', `id2` int(11) NOT NULL default '0', `id3` int(11) NOT NULL default '0', `dummy1` char(30) default NULL, PRIMARY KEY (`id`,`id2`), KEY `index_id3` (`id3`))") + tk.MustExec("insert into t1 (id,id2) values (1,1),(1,2),(1,3)") + tk.MustExec("LOCK TABLE t1 WRITE") + tk.MustExec("select dummy1,count(distinct id) from t1 group by dummy1") + tk.MustExec("update t1 set id=-1 where id=1") + tk.MustExec("LOCK TABLE t1 READ") + _, err := tk.Exec("update t1 set id=1 where id=1") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableNotLockedForWrite), IsTrue) + tk.MustExec("unlock tables") + tk.MustExec("update t1 set id=1 where id=-1") + tk.MustExec("drop table t1") +} + +// port from mysql +// https://github.com/mysql/mysql-server/blob/4f1d7cf5fcb11a3f84cff27e37100d7295e7d5ca/mysql-test/t/tablelock.test +func (s *testDBSuite2) TestTableLock(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + tk := s.tk + tk.MustExec("use test") + tk.MustExec("drop table if exists t1,t2") + + /* Test of lock tables */ + tk.MustExec("create table t1 ( n int auto_increment primary key)") + tk.MustExec("lock tables t1 write") + tk.MustExec("insert into t1 values(NULL)") + tk.MustExec("unlock tables") + checkTableLock(c, tk.Se, "test", "t1", model.TableLockNone) + + tk.MustExec("lock tables t1 write") + tk.MustExec("insert into t1 values(NULL)") + tk.MustExec("unlock tables") + checkTableLock(c, tk.Se, "test", "t1", model.TableLockNone) + + tk.MustExec("drop table if exists t1") + + /* Test of locking and delete of files */ + tk.MustExec("drop table if exists t1,t2") + tk.MustExec("CREATE TABLE t1 (a int)") + tk.MustExec("CREATE TABLE t2 (a int)") + tk.MustExec("lock tables t1 write, t2 write") + tk.MustExec("drop table t1,t2") + + tk.MustExec("CREATE TABLE t1 (a int)") + tk.MustExec("CREATE TABLE t2 (a int)") + tk.MustExec("lock tables t1 write, t2 write") + tk.MustExec("drop table t2,t1") +} + +// port from mysql +// https://github.com/mysql/mysql-server/blob/4f1d7cf5fcb11a3f84cff27e37100d7295e7d5ca/mysql-test/t/lock_tables_lost_commit.test +func (s *testDBSuite2) TestTableLocksLostCommit(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + tk2 := testkit.NewTestKit(c, s.store) + tk := s.tk + tk.MustExec("use test") + tk2.MustExec("use test") + + tk.MustExec("DROP TABLE IF EXISTS t1") + tk.MustExec("CREATE TABLE t1(a INT)") + tk.MustExec("LOCK TABLES t1 WRITE") + tk.MustExec("INSERT INTO t1 VALUES(10)") + + _, err := tk2.Exec("SELECT * FROM t1") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + + tk.Se.Close() + + tk2.MustExec("SELECT * FROM t1") + tk2.MustExec("DROP TABLE t1") + + tk.MustExec("unlock tables") +} + +// test write local lock +func (s *testDBSuite2) TestWriteLocal(c *C) { + s.tk = testkit.NewTestKit(c, s.store) + tk2 := testkit.NewTestKit(c, s.store) + tk := s.tk + tk.MustExec("use test") + tk2.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 ( n int auto_increment primary key)") + + // Test: allow read + tk.MustExec("lock tables t1 write local") + tk.MustExec("insert into t1 values(NULL)") + tk2.MustQuery("select count(*) from t1") + tk.MustExec("unlock tables") + tk2.MustExec("unlock tables") + + // Test: forbid write + tk.MustExec("lock tables t1 write local") + _, err := tk2.Exec("insert into t1 values(NULL)") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + tk.MustExec("unlock tables") + tk2.MustExec("unlock tables") + + // Test mutex: lock write local first + tk.MustExec("lock tables t1 write local") + _, err = tk2.Exec("lock tables t1 write local") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + _, err = tk2.Exec("lock tables t1 write") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + _, err = tk2.Exec("lock tables t1 read") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + tk.MustExec("unlock tables") + tk2.MustExec("unlock tables") + + // Test mutex: lock write first + tk.MustExec("lock tables t1 write") + _, err = tk2.Exec("lock tables t1 write local") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + tk.MustExec("unlock tables") + tk2.MustExec("unlock tables") + + // Test mutex: lock read first + tk.MustExec("lock tables t1 read") + _, err = tk2.Exec("lock tables t1 write local") + c.Assert(terror.ErrorEqual(err, infoschema.ErrTableLocked), IsTrue) + tk.MustExec("unlock tables") + tk2.MustExec("unlock tables") +} + func (s *testDBSuite2) TestLockTables(c *C) { if israce.RaceEnabled { c.Skip("skip race test") From ca70d74a284d96e1206021ad7de1fa5aa62d8556 Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Tue, 23 Jul 2019 17:02:24 +0800 Subject: [PATCH 072/196] =?UTF-8?q?executor,=20expression:=20fix=20current?= =?UTF-8?q?=5Ftimestamp/now=20not=20consistent=E2=80=A6=20(#11342)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/write_test.go | 12 +++++++ expression/builtin_time.go | 62 +++++++++++++++------------------ expression/builtin_time_test.go | 3 +- expression/helper.go | 27 +++++++------- sessionctx/stmtctx/stmtctx.go | 19 ++++++++-- 5 files changed, 74 insertions(+), 49 deletions(-) diff --git a/executor/write_test.go b/executor/write_test.go index 3a60e95c97bd0..43408d57ab03a 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -2538,3 +2538,15 @@ func (s *testSuite4) TestSetWithRefGenCol(c *C) { _, err = tk.Exec(`insert into t3 set j = i + 1`) c.Assert(err, NotNil) } + +func (s *testSuite4) TestSetWithCurrentTimestampAndNow(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`drop table if exists tbl;`) + tk.MustExec(`create table t1(c1 timestamp default current_timestamp, c2 int, c3 timestamp default current_timestamp);`) + //c1 insert using now() function result, c3 using default value calculation, should be same + tk.MustExec(`insert into t1 set c1 = current_timestamp, c2 = sleep(2);`) + tk.MustQuery("select c1 = c3 from t1").Check(testkit.Rows("1")) + tk.MustExec(`insert into t1 set c1 = current_timestamp, c2 = sleep(1);`) + tk.MustQuery("select c1 = c3 from t1").Check(testkit.Rows("1", "1")) +} diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 9b302873eb852..7a085453fdb06 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -2028,9 +2028,9 @@ func (b *builtinCurrentDateSig) Clone() builtinFunc { // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_curdate func (b *builtinCurrentDateSig) evalTime(row chunk.Row) (d types.Time, isNull bool, err error) { tz := b.ctx.GetSessionVars().Location() - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Time{}, true, err } year, month, day := nowTs.In(tz).Date() result := types.Time{ @@ -2087,9 +2087,9 @@ func (b *builtinCurrentTime0ArgSig) Clone() builtinFunc { func (b *builtinCurrentTime0ArgSig) evalDuration(row chunk.Row) (types.Duration, bool, error) { tz := b.ctx.GetSessionVars().Location() - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Duration{}, true, err } dur := nowTs.In(tz).Format(types.TimeFormat) res, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, dur, types.MinFsp) @@ -2115,9 +2115,9 @@ func (b *builtinCurrentTime1ArgSig) evalDuration(row chunk.Row) (types.Duration, return types.Duration{}, true, err } tz := b.ctx.GetSessionVars().Location() - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Duration{}, true, err } dur := nowTs.In(tz).Format(types.TimeFSPFormat) res, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, dur, int(fsp)) @@ -2257,9 +2257,9 @@ func (b *builtinUTCDateSig) Clone() builtinFunc { // evalTime evals UTC_DATE, UTC_DATE(). // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_utc-date func (b *builtinUTCDateSig) evalTime(row chunk.Row) (types.Time, bool, error) { - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Time{}, true, err } year, month, day := nowTs.UTC().Date() result := types.Time{ @@ -2318,9 +2318,9 @@ func (c *utcTimestampFunctionClass) getFunction(ctx sessionctx.Context, args []E } func evalUTCTimestampWithFsp(ctx sessionctx.Context, fsp int) (types.Time, bool, error) { - var nowTs = &ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(ctx) + if err != nil { + return types.Time{}, true, err } result, err := convertTimeToMysqlTime(nowTs.UTC(), fsp, types.ModeHalfEven) if err != nil { @@ -2405,13 +2405,9 @@ func (c *nowFunctionClass) getFunction(ctx sessionctx.Context, args []Expression } func evalNowWithFsp(ctx sessionctx.Context, fsp int) (types.Time, bool, error) { - var sysTs = &ctx.GetSessionVars().StmtCtx.SysTs - if sysTs.Equal(time.Time{}) { - var err error - *sysTs, err = getSystemTimestamp(ctx) - if err != nil { - return types.Time{}, true, err - } + nowTs, err := getStmtTimestamp(ctx) + if err != nil { + return types.Time{}, true, err } // In MySQL's implementation, now() will truncate the result instead of rounding it. @@ -2422,7 +2418,7 @@ func evalNowWithFsp(ctx sessionctx.Context, fsp int) (types.Time, bool, error) { // +----------------------------+-------------------------+---------------------+ // | 2019-03-25 15:57:56.612966 | 2019-03-25 15:57:56.612 | 2019-03-25 15:57:56 | // +----------------------------+-------------------------+---------------------+ - result, err := convertTimeToMysqlTime(*sysTs, fsp, types.ModeTruncate) + result, err := convertTimeToMysqlTime(nowTs, fsp, types.ModeTruncate) if err != nil { return types.Time{}, true, err } @@ -4278,11 +4274,11 @@ func (b *builtinUnixTimestampCurrentSig) Clone() builtinFunc { // evalInt evals a UNIX_TIMESTAMP(). // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_unix-timestamp func (b *builtinUnixTimestampCurrentSig) evalInt(row chunk.Row) (int64, bool, error) { - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return 0, true, err } - dec, err := goTimeToMysqlUnixTimestamp(*nowTs, 1) + dec, err := goTimeToMysqlUnixTimestamp(nowTs, 1) if err != nil { return 0, true, err } @@ -6340,9 +6336,9 @@ func (b *builtinUTCTimeWithoutArgSig) Clone() builtinFunc { // evalDuration evals a builtinUTCTimeWithoutArgSig. // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_utc-time func (b *builtinUTCTimeWithoutArgSig) evalDuration(row chunk.Row) (types.Duration, bool, error) { - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Duration{}, true, err } v, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, nowTs.UTC().Format(types.TimeFormat), 0) return v, false, err @@ -6371,9 +6367,9 @@ func (b *builtinUTCTimeWithArgSig) evalDuration(row chunk.Row) (types.Duration, if fsp < int64(types.MinFsp) { return types.Duration{}, true, errors.Errorf("Invalid negative %d specified, must in [0, 6].", fsp) } - var nowTs = &b.ctx.GetSessionVars().StmtCtx.NowTs - if nowTs.Equal(time.Time{}) { - *nowTs = time.Now() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return types.Duration{}, true, err } v, err := types.ParseDuration(b.ctx.GetSessionVars().StmtCtx, nowTs.UTC().Format(types.TimeFSPFormat), int(fsp)) return v, false, err diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index eebb80b524d51..c225ba7bd28c5 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -764,8 +764,7 @@ func (s *testEvaluatorSuite) TestTime(c *C) { } func resetStmtContext(ctx sessionctx.Context) { - ctx.GetSessionVars().StmtCtx.NowTs = time.Time{} - ctx.GetSessionVars().StmtCtx.SysTs = time.Time{} + ctx.GetSessionVars().StmtCtx.ResetNowTs() } func (s *testEvaluatorSuite) TestNowAndUTCTimestamp(c *C) { diff --git a/expression/helper.go b/expression/helper.go index cd150fcb6cc84..7b0f49e71a71d 100644 --- a/expression/helper.go +++ b/expression/helper.go @@ -54,7 +54,7 @@ func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int) (d ty case string: upperX := strings.ToUpper(x) if upperX == strings.ToUpper(ast.CurrentTimestamp) { - defaultTime, err := getSystemTimestamp(ctx) + defaultTime, err := getStmtTimestamp(ctx) if err != nil { return d, err } @@ -120,7 +120,9 @@ func GetTimeValue(ctx sessionctx.Context, v interface{}, tp byte, fsp int) (d ty return d, nil } -func getSystemTimestamp(ctx sessionctx.Context) (time.Time, error) { +// if timestamp session variable set, use session variable as current time, otherwise use cached time +// during one sql statement, the "current_time" should be the same +func getStmtTimestamp(ctx sessionctx.Context) (time.Time, error) { now := time.Now() if ctx == nil { @@ -133,15 +135,16 @@ func getSystemTimestamp(ctx sessionctx.Context) (time.Time, error) { return now, err } - if timestampStr == "" { - return now, nil - } - timestamp, err := types.StrToInt(sessionVars.StmtCtx, timestampStr) - if err != nil { - return time.Time{}, err - } - if timestamp <= 0 { - return now, nil + if timestampStr != "" { + timestamp, err := types.StrToInt(sessionVars.StmtCtx, timestampStr) + if err != nil { + return time.Time{}, err + } + if timestamp <= 0 { + return now, nil + } + return time.Unix(timestamp, 0), nil } - return time.Unix(timestamp, 0), nil + stmtCtx := ctx.GetSessionVars().StmtCtx + return stmtCtx.GetNowTsCached(), nil } diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index fdbc0f9ec6653..e19345d469ffd 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -120,8 +120,8 @@ type StatementContext struct { RuntimeStatsColl *execdetails.RuntimeStatsColl TableIDs []int64 IndexIDs []int64 - NowTs time.Time - SysTs time.Time + nowTs time.Time // use this variable for now/current_timestamp calculation/cache for one stmt + stmtTimeCached bool StmtType string OriginalSQL string digestMemo struct { @@ -132,6 +132,21 @@ type StatementContext struct { Tables []TableEntry } +// GetNowTsCached getter for nowTs, if not set get now time and cache it +func (sc *StatementContext) GetNowTsCached() time.Time { + if !sc.stmtTimeCached { + now := time.Now() + sc.nowTs = now + sc.stmtTimeCached = true + } + return sc.nowTs +} + +// ResetNowTs resetter for nowTs, clear cached time flag +func (sc *StatementContext) ResetNowTs() { + sc.stmtTimeCached = false +} + // SQLDigest gets normalized and digest for provided sql. // it will cache result after first calling. func (sc *StatementContext) SQLDigest() (normalized, sqlDigest string) { From 3b6d2f475f072da53d4c438c0d312fb0f1675a0c Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 23 Jul 2019 17:35:13 +0800 Subject: [PATCH 073/196] executor/split: return split result when do split region and refine split timeout logic. (#11259) --- ddl/split_region.go | 2 +- executor/builder.go | 5 +- executor/executor_test.go | 19 ++-- executor/split.go | 153 ++++++++++++++++++++-------- kv/kv.go | 2 +- planner/core/planbuilder.go | 9 ++ store/mockstore/mocktikv/cluster.go | 3 +- store/mockstore/mocktikv/rpc.go | 4 +- store/tikv/split_region.go | 9 +- 9 files changed, 143 insertions(+), 63 deletions(-) mode change 100644 => 100755 executor/split.go diff --git a/ddl/split_region.go b/ddl/split_region.go index 1c1a6263be69f..a7c710b381315 100644 --- a/ddl/split_region.go +++ b/ddl/split_region.go @@ -118,7 +118,7 @@ func splitIndexRegion(store kv.SplitableStore, tblInfo *model.TableInfo, scatter func waitScatterRegionFinish(store kv.SplitableStore, regionIDs ...uint64) { for _, regionID := range regionIDs { - err := store.WaitScatterRegionFinish(regionID) + err := store.WaitScatterRegionFinish(regionID, 0) if err != nil { logutil.BgLogger().Warn("[ddl] wait scatter region failed", zap.Uint64("regionID", regionID), zap.Error(err)) } diff --git a/executor/builder.go b/executor/builder.go index 20c4f9525e92a..bc41b4b25fc64 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1265,8 +1265,9 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo } func (b *executorBuilder) buildSplitRegion(v *plannercore.SplitRegion) Executor { - base := newBaseExecutor(b.ctx, nil, v.ExplainID()) - base.initCap = chunk.ZeroCapacity + base := newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()) + base.initCap = 1 + base.maxChunkSize = 1 if v.IndexInfo != nil { return &SplitIndexRegionExec{ baseExecutor: base, diff --git a/executor/executor_test.go b/executor/executor_test.go index 3eda72e7bbe07..762d4fd044d90 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2074,16 +2074,9 @@ func (s *testSuite4) TestSplitRegionTimeout(c *C) { tk.MustExec("create table t(a varchar(100),b int, index idx1(b,a))") tk.MustExec(`split table t index idx1 by (10000,"abcd"),(10000000);`) tk.MustExec(`set @@tidb_wait_split_region_timeout=1`) - _, err := tk.Exec(`split table t between (0) and (10000) regions 10`) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "split region timeout(1s)") + // result 0 0 means split 0 region and 0 region finish scatter regions before timeout. + tk.MustQuery(`split table t between (0) and (10000) regions 10`).Check(testkit.Rows("0 0")) c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/mockSplitRegionTimeout"), IsNil) - - c.Assert(failpoint.Enable("github.com/pingcap/tidb/executor/mockScatterRegionTimeout", `return(true)`), IsNil) - _, err = tk.Exec(`split table t between (0) and (10000) regions 10`) - c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "wait split region scatter timeout(1s)") - c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/mockScatterRegionTimeout"), IsNil) } func (s *testSuiteP1) TestRow(c *C) { @@ -4059,7 +4052,7 @@ func (s *testSuite) TestShowTableRegion(c *C) { // Test show table regions. tk.MustExec(`split table t_regions1 by (0)`) - tk.MustExec(`split table t_regions between (-10000) and (10000) regions 4;`) + tk.MustQuery(`split table t_regions between (-10000) and (10000) regions 4;`).Check(testkit.Rows("3 1")) re := tk.MustQuery("show table t_regions regions") rows := re.Rows() // Table t_regions should have 4 regions now. @@ -4074,7 +4067,7 @@ func (s *testSuite) TestShowTableRegion(c *C) { c.Assert(rows[3][1], Equals, fmt.Sprintf("t_%d_r_5000", tbl.Meta().ID)) // Test show table index regions. - tk.MustExec(`split table t_regions index idx between (-1000) and (1000) regions 4;`) + tk.MustQuery(`split table t_regions index idx between (-1000) and (1000) regions 4;`).Check(testkit.Rows("4 1")) re = tk.MustQuery("show table t_regions index idx regions") rows = re.Rows() // The index `idx` of table t_regions should have 4 regions now. @@ -4104,7 +4097,7 @@ func (s *testSuite) TestShowTableRegion(c *C) { // Test show table regions. tk.MustExec(`set @@session.tidb_wait_split_region_finish=1;`) - tk.MustExec(`split table t_regions between (0) and (10000) regions 4;`) + tk.MustQuery(`split table t_regions by (2500),(5000),(7500);`).Check(testkit.Rows("3 1")) re = tk.MustQuery("show table t_regions regions") rows = re.Rows() // Table t_regions should have 4 regions now. @@ -4117,7 +4110,7 @@ func (s *testSuite) TestShowTableRegion(c *C) { c.Assert(rows[3][1], Equals, fmt.Sprintf("t_%d_r_7500", tbl.Meta().ID)) // Test show table index regions. - tk.MustExec(`split table t_regions index idx between (0) and (1000) regions 4;`) + tk.MustQuery(`split table t_regions index idx by (250),(500),(750);`).Check(testkit.Rows("4 1")) re = tk.MustQuery("show table t_regions index idx regions") rows = re.Rows() // The index `idx` of table t_regions should have 4 regions now. diff --git a/executor/split.go b/executor/split.go old mode 100644 new mode 100755 index c004d4a77831a..eefda2fa65601 --- a/executor/split.go +++ b/executor/split.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -48,10 +49,37 @@ type SplitIndexRegionExec struct { upper []types.Datum num int valueLists [][]types.Datum + + done bool + splitRegionResult +} + +type splitRegionResult struct { + splitRegions int + finishScatterNum int +} + +// Open implements the Executor Open interface. +func (e *SplitIndexRegionExec) Open(ctx context.Context) error { + return e.splitIndexRegion(ctx) } // Next implements the Executor Next interface. -func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { +func (e *SplitIndexRegionExec) Next(ctx context.Context, chk *chunk.Chunk) error { + chk.Reset() + if e.done { + return nil + } + appendSplitRegionResultToChunk(chk, e.splitRegions, e.finishScatterNum) + e.done = true + return nil +} + +// checkScatterRegionFinishBackOff is the back off time that used to check if a region has finished scattering before split region timeout. +const checkScatterRegionFinishBackOff = 50 + +// splitIndexRegion is used to split index regions. +func (e *SplitIndexRegionExec) splitIndexRegion(ctx context.Context) error { store := e.ctx.GetStore() s, ok := store.(kv.SplitableStore) if !ok { @@ -62,10 +90,15 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { return err } + start := time.Now() ctxWithTimeout, cancel := context.WithTimeout(ctx, e.ctx.GetSessionVars().GetSplitRegionTimeout()) defer cancel() regionIDs := make([]uint64, 0, len(splitIdxKeys)) for _, idxKey := range splitIdxKeys { + if isCtxDone(ctxWithTimeout) { + break + } + regionID, err := s.SplitRegion(idxKey, true) if err != nil { logutil.BgLogger().Warn("split table index region failed", @@ -74,28 +107,17 @@ func (e *SplitIndexRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { zap.Error(err)) continue } + if regionID == 0 { + continue + } regionIDs = append(regionIDs, regionID) - if isCtxDone(ctxWithTimeout) { - return errors.Errorf("wait split region timeout(%v)", e.ctx.GetSessionVars().GetSplitRegionTimeout()) - } } + e.splitRegions = len(regionIDs) if !e.ctx.GetSessionVars().WaitSplitRegionFinish { return nil } - for _, regionID := range regionIDs { - err := s.WaitScatterRegionFinish(regionID) - if err != nil { - logutil.BgLogger().Warn("wait scatter region failed", - zap.Uint64("regionID", regionID), - zap.String("table", e.tableInfo.Name.L), - zap.String("index", e.indexInfo.Name.L), - zap.Error(err)) - } - if isCtxDone(ctxWithTimeout) { - return errors.Errorf("wait split region timeout(%v)", e.ctx.GetSessionVars().GetSplitRegionTimeout()) - } - } + e.finishScatterNum = waitScatterRegionFinish(ctxWithTimeout, e.ctx, start, s, regionIDs, e.tableInfo.Name.L, e.indexInfo.Name.L) return nil } @@ -225,16 +247,35 @@ type SplitTableRegionExec struct { upper types.Datum num int valueLists [][]types.Datum + + done bool + splitRegionResult +} + +// Open implements the Executor Open interface. +func (e *SplitTableRegionExec) Open(ctx context.Context) error { + return e.splitTableRegion(ctx) } // Next implements the Executor Next interface. -func (e *SplitTableRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { +func (e *SplitTableRegionExec) Next(ctx context.Context, chk *chunk.Chunk) error { + chk.Reset() + if e.done { + return nil + } + appendSplitRegionResultToChunk(chk, e.splitRegions, e.finishScatterNum) + e.done = true + return nil +} + +func (e *SplitTableRegionExec) splitTableRegion(ctx context.Context) error { store := e.ctx.GetStore() s, ok := store.(kv.SplitableStore) if !ok { return nil } + start := time.Now() ctxWithTimeout, cancel := context.WithTimeout(ctx, e.ctx.GetSessionVars().GetSplitRegionTimeout()) defer cancel() @@ -244,6 +285,14 @@ func (e *SplitTableRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { } regionIDs := make([]uint64, 0, len(splitKeys)) for _, key := range splitKeys { + failpoint.Inject("mockSplitRegionTimeout", func(val failpoint.Value) { + if val.(bool) { + time.Sleep(time.Second*1 + time.Millisecond*10) + } + }) + if isCtxDone(ctxWithTimeout) { + break + } regionID, err := s.SplitRegion(key, true) if err != nil { logutil.BgLogger().Warn("split table region failed", @@ -251,41 +300,63 @@ func (e *SplitTableRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { zap.Error(err)) continue } + if regionID == 0 { + continue + } regionIDs = append(regionIDs, regionID) - failpoint.Inject("mockSplitRegionTimeout", func(val failpoint.Value) { - if val.(bool) { - time.Sleep(time.Second * 1) - } - }) - - if isCtxDone(ctxWithTimeout) { - return errors.Errorf("split region timeout(%v)", e.ctx.GetSessionVars().GetSplitRegionTimeout()) - } } + e.splitRegions = len(regionIDs) if !e.ctx.GetSessionVars().WaitSplitRegionFinish { return nil } + + e.finishScatterNum = waitScatterRegionFinish(ctxWithTimeout, e.ctx, start, s, regionIDs, e.tableInfo.Name.L, "") + return nil +} + +func waitScatterRegionFinish(ctxWithTimeout context.Context, sctx sessionctx.Context, startTime time.Time, store kv.SplitableStore, regionIDs []uint64, tableName, indexName string) int { + remainMillisecond := 0 + finishScatterNum := 0 for _, regionID := range regionIDs { - err := s.WaitScatterRegionFinish(regionID) - if err != nil { - logutil.BgLogger().Warn("wait scatter region failed", - zap.Uint64("regionID", regionID), - zap.String("table", e.tableInfo.Name.L), - zap.Error(err)) + if isCtxDone(ctxWithTimeout) { + // Do not break here for checking remain regions scatter finished with a very short backoff time. + // Consider this situation - Regions 1, 2, and 3 are to be split. + // Region 1 times out before scattering finishes, while Region 2 and Region 3 have finished scattering. + // In this case, we should return 2 Regions, instead of 0, have finished scattering. + remainMillisecond = checkScatterRegionFinishBackOff + } else { + remainMillisecond = int((sctx.GetSessionVars().GetSplitRegionTimeout().Seconds() - time.Since(startTime).Seconds()) * 1000) } - failpoint.Inject("mockScatterRegionTimeout", func(val failpoint.Value) { - if val.(bool) { - time.Sleep(time.Second * 1) + err := store.WaitScatterRegionFinish(regionID, remainMillisecond) + if err == nil { + finishScatterNum++ + } else { + if len(indexName) == 0 { + logutil.BgLogger().Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", tableName), + zap.Error(err)) + } else { + logutil.BgLogger().Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", tableName), + zap.String("index", indexName), + zap.Error(err)) } - }) - - if isCtxDone(ctxWithTimeout) { - return errors.Errorf("wait split region scatter timeout(%v)", e.ctx.GetSessionVars().GetSplitRegionTimeout()) } } - return nil + return finishScatterNum +} + +func appendSplitRegionResultToChunk(chk *chunk.Chunk, totalRegions, finishScatterNum int) { + chk.AppendInt64(0, int64(totalRegions)) + if finishScatterNum > 0 && totalRegions > 0 { + chk.AppendFloat64(1, float64(finishScatterNum)/float64(totalRegions)) + } else { + chk.AppendFloat64(1, float64(0)) + } } func isCtxDone(ctx context.Context) bool { diff --git a/kv/kv.go b/kv/kv.go index ff615b521a089..67daa736110ef 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -307,6 +307,6 @@ type Iterator interface { // SplitableStore is the kv store which supports split regions. type SplitableStore interface { SplitRegion(splitKey Key, scatter bool) (regionID uint64, err error) - WaitScatterRegionFinish(regionID uint64) error + WaitScatterRegionFinish(regionID uint64, backOff int) error CheckRegionInScattering(regionID uint64) (bool, error) } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index ebae4fc1b4dee..1f4ecea4fa3e8 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1002,6 +1002,13 @@ func buildTableRegionsSchema() *expression.Schema { return schema } +func buildSplitRegionsSchema() *expression.Schema { + schema := expression.NewSchema(make([]*expression.Column, 0, 2)...) + schema.Append(buildColumn("", "TOTAL_SPLIT_REGION", mysql.TypeLonglong, 4)) + schema.Append(buildColumn("", "SCATTER_FINISH_RATIO", mysql.TypeDouble, 8)) + return schema +} + func buildShowDDLJobQueriesFields() *expression.Schema { schema := expression.NewSchema(make([]*expression.Column, 0, 1)...) schema.Append(buildColumn("", "QUERY", mysql.TypeVarchar, 256)) @@ -1677,6 +1684,7 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, er TableInfo: tblInfo, IndexInfo: indexInfo, } + p.SetSchema(buildSplitRegionsSchema()) // Split index regions by user specified value lists. if len(node.SplitOpt.ValueLists) > 0 { indexValues := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) @@ -1791,6 +1799,7 @@ func (b *PlanBuilder) buildSplitTableRegion(node *ast.SplitRegionStmt) (Plan, er p := &SplitRegion{ TableInfo: tblInfo, } + p.SetSchema(buildSplitRegionsSchema()) if len(node.SplitOpt.ValueLists) > 0 { values := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) for i, valuesItem := range node.SplitOpt.ValueLists { diff --git a/store/mockstore/mocktikv/cluster.go b/store/mockstore/mocktikv/cluster.go index f4deaca856c4c..3196e1374be91 100644 --- a/store/mockstore/mocktikv/cluster.go +++ b/store/mockstore/mocktikv/cluster.go @@ -363,12 +363,13 @@ func (c *Cluster) Split(regionID, newRegionID uint64, key []byte, peerIDs []uint } // SplitRaw splits a Region at the key (not encoded) and creates new Region. -func (c *Cluster) SplitRaw(regionID, newRegionID uint64, rawKey []byte, peerIDs []uint64, leaderPeerID uint64) { +func (c *Cluster) SplitRaw(regionID, newRegionID uint64, rawKey []byte, peerIDs []uint64, leaderPeerID uint64) *Region { c.Lock() defer c.Unlock() newRegion := c.regions[regionID].split(newRegionID, rawKey, peerIDs, leaderPeerID) c.regions[newRegionID] = newRegion + return newRegion } // Merge merges 2 regions, their key ranges should be adjacent. diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index 99866195b17a9..2081b7f0ece39 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -609,8 +609,8 @@ func (h *rpcHandler) handleSplitRegion(req *kvrpcpb.SplitRegionRequest) *kvrpcpb return &kvrpcpb.SplitRegionResponse{} } newRegionID, newPeerIDs := h.cluster.AllocID(), h.cluster.AllocIDs(len(region.Peers)) - h.cluster.SplitRaw(region.GetId(), newRegionID, key, newPeerIDs, newPeerIDs[0]) - return &kvrpcpb.SplitRegionResponse{} + newRegion := h.cluster.SplitRaw(region.GetId(), newRegionID, key, newPeerIDs, newPeerIDs[0]) + return &kvrpcpb.SplitRegionResponse{Left: newRegion.Meta} } // RPCClient sends kv RPC calls to mock cluster. RPCClient mocks the behavior of diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index a232573575682..0c20f2058d3c2 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -103,10 +103,15 @@ func (s *tikvStore) scatterRegion(regionID uint64) error { } // WaitScatterRegionFinish implements SplitableStore interface. -func (s *tikvStore) WaitScatterRegionFinish(regionID uint64) error { +// backOff is the back off time of the wait scatter region.(Milliseconds) +// if backOff <= 0, the default wait scatter back off time will be used. +func (s *tikvStore) WaitScatterRegionFinish(regionID uint64, backOff int) error { logutil.BgLogger().Info("wait scatter region", zap.Uint64("regionID", regionID)) - bo := NewBackoffer(context.Background(), waitScatterRegionFinishBackoff) + if backOff <= 0 { + backOff = waitScatterRegionFinishBackoff + } + bo := NewBackoffer(context.Background(), backOff) logFreq := 0 for { resp, err := s.pdClient.GetOperator(context.Background(), regionID) From 119d5329798b6ff7bb7894f9b6977ec17139174a Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Tue, 23 Jul 2019 18:41:52 +0800 Subject: [PATCH 074/196] =?UTF-8?q?executor:=20load=20data/batch=20insert?= =?UTF-8?q?=20improvement=20reducing=20memory=20a=E2=80=A6=20(#11284)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/builder.go | 3 +++ executor/insert_common.go | 21 ++++++++++++++++++--- executor/load_data.go | 23 +++++++++++++++-------- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index bc41b4b25fc64..0b4de52b007a5 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -681,6 +681,9 @@ func (b *executorBuilder) buildLoadData(v *plannercore.LoadData) Executor { }, } + var defaultLoadDataBatchCnt uint64 = 20000 // TODO this will be changed to variable in another pr + loadDataExec.loadDataInfo.SetMaxRowsInBatch(defaultLoadDataBatchCnt) + return loadDataExec } diff --git a/executor/insert_common.go b/executor/insert_common.go index f2ef0ed89a68c..71ad672830378 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -36,6 +36,7 @@ type InsertValues struct { batchChecker rowCount uint64 + curBatchCnt uint64 maxRowsInBatch uint64 lastInsertID uint64 hasRefCols bool @@ -379,6 +380,20 @@ func (e *InsertValues) getRow(ctx context.Context, vals []types.Datum) ([]types. return e.fillRow(ctx, row, hasValue) } +func (e *InsertValues) getRowInPlace(ctx context.Context, vals []types.Datum, rowBuf []types.Datum) ([]types.Datum, error) { + hasValue := make([]bool, len(e.Table.Cols())) + for i, v := range vals { + casted, err := table.CastValue(e.ctx, v, e.insertColumns[i].ToInfo()) + if e.filterErr(err) != nil { + return nil, err + } + offset := e.insertColumns[i].Offset + rowBuf[offset] = casted + hasValue[offset] = true + } + return e.fillRow(ctx, rowBuf, hasValue) +} + func (e *InsertValues) filterErr(err error) error { if err == nil { return nil @@ -565,9 +580,9 @@ func (e *InsertValues) batchCheckAndInsert(rows [][]types.Datum, addRecord func( } // append warnings and get no duplicated error rows for i, r := range e.toBeCheckedRows { + skip := false if r.handleKey != nil { if _, found := e.dupKVs[string(r.handleKey.newKV.key)]; found { - rows[i] = nil e.ctx.GetSessionVars().StmtCtx.AppendWarning(r.handleKey.dupErr) continue } @@ -575,15 +590,15 @@ func (e *InsertValues) batchCheckAndInsert(rows [][]types.Datum, addRecord func( for _, uk := range r.uniqueKeys { if _, found := e.dupKVs[string(uk.newKV.key)]; found { // If duplicate keys were found in BatchGet, mark row = nil. - rows[i] = nil e.ctx.GetSessionVars().StmtCtx.AppendWarning(uk.dupErr) + skip = true break } } // If row was checked with no duplicate keys, // it should be add to values map for the further row check. // There may be duplicate keys inside the insert statement. - if rows[i] != nil { + if !skip { e.ctx.GetSessionVars().StmtCtx.AddCopiedRows(1) _, err = addRecord(rows[i]) if err != nil { diff --git a/executor/load_data.go b/executor/load_data.go index 16d55d5f36114..dc4c4f4024444 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" @@ -112,7 +113,12 @@ type LoadDataInfo struct { // SetMaxRowsInBatch sets the max number of rows to insert in a batch. func (e *LoadDataInfo) SetMaxRowsInBatch(limit uint64) { e.maxRowsInBatch = limit - e.rows = make([][]types.Datum, 0, limit) + if uint64(cap(e.rows)) < limit { + e.rows = make([][]types.Datum, 0, limit) + for i := 0; uint64(i) < limit; i++ { + e.rows = append(e.rows, make([]types.Datum, len(e.Table.Cols()))) + } + } } // getValidData returns prevData and curData that starts from starting symbol. @@ -174,7 +180,7 @@ func (e *LoadDataInfo) getLine(prevData, curData []byte) ([]byte, []byte, bool) } endIdx := -1 if len(curData) >= curStartIdx { - endIdx = strings.Index(string(curData[curStartIdx:]), e.LinesInfo.Terminated) + endIdx = strings.Index(string(hack.String(curData[curStartIdx:])), e.LinesInfo.Terminated) } if endIdx == -1 { // no terminated symbol @@ -253,8 +259,9 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) if err != nil { return nil, false, err } - e.rows = append(e.rows, e.colsToRow(ctx, cols)) + e.colsToRow(ctx, cols) e.rowCount++ + e.curBatchCnt++ if e.maxRowsInBatch != 0 && e.rowCount%e.maxRowsInBatch == 0 { reachLimit = true logutil.BgLogger().Info("batch limit hit when inserting rows", zap.Int("maxBatchRows", e.maxChunkSize), @@ -268,15 +275,15 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) // CheckAndInsertOneBatch is used to commit one transaction batch full filled data func (e *LoadDataInfo) CheckAndInsertOneBatch() error { var err error - if len(e.rows) == 0 { + if e.curBatchCnt == 0 { return err } - e.ctx.GetSessionVars().StmtCtx.AddRecordRows(uint64(len(e.rows))) - err = e.batchCheckAndInsert(e.rows, e.addRecordLD) + e.ctx.GetSessionVars().StmtCtx.AddRecordRows(e.curBatchCnt) + err = e.batchCheckAndInsert(e.rows[0:e.curBatchCnt], e.addRecordLD) if err != nil { return err } - e.rows = e.rows[:0] + e.curBatchCnt = 0 return err } @@ -312,7 +319,7 @@ func (e *LoadDataInfo) colsToRow(ctx context.Context, cols []field) []types.Datu e.row[i].SetString(string(cols[i].str)) } } - row, err := e.getRow(ctx, e.row) + row, err := e.getRowInPlace(ctx, e.row, e.rows[e.curBatchCnt]) if err != nil { e.handleWarning(err) return nil From 3df70a7c2066a9a203ecf671855c021788848d8a Mon Sep 17 00:00:00 2001 From: lysu Date: Tue, 23 Jul 2019 20:00:38 +0800 Subject: [PATCH 075/196] codec: pre-alloc encode buffer in row level (#11218) --- util/codec/codec.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/util/codec/codec.go b/util/codec/codec.go index 19cbfff6c7977..3ea30921b0761 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -42,9 +42,33 @@ const ( maxFlag byte = 250 ) +func preRealloc(b []byte, vals []types.Datum, comparable bool) []byte { + var size int + for i := range vals { + switch vals[i].Kind() { + case types.KindInt64, types.KindUint64, types.KindMysqlEnum, types.KindMysqlSet, types.KindMysqlBit, types.KindBinaryLiteral: + size += sizeInt(comparable) + case types.KindString, types.KindBytes: + size += sizeBytes(vals[i].GetBytes(), comparable) + case types.KindMysqlTime, types.KindMysqlDuration, types.KindFloat32, types.KindFloat64: + size += 9 + case types.KindNull, types.KindMinNotNull, types.KindMaxValue: + size += 1 + case types.KindMysqlJSON: + size += 2 + len(vals[i].GetBytes()) + case types.KindMysqlDecimal: + size += 1 + types.MyDecimalStructSize + default: + return b + } + } + return reallocBytes(b, size) +} + // encode will encode a datum and append it to a byte slice. If comparable is true, the encoded bytes can be sorted as it's original order. // If hash is true, the encoded bytes can be checked equal as it's original value. func encode(sc *stmtctx.StatementContext, b []byte, vals []types.Datum, comparable bool, hash bool) (_ []byte, err error) { + b = preRealloc(b, vals, comparable) for i, length := 0, len(vals); i < length; i++ { switch vals[i].Kind() { case types.KindInt64: @@ -157,6 +181,15 @@ func encodeBytes(b []byte, v []byte, comparable bool) []byte { return b } +func sizeBytes(v []byte, comparable bool) int { + if comparable { + reallocSize := (len(v)/encGroupSize + 1) * (encGroupSize + 1) + return 1 + reallocSize + } + reallocSize := binary.MaxVarintLen64 + len(v) + return 1 + reallocSize +} + func encodeSignedInt(b []byte, v int64, comparable bool) []byte { if comparable { b = append(b, intFlag) @@ -179,6 +212,13 @@ func encodeUnsignedInt(b []byte, v uint64, comparable bool) []byte { return b } +func sizeInt(comparable bool) int { + if comparable { + return 9 + } + return 1 + binary.MaxVarintLen64 +} + // EncodeKey appends the encoded values to byte slice b, returns the appended // slice. It guarantees the encoded value is in ascending order for comparison. // For Decimal type, datum must set datum's length and frac. From 899ff96e8685ebb4482fe3c456346ad1b1b07807 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 23 Jul 2019 21:00:30 +0800 Subject: [PATCH 076/196] =?UTF-8?q?executor:=20introduce=20`Sel`=20to=20`C?= =?UTF-8?q?hunk`=20to=20indicate=20which=20rows=20ar=E2=80=A6=20(#11384)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/joiner.go | 2 +- executor/window.go | 13 +++-- expression/evaluator.go | 9 ++- util/chunk/chunk.go | 83 +++++++++++++++++++++++++- util/chunk/chunk_test.go | 18 ++++++ util/chunk/chunk_util.go | 11 +++- util/chunk/column.go | 52 +++++++++++++++++ util/chunk/column_test.go | 113 ++++++++++++++++++++++++++++++++++++ util/chunk/iterator_test.go | 20 +++++++ 9 files changed, 308 insertions(+), 13 deletions(-) diff --git a/executor/joiner.go b/executor/joiner.go index 8795e4de095b1..75f3b39a1f670 100644 --- a/executor/joiner.go +++ b/executor/joiner.go @@ -179,7 +179,7 @@ func (j *baseJoiner) filter(input, output *chunk.Chunk, outerColsLen int) (bool, if !j.outerIsRight { innerColOffset, outerColOffset = outerColsLen, 0 } - return chunk.CopySelectedJoinRows(input, innerColOffset, outerColOffset, j.selected, output), nil + return chunk.CopySelectedJoinRows(input, innerColOffset, outerColOffset, j.selected, output) } type semiJoiner struct { diff --git a/executor/window.go b/executor/window.go index d1237ee41f0ce..ee151cfd80fe6 100644 --- a/executor/window.go +++ b/executor/window.go @@ -140,7 +140,9 @@ func (e *WindowExec) fetchChildIfNecessary(ctx context.Context, chk *chunk.Chunk // appendResult2Chunk appends result of the window function to the result chunk. func (e *WindowExec) appendResult2Chunk(chk *chunk.Chunk) (err error) { - e.copyChk(chk) + if err := e.copyChk(chk); err != nil { + return err + } remained := mathutil.Min(e.remainingRowsInChunk, e.remainingRowsInGroup) e.groupRows, err = e.processor.appendResult2Chunk(e.ctx, e.groupRows, chk, remained) if err != nil { @@ -155,17 +157,20 @@ func (e *WindowExec) appendResult2Chunk(chk *chunk.Chunk) (err error) { return nil } -func (e *WindowExec) copyChk(chk *chunk.Chunk) { +func (e *WindowExec) copyChk(chk *chunk.Chunk) error { if len(e.childResults) == 0 || chk.NumRows() > 0 { - return + return nil } childResult := e.childResults[0] e.childResults = e.childResults[1:] e.remainingRowsInChunk = childResult.NumRows() columns := e.Schema().Columns[:len(e.Schema().Columns)-e.numWindowFuncs] for i, col := range columns { - chk.MakeRefTo(i, childResult, col.Index) + if err := chk.MakeRefTo(i, childResult, col.Index); err != nil { + return err + } } + return nil } // windowProcessor is the interface for processing different kinds of windows. diff --git a/expression/evaluator.go b/expression/evaluator.go index 5c4cf59eb202d..c0e401d69ffbb 100644 --- a/expression/evaluator.go +++ b/expression/evaluator.go @@ -25,13 +25,16 @@ type columnEvaluator struct { // run evaluates "Column" expressions. // NOTE: It should be called after all the other expressions are evaluated // since it will change the content of the input Chunk. -func (e *columnEvaluator) run(ctx sessionctx.Context, input, output *chunk.Chunk) { +func (e *columnEvaluator) run(ctx sessionctx.Context, input, output *chunk.Chunk) error { for inputIdx, outputIdxes := range e.inputIdxToOutputIdxes { - output.SwapColumn(outputIdxes[0], input, inputIdx) + if err := output.SwapColumn(outputIdxes[0], input, inputIdx); err != nil { + return err + } for i, length := 1, len(outputIdxes); i < length; i++ { output.MakeRef(outputIdxes[0], outputIdxes[i]) } } + return nil } type defaultEvaluator struct { @@ -117,7 +120,7 @@ func (e *EvaluatorSuite) Run(ctx sessionctx.Context, input, output *chunk.Chunk) } if e.columnEvaluator != nil { - e.columnEvaluator.run(ctx, input, output) + return e.columnEvaluator.run(ctx, input, output) } return nil } diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 4b78ebdcd4de2..8f6d166787905 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -19,15 +19,22 @@ import ( "unsafe" "github.com/cznic/mathutil" + "github.com/pingcap/errors" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" ) +var msgErrSelNotNil = "The selection vector of Chunk is not nil. Please file a bug to the TiDB Team" + // Chunk stores multiple rows of data in Apache Arrow format. // See https://arrow.apache.org/docs/memory_layout.html // Values are appended in compact format and can be directly accessed without decoding. // When the chunk is done processing, we can reuse the allocated memory by resetting it. type Chunk struct { + // sel indicates which rows are selected. + // If it is nil, all rows are selected. + sel []int + columns []*Column // numVirtualRows indicates the number of virtual rows, which have zero Column. // It is used only when this Chunk doesn't hold any data, i.e. "len(columns)==0". @@ -165,14 +172,21 @@ func (c *Chunk) MakeRef(srcColIdx, dstColIdx int) { } // MakeRefTo copies columns `src.columns[srcColIdx]` to `c.columns[dstColIdx]`. -func (c *Chunk) MakeRefTo(dstColIdx int, src *Chunk, srcColIdx int) { +func (c *Chunk) MakeRefTo(dstColIdx int, src *Chunk, srcColIdx int) error { + if c.sel != nil || src.sel != nil { + return errors.New(msgErrSelNotNil) + } c.columns[dstColIdx] = src.columns[srcColIdx] + return nil } // SwapColumn swaps Column "c.columns[colIdx]" with Column // "other.columns[otherIdx]". If there exists columns refer to the Column to be // swapped, we need to re-build the reference. -func (c *Chunk) SwapColumn(colIdx int, other *Chunk, otherIdx int) { +func (c *Chunk) SwapColumn(colIdx int, other *Chunk, otherIdx int) error { + if c.sel != nil || other.sel != nil { + return errors.New(msgErrSelNotNil) + } // Find the leftmost Column of the reference which is the actual Column to // be swapped. for i := 0; i < colIdx; i++ { @@ -210,10 +224,12 @@ func (c *Chunk) SwapColumn(colIdx int, other *Chunk, otherIdx int) { for _, i := range refColsIdx4Other { other.MakeRef(otherIdx, i) } + return nil } // SwapColumns swaps columns with another Chunk. func (c *Chunk) SwapColumns(other *Chunk) { + c.sel, other.sel = other.sel, c.sel c.columns, other.columns = other.columns, c.columns c.numVirtualRows, other.numVirtualRows = other.numVirtualRows, c.numVirtualRows } @@ -227,6 +243,7 @@ func (c *Chunk) SetNumVirtualRows(numVirtualRows int) { // Reset resets the chunk, so the memory it allocated can be reused. // Make sure all the data in the chunk is not used anymore before you reuse this chunk. func (c *Chunk) Reset() { + c.sel = nil if c.columns == nil { return } @@ -242,6 +259,10 @@ func (c *Chunk) CopyConstruct() *Chunk { for i := range c.columns { newChk.columns[i] = c.columns[i].copyConstruct() } + if c.sel != nil { + newChk.sel = make([]int, len(c.sel)) + copy(newChk.sel, c.sel) + } return newChk } @@ -249,6 +270,7 @@ func (c *Chunk) CopyConstruct() *Chunk { // The doubled capacity should not be larger than maxChunkSize. // TODO: this method will be used in following PR. func (c *Chunk) GrowAndReset(maxChunkSize int) { + c.sel = nil if c.columns == nil { return } @@ -284,6 +306,9 @@ func (c *Chunk) NumCols() int { // NumRows returns the number of rows in the chunk. func (c *Chunk) NumRows() int { + if c.sel != nil { + return len(c.sel) + } if c.NumCols() == 0 { return c.numVirtualRows } @@ -292,6 +317,16 @@ func (c *Chunk) NumRows() int { // GetRow gets the Row in the chunk with the row index. func (c *Chunk) GetRow(idx int) Row { + if c.sel != nil { + // mapping the logical RowIdx to the actual physical RowIdx; + // for example, if the Sel is [1, 5, 6], then + // logical 0 -> physical 1, + // logical 1 -> physical 5, + // logical 2 -> physical 6. + // Then when we iterate this Chunk according to Row, only selected rows will be + // accessed while all filtered rows will be ignored. + return Row{c: c, idx: int(c.sel[idx])} + } return Row{c: c, idx: idx} } @@ -303,6 +338,7 @@ func (c *Chunk) AppendRow(row Row) { // AppendPartialRow appends a row to the chunk. func (c *Chunk) AppendPartialRow(colIdx int, row Row) { + c.appendSel(colIdx) for i, rowCol := range row.c.columns { chkCol := c.columns[colIdx+i] chkCol.appendNullBitmap(!rowCol.IsNull(row.idx)) @@ -417,6 +453,7 @@ func (c *Chunk) Append(other *Chunk, begin, end int) { } } for i := begin; i < end; i++ { + c.appendSel(colID) dst.appendNullBitmap(!src.IsNull(i)) dst.length++ } @@ -426,6 +463,7 @@ func (c *Chunk) Append(other *Chunk, begin, end int) { // TruncateTo truncates rows from tail to head in a Chunk to "numRows" rows. func (c *Chunk) TruncateTo(numRows int) { + c.Reconstruct() for _, col := range c.columns { if col.isFixed() { elemLen := len(col.elemBuf) @@ -457,70 +495,89 @@ func (c *Chunk) TruncateTo(numRows int) { // AppendNull appends a null value to the chunk. func (c *Chunk) AppendNull(colIdx int) { + c.appendSel(colIdx) c.columns[colIdx].AppendNull() } // AppendInt64 appends a int64 value to the chunk. func (c *Chunk) AppendInt64(colIdx int, i int64) { + c.appendSel(colIdx) c.columns[colIdx].AppendInt64(i) } // AppendUint64 appends a uint64 value to the chunk. func (c *Chunk) AppendUint64(colIdx int, u uint64) { + c.appendSel(colIdx) c.columns[colIdx].AppendUint64(u) } // AppendFloat32 appends a float32 value to the chunk. func (c *Chunk) AppendFloat32(colIdx int, f float32) { + c.appendSel(colIdx) c.columns[colIdx].AppendFloat32(f) } // AppendFloat64 appends a float64 value to the chunk. func (c *Chunk) AppendFloat64(colIdx int, f float64) { + c.appendSel(colIdx) c.columns[colIdx].AppendFloat64(f) } // AppendString appends a string value to the chunk. func (c *Chunk) AppendString(colIdx int, str string) { + c.appendSel(colIdx) c.columns[colIdx].AppendString(str) } // AppendBytes appends a bytes value to the chunk. func (c *Chunk) AppendBytes(colIdx int, b []byte) { + c.appendSel(colIdx) c.columns[colIdx].AppendBytes(b) } // AppendTime appends a Time value to the chunk. // TODO: change the time structure so it can be directly written to memory. func (c *Chunk) AppendTime(colIdx int, t types.Time) { + c.appendSel(colIdx) c.columns[colIdx].AppendTime(t) } // AppendDuration appends a Duration value to the chunk. func (c *Chunk) AppendDuration(colIdx int, dur types.Duration) { + c.appendSel(colIdx) c.columns[colIdx].AppendDuration(dur) } // AppendMyDecimal appends a MyDecimal value to the chunk. func (c *Chunk) AppendMyDecimal(colIdx int, dec *types.MyDecimal) { + c.appendSel(colIdx) c.columns[colIdx].AppendMyDecimal(dec) } // AppendEnum appends an Enum value to the chunk. func (c *Chunk) AppendEnum(colIdx int, enum types.Enum) { + c.appendSel(colIdx) c.columns[colIdx].appendNameValue(enum.Name, enum.Value) } // AppendSet appends a Set value to the chunk. func (c *Chunk) AppendSet(colIdx int, set types.Set) { + c.appendSel(colIdx) c.columns[colIdx].appendNameValue(set.Name, set.Value) } // AppendJSON appends a JSON value to the chunk. func (c *Chunk) AppendJSON(colIdx int, j json.BinaryJSON) { + c.appendSel(colIdx) c.columns[colIdx].AppendJSON(j) } +func (c *Chunk) appendSel(colIdx int) { + if colIdx == 0 && c.sel != nil { // use column 0 as standard + c.sel = append(c.sel, c.columns[0].length) + } +} + // AppendDatum appends a datum into the chunk. func (c *Chunk) AppendDatum(colIdx int, d *types.Datum) { switch d.Kind() { @@ -556,6 +613,28 @@ func (c *Chunk) Column(colIdx int) *Column { return c.columns[colIdx] } +// Sel returns Sel of this Chunk. +func (c *Chunk) Sel() []int { + return c.sel +} + +// SetSel sets a Sel for this Chunk. +func (c *Chunk) SetSel(sel []int) { + c.sel = sel +} + +// Reconstruct removes all filtered rows in this Chunk. +func (c *Chunk) Reconstruct() { + if c.sel == nil { + return + } + for _, col := range c.columns { + col.reconstruct(c.sel) + } + c.numVirtualRows = len(c.sel) + c.sel = nil +} + func writeTime(buf []byte, t types.Time) { binary.BigEndian.PutUint16(buf, uint16(t.Time.Year())) buf[2] = uint8(t.Time.Month()) diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 769b4f00b804f..08a34f439c204 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -727,6 +727,24 @@ func (s *testChunkSuite) TestPreAlloc4RowAndInsert(c *check.C) { } } +func (s *testChunkSuite) TestAppendSel(c *check.C) { + tll := &types.FieldType{Tp: mysql.TypeLonglong} + chk := NewChunkWithCapacity([]*types.FieldType{tll}, 1024) + sel := make([]int, 0, 1024/2) + for i := 0; i < 1024/2; i++ { + chk.AppendInt64(0, int64(i)) + if i%2 == 0 { + sel = append(sel, i) + } + } + chk.SetSel(sel) + c.Assert(chk.NumRows(), check.Equals, 1024/2/2) + chk.AppendInt64(0, int64(1)) + c.Assert(chk.NumRows(), check.Equals, 1024/2/2+1) + sel = chk.Sel() + c.Assert(sel[len(sel)-1], check.Equals, 1024/2) +} + func (s *testChunkSuite) TestMakeRefTo(c *check.C) { fieldTypes := make([]*types.FieldType, 0, 2) fieldTypes = append(fieldTypes, &types.FieldType{Tp: mysql.TypeFloat}) diff --git a/util/chunk/chunk_util.go b/util/chunk/chunk_util.go index be15dafe44a87..688bfb44c54e8 100644 --- a/util/chunk/chunk_util.go +++ b/util/chunk/chunk_util.go @@ -13,20 +13,25 @@ package chunk +import "github.com/pingcap/errors" + // CopySelectedJoinRows copies the selected joined rows from the source Chunk // to the destination Chunk. // Return true if at least one joined row was selected. // // NOTE: All the outer rows in the source Chunk should be the same. -func CopySelectedJoinRows(src *Chunk, innerColOffset, outerColOffset int, selected []bool, dst *Chunk) bool { +func CopySelectedJoinRows(src *Chunk, innerColOffset, outerColOffset int, selected []bool, dst *Chunk) (bool, error) { if src.NumRows() == 0 { - return false + return false, nil + } + if src.sel != nil || dst.sel != nil { + return false, errors.New(msgErrSelNotNil) } numSelected := copySelectedInnerRows(innerColOffset, outerColOffset, src, selected, dst) copyOuterRows(innerColOffset, outerColOffset, src, numSelected, dst) dst.numVirtualRows += numSelected - return numSelected > 0 + return numSelected > 0, nil } // copySelectedInnerRows copies the selected inner rows from the source Chunk diff --git a/util/chunk/column.go b/util/chunk/column.go index a10a699260db4..bd8df6eeb2726 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -334,3 +334,55 @@ func (c *Column) getNameValue(rowID int) (string, uint64) { } return string(hack.String(c.data[start+8 : end])), *(*uint64)(unsafe.Pointer(&c.data[start])) } + +// reconstruct reconstructs this Column by removing all filtered rows in it according to sel. +func (c *Column) reconstruct(sel []int) { + if sel == nil { + return + } + nullCnt := 0 + if c.isFixed() { + elemLen := len(c.elemBuf) + for dst, src := range sel { + idx := dst >> 3 + pos := uint16(dst & 7) + if c.IsNull(src) { + nullCnt++ + c.nullBitmap[idx] &= ^byte(1 << pos) + } else { + copy(c.data[dst*elemLen:dst*elemLen+elemLen], c.data[src*elemLen:src*elemLen+elemLen]) + c.nullBitmap[idx] |= byte(1 << pos) + } + } + c.data = c.data[:len(sel)*elemLen] + } else { + tail := 0 + for dst, src := range sel { + idx := dst >> 3 + pos := uint(dst & 7) + if c.IsNull(src) { + nullCnt++ + c.nullBitmap[idx] &= ^byte(1 << pos) + c.offsets[dst+1] = int64(tail) + } else { + start, end := c.offsets[src], c.offsets[src+1] + copy(c.data[tail:], c.data[start:end]) + tail += int(end - start) + c.offsets[dst+1] = int64(tail) + c.nullBitmap[idx] |= byte(1 << pos) + } + } + c.data = c.data[:tail] + c.offsets = c.offsets[:len(sel)+1] + } + c.length = len(sel) + c.nullCount = nullCnt + + // clean nullBitmap + c.nullBitmap = c.nullBitmap[:(len(sel)+7)>>3] + idx := len(sel) >> 3 + if idx < len(c.nullBitmap) { + pos := uint16(len(sel) & 7) + c.nullBitmap[idx] &= byte((1 << pos) - 1) + } +} diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index e3748ad7583b3..39e6c1cca4fbf 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -15,6 +15,7 @@ package chunk import ( "fmt" + "math/rand" "time" "github.com/pingcap/check" @@ -324,3 +325,115 @@ func (s *testChunkSuite) TestNullsColumn(c *check.C) { i++ } } + +func (s *testChunkSuite) TestReconstructFixedLen(c *check.C) { + col := NewColumn(types.NewFieldType(mysql.TypeLonglong), 1024) + results := make([]int64, 0, 1024) + nulls := make([]bool, 0, 1024) + sel := make([]int, 0, 1024) + for i := 0; i < 1024; i++ { + if rand.Intn(10) < 6 { + sel = append(sel, i) + } + + if rand.Intn(10) < 2 { + col.AppendNull() + nulls = append(nulls, true) + results = append(results, 0) + continue + } + + v := rand.Int63() + col.AppendInt64(v) + results = append(results, v) + nulls = append(nulls, false) + } + + col.reconstruct(sel) + nullCnt := 0 + for n, i := range sel { + if nulls[i] { + nullCnt++ + c.Assert(col.IsNull(n), check.Equals, true) + } else { + c.Assert(col.GetInt64(n), check.Equals, results[i]) + } + } + c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(col.length, check.Equals, len(sel)) + + for i := 0; i < 128; i++ { + if i%2 == 0 { + col.AppendNull() + } else { + col.AppendInt64(int64(i * i * i)) + } + } + + c.Assert(col.length, check.Equals, len(sel)+128) + c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + for i := 0; i < 128; i++ { + if i%2 == 0 { + c.Assert(col.IsNull(len(sel)+i), check.Equals, true) + } else { + c.Assert(col.GetInt64(len(sel)+i), check.Equals, int64(i*i*i)) + c.Assert(col.IsNull(len(sel)+i), check.Equals, false) + } + } +} + +func (s *testChunkSuite) TestReconstructVarLen(c *check.C) { + col := NewColumn(types.NewFieldType(mysql.TypeVarString), 1024) + results := make([]string, 0, 1024) + nulls := make([]bool, 0, 1024) + sel := make([]int, 0, 1024) + for i := 0; i < 1024; i++ { + if rand.Intn(10) < 6 { + sel = append(sel, i) + } + + if rand.Intn(10) < 2 { + col.AppendNull() + nulls = append(nulls, true) + results = append(results, "") + continue + } + + v := fmt.Sprintf("%v", rand.Int63()) + col.AppendString(v) + results = append(results, v) + nulls = append(nulls, false) + } + + col.reconstruct(sel) + nullCnt := 0 + for n, i := range sel { + if nulls[i] { + nullCnt++ + c.Assert(col.IsNull(n), check.Equals, true) + } else { + c.Assert(col.GetString(n), check.Equals, results[i]) + } + } + c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(col.length, check.Equals, len(sel)) + + for i := 0; i < 128; i++ { + if i%2 == 0 { + col.AppendNull() + } else { + col.AppendString(fmt.Sprintf("%v", i*i*i)) + } + } + + c.Assert(col.length, check.Equals, len(sel)+128) + c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + for i := 0; i < 128; i++ { + if i%2 == 0 { + c.Assert(col.IsNull(len(sel)+i), check.Equals, true) + } else { + c.Assert(col.GetString(len(sel)+i), check.Equals, fmt.Sprintf("%v", i*i*i)) + c.Assert(col.IsNull(len(sel)+i), check.Equals, false) + } + } +} diff --git a/util/chunk/iterator_test.go b/util/chunk/iterator_test.go index 5438f062a1383..2c937447b1909 100644 --- a/util/chunk/iterator_test.go +++ b/util/chunk/iterator_test.go @@ -19,6 +19,26 @@ import ( "github.com/pingcap/tidb/types" ) +func (s *testChunkSuite) TestIteratorOnSel(c *check.C) { + fields := []*types.FieldType{types.NewFieldType(mysql.TypeLonglong)} + chk := New(fields, 32, 1024) + sel := make([]int, 0, 1024) + for i := 0; i < 1024; i++ { + chk.AppendInt64(0, int64(i)) + if i%2 == 0 { + sel = append(sel, i) + } + } + chk.SetSel(sel) + it := NewIterator4Chunk(chk) + cnt := 0 + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetInt64(0)%2, check.Equals, int64(0)) + cnt++ + } + c.Assert(cnt, check.Equals, 1024/2) +} + func (s *testChunkSuite) TestIterator(c *check.C) { fields := []*types.FieldType{types.NewFieldType(mysql.TypeLonglong)} chk := New(fields, 32, 1024) From 18724b950bb556a308796f513c7fc702a34a2dd6 Mon Sep 17 00:00:00 2001 From: disksing Date: Tue, 23 Jul 2019 21:22:24 +0800 Subject: [PATCH 077/196] codec: improve buffer reuse (#10801) --- store/mockstore/mocktikv/executor.go | 2 +- tablecodec/tablecodec.go | 4 ++-- util/codec/codec.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/store/mockstore/mocktikv/executor.go b/store/mockstore/mocktikv/executor.go index d12206a48a7fc..f4f0e7a1d8eba 100644 --- a/store/mockstore/mocktikv/executor.go +++ b/store/mockstore/mocktikv/executor.go @@ -396,7 +396,7 @@ func (e *indexScanExec) decodeIndexKV(pair Pair) ([][]byte, error) { } else { handleDatum = types.NewIntDatum(handle) } - handleBytes, err := codec.EncodeValue(nil, b, handleDatum) + handleBytes, err := codec.EncodeValue(nil, nil, handleDatum) if err != nil { return nil, errors.Trace(err) } diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index 0666427666fe0..fbbf6b232e736 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -235,12 +235,12 @@ func EncodeRow(sc *stmtctx.StatementContext, row []types.Datum, colIDs []int64, values[2*i].SetInt64(id) err := flatten(sc, c, &values[2*i+1]) if err != nil { - return nil, errors.Trace(err) + return valBuf, errors.Trace(err) } } if len(values) == 0 { // We could not set nil value into kv. - return []byte{codec.NilFlag}, nil + return append(valBuf, codec.NilFlag), nil } return codec.EncodeValue(sc, valBuf, values...) } diff --git a/util/codec/codec.go b/util/codec/codec.go index 3ea30921b0761..bec5fc74123ba 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -93,7 +93,7 @@ func encode(sc *stmtctx.StatementContext, b []byte, vals []types.Datum, comparab b = append(b, uintFlag) b, err = EncodeMySQLTime(sc, vals[i], mysql.TypeUnspecified, b) if err != nil { - return nil, err + return b, err } case types.KindMysqlDuration: // duration may have negative value, so we cannot use String to encode directly. @@ -107,7 +107,7 @@ func encode(sc *stmtctx.StatementContext, b []byte, vals []types.Datum, comparab var bin []byte bin, err = dec.ToHashKey() if err != nil { - return nil, errors.Trace(err) + return b, errors.Trace(err) } b = append(b, bin...) } else { @@ -140,7 +140,7 @@ func encode(sc *stmtctx.StatementContext, b []byte, vals []types.Datum, comparab case types.KindMaxValue: b = append(b, maxFlag) default: - return nil, errors.Errorf("unsupport encode type %d", vals[i].Kind()) + return b, errors.Errorf("unsupport encode type %d", vals[i].Kind()) } } From 1ad073bf80cfddf8af8a2d4dc696a07ed0896740 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 24 Jul 2019 10:53:02 +0800 Subject: [PATCH 078/196] executor: show operators' memory consumption in results of `EXPLAIN ANALYZE` (#11334) --- distsql/distsql_test.go | 4 ++- distsql/request_builder.go | 8 ++---- executor/distsql.go | 19 ++++++++------ executor/explain_test.go | 49 +++++++++++++++++++++++++++++++++++ executor/index_lookup_join.go | 4 --- executor/join.go | 2 -- executor/merge_join.go | 1 - executor/sort.go | 1 - executor/table_reader.go | 11 +++++--- planner/core/cbo_test.go | 4 +-- planner/core/common_plans.go | 9 ++++++- util/memory/tracker.go | 36 ++++++++++++++----------- 12 files changed, 103 insertions(+), 45 deletions(-) diff --git a/distsql/distsql_test.go b/distsql/distsql_test.go index 470f2e78bab67..4a145717d0fce 100644 --- a/distsql/distsql_test.go +++ b/distsql/distsql_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/execdetails" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/stringutil" "github.com/pingcap/tipb/go-tipb" ) @@ -42,7 +43,8 @@ func (s *testSuite) createSelectNormal(batch, totalRows int, c *C, planIDs []str SetDesc(false). SetKeepOrder(false). SetFromSessionVars(variable.NewSessionVars()). - SetMemTracker(s.sctx, stringutil.StringerStr("testSuite.createSelectNormal")). + SetMemTracker(memory.NewTracker(stringutil.StringerStr("testSuite.createSelectNormal"), + s.sctx.GetSessionVars().MemQuotaDistSQL)). Build() c.Assert(err, IsNil) diff --git a/distsql/request_builder.go b/distsql/request_builder.go index 9e1ef6c624f05..10c91e93598d1 100644 --- a/distsql/request_builder.go +++ b/distsql/request_builder.go @@ -14,12 +14,10 @@ package distsql import ( - "fmt" "math" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics" @@ -44,10 +42,8 @@ func (builder *RequestBuilder) Build() (*kv.Request, error) { } // SetMemTracker sets a memTracker for this request. -func (builder *RequestBuilder) SetMemTracker(sctx sessionctx.Context, label fmt.Stringer) *RequestBuilder { - t := memory.NewTracker(label, sctx.GetSessionVars().MemQuotaDistSQL) - t.AttachTo(sctx.GetSessionVars().StmtCtx.MemTracker) - builder.Request.MemTracker = t +func (builder *RequestBuilder) SetMemTracker(tracker *memory.Tracker) *RequestBuilder { + builder.Request.MemTracker = tracker return builder } diff --git a/executor/distsql.go b/executor/distsql.go index fcfbc110fcad3..c4cf91124d696 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -19,6 +19,7 @@ import ( "math" "runtime" "sort" + "strconv" "sync" "sync/atomic" "time" @@ -220,6 +221,8 @@ type IndexReaderExecutor struct { colLens []int plans []plannercore.PhysicalPlan + memTracker *memory.Tracker + selectResultHook // for testing } @@ -261,8 +264,6 @@ func (e *IndexReaderExecutor) Open(ctx context.Context) error { return e.open(ctx, kvRanges) } -var indexReaderDistSQLTrackerLabel fmt.Stringer = stringutil.StringerStr("IndexReaderDistSQLTracker") - func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) error { var err error if e.corColInFilter { @@ -277,6 +278,8 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) e.dagPB.CollectExecutionSummaries = &collExec } + e.memTracker = memory.NewTracker(e.id, e.ctx.GetSessionVars().MemQuotaDistSQL) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) var builder distsql.RequestBuilder kvReq, err := builder.SetKeyRanges(kvRanges). SetDAGRequest(e.dagPB). @@ -284,7 +287,7 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetMemTracker(e.ctx, indexReaderDistSQLTrackerLabel). + SetMemTracker(e.memTracker). Build() if err != nil { e.feedback.Invalidate() @@ -415,6 +418,8 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, kvRanges []k e.dagPB.CollectExecutionSummaries = &collExec } + tracker := memory.NewTracker(stringutil.StringerStr("IndexWorker"), e.ctx.GetSessionVars().MemQuotaIndexLookupReader) + tracker.AttachTo(e.memTracker) var builder distsql.RequestBuilder kvReq, err := builder.SetKeyRanges(kvRanges). SetDAGRequest(e.dagPB). @@ -422,7 +427,7 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, kvRanges []k SetKeepOrder(e.keepOrder). SetStreaming(e.indexStreaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetMemTracker(e.ctx, indexLookupDistSQLTrackerLabel). + SetMemTracker(tracker). Build() if err != nil { return err @@ -471,8 +476,6 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, kvRanges []k return nil } -var tableWorkerLabel fmt.Stringer = stringutil.StringerStr("tableWorker") - // startTableWorker launchs some background goroutines which pick tasks from workCh and execute the task. func (e *IndexLookUpExecutor) startTableWorker(ctx context.Context, workCh <-chan *lookupTableTask) { lookupConcurrencyLimit := e.ctx.GetSessionVars().IndexLookupConcurrency @@ -486,7 +489,8 @@ func (e *IndexLookUpExecutor) startTableWorker(ctx context.Context, workCh <-cha keepOrder: e.keepOrder, handleIdx: e.handleIdx, isCheckOp: e.isCheckOp, - memTracker: memory.NewTracker(tableWorkerLabel, -1), + memTracker: memory.NewTracker(stringutil.MemoizeStr(func() string { return "TableWorker_" + strconv.Itoa(i) }), + e.ctx.GetSessionVars().MemQuotaIndexLookupReader), } worker.memTracker.AttachTo(e.memTracker) ctx1, cancel := context.WithCancel(ctx) @@ -531,7 +535,6 @@ func (e *IndexLookUpExecutor) Close() error { e.tblWorkerWg.Wait() e.finished = nil e.workerStarted = false - e.memTracker.Detach() e.memTracker = nil if e.runtimeStats != nil { copStats := e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetRootStats(e.idxPlans[0].ExplainID().String()) diff --git a/executor/explain_test.go b/executor/explain_test.go index c36179a283e52..47c634f3f454e 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -14,6 +14,8 @@ package executor_test import ( + "strings" + . "github.com/pingcap/check" "github.com/pingcap/parser/auth" plannercore "github.com/pingcap/tidb/planner/core" @@ -74,3 +76,50 @@ func (s *testSuite1) TestExplainWrite(c *C) { tk.MustExec("explain analyze insert into t select 1") tk.MustQuery("select * from t order by a").Check(testkit.Rows("1", "2")) } + +func (s *testSuite1) TestExplainAnalyzeMemory(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (v int, k int, key(k))") + tk.MustExec("insert into t values (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)") + + s.checkMemoryInfo(c, tk, "explain analyze select * from t order by v") + s.checkMemoryInfo(c, tk, "explain analyze select * from t order by v limit 5") + s.checkMemoryInfo(c, tk, "explain analyze select /*+ TIDB_HJ(t1, t2) */ t1.k from t t1, t t2 where t1.v = t2.v+1") + s.checkMemoryInfo(c, tk, "explain analyze select /*+ TIDB_SMJ(t1, t2) */ t1.k from t t1, t t2 where t1.k = t2.k+1") + s.checkMemoryInfo(c, tk, "explain analyze select /*+ TIDB_INLJ(t1, t2) */ t1.k from t t1, t t2 where t1.k = t2.k and t1.v=1") + s.checkMemoryInfo(c, tk, "explain analyze select sum(k) from t group by v") + s.checkMemoryInfo(c, tk, "explain analyze select sum(v) from t group by k") + s.checkMemoryInfo(c, tk, "explain analyze select * from t") + s.checkMemoryInfo(c, tk, "explain analyze select k from t use index(k)") + s.checkMemoryInfo(c, tk, "explain analyze select * from t use index(k)") +} + +func (s *testSuite1) checkMemoryInfo(c *C, tk *testkit.TestKit, sql string) { + memCol := 5 + ops := []string{"Join", "Reader", "Top", "Sort", "LookUp"} + rows := tk.MustQuery(sql).Rows() + for _, row := range rows { + strs := make([]string, len(row)) + for i, c := range row { + strs[i] = c.(string) + } + if strings.Contains(strs[2], "cop") { + continue + } + + shouldHasMem := false + for _, op := range ops { + if strings.Contains(strs[0], op) { + shouldHasMem = true + break + } + } + + if shouldHasMem { + c.Assert(strs[memCol], Not(Equals), "N/A") + } else { + c.Assert(strs[memCol], Equals, "N/A") + } + } +} diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 7f90e85303fe5..5f596a69b168d 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -295,9 +295,6 @@ func (e *IndexLookUpJoin) getFinishedTask(ctx context.Context) (*lookUpJoinTask, return nil, nil } - if e.task != nil { - e.task.memTracker.Detach() - } e.task = task return task, nil } @@ -650,7 +647,6 @@ func (e *IndexLookUpJoin) Close() error { e.cancelFunc() } e.workerWg.Wait() - e.memTracker.Detach() e.memTracker = nil return e.children[0].Close() } diff --git a/executor/join.go b/executor/join.go index 12cd377271ade..d6f8e9042ca4a 100644 --- a/executor/join.go +++ b/executor/join.go @@ -134,7 +134,6 @@ func (e *HashJoinExec) Close() error { e.outerChkResourceCh = nil e.joinChkResourceCh = nil } - e.memTracker.Detach() e.memTracker = nil err := e.baseExecutor.Close() @@ -633,7 +632,6 @@ type NestedLoopApplyExec struct { func (e *NestedLoopApplyExec) Close() error { e.innerRows = nil - e.memTracker.Detach() e.memTracker = nil return e.outerExec.Close() } diff --git a/executor/merge_join.go b/executor/merge_join.go index 9a7a09912d440..f7415d6bf1444 100644 --- a/executor/merge_join.go +++ b/executor/merge_join.go @@ -200,7 +200,6 @@ func (t *mergeJoinInnerTable) reallocReaderResult() { // Close implements the Executor Close interface. func (e *MergeJoinExec) Close() error { - e.memTracker.Detach() e.childrenResults = nil e.memTracker = nil diff --git a/executor/sort.go b/executor/sort.go index 4209dafd91d4c..aa895015b1eec 100644 --- a/executor/sort.go +++ b/executor/sort.go @@ -54,7 +54,6 @@ type SortExec struct { // Close implements the Executor Close interface. func (e *SortExec) Close() error { - e.memTracker.Detach() e.memTracker = nil return e.children[0].Close() } diff --git a/executor/table_reader.go b/executor/table_reader.go index 3f6bea659295a..019957e678247 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -26,8 +26,8 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/ranger" - "github.com/pingcap/tidb/util/stringutil" "github.com/pingcap/tipb/go-tipb" ) @@ -72,11 +72,16 @@ type TableReaderExecutor struct { corColInAccess bool plans []plannercore.PhysicalPlan + memTracker *memory.Tracker + selectResultHook // for testing } // Open initialzes necessary variables for using this executor. func (e *TableReaderExecutor) Open(ctx context.Context) error { + e.memTracker = memory.NewTracker(e.id, e.ctx.GetSessionVars().MemQuotaDistSQL) + e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) + var err error if e.corColInFilter { e.dagPB.Executors, _, err = constructDistExec(e.ctx, e.plans) @@ -148,8 +153,6 @@ func (e *TableReaderExecutor) Close() error { return err } -var tableReaderDistSQLTrackerLabel fmt.Stringer = stringutil.StringerStr("TableReaderDistSQLTracker") - // buildResp first builds request and sends it to tikv using distsql.Select. It uses SelectResut returned by the callee // to fetch all results. func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Range) (distsql.SelectResult, error) { @@ -160,7 +163,7 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra SetKeepOrder(e.keepOrder). SetStreaming(e.streaming). SetFromSessionVars(e.ctx.GetSessionVars()). - SetMemTracker(e.ctx, tableReaderDistSQLTrackerLabel). + SetMemTracker(e.memTracker). Build() if err != nil { return nil, err diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index cdce14d6e8f73..222512c2419c0 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -80,7 +80,7 @@ func (s *testAnalyzeSuite) TestExplainAnalyze(c *C) { rs := tk.MustQuery("explain analyze select t1.a, t1.b, sum(t1.c) from t1 join t2 on t1.a = t2.b where t1.a > 1") c.Assert(len(rs.Rows()), Equals, 10) for _, row := range rs.Rows() { - c.Assert(len(row), Equals, 5) + c.Assert(len(row), Equals, 6) execInfo := row[4].(string) c.Assert(strings.Contains(execInfo, "time"), Equals, true) c.Assert(strings.Contains(execInfo, "loops"), Equals, true) @@ -977,7 +977,7 @@ func (s *testAnalyzeSuite) TestIssue9805(c *C) { c.Assert(rs.Rows(), HasLen, 10) hasIndexLookUp12 := false for _, row := range rs.Rows() { - c.Assert(row, HasLen, 5) + c.Assert(row, HasLen, 6) if strings.HasSuffix(row[0].(string), "IndexLookUp_12") { hasIndexLookUp12 = true c.Assert(row[4], Equals, "time:0ns, loops:0, rows:0") diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 96a33507287f2..c43f03bf67b50 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -563,7 +563,7 @@ func (e *Explain) prepareSchema() error { case ast.ExplainFormatROW: retFields := []string{"id", "count", "task", "operator info"} if e.Analyze { - retFields = append(retFields, "execution info") + retFields = append(retFields, "execution info", "memory") } schema := expression.NewSchema(make([]*expression.Column, 0, len(retFields))...) for _, fieldName := range retFields { @@ -643,6 +643,13 @@ func (e *Explain) prepareOperatorInfo(p PhysicalPlan, taskType string, indent st } else { row = append(row, "time:0ns, loops:0, rows:0") } + + tracker := e.ctx.GetSessionVars().StmtCtx.MemTracker.SearchTracker(p.ExplainID().String()) + if tracker != nil { + row = append(row, tracker.BytesToString(tracker.MaxConsumed())) + } else { + row = append(row, "N/A") + } } e.Rows = append(e.Rows, row) } diff --git a/util/memory/tracker.go b/util/memory/tracker.go index befbab5ce3e76..3b935360f0fce 100644 --- a/util/memory/tracker.go +++ b/util/memory/tracker.go @@ -87,11 +87,6 @@ func (t *Tracker) AttachTo(parent *Tracker) { t.parent.Consume(t.BytesConsumed()) } -// Detach detaches this Tracker from its parent. -func (t *Tracker) Detach() { - t.parent.remove(t) -} - func (t *Tracker) remove(oldChild *Tracker) { t.mu.Lock() defer t.mu.Unlock() @@ -144,17 +139,13 @@ func (t *Tracker) Consume(bytes int64) { rootExceed = tracker } - if tracker.parent == nil { - // since we only need a total memory usage during execution, - // we only record max consumed bytes in root(statement-level) for performance. - for { - maxNow := atomic.LoadInt64(&tracker.maxConsumed) - consumed := atomic.LoadInt64(&tracker.bytesConsumed) - if consumed > maxNow && !atomic.CompareAndSwapInt64(&tracker.maxConsumed, maxNow, consumed) { - continue - } - break + for { + maxNow := atomic.LoadInt64(&tracker.maxConsumed) + consumed := atomic.LoadInt64(&tracker.bytesConsumed) + if consumed > maxNow && !atomic.CompareAndSwapInt64(&tracker.maxConsumed, maxNow, consumed) { + continue } + break } } if rootExceed != nil { @@ -172,6 +163,21 @@ func (t *Tracker) MaxConsumed() int64 { return atomic.LoadInt64(&t.maxConsumed) } +// SearchTracker searches the specific tracker under this tracker. +func (t *Tracker) SearchTracker(label string) *Tracker { + if t.label.String() == label { + return t + } + t.mu.Lock() + defer t.mu.Unlock() + for _, child := range t.mu.children { + if result := child.SearchTracker(label); result != nil { + return result + } + } + return nil +} + // String returns the string representation of this Tracker tree. func (t *Tracker) String() string { buffer := bytes.NewBufferString("\n") From 05f66f4384e66f7c11f0bb530fabc05c4b32b72b Mon Sep 17 00:00:00 2001 From: Tanner Date: Wed, 24 Jul 2019 11:15:22 +0800 Subject: [PATCH 079/196] ddl: fix invalid index on multi-layer virtual columns (#11260) --- ddl/db_integration_test.go | 63 +++++++++++++++++++++ ddl/index.go | 38 ++++++------- util/admin/admin.go | 29 ++++------ util/rowDecoder/decoder.go | 113 +++++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 40 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 1059930f69bc7..477961ffb7320 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -799,6 +799,69 @@ func (s *testIntegrationSuite5) TestModifyingColumnOption(c *C) { assertErrCode("alter table t2 modify column c int references t1(a)", errMsg) } +func (s *testIntegrationSuite1) TestIndexOnMultipleGeneratedColumn(c *C) { + tk := testkit.NewTestKit(c, s.store) + + tk.MustExec("create database if not exists test") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int as (a + 1), c int as (b + 1))") + tk.MustExec("insert into t (a) values (1)") + tk.MustExec("create index idx on t (c)") + tk.MustQuery("select * from t where c > 1").Check(testkit.Rows("1 2 3")) + res := tk.MustQuery("select * from t use index(idx) where c > 1") + tk.MustQuery("select * from t ignore index(idx) where c > 1").Check(res.Rows()) + tk.MustExec("admin check table t") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int as (a + 1), c int as (b + 1), d int as (c + 1))") + tk.MustExec("insert into t (a) values (1)") + tk.MustExec("create index idx on t (d)") + tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 3 4")) + res = tk.MustQuery("select * from t use index(idx) where d > 2") + tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) + tk.MustExec("admin check table t") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a bigint, b decimal as (a+1), c varchar(20) as (b*2), d float as (a*23+b-1+length(c)))") + tk.MustExec("insert into t (a) values (1)") + tk.MustExec("create index idx on t (d)") + tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 4 25")) + res = tk.MustQuery("select * from t use index(idx) where d > 2") + tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) + tk.MustExec("admin check table t") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a varchar(10), b float as (length(a)+123), c varchar(20) as (right(a, 2)), d float as (b+b-7+1-3+3*ASCII(c)))") + tk.MustExec("insert into t (a) values ('adorable')") + tk.MustExec("create index idx on t (d)") + tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("adorable 131 le 577")) // 131+131-7+1-3+3*108 + res = tk.MustQuery("select * from t use index(idx) where d > 2") + tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) + tk.MustExec("admin check table t") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a bigint, b decimal as (a), c int(10) as (a+b), d float as (a+b+c), e decimal as (a+b+c+d))") + tk.MustExec("insert into t (a) values (1)") + tk.MustExec("create index idx on t (d)") + tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 1 2 4 8")) + res = tk.MustQuery("select * from t use index(idx) where d > 2") + tk.MustQuery("select * from t ignore index(idx) where d > 2").Check(res.Rows()) + tk.MustExec("admin check table t") + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a bigint, b bigint as (a+1) virtual, c bigint as (b+1) virtual)") + tk.MustExec("alter table t add index idx_b(b)") + tk.MustExec("alter table t add index idx_c(c)") + tk.MustExec("insert into t(a) values(1)") + tk.MustExec("alter table t add column(d bigint as (c+1) virtual)") + tk.MustExec("alter table t add index idx_d(d)") + tk.MustQuery("select * from t where d > 2").Check(testkit.Rows("1 2 3 4")) + res = tk.MustQuery("select * from t use index(idx_d) where d > 2") + tk.MustQuery("select * from t ignore index(idx_d) where d > 2").Check(res.Rows()) + tk.MustExec("admin check table t") +} + func (s *testIntegrationSuite2) TestCaseInsensitiveCharsetAndCollate(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/ddl/index.go b/ddl/index.go index c6ffa9e3a1bb1..3c58ffb79e560 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -943,27 +943,23 @@ func (w *addIndexWorker) run(d *ddlCtx) { func makeupDecodeColMap(sessCtx sessionctx.Context, t table.Table, indexInfo *model.IndexInfo) (map[int64]decoder.Column, error) { cols := t.Cols() - decodeColMap := make(map[int64]decoder.Column, len(indexInfo.Columns)) - for _, v := range indexInfo.Columns { - col := cols[v.Offset] - tpExpr := decoder.Column{ - Col: col, - } - if col.IsGenerated() && !col.GeneratedStored { - for _, c := range cols { - if _, ok := col.Dependences[c.Name.L]; ok { - decodeColMap[c.ID] = decoder.Column{ - Col: c, - } - } - } - e, err := expression.ParseSimpleExprCastWithTableInfo(sessCtx, col.GeneratedExprString, t.Meta(), &col.FieldType) - if err != nil { - return nil, errors.Trace(err) - } - tpExpr.GenExpr = e - } - decodeColMap[col.ID] = tpExpr + indexedCols := make([]*table.Column, len(indexInfo.Columns)) + for i, v := range indexInfo.Columns { + indexedCols[i] = cols[v.Offset] + } + + var containsVirtualCol bool + decodeColMap, err := decoder.BuildFullDecodeColMap(indexedCols, t, func(genCol *table.Column) (expression.Expression, error) { + containsVirtualCol = true + return expression.ParseSimpleExprCastWithTableInfo(sessCtx, genCol.GeneratedExprString, t.Meta(), &genCol.FieldType) + }) + if err != nil { + return nil, err + } + + if containsVirtualCol { + decoder.SubstituteGenColsInDecodeColMap(decodeColMap) + decoder.RemoveUnusedVirtualCols(decodeColMap, indexedCols) } return decodeColMap, nil } diff --git a/util/admin/admin.go b/util/admin/admin.go index 533e5d82cf9bd..ad6f0774465ce 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -589,25 +589,16 @@ func CompareTableRecord(sessCtx sessionctx.Context, txn kv.Transaction, t table. } func makeRowDecoder(t table.Table, decodeCol []*table.Column, genExpr map[model.TableColumnID]expression.Expression) *decoder.RowDecoder { - cols := t.Cols() - tblInfo := t.Meta() - decodeColsMap := make(map[int64]decoder.Column, len(decodeCol)) - for _, v := range decodeCol { - col := cols[v.Offset] - tpExpr := decoder.Column{ - Col: col, - } - if col.IsGenerated() && !col.GeneratedStored { - for _, c := range cols { - if _, ok := col.Dependences[c.Name.L]; ok { - decodeColsMap[c.ID] = decoder.Column{ - Col: c, - } - } - } - tpExpr.GenExpr = genExpr[model.TableColumnID{TableID: tblInfo.ID, ColumnID: col.ID}] - } - decodeColsMap[col.ID] = tpExpr + var containsVirtualCol bool + decodeColsMap, ignored := decoder.BuildFullDecodeColMap(decodeCol, t, func(genCol *table.Column) (expression.Expression, error) { + containsVirtualCol = true + return genExpr[model.TableColumnID{TableID: t.Meta().ID, ColumnID: genCol.ID}], nil + }) + _ = ignored + + if containsVirtualCol { + decoder.SubstituteGenColsInDecodeColMap(decodeColsMap) + decoder.RemoveUnusedVirtualCols(decodeColsMap, decodeCol) } return decoder.NewRowDecoder(t, decodeColsMap) } diff --git a/util/rowDecoder/decoder.go b/util/rowDecoder/decoder.go index e6c91425a2778..68ef00c9961ae 100644 --- a/util/rowDecoder/decoder.go +++ b/util/rowDecoder/decoder.go @@ -14,8 +14,10 @@ package decoder import ( + "sort" "time" + "github.com/pingcap/errors" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" @@ -134,3 +136,114 @@ func (rd *RowDecoder) DecodeAndEvalRowWithMap(ctx sessionctx.Context, handle int } return row, nil } + +// BuildFullDecodeColMap build a map that contains [columnID -> struct{*table.Column, expression.Expression}] from +// indexed columns and all of its depending columns. `genExprProducer` is used to produce a generated expression based on a table.Column. +func BuildFullDecodeColMap(indexedCols []*table.Column, t table.Table, genExprProducer func(*table.Column) (expression.Expression, error)) (map[int64]Column, error) { + pendingCols := make([]*table.Column, len(indexedCols)) + copy(pendingCols, indexedCols) + decodeColMap := make(map[int64]Column, len(pendingCols)) + + for i := 0; i < len(pendingCols); i++ { + col := pendingCols[i] + if _, ok := decodeColMap[col.ID]; ok { + continue // already discovered + } + + if col.IsGenerated() && !col.GeneratedStored { + // Find depended columns and put them into pendingCols. For example, idx(c) with column definition `c int as (a + b)`, + // depended columns of `c` is `a` and `b`, and both of them will be put into the pendingCols, waiting for next traversal. + for _, c := range t.Cols() { + if _, ok := col.Dependences[c.Name.L]; ok { + pendingCols = append(pendingCols, c) + } + } + + e, err := genExprProducer(col) + if err != nil { + return nil, errors.Trace(err) + } + decodeColMap[col.ID] = Column{ + Col: col, + GenExpr: e, + } + } else { + decodeColMap[col.ID] = Column{ + Col: col, + } + } + } + return decodeColMap, nil +} + +// SubstituteGenColsInDecodeColMap substitutes generated columns in every expression +// with non-generated one by looking up decodeColMap. +func SubstituteGenColsInDecodeColMap(decodeColMap map[int64]Column) { + // Sort columns by table.Column.Offset in ascending order. + type Pair struct { + colID int64 + colOffset int + } + var orderedCols []Pair + for colID, col := range decodeColMap { + orderedCols = append(orderedCols, Pair{colID, col.Col.Offset}) + } + sort.Slice(orderedCols, func(i, j int) bool { return orderedCols[i].colOffset < orderedCols[j].colOffset }) + + // Iterate over decodeColMap, the substitution only happens once for each virtual column because + // columns with smaller offset can not refer to those with larger ones. https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html. + for _, pair := range orderedCols { + colID := pair.colID + decCol := decodeColMap[colID] + if decCol.GenExpr != nil { + decodeColMap[colID] = Column{ + Col: decCol.Col, + GenExpr: substituteGeneratedColumn(decCol.GenExpr, decodeColMap), + } + } else { + decodeColMap[colID] = Column{ + Col: decCol.Col, + } + } + } +} + +// substituteGeneratedColumn substitutes generated columns in an expression with non-generated one by looking up decodeColMap. +func substituteGeneratedColumn(expr expression.Expression, decodeColMap map[int64]Column) expression.Expression { + switch v := expr.(type) { + case *expression.Column: + if c, ok := decodeColMap[v.ID]; c.GenExpr != nil && ok { + return c.GenExpr + } + return v + case *expression.ScalarFunction: + newArgs := make([]expression.Expression, 0, len(v.GetArgs())) + for _, arg := range v.GetArgs() { + newArgs = append(newArgs, substituteGeneratedColumn(arg, decodeColMap)) + } + return expression.NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, newArgs...) + } + return expr +} + +// RemoveUnusedVirtualCols removes all virtual columns in decodeColMap that cannot found in indexedCols. +func RemoveUnusedVirtualCols(decodeColMap map[int64]Column, indexedCols []*table.Column) { + for colID, decCol := range decodeColMap { + col := decCol.Col + if !col.IsGenerated() || col.GeneratedStored { + continue + } + + found := false + for _, v := range indexedCols { + if v.Offset == col.Offset { + found = true + break + } + } + + if !found { + delete(decodeColMap, colID) + } + } +} From 99e4dd4c3c99fed9a06c957580836008787fac2e Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Wed, 24 Jul 2019 11:34:28 +0800 Subject: [PATCH 080/196] fix a coding mistake (#11393) --- server/http_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/http_handler.go b/server/http_handler.go index c3507fed709e0..296031ef393d6 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1155,7 +1155,7 @@ func NewFrameItemFromRegionKey(key []byte) (frame *FrameItem, err error) { } // bigger than tablePrefix, means is bigger than all tables. frame.TableID = math.MaxInt64 - frame.TableID = math.MaxInt64 + frame.IndexID = math.MaxInt64 frame.IsRecord = true return } From 2225d5ff6829f68e67af6c5eb9a8cb0bf4d59f8b Mon Sep 17 00:00:00 2001 From: qupeng Date: Wed, 24 Jul 2019 12:10:37 +0800 Subject: [PATCH 081/196] tikv: forbid to try to get a connection forever (#11391) --- store/tikv/client_batch.go | 19 +++++++++++++++---- store/tikv/client_test.go | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index bb4da4a8b71e3..2721c34e99586 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -408,15 +408,26 @@ func (a *batchConn) batchSendLoop(cfg config.TiKVClient) { func (a *batchConn) getClientAndSend(entries []*batchCommandsEntry, requests []*tikvpb.BatchCommandsRequest_Request, requestIDs []uint64) { // Choose a connection by round-robbin. - var cli *batchCommandsClient - for { + var cli *batchCommandsClient = nil + var target string = "" + for i := 0; i < len(a.batchCommandsClients); i++ { a.index = (a.index + 1) % uint32(len(a.batchCommandsClients)) - cli = a.batchCommandsClients[a.index] + target = a.batchCommandsClients[a.index].target // The lock protects the batchCommandsClient from been closed while it's inuse. - if cli.tryLockForSend() { + if a.batchCommandsClients[a.index].tryLockForSend() { + cli = a.batchCommandsClients[a.index] break } } + if cli == nil { + logutil.BgLogger().Warn("no available connections", zap.String("target", target)) + for _, entry := range entries { + // Please ensure the error is handled in region cache correctly. + entry.err = errors.New("no available connections") + close(entry.res) + } + return + } defer cli.unlockForSend() maxBatchID := atomic.AddUint64(&cli.idAlloc, uint64(len(requests))) diff --git a/store/tikv/client_test.go b/store/tikv/client_test.go index 4871a78afa557..00bc0bbee0f95 100644 --- a/store/tikv/client_test.go +++ b/store/tikv/client_test.go @@ -15,6 +15,7 @@ package tikv import ( "context" + "fmt" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/tikvpb" "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/store/tikv/tikvrpc" ) func TestT(t *testing.T) { @@ -98,3 +100,24 @@ func (s *testClientSuite) TestCancelTimeoutRetErr(c *C) { _, err = sendBatchRequest(context.Background(), "", a, req, 0) c.Assert(errors.Cause(err), Equals, context.DeadlineExceeded) } + +func (s *testClientSuite) TestSendWhenReconnect(c *C) { + server, port := startMockTikvService() + c.Assert(port > 0, IsTrue) + + rpcClient := newRPCClient(config.Security{}) + addr := fmt.Sprintf("%s:%d", "127.0.0.1", port) + conn, err := rpcClient.getConnArray(addr) + c.Assert(err, IsNil) + + // Suppose all connections are re-establishing. + for _, client := range conn.batchConn.batchCommandsClients { + client.lockForRecreate() + } + + req := tikvrpc.NewRequest(tikvrpc.CmdEmpty, &tikvpb.BatchCommandsEmptyRequest{}) + _, err = rpcClient.SendRequest(context.Background(), addr, req, 100*time.Second) + c.Assert(err.Error() == "no available connections", IsTrue) + conn.Close() + server.Stop() +} From 2b251d194e9a75893d80c16f0eef9b38fa674d15 Mon Sep 17 00:00:00 2001 From: lysu Date: Wed, 24 Jul 2019 13:18:28 +0800 Subject: [PATCH 082/196] tikv: invalidate store's regions when send store fail (#11344) --- store/tikv/region_cache.go | 43 +++++++++++++++++++++++++------ store/tikv/region_cache_test.go | 28 +++++++++++++++++++- store/tikv/region_request_test.go | 10 +++++-- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index 762e0f4f6f748..04950d0ccafc6 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -69,13 +69,19 @@ type Region struct { type RegionStore struct { workStoreIdx int32 // point to current work peer in meta.Peers and work store in stores(same idx) stores []*Store // stores in this region + storeFails []uint32 // snapshots of store's fail, need reload when `storeFails[curr] != stores[cur].fail` } // clone clones region store struct. func (r *RegionStore) clone() *RegionStore { + storeFails := make([]uint32, len(r.stores)) + for i, e := range r.storeFails { + storeFails[i] = e + } return &RegionStore{ workStoreIdx: r.workStoreIdx, stores: r.stores, + storeFails: storeFails, } } @@ -86,6 +92,7 @@ func (r *Region) init(c *RegionCache) { rs := &RegionStore{ workStoreIdx: 0, stores: make([]*Store, 0, len(r.meta.Peers)), + storeFails: make([]uint32, 0, len(r.meta.Peers)), } for _, p := range r.meta.Peers { c.storeMu.RLock() @@ -95,6 +102,7 @@ func (r *Region) init(c *RegionCache) { store = c.getStoreByStoreID(p.StoreId) } rs.stores = append(rs.stores, store) + rs.storeFails = append(rs.storeFails, atomic.LoadUint32(&store.fail)) } atomic.StorePointer(&r.store, unsafe.Pointer(rs)) @@ -272,6 +280,15 @@ func (c *RegionCache) GetRPCContext(bo *Backoffer, id RegionVerID) (*RPCContext, return nil, nil } + storeFailEpoch := atomic.LoadUint32(&store.fail) + if storeFailEpoch != regionStore.storeFails[regionStore.workStoreIdx] { + cachedRegion.invalidate() + logutil.BgLogger().Info("invalidate current region, because others failed on same store", + zap.Uint64("region", id.GetID()), + zap.String("store", store.addr)) + return nil, nil + } + return &RPCContext{ Region: id, Meta: cachedRegion.meta, @@ -368,7 +385,7 @@ func (c *RegionCache) OnSendFail(bo *Backoffer, ctx *RPCContext, scheduleReload tikvRegionCacheCounterWithSendFail.Inc() r := c.getCachedRegionWithRLock(ctx.Region) if r != nil { - c.switchNextPeer(r, ctx.PeerIdx) + c.switchNextPeer(r, ctx.PeerIdx, err) if scheduleReload { r.scheduleReload() } @@ -523,7 +540,7 @@ func (c *RegionCache) UpdateLeader(regionID RegionVerID, leaderStoreID uint64, c } if leaderStoreID == 0 { - c.switchNextPeer(r, currentPeerIdx) + c.switchNextPeer(r, currentPeerIdx, nil) logutil.BgLogger().Info("switch region peer to next due to NotLeader with NULL leader", zap.Int("currIdx", currentPeerIdx), zap.Uint64("regionID", regionID.GetID())) @@ -939,15 +956,24 @@ func (c *RegionCache) switchToPeer(r *Region, targetStoreID uint64) (found bool) return } -func (c *RegionCache) switchNextPeer(r *Region, currentPeerIdx int) { - regionStore := r.getStore() - if int(regionStore.workStoreIdx) != currentPeerIdx { +func (c *RegionCache) switchNextPeer(r *Region, currentPeerIdx int, err error) { + rs := r.getStore() + if int(rs.workStoreIdx) != currentPeerIdx { return } - nextIdx := (currentPeerIdx + 1) % len(regionStore.stores) - newRegionStore := regionStore.clone() + + if err != nil { // TODO: refine err, only do this for some errors. + s := rs.stores[rs.workStoreIdx] + epoch := rs.storeFails[rs.workStoreIdx] + if atomic.CompareAndSwapUint32(&s.fail, epoch, epoch+1) { + logutil.BgLogger().Info("mark store's regions need be refill", zap.String("store", s.addr)) + } + } + + nextIdx := (currentPeerIdx + 1) % len(rs.stores) + newRegionStore := rs.clone() newRegionStore.workStoreIdx = int32(nextIdx) - r.compareAndSwapStore(regionStore, newRegionStore) + r.compareAndSwapStore(rs, newRegionStore) } func (c *RegionCache) getPeerStoreIndex(r *Region, id uint64) (idx int, found bool) { @@ -1000,6 +1026,7 @@ type Store struct { storeID uint64 // store's id state uint64 // unsafe store storeState resolveMutex sync.Mutex // protect pd from concurrent init requests + fail uint32 // store fail count, see RegionStore.storeFails } type resolveState uint64 diff --git a/store/tikv/region_cache_test.go b/store/tikv/region_cache_test.go index 6437fd7c7532e..68788fd6c897a 100644 --- a/store/tikv/region_cache_test.go +++ b/store/tikv/region_cache_test.go @@ -15,6 +15,7 @@ package tikv import ( "context" + "errors" "fmt" "testing" "time" @@ -289,6 +290,31 @@ func (s *testRegionCacheSuite) TestSendFailedInHibernateRegion(c *C) { c.Assert(ctx.Peer.Id, Equals, s.peer1) } +func (s *testRegionCacheSuite) TestSendFailInvalidateRegionsInSameStore(c *C) { + // key range: ['' - 'm' - 'z'] + region2 := s.cluster.AllocID() + newPeers := s.cluster.AllocIDs(2) + s.cluster.Split(s.region1, region2, []byte("m"), newPeers, newPeers[0]) + + // Check the two regions. + loc1, err := s.cache.LocateKey(s.bo, []byte("a")) + c.Assert(err, IsNil) + c.Assert(loc1.Region.id, Equals, s.region1) + loc2, err := s.cache.LocateKey(s.bo, []byte("x")) + c.Assert(err, IsNil) + c.Assert(loc2.Region.id, Equals, region2) + + // Send fail on region1 + ctx, _ := s.cache.GetRPCContext(s.bo, loc1.Region) + s.checkCache(c, 2) + s.cache.OnSendFail(s.bo, ctx, false, errors.New("test error")) + + // Get region2 cache will get nil then reload. + ctx2, err := s.cache.GetRPCContext(s.bo, loc2.Region) + c.Assert(ctx2, IsNil) + c.Assert(err, IsNil) +} + func (s *testRegionCacheSuite) TestSendFailedInMultipleNode(c *C) { // 3 nodes and no.1 is leader. store3 := s.cluster.AllocID() @@ -639,7 +665,7 @@ func BenchmarkOnRequestFail(b *testing.B) { } r := cache.getCachedRegionWithRLock(rpcCtx.Region) if r == nil { - cache.switchNextPeer(r, rpcCtx.PeerIdx) + cache.switchNextPeer(r, rpcCtx.PeerIdx, nil) } } }) diff --git a/store/tikv/region_request_test.go b/store/tikv/region_request_test.go index 9d80af4c67efa..444e5410a1c8f 100644 --- a/store/tikv/region_request_test.go +++ b/store/tikv/region_request_test.go @@ -112,7 +112,10 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOn // send to failed store resp, err = s.regionRequestSender.SendReq(NewBackoffer(context.Background(), 100), req, region.Region, time.Second) - c.Assert(err, NotNil) + c.Assert(err, IsNil) + regionErr, err := resp.GetRegionError() + c.Assert(err, IsNil) + c.Assert(regionErr, NotNil) // retry to send store by old region info region, err = s.cache.LocateRegionByID(s.bo, s.region) @@ -121,7 +124,10 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOn // retry again, reload region info and send to new store. resp, err = s.regionRequestSender.SendReq(NewBackoffer(context.Background(), 100), req, region.Region, time.Second) - c.Assert(err, NotNil) + c.Assert(err, IsNil) + regionErr, err = resp.GetRegionError() + c.Assert(err, IsNil) + c.Assert(regionErr, NotNil) } func (s *testRegionRequestSuite) TestSendReqCtx(c *C) { From f8912049f52e94d2ec09001395f7f491630d5c29 Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Wed, 24 Jul 2019 13:36:29 +0800 Subject: [PATCH 083/196] *:add opt_rule_blacklist in mysql tables. (#11096) --- cmd/explaintest/r/black_list.result | 35 +++++++++++++ cmd/explaintest/t/black_list.test | 27 ++++++++++ executor/analyze_test.go | 2 +- executor/builder.go | 6 +++ executor/opt_rule_blacklist.go | 50 +++++++++++++++++++ executor/reload_expr_pushdown_blacklist.go | 3 +- .../reload_expr_pushdown_blacklist_test.go | 42 ---------------- expression/integration_test.go | 5 ++ go.mod | 1 - go.sum | 3 +- planner/core/common_plans.go | 5 ++ planner/core/optimizer.go | 14 +++++- planner/core/planbuilder.go | 2 + planner/core/rule_aggregation_elimination.go | 4 ++ planner/core/rule_aggregation_push_down.go | 4 ++ planner/core/rule_build_key_info.go | 4 ++ planner/core/rule_column_pruning.go | 4 ++ planner/core/rule_decorrelate.go | 4 ++ planner/core/rule_eliminate_projection.go | 4 ++ planner/core/rule_join_elimination.go | 4 ++ planner/core/rule_join_reorder.go | 4 ++ planner/core/rule_max_min_eliminate.go | 4 ++ planner/core/rule_partition_processor.go | 4 ++ planner/core/rule_predicate_push_down.go | 4 ++ planner/core/rule_topn_push_down.go | 4 ++ session/bootstrap.go | 16 ++++++ session/session.go | 7 ++- 27 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 cmd/explaintest/r/black_list.result create mode 100644 cmd/explaintest/t/black_list.test create mode 100644 executor/opt_rule_blacklist.go delete mode 100644 executor/reload_expr_pushdown_blacklist_test.go diff --git a/cmd/explaintest/r/black_list.result b/cmd/explaintest/r/black_list.result new file mode 100644 index 0000000000000..465490c35bfdb --- /dev/null +++ b/cmd/explaintest/r/black_list.result @@ -0,0 +1,35 @@ +use test; +drop table if exists t; +create table t (a int); +explain select * from t where a < 1; +id count task operator info +TableReader_7 3323.33 root data:Selection_6 +└─Selection_6 3323.33 cop lt(test.t.a, 1) + └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo +insert into mysql.opt_rule_blacklist values('predicate_push_down'); +admin reload opt_rule_blacklist; + +explain select * from t where a < 1; +id count task operator info +Selection_5 8000.00 root lt(test.t.a, 1) +└─TableReader_7 10000.00 root data:TableScan_6 + └─TableScan_6 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo +delete from mysql.opt_rule_blacklist where name='predicate_push_down'; +admin reload opt_rule_blacklist; + +explain select * from t where a < 1; +id count task operator info +TableReader_7 3323.33 root data:Selection_6 +└─Selection_6 3323.33 cop lt(test.t.a, 1) + └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo +insert into mysql.expr_pushdown_blacklist values('lt'); +admin reload expr_pushdown_blacklist; + +explain select * from t where a < 1; +id count task operator info +Selection_5 8000.00 root lt(test.t.a, 1) +└─TableReader_7 10000.00 root data:TableScan_6 + └─TableScan_6 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo +delete from mysql.expr_pushdown_blacklist where name='lt'; +admin reload expr_pushdown_blacklist; + diff --git a/cmd/explaintest/t/black_list.test b/cmd/explaintest/t/black_list.test new file mode 100644 index 0000000000000..27a41226295cf --- /dev/null +++ b/cmd/explaintest/t/black_list.test @@ -0,0 +1,27 @@ +use test; +drop table if exists t; +create table t (a int); + +explain select * from t where a < 1; + +insert into mysql.opt_rule_blacklist values('predicate_push_down'); + +admin reload opt_rule_blacklist; + +explain select * from t where a < 1; + +delete from mysql.opt_rule_blacklist where name='predicate_push_down'; + +admin reload opt_rule_blacklist; + +explain select * from t where a < 1; + +insert into mysql.expr_pushdown_blacklist values('lt'); + +admin reload expr_pushdown_blacklist; + +explain select * from t where a < 1; + +delete from mysql.expr_pushdown_blacklist where name='lt'; + +admin reload expr_pushdown_blacklist; \ No newline at end of file diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 50577ca7c50ba..ce0f579e56757 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -265,7 +265,7 @@ func (s *testSuite1) TestFastAnalyze(c *C) { c.Assert(err, IsNil) tableInfo := table.Meta() tbl := dom.StatsHandle().GetTableStats(tableInfo) - c.Assert(tbl.String(), Equals, "Table:41 Count:20\n"+ + c.Assert(tbl.String(), Equals, "Table:43 Count:20\n"+ "column:1 ndv:20 totColSize:0\n"+ "num: 6 lower_bound: 3 upper_bound: 15 repeats: 1\n"+ "num: 7 lower_bound: 18 upper_bound: 33 repeats: 1\n"+ diff --git a/executor/builder.go b/executor/builder.go index 0b4de52b007a5..d2d048616bd93 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -108,6 +108,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildChecksumTable(v) case *plannercore.ReloadExprPushdownBlacklist: return b.buildReloadExprPushdownBlacklist(v) + case *plannercore.ReloadOptRuleBlacklist: + return b.buildReloadOptRuleBlacklist(v) case *plannercore.AdminPlugins: return b.buildAdminPlugins(v) case *plannercore.DDL: @@ -469,6 +471,10 @@ func (b *executorBuilder) buildReloadExprPushdownBlacklist(v *plannercore.Reload return &ReloadExprPushdownBlacklistExec{baseExecutor{ctx: b.ctx}} } +func (b *executorBuilder) buildReloadOptRuleBlacklist(v *plannercore.ReloadOptRuleBlacklist) Executor { + return &ReloadOptRuleBlacklistExec{baseExecutor{ctx: b.ctx}} +} + func (b *executorBuilder) buildAdminPlugins(v *plannercore.AdminPlugins) Executor { return &AdminPluginsExec{baseExecutor: baseExecutor{ctx: b.ctx}, Action: v.Action, Plugins: v.Plugins} } diff --git a/executor/opt_rule_blacklist.go b/executor/opt_rule_blacklist.go new file mode 100644 index 0000000000000..a6a4d37ec28ef --- /dev/null +++ b/executor/opt_rule_blacklist.go @@ -0,0 +1,50 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + + plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/set" + "github.com/pingcap/tidb/util/sqlexec" +) + +// ReloadOptRuleBlacklistExec indicates ReloadOptRuleBlacklist executor. +type ReloadOptRuleBlacklistExec struct { + baseExecutor +} + +// Next implements the Executor Next interface. +func (e *ReloadOptRuleBlacklistExec) Next(ctx context.Context, _ *chunk.Chunk) error { + return LoadOptRuleBlacklist(e.ctx) +} + +// LoadOptRuleBlacklist loads the latest data from table mysql.opt_rule_blacklist. +func LoadOptRuleBlacklist(ctx sessionctx.Context) (err error) { + sql := "select HIGH_PRIORITY name from mysql.opt_rule_blacklist" + rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, sql) + if err != nil { + return err + } + newDisabledLogicalRules := set.NewStringSet() + for _, row := range rows { + name := row.GetString(0) + newDisabledLogicalRules.Insert(name) + } + plannercore.DefaultDisabledLogicalRulesList.Store(newDisabledLogicalRules) + return nil +} diff --git a/executor/reload_expr_pushdown_blacklist.go b/executor/reload_expr_pushdown_blacklist.go index e80a9ea9cf65e..90bbcae1362fa 100644 --- a/executor/reload_expr_pushdown_blacklist.go +++ b/executor/reload_expr_pushdown_blacklist.go @@ -15,7 +15,6 @@ package executor import ( "context" - "strings" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" @@ -43,7 +42,7 @@ func LoadExprPushdownBlacklist(ctx sessionctx.Context) (err error) { newBlacklist := make(map[string]struct{}) for _, row := range rows { name := row.GetString(0) - newBlacklist[strings.ToLower(name)] = struct{}{} + newBlacklist[name] = struct{}{} } expression.DefaultExprPushdownBlacklist.Store(newBlacklist) return nil diff --git a/executor/reload_expr_pushdown_blacklist_test.go b/executor/reload_expr_pushdown_blacklist_test.go deleted file mode 100644 index a31c444511aec..0000000000000 --- a/executor/reload_expr_pushdown_blacklist_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package executor_test - -import ( - . "github.com/pingcap/check" - "github.com/pingcap/tidb/util/testkit" -) - -func (s *testSuite2) TestReloadExprPushdownBlacklist(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("create database expr_pushdown_blacklist") - tk.MustExec("use expr_pushdown_blacklist") - tk.MustExec("create table t (a int)") - tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( - "TableReader_7 3323.33 root data:Selection_6", - "└─Selection_6 3323.33 cop lt(expr_pushdown_blacklist.t.a, 1)", - " └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) - - tk.MustExec("insert into mysql.expr_pushdown_blacklist values('lt')") - tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( - "TableReader_7 3323.33 root data:Selection_6", - "└─Selection_6 3323.33 cop lt(expr_pushdown_blacklist.t.a, 1)", - " └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) - - tk.MustExec("admin reload expr_pushdown_blacklist") - tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( - "Selection_5 8000.00 root lt(expr_pushdown_blacklist.t.a, 1)", - "└─TableReader_7 10000.00 root data:TableScan_6", - " └─TableScan_6 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) -} diff --git a/expression/integration_test.go b/expression/integration_test.go index 3e551f0199e2f..4b55bc284bc8e 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4458,6 +4458,11 @@ func (s *testIntegrationSuite) TestExprPushdownBlacklist(c *C) { tk.MustQuery(`select * from mysql.expr_pushdown_blacklist`).Check(testkit.Rows()) } +func (s *testIntegrationSuite) TestOptRuleBlacklist(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustQuery(`select * from mysql.opt_rule_blacklist`).Check(testkit.Rows()) +} + func (s *testIntegrationSuite) TestIssue10804(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustQuery(`SELECT @@information_schema_stats_expiry`).Check(testkit.Rows(`86400`)) diff --git a/go.mod b/go.mod index 5950f1dc2f5c0..ba19c42dd4d89 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,6 @@ require ( golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0 google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect google.golang.org/grpc v1.17.0 diff --git a/go.sum b/go.sum index 10bd3f3646065..b9ccf000166a7 100644 --- a/go.sum +++ b/go.sum @@ -277,9 +277,8 @@ golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYb golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0 h1:iRpjPej1fPzmfoBhMFkp3HdqzF+ytPmAwiQhJGV0zGw= golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index c43f03bf67b50..8a458ffc90bc6 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -133,6 +133,11 @@ type ReloadExprPushdownBlacklist struct { baseSchemaProducer } +// ReloadOptRuleBlacklist reloads the data from opt_rule_blacklist table. +type ReloadOptRuleBlacklist struct { + baseSchemaProducer +} + // AdminPluginsAction indicate action will be taken on plugins. type AdminPluginsAction int diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index ab7d09117fe19..f164dbb5a92a4 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/set" "go.uber.org/atomic" ) @@ -68,6 +69,7 @@ var optRuleList = []logicalOptRule{ // logicalOptRule means a logical optimizing rule, which contains decorrelate, ppd, column pruning, etc. type logicalOptRule interface { optimize(LogicalPlan) (LogicalPlan, error) + name() string } // BuildLogicalPlan used to build logical plan from ast.Node. @@ -139,7 +141,7 @@ func logicalOptimize(flag uint64, logic LogicalPlan) (LogicalPlan, error) { // The order of flags is same as the order of optRule in the list. // We use a bitmask to record which opt rules should be used. If the i-th bit is 1, it means we should // apply i-th optimizing rule. - if flag&(1< Date: Wed, 24 Jul 2019 14:04:38 +0800 Subject: [PATCH 084/196] planner: reject invalid conversion from like to = (#11320) --- go.sum | 1 + planner/core/expression_rewriter.go | 2 +- planner/core/expression_rewriter_test.go | 18 ++++++++++++++++++ planner/core/logical_plan_test.go | 2 +- planner/core/physical_plan_test.go | 6 ------ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/go.sum b/go.sum index b9ccf000166a7..6d6d387061618 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index a241b1133a27a..6d5dbbea7610b 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -1237,7 +1237,7 @@ func (er *expressionRewriter) patternLikeToExpression(v *ast.PatternLikeExpr) { } if !isNull { patValue, patTypes := stringutil.CompilePattern(patString, v.Escape) - if stringutil.IsExactMatch(patTypes) { + if stringutil.IsExactMatch(patTypes) && er.ctxStack[l-2].GetType().EvalType() == types.ETString { op := ast.EQ if v.Not { op = ast.NE diff --git a/planner/core/expression_rewriter_test.go b/planner/core/expression_rewriter_test.go index 034634685e237..c180b6aea50cc 100644 --- a/planner/core/expression_rewriter_test.go +++ b/planner/core/expression_rewriter_test.go @@ -242,3 +242,21 @@ func (s *testExpressionRewriterSuite) TestCheckFullGroupBy(c *C) { err = tk.ExecToErr("select t1.a, (select t2.a, max(t2.b) from t t2) from t t1") c.Assert(terror.ErrorEqual(err, core.ErrMixOfGroupFuncAndFields), IsTrue, Commentf("err %v", err)) } + +func (s *testExpressionRewriterSuite) TestPatternLikeToExpression(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + defer func() { + dom.Close() + store.Close() + }() + tk.MustQuery("select 0 like 'a string';").Check(testkit.Rows("0")) + tk.MustQuery("select 0.0 like 'a string';").Check(testkit.Rows("0")) + tk.MustQuery("select 0 like '0.00';").Check(testkit.Rows("0")) + tk.MustQuery("select cast(\"2011-5-3\" as datetime) like \"2011-05-03\";").Check(testkit.Rows("0")) + tk.MustQuery("select 1 like '1';").Check(testkit.Rows("1")) + tk.MustQuery("select 0 like '0';").Check(testkit.Rows("1")) + tk.MustQuery("select 0.00 like '0.00';").Check(testkit.Rows("1")) +} diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index b9faf05e017fa..a9839e100d6e5 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -83,7 +83,7 @@ func (s *testPlanSuite) TestPredicatePushDown(c *C) { }, { sql: "select * from t t1, t t2 where t1.a = t2.b and t2.b > 0 and t1.a = t1.c and t1.d like 'abc' and t2.d = t1.d", - best: "Join{DataScan(t1)->Sel([eq(cast(test.t1.d), cast(abc))])->DataScan(t2)->Sel([eq(cast(test.t2.d), cast(abc))])}(test.t1.a,test.t2.b)(test.t1.d,test.t2.d)->Projection", + best: "Join{DataScan(t1)->Sel([like(cast(test.t1.d), abc, 92)])->DataScan(t2)->Sel([like(cast(test.t2.d), abc, 92)])}(test.t1.a,test.t2.b)(test.t1.d,test.t2.d)->Projection", }, { sql: "select * from t ta join t tb on ta.d = tb.d and ta.d > 1 where tb.a = 0", diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 459aa9cc5d113..fc3de15294eec 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1170,12 +1170,6 @@ func (s *testPlanSuite) TestRefine(c *C) { sql: `select a from t where c_str like 123`, best: "IndexReader(Index(t.c_d_e_str)[[\"123\",\"123\"]])->Projection", }, - // c is type int which will be added cast to specified type when building function signature, - // and rewrite predicate like to predicate '=' when exact match , index still can be used. - { - sql: `select a from t where c like '1'`, - best: "IndexReader(Index(t.c_d_e)[[1,1]])->Projection", - }, { sql: `select a from t where c = 1.9 and d > 3`, best: "Dual", From 29970ca353f9611e56e6538163ed920c90827b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B0=E5=87=8C=E7=BF=94?= Date: Wed, 24 Jul 2019 16:06:37 +0800 Subject: [PATCH 085/196] DDL: fix a bug in column charset and collate when create table and modify column (#11300) * fix a bug in column charset and collate when create table and modify column * extract the fix code logic to a func * fix multiple collate specified in column * add more test for multiple column collate * add more Err case for multiple column collate * fix comment format --- ddl/ddl_api.go | 79 ++++++++++++++++++++++++++++++++++++++------ executor/ddl_test.go | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 11 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index f24102e713d12..f96c1723dac62 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -312,15 +312,34 @@ func typesNeedCharset(tp byte) bool { return false } -func setCharsetCollationFlenDecimal(tp *types.FieldType, tblCharset string, dbCharset string) error { +func setCharsetCollationFlenDecimal(tp *types.FieldType, specifiedCollates []string, tblCharset string, dbCharset string) error { tp.Charset = strings.ToLower(tp.Charset) tp.Collate = strings.ToLower(tp.Collate) if len(tp.Charset) == 0 { if typesNeedCharset(tp.Tp) { - var err error - tp.Charset, tp.Collate, err = ResolveCharsetCollation(tblCharset, dbCharset) - if err != nil { - return errors.Trace(err) + if len(specifiedCollates) == 0 { + // Both the charset and collate are not specified. + var err error + tp.Charset, tp.Collate, err = ResolveCharsetCollation(tblCharset, dbCharset) + if err != nil { + return errors.Trace(err) + } + } else { + // The charset is not specified but the collate is. + // We should derive charset from it's collate specified rather than getting from table and db. + // It is handled like mysql's logic, use derived charset to judge conflict with next collate. + for _, spc := range specifiedCollates { + derivedCollation, err := charset.GetCollationByName(spc) + if err != nil { + return errors.Trace(err) + } + if len(tp.Charset) == 0 { + tp.Charset = derivedCollation.CharsetName + } else if tp.Charset != derivedCollation.CharsetName { + return ErrCollationCharsetMismatch.GenWithStackByArgs(derivedCollation.Name, tp.Charset) + } + tp.Collate = derivedCollation.Name + } } } else { tp.Charset = charset.CharsetBin @@ -331,10 +350,25 @@ func setCharsetCollationFlenDecimal(tp *types.FieldType, tblCharset string, dbCh return errUnsupportedCharset.GenWithStackByArgs(tp.Charset, tp.Collate) } if len(tp.Collate) == 0 { - var err error - tp.Collate, err = charset.GetDefaultCollation(tp.Charset) - if err != nil { - return errors.Trace(err) + if len(specifiedCollates) == 0 { + // The charset is specified, but the collate is not. + var err error + tp.Collate, err = charset.GetDefaultCollation(tp.Charset) + if err != nil { + return errors.Trace(err) + } + } else { + // Both the charset and collate are specified. + for _, spc := range specifiedCollates { + derivedCollation, err := charset.GetCollationByName(spc) + if err != nil { + return errors.Trace(err) + } + if tp.Charset != derivedCollation.CharsetName { + return ErrCollationCharsetMismatch.GenWithStackByArgs(derivedCollation.Name, tp.Charset) + } + tp.Collate = derivedCollation.Name + } } } } @@ -358,7 +392,10 @@ func setCharsetCollationFlenDecimal(tp *types.FieldType, tblCharset string, dbCh // outPriKeyConstraint is the primary key constraint out of column definition. such as: create table t1 (id int , age int, primary key(id)); func buildColumnAndConstraint(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, outPriKeyConstraint *ast.Constraint, tblCharset, dbCharset string) (*table.Column, []*ast.Constraint, error) { - if err := setCharsetCollationFlenDecimal(colDef.Tp, tblCharset, dbCharset); err != nil { + // specifiedCollates refers to collates in colDef.Options, should handle them together. + specifiedCollates := extractCollateFromOption(colDef) + + if err := setCharsetCollationFlenDecimal(colDef.Tp, specifiedCollates, tblCharset, dbCharset); err != nil { return nil, nil, errors.Trace(err) } col, cts, err := columnDefToCol(ctx, offset, colDef, outPriKeyConstraint) @@ -2596,7 +2633,11 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or newCol.FieldType.Charset = col.FieldType.Charset newCol.FieldType.Collate = col.FieldType.Collate } - err = setCharsetCollationFlenDecimal(&newCol.FieldType, t.Meta().Charset, schema.Charset) + // specifiedCollates refers to collates in colDef.Option. When setting charset and collate here we + // should take the collate in colDef.Option into consideration rather than handling it separately + specifiedCollates := extractCollateFromOption(specNewColumn) + + err = setCharsetCollationFlenDecimal(&newCol.FieldType, specifiedCollates, t.Meta().Charset, schema.Charset) if err != nil { return nil, errors.Trace(err) } @@ -3582,3 +3623,19 @@ type lockTablesArg struct { SessionInfo model.SessionInfo IsCleanup bool } + +// extractCollateFromOption take collates(may multiple) in option into consideration +// when handle charset and collate of a column, rather than handling it separately. +func extractCollateFromOption(def *ast.ColumnDef) []string { + specifiedCollates := make([]string, 0, 0) + for i := 0; i < len(def.Options); i++ { + op := def.Options[i] + if op.Tp == ast.ColumnOptionCollate { + specifiedCollates = append(specifiedCollates, op.StrValue) + def.Options = append(def.Options[:i], def.Options[i+1:]...) + // maintain the correct index + i-- + } + } + return specifiedCollates +} diff --git a/executor/ddl_test.go b/executor/ddl_test.go index c29dce7a8acc8..2590b72af428d 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -121,6 +121,36 @@ func (s *testSuite3) TestCreateTable(c *C) { } } + // test multiple collate specified in column when create. + tk.MustExec("drop table if exists test_multiple_column_collate;") + tk.MustExec("create table test_multiple_column_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + t, err := domain.GetDomain(tk.Se).InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("test_multiple_column_collate")) + c.Assert(err, IsNil) + c.Assert(t.Cols()[0].Charset, Equals, "utf8") + c.Assert(t.Cols()[0].Collate, Equals, "utf8_general_ci") + c.Assert(t.Meta().Charset, Equals, "utf8mb4") + c.Assert(t.Meta().Collate, Equals, "utf8mb4_bin") + + tk.MustExec("drop table if exists test_multiple_column_collate;") + tk.MustExec("create table test_multiple_column_collate (a char(1) charset utf8 collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + t, err = domain.GetDomain(tk.Se).InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("test_multiple_column_collate")) + c.Assert(err, IsNil) + c.Assert(t.Cols()[0].Charset, Equals, "utf8") + c.Assert(t.Cols()[0].Collate, Equals, "utf8_general_ci") + c.Assert(t.Meta().Charset, Equals, "utf8mb4") + c.Assert(t.Meta().Collate, Equals, "utf8mb4_bin") + + // test Err case for multiple collate specified in column when create. + tk.MustExec("drop table if exists test_err_multiple_collate;") + _, err = tk.Exec("create table test_err_multiple_collate (a char(1) charset utf8mb4 collate utf8_unicode_ci collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, ddl.ErrCollationCharsetMismatch.GenWithStackByArgs("utf8_unicode_ci", "utf8mb4").Error()) + + tk.MustExec("drop table if exists test_err_multiple_collate;") + _, err = tk.Exec("create table test_err_multiple_collate (a char(1) collate utf8_unicode_ci collate utf8mb4_general_ci) charset utf8mb4 collate utf8mb4_bin") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, ddl.ErrCollationCharsetMismatch.GenWithStackByArgs("utf8mb4_general_ci", "utf8").Error()) + // table option is auto-increment tk.MustExec("drop table if exists create_auto_increment_test;") tk.MustExec("create table create_auto_increment_test (id int not null auto_increment, name varchar(255), primary key(id)) auto_increment = 999;") @@ -322,6 +352,43 @@ func (s *testSuite3) TestAlterTableModifyColumn(c *C) { _, err = tk.Exec("alter table alter_view modify column c2 text") c.Assert(err.Error(), Equals, ddl.ErrWrongObject.GenWithStackByArgs("test", "alter_view", "BASE TABLE").Error()) tk.MustExec("drop view alter_view") + + // test multiple collate modification in column. + tk.MustExec("drop table if exists modify_column_multiple_collate") + tk.MustExec("create table modify_column_multiple_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + _, err = tk.Exec("alter table modify_column_multiple_collate modify column a char(1) collate utf8mb4_bin;") + c.Assert(err, IsNil) + t, err := domain.GetDomain(tk.Se).InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("modify_column_multiple_collate")) + c.Assert(err, IsNil) + c.Assert(t.Cols()[0].Charset, Equals, "utf8mb4") + c.Assert(t.Cols()[0].Collate, Equals, "utf8mb4_bin") + c.Assert(t.Meta().Charset, Equals, "utf8mb4") + c.Assert(t.Meta().Collate, Equals, "utf8mb4_bin") + + tk.MustExec("drop table if exists modify_column_multiple_collate;") + tk.MustExec("create table modify_column_multiple_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + _, err = tk.Exec("alter table modify_column_multiple_collate modify column a char(1) charset utf8mb4 collate utf8mb4_bin;") + c.Assert(err, IsNil) + t, err = domain.GetDomain(tk.Se).InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("modify_column_multiple_collate")) + c.Assert(err, IsNil) + c.Assert(t.Cols()[0].Charset, Equals, "utf8mb4") + c.Assert(t.Cols()[0].Collate, Equals, "utf8mb4_bin") + c.Assert(t.Meta().Charset, Equals, "utf8mb4") + c.Assert(t.Meta().Collate, Equals, "utf8mb4_bin") + + // test Err case for multiple collate modification in column. + tk.MustExec("drop table if exists err_modify_multiple_collate;") + tk.MustExec("create table err_modify_multiple_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + _, err = tk.Exec("alter table err_modify_multiple_collate modify column a char(1) charset utf8mb4 collate utf8_bin;") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, ddl.ErrCollationCharsetMismatch.GenWithStackByArgs("utf8_bin", "utf8mb4").Error()) + + tk.MustExec("drop table if exists err_modify_multiple_collate;") + tk.MustExec("create table err_modify_multiple_collate (a char(1) collate utf8_bin collate utf8_general_ci) charset utf8mb4 collate utf8mb4_bin") + _, err = tk.Exec("alter table err_modify_multiple_collate modify column a char(1) collate utf8_bin collate utf8mb4_bin;") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, ddl.ErrCollationCharsetMismatch.GenWithStackByArgs("utf8mb4_bin", "utf8").Error()) + } func (s *testSuite3) TestDefaultDBAfterDropCurDB(c *C) { From be431dad25ab9bcbc74219103324d1edbd16be87 Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Wed, 24 Jul 2019 16:33:29 +0800 Subject: [PATCH 086/196] make time func now related unit tests stable (#11412) --- expression/builtin_time_test.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index c225ba7bd28c5..88566b9fbd584 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -783,9 +783,9 @@ func (s *testEvaluatorSuite) TestNowAndUTCTimestamp(c *C) { {funcs[ast.Now], func() time.Time { return time.Now() }}, {funcs[ast.UTCTimestamp], func() time.Time { return time.Now().UTC() }}, } { - resetStmtContext(s.ctx) f, err := x.fc.getFunction(s.ctx, s.datumsToConstants(nil)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) ts := x.now() c.Assert(err, IsNil) @@ -795,9 +795,9 @@ func (s *testEvaluatorSuite) TestNowAndUTCTimestamp(c *C) { c.Assert(strings.Contains(t.String(), "."), IsFalse) c.Assert(ts.Sub(gotime(t, ts.Location())), LessEqual, time.Second) - resetStmtContext(s.ctx) f, err = x.fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(6))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err = evalBuiltinFunc(f, chunk.Row{}) ts = x.now() c.Assert(err, IsNil) @@ -1071,6 +1071,7 @@ func (s *testEvaluatorSuite) TestSysDate(c *C) { variable.SetSessionSystemVar(ctx.GetSessionVars(), "timestamp", timezone) f, err := fc.getFunction(ctx, s.datumsToConstants(nil)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) last := time.Now() c.Assert(err, IsNil) @@ -1081,6 +1082,7 @@ func (s *testEvaluatorSuite) TestSysDate(c *C) { last := time.Now() f, err := fc.getFunction(ctx, s.datumsToConstants(types.MakeDatums(6))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n := v.GetMysqlTime() @@ -1213,6 +1215,7 @@ func (s *testEvaluatorSuite) TestCurrentDate(c *C) { fc := funcs[ast.CurrentDate] f, err := fc.getFunction(mock.NewContext(), s.datumsToConstants(nil)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n := v.GetMysqlTime() @@ -1227,6 +1230,7 @@ func (s *testEvaluatorSuite) TestCurrentTime(c *C) { fc := funcs[ast.CurrentTime] f, err := fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(nil))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n := v.GetMysqlDuration() @@ -1235,6 +1239,7 @@ func (s *testEvaluatorSuite) TestCurrentTime(c *C) { f, err = fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(3))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err = evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n = v.GetMysqlDuration() @@ -1243,6 +1248,7 @@ func (s *testEvaluatorSuite) TestCurrentTime(c *C) { f, err = fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(6))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err = evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n = v.GetMysqlDuration() @@ -1269,9 +1275,9 @@ func (s *testEvaluatorSuite) TestUTCTime(c *C) { }{{0, 8}, {3, 12}, {6, 15}, {-1, 0}, {7, 0}} for _, test := range tests { - resetStmtContext(s.ctx) f, err := fc.getFunction(s.ctx, s.datumsToConstants(types.MakeDatums(test.param))) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) if test.expect > 0 { c.Assert(err, IsNil) @@ -1285,6 +1291,7 @@ func (s *testEvaluatorSuite) TestUTCTime(c *C) { f, err := fc.getFunction(s.ctx, make([]Expression, 0)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) v, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n := v.GetMysqlDuration() @@ -1296,9 +1303,9 @@ func (s *testEvaluatorSuite) TestUTCDate(c *C) { defer testleak.AfterTest(c)() last := time.Now().UTC() fc := funcs[ast.UTCDate] - resetStmtContext(mock.NewContext()) f, err := fc.getFunction(mock.NewContext(), s.datumsToConstants(nil)) c.Assert(err, IsNil) + resetStmtContext(mock.NewContext()) v, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) n := v.GetMysqlTime() @@ -1635,9 +1642,9 @@ func (s *testEvaluatorSuite) TestTimestampDiff(c *C) { func (s *testEvaluatorSuite) TestUnixTimestamp(c *C) { // Test UNIX_TIMESTAMP(). fc := funcs[ast.UnixTimestamp] - resetStmtContext(s.ctx) f, err := fc.getFunction(s.ctx, nil) c.Assert(err, IsNil) + resetStmtContext(s.ctx) d, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) c.Assert(d.GetInt64()-time.Now().Unix(), GreaterEqual, int64(-1)) @@ -1652,9 +1659,9 @@ func (s *testEvaluatorSuite) TestUnixTimestamp(c *C) { n := types.Datum{} n.SetMysqlTime(now) args := []types.Datum{n} - resetStmtContext(s.ctx) f, err = fc.getFunction(s.ctx, s.datumsToConstants(args)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) d, err = evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) val, _ := d.GetMysqlDecimal().ToInt() @@ -2685,9 +2692,9 @@ func (s *testEvaluatorSuite) TestWithTimeZone(c *C) { for _, t := range tests { now := time.Now().In(sv.TimeZone) - resetStmtContext(s.ctx) f, err := funcs[t.method].getFunction(s.ctx, s.datumsToConstants(t.Input)) c.Assert(err, IsNil) + resetStmtContext(s.ctx) d, err := evalBuiltinFunc(f, chunk.Row{}) c.Assert(err, IsNil) result := t.convertToTime(d, sv.TimeZone) From 1cca2d1455ffbe9813d1f87006be1175e8cefd12 Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Wed, 24 Jul 2019 16:40:13 +0800 Subject: [PATCH 087/196] improvement. change unessesary mem copy in for index substring call (#11405) Test pass, auto merge by Bot --- executor/load_data.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/executor/load_data.go b/executor/load_data.go index dc4c4f4024444..eb612451d36ed 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -133,7 +133,7 @@ func (e *LoadDataInfo) getValidData(prevData, curData []byte) ([]byte, []byte) { prevLen := len(prevData) if prevLen > 0 { // starting symbol in the prevData - idx := strings.Index(string(prevData), e.LinesInfo.Starting) + idx := strings.Index(string(hack.String(prevData)), e.LinesInfo.Starting) if idx != -1 { return prevData[idx:], curData } @@ -144,14 +144,14 @@ func (e *LoadDataInfo) getValidData(prevData, curData []byte) ([]byte, []byte) { restStart = curData[:startingLen-1] } prevData = append(prevData, restStart...) - idx = strings.Index(string(prevData), e.LinesInfo.Starting) + idx = strings.Index(string(hack.String(prevData)), e.LinesInfo.Starting) if idx != -1 { return prevData[idx:prevLen], curData } } // starting symbol in the curData - idx := strings.Index(string(curData), e.LinesInfo.Starting) + idx := strings.Index(string(hack.String(curData)), e.LinesInfo.Starting) if idx != -1 { return nil, curData[idx:] } @@ -190,7 +190,7 @@ func (e *LoadDataInfo) getLine(prevData, curData []byte) ([]byte, []byte, bool) // terminated symbol in the middle of prevData and curData curData = append(prevData, curData...) - endIdx = strings.Index(string(curData[startingLen:]), e.LinesInfo.Terminated) + endIdx = strings.Index(string(hack.String(curData[startingLen:])), e.LinesInfo.Terminated) if endIdx != -1 { nextDataIdx := startingLen + endIdx + terminatedLen return curData[startingLen : startingLen+endIdx], curData[nextDataIdx:], true @@ -207,7 +207,7 @@ func (e *LoadDataInfo) getLine(prevData, curData []byte) ([]byte, []byte, bool) // terminated symbol in the curData prevData = append(prevData, curData[:nextDataIdx]...) - endIdx = strings.Index(string(prevData[startingLen:]), e.LinesInfo.Terminated) + endIdx = strings.Index(string(hack.String(prevData[startingLen:])), e.LinesInfo.Terminated) if endIdx >= prevLen { return prevData[startingLen : startingLen+endIdx], curData[nextDataIdx:], true } From b97c0434db2ee38b541ec578b4db57cc218af180 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 24 Jul 2019 16:57:51 +0800 Subject: [PATCH 088/196] executor: show `CARTESIAN Join` explicitly in the results of `Explain` (#11415) --- cmd/explaintest/r/explain_easy.result | 26 ++++++++-------- cmd/explaintest/r/explain_easy_stats.result | 4 +-- cmd/explaintest/r/select.result | 6 ++-- cmd/explaintest/r/subquery.result | 4 +-- executor/explain_test.go | 29 ++++++++++++++++++ expression/constant_propagation_test.go | 34 ++++++++++----------- planner/core/cbo_test.go | 20 ++++++------ planner/core/explain.go | 8 ++++- util/ranger/ranger_test.go | 2 +- 9 files changed, 84 insertions(+), 49 deletions(-) diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 5da87ec057f28..9699f6c7a5c95 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -89,7 +89,7 @@ explain select sum(t1.c1 in (select c1 from t2)) from t1; id count task operator info StreamAgg_12 1.00 root funcs:sum(col_0) └─Projection_19 10000.00 root cast(5_aux_0) - └─HashLeftJoin_18 10000.00 root left outer semi join, inner:TableReader_17, other cond:eq(test.t1.c1, test.t2.c1) + └─HashLeftJoin_18 10000.00 root CARTESIAN left outer semi join, inner:TableReader_17, other cond:eq(test.t1.c1, test.t2.c1) ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_17 10000.00 root data:TableScan_16 @@ -121,7 +121,7 @@ MemTableScan_4 10000.00 root explain select c2 = (select c2 from t2 where t1.c1 = t2.c1 order by c1 limit 1) from t1; id count task operator info Projection_12 10000.00 root eq(test.t1.c2, test.t2.c2) -└─Apply_14 10000.00 root left outer join, inner:Limit_21 +└─Apply_14 10000.00 root CARTESIAN left outer join, inner:Limit_21 ├─TableReader_16 10000.00 root data:TableScan_15 │ └─TableScan_15 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─Limit_21 1.00 root offset:0, count:1 @@ -225,7 +225,7 @@ explain select sum(t1.c1 in (select c1 from t2)) from t1; id count task operator info StreamAgg_12 1.00 root funcs:sum(col_0) └─Projection_19 10000.00 root cast(5_aux_0) - └─HashLeftJoin_18 10000.00 root left outer semi join, inner:TableReader_17, other cond:eq(test.t1.c1, test.t2.c1) + └─HashLeftJoin_18 10000.00 root CARTESIAN left outer semi join, inner:TableReader_17, other cond:eq(test.t1.c1, test.t2.c1) ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_17 10000.00 root data:TableScan_16 @@ -233,7 +233,7 @@ StreamAgg_12 1.00 root funcs:sum(col_0) explain select 1 in (select c2 from t2) from t1; id count task operator info Projection_6 10000.00 root 5_aux_0 -└─HashLeftJoin_7 10000.00 root left outer semi join, inner:TableReader_12 +└─HashLeftJoin_7 10000.00 root CARTESIAN left outer semi join, inner:TableReader_12 ├─TableReader_9 10000.00 root data:TableScan_8 │ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_12 10.00 root data:Selection_11 @@ -243,7 +243,7 @@ explain select sum(6 in (select c2 from t2)) from t1; id count task operator info StreamAgg_12 1.00 root funcs:sum(col_0) └─Projection_20 10000.00 root cast(5_aux_0) - └─HashLeftJoin_19 10000.00 root left outer semi join, inner:TableReader_18 + └─HashLeftJoin_19 10000.00 root CARTESIAN left outer semi join, inner:TableReader_18 ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_18 10.00 root data:Selection_17 @@ -312,7 +312,7 @@ create table t(a int primary key, b int, c int, index idx(b)); explain select t.c in (select count(*) from t s ignore index(idx), t t1 where s.a = t.a and s.a = t1.a) from t; id count task operator info Projection_11 10000.00 root 9_aux_0 -└─Apply_13 10000.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) +└─Apply_13 10000.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_20 1.00 root funcs:count(1) @@ -325,7 +325,7 @@ Projection_11 10000.00 root 9_aux_0 explain select t.c in (select count(*) from t s use index(idx), t t1 where s.b = t.a and s.a = t1.a) from t; id count task operator info Projection_11 10000.00 root 9_aux_0 -└─Apply_13 10000.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) +└─Apply_13 10000.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_20 1.00 root funcs:count(1) @@ -337,7 +337,7 @@ Projection_11 10000.00 root 9_aux_0 explain select t.c in (select count(*) from t s use index(idx), t t1 where s.b = t.a and s.c = t1.a) from t; id count task operator info Projection_11 10000.00 root 9_aux_0 -└─Apply_13 10000.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) +└─Apply_13 10000.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_20 1.00 root funcs:count(1) @@ -353,7 +353,7 @@ analyze table t; explain select t.c in (select count(*) from t s, t t1 where s.b = t.a and s.b = 3 and s.a = t1.a) from t; id count task operator info Projection_11 5.00 root 9_aux_0 -└─Apply_13 5.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) +└─Apply_13 5.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) ├─TableReader_15 5.00 root data:TableScan_14 │ └─TableScan_14 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_20 1.00 root funcs:count(1) @@ -367,7 +367,7 @@ Projection_11 5.00 root 9_aux_0 explain select t.c in (select count(*) from t s left join t t1 on s.a = t1.a where 3 = t.a and s.b = 3) from t; id count task operator info Projection_10 5.00 root 9_aux_0 -└─Apply_12 5.00 root left outer semi join, inner:StreamAgg_19, other cond:eq(test.t.c, 7_col_0) +└─Apply_12 5.00 root CARTESIAN left outer semi join, inner:StreamAgg_19, other cond:eq(test.t.c, 7_col_0) ├─TableReader_14 5.00 root data:TableScan_13 │ └─TableScan_13 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_19 1.00 root funcs:count(1) @@ -381,7 +381,7 @@ Projection_10 5.00 root 9_aux_0 explain select t.c in (select count(*) from t s right join t t1 on s.a = t1.a where 3 = t.a and t1.b = 3) from t; id count task operator info Projection_10 5.00 root 9_aux_0 -└─Apply_12 5.00 root left outer semi join, inner:StreamAgg_19, other cond:eq(test.t.c, 7_col_0) +└─Apply_12 5.00 root CARTESIAN left outer semi join, inner:StreamAgg_19, other cond:eq(test.t.c, 7_col_0) ├─TableReader_14 5.00 root data:TableScan_13 │ └─TableScan_13 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_19 1.00 root funcs:count(1) @@ -559,7 +559,7 @@ HashRightJoin_9 4166.67 root inner join, inner:TableReader_12, equal:[eq(test.ta explain select ifnull(t.nc, 1) in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t; id count task operator info Projection_12 10000.00 root 9_aux_0 -└─Apply_14 10000.00 root left outer semi join, inner:HashAgg_19, other cond:eq(test.t.nc, 7_col_0) +└─Apply_14 10000.00 root CARTESIAN left outer semi join, inner:HashAgg_19, other cond:eq(test.t.nc, 7_col_0) ├─TableReader_16 10000.00 root data:TableScan_15 │ └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─HashAgg_19 1.00 root funcs:count(join_agg_0) @@ -592,7 +592,7 @@ HashRightJoin_7 8000.00 root right outer join, inner:TableReader_10, equal:[eq(t explain select ifnull(t.a, 1) in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t; id count task operator info Projection_12 10000.00 root 9_aux_0 -└─Apply_14 10000.00 root left outer semi join, inner:HashAgg_19, other cond:eq(ifnull(test.t.a, 1), 7_col_0) +└─Apply_14 10000.00 root CARTESIAN left outer semi join, inner:HashAgg_19, other cond:eq(ifnull(test.t.a, 1), 7_col_0) ├─TableReader_16 10000.00 root data:TableScan_15 │ └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─HashAgg_19 1.00 root funcs:count(join_agg_0) diff --git a/cmd/explaintest/r/explain_easy_stats.result b/cmd/explaintest/r/explain_easy_stats.result index 787ae045aa54c..4945dd14c628d 100644 --- a/cmd/explaintest/r/explain_easy_stats.result +++ b/cmd/explaintest/r/explain_easy_stats.result @@ -107,7 +107,7 @@ MemTableScan_4 10000.00 root explain select c2 = (select c2 from t2 where t1.c1 = t2.c1 order by c1 limit 1) from t1; id count task operator info Projection_12 1999.00 root eq(test.t1.c2, test.t2.c2) -└─Apply_14 1999.00 root left outer join, inner:Limit_21 +└─Apply_14 1999.00 root CARTESIAN left outer join, inner:Limit_21 ├─TableReader_16 1999.00 root data:TableScan_15 │ └─TableScan_15 1999.00 cop table:t1, range:[-inf,+inf], keep order:false └─Limit_21 1.00 root offset:0, count:1 @@ -126,7 +126,7 @@ set @@session.tidb_opt_insubq_to_join_and_agg=0; explain select 1 in (select c2 from t2) from t1; id count task operator info Projection_6 1999.00 root 5_aux_0 -└─HashLeftJoin_7 1999.00 root left outer semi join, inner:TableReader_12 +└─HashLeftJoin_7 1999.00 root CARTESIAN left outer semi join, inner:TableReader_12 ├─TableReader_9 1999.00 root data:TableScan_8 │ └─TableScan_8 1999.00 cop table:t1, range:[-inf,+inf], keep order:false └─TableReader_12 0.00 root data:Selection_11 diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result index 5f56ff0e3c775..143cf7cc28a9f 100644 --- a/cmd/explaintest/r/select.result +++ b/cmd/explaintest/r/select.result @@ -325,7 +325,7 @@ drop table if exists t; create table t (id int primary key, a int, b int); explain select * from (t t1 left join t t2 on t1.a = t2.a) left join (t t3 left join t t4 on t3.a = t4.a) on t2.b = 1; id count task operator info -HashLeftJoin_10 155937656.25 root left outer join, inner:HashLeftJoin_17, left cond:[eq(test.t2.b, 1)] +HashLeftJoin_10 155937656.25 root CARTESIAN left outer join, inner:HashLeftJoin_17, left cond:[eq(test.t2.b, 1)] ├─HashLeftJoin_11 12487.50 root left outer join, inner:TableReader_16, equal:[eq(test.t1.a, test.t2.a)] │ ├─TableReader_13 10000.00 root data:TableScan_12 │ │ └─TableScan_12 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo @@ -381,7 +381,7 @@ create table t(a int, b int); explain select a != any (select a from t t2) from t t1; id count task operator info Projection_9 10000.00 root and(or(or(gt(col_count, 1), ne(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 0)), and(ne(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 1))) -└─HashLeftJoin_10 10000.00 root inner join, inner:StreamAgg_17 +└─HashLeftJoin_10 10000.00 root CARTESIAN inner join, inner:StreamAgg_17 ├─TableReader_13 10000.00 root data:TableScan_12 │ └─TableScan_12 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_17 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1) @@ -391,7 +391,7 @@ Projection_9 10000.00 root and(or(or(gt(col_count, 1), ne(test.t1.a, col_firstro explain select a = all (select a from t t2) from t t1; id count task operator info Projection_9 10000.00 root or(and(and(le(col_count, 1), eq(test.t1.a, col_firstrow)), if(ne(agg_col_sum, 0), NULL, 1)), or(eq(agg_col_cnt, 0), if(isnull(test.t1.a), NULL, 0))) -└─HashLeftJoin_10 10000.00 root inner join, inner:StreamAgg_17 +└─HashLeftJoin_10 10000.00 root CARTESIAN inner join, inner:StreamAgg_17 ├─TableReader_13 10000.00 root data:TableScan_12 │ └─TableScan_12 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_17 1.00 root funcs:firstrow(col_0), count(distinct col_1), sum(col_2), count(1) diff --git a/cmd/explaintest/r/subquery.result b/cmd/explaintest/r/subquery.result index f0bad21a8d2e0..3d367f8bbeb07 100644 --- a/cmd/explaintest/r/subquery.result +++ b/cmd/explaintest/r/subquery.result @@ -4,7 +4,7 @@ create table t1(a bigint, b bigint); create table t2(a bigint, b bigint); explain select * from t1 where t1.a in (select t1.b + t2.b from t2); id count task operator info -HashLeftJoin_8 8000.00 root semi join, inner:TableReader_12, other cond:eq(test.t1.a, plus(test.t1.b, test.t2.b)) +HashLeftJoin_8 8000.00 root CARTESIAN semi join, inner:TableReader_12, other cond:eq(test.t1.a, plus(test.t1.b, test.t2.b)) ├─TableReader_10 10000.00 root data:TableScan_9 │ └─TableScan_9 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─TableReader_12 10000.00 root data:TableScan_11 @@ -16,7 +16,7 @@ analyze table t; explain select t.c in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c = 1 and s.d = t.a and s.a = t1.a) from t; id count task operator info Projection_11 5.00 root 9_aux_0 -└─Apply_13 5.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) +└─Apply_13 5.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0) ├─TableReader_15 5.00 root data:TableScan_14 │ └─TableScan_14 5.00 cop table:t, range:[-inf,+inf], keep order:false └─StreamAgg_20 1.00 root funcs:count(1) diff --git a/executor/explain_test.go b/executor/explain_test.go index 47c634f3f454e..4af5141e71779 100644 --- a/executor/explain_test.go +++ b/executor/explain_test.go @@ -14,6 +14,7 @@ package executor_test import ( + "fmt" "strings" . "github.com/pingcap/check" @@ -63,6 +64,34 @@ func (s *testSuite1) TestExplainPriviliges(c *C) { c.Assert(err.Error(), Equals, plannercore.ErrTableaccessDenied.GenWithStackByArgs("SELECT", "explain", "%", "v").Error()) } +func (s *testSuite1) TestExplainCartesianJoin(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (v int)") + + cases := []struct { + sql string + isCartesianJoin bool + }{ + {"explain select * from t t1, t t2", true}, + {"explain select * from t t1 where exists (select 1 from t t2 where t2.v > t1.v)", true}, + {"explain select * from t t1 where exists (select 1 from t t2 where t2.v in (t1.v+1, t1.v+2))", true}, + {"explain select * from t t1, t t2 where t1.v = t2.v", false}, + } + for _, ca := range cases { + rows := tk.MustQuery(ca.sql).Rows() + ok := false + for _, row := range rows { + str := fmt.Sprintf("%v", row) + if strings.Contains(str, "CARTESIAN") { + ok = true + } + } + + c.Assert(ok, Equals, ca.isCartesianJoin) + } +} + func (s *testSuite1) TestExplainWrite(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t") diff --git a/expression/constant_propagation_test.go b/expression/constant_propagation_test.go index d457e85d9eb17..f8a9ab352dca2 100644 --- a/expression/constant_propagation_test.go +++ b/expression/constant_propagation_test.go @@ -66,7 +66,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { // Positive tests. tk.MustQuery("explain select * from t1 left join t2 on t1.a > t2.a and t1.a = 1;").Check(testkit.Rows( - "HashLeftJoin_6 33233333.33 root left outer join, inner:TableReader_11, left cond:[eq(test.t1.a, 1)]", + "HashLeftJoin_6 33233333.33 root CARTESIAN left outer join, inner:TableReader_11, left cond:[eq(test.t1.a, 1)]", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_11 3323.33 root data:Selection_10", @@ -74,7 +74,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_9 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on t1.a > t2.a where t1.a = 1;").Check(testkit.Rows( - "HashLeftJoin_7 33233.33 root left outer join, inner:TableReader_13", + "HashLeftJoin_7 33233.33 root CARTESIAN left outer join, inner:TableReader_13", "├─TableReader_10 10.00 root data:Selection_9", "│ └─Selection_9 10.00 cop eq(test.t1.a, 1)", "│ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -100,7 +100,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_11 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 right join t2 on t1.a > t2.a where t2.a = 1;").Check(testkit.Rows( - "HashRightJoin_7 33333.33 root right outer join, inner:TableReader_10", + "HashRightJoin_7 33333.33 root CARTESIAN right outer join, inner:TableReader_10", "├─TableReader_10 3333.33 root data:Selection_9", "│ └─Selection_9 3333.33 cop gt(test.t1.a, 1)", "│ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -126,7 +126,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_10 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 right join t2 on t1.a > t2.a and t2.a = 1;").Check(testkit.Rows( - "HashRightJoin_6 33333333.33 root right outer join, inner:TableReader_9, right cond:eq(test.t2.a, 1)", + "HashRightJoin_6 33333333.33 root CARTESIAN right outer join, inner:TableReader_9, right cond:eq(test.t2.a, 1)", "├─TableReader_9 3333.33 root data:Selection_8", "│ └─Selection_8 3333.33 cop gt(test.t1.a, 1)", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -143,7 +143,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_9 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on t1.a > t2.a and t2.a = 1;").Check(testkit.Rows( - "HashLeftJoin_6 100000.00 root left outer join, inner:TableReader_11, other cond:gt(test.t1.a, test.t2.a)", + "HashLeftJoin_6 100000.00 root CARTESIAN left outer join, inner:TableReader_11, other cond:gt(test.t1.a, test.t2.a)", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_11 10.00 root data:Selection_10", @@ -151,7 +151,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_9 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 right join t2 on t1.a > t2.a and t1.a = 1;").Check(testkit.Rows( - "HashRightJoin_6 100000.00 root right outer join, inner:TableReader_9, other cond:gt(test.t1.a, test.t2.a)", + "HashRightJoin_6 100000.00 root CARTESIAN right outer join, inner:TableReader_9, other cond:gt(test.t1.a, test.t2.a)", "├─TableReader_9 10.00 root data:Selection_8", "│ └─Selection_8 10.00 cop eq(test.t1.a, 1), not(isnull(test.t1.a))", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -167,14 +167,14 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_10 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on t1.a = t1.b and t1.a > 1;").Check(testkit.Rows( - "HashLeftJoin_6 100000000.00 root left outer join, inner:TableReader_10, left cond:[eq(test.t1.a, test.t1.b) gt(test.t1.a, 1)]", + "HashLeftJoin_6 100000000.00 root CARTESIAN left outer join, inner:TableReader_10, left cond:[eq(test.t1.a, test.t1.b) gt(test.t1.a, 1)]", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_10 10000.00 root data:TableScan_9", " └─TableScan_9 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on t2.a = t2.b and t2.a > 1;").Check(testkit.Rows( - "HashLeftJoin_6 26666666.67 root left outer join, inner:TableReader_11", + "HashLeftJoin_6 26666666.67 root CARTESIAN left outer join, inner:TableReader_11", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_11 2666.67 root data:Selection_10", @@ -195,7 +195,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { "TableDual_8 0.00 root rows:0", )) tk.MustQuery("explain select * from t1 left join t2 on true where t1.a = 1 and t1.a = 1;").Check(testkit.Rows( - "HashLeftJoin_7 80000.00 root left outer join, inner:TableReader_12", + "HashLeftJoin_7 80000.00 root CARTESIAN left outer join, inner:TableReader_12", "├─TableReader_10 10.00 root data:Selection_9", "│ └─Selection_9 10.00 cop eq(test.t1.a, 1)", "│ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -203,32 +203,32 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { " └─TableScan_11 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on false;").Check(testkit.Rows( - "HashLeftJoin_6 80000000.00 root left outer join, inner:TableDual_9", + "HashLeftJoin_6 80000000.00 root CARTESIAN left outer join, inner:TableDual_9", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableDual_9 8000.00 root rows:0", )) tk.MustQuery("explain select * from t1 right join t2 on false;").Check(testkit.Rows( - "HashRightJoin_6 80000000.00 root right outer join, inner:TableDual_7", + "HashRightJoin_6 80000000.00 root CARTESIAN right outer join, inner:TableDual_7", "├─TableDual_7 8000.00 root rows:0", "└─TableReader_9 10000.00 root data:TableScan_8", " └─TableScan_8 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on t1.a = 1 and t1.a = 2;").Check(testkit.Rows( - "HashLeftJoin_6 80000000.00 root left outer join, inner:TableDual_9", + "HashLeftJoin_6 80000000.00 root CARTESIAN left outer join, inner:TableDual_9", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableDual_9 8000.00 root rows:0", )) tk.MustQuery("explain select * from t1 left join t2 on t1.a =1 where t1.a = 2;").Check(testkit.Rows( - "HashLeftJoin_7 80000.00 root left outer join, inner:TableDual_11", + "HashLeftJoin_7 80000.00 root CARTESIAN left outer join, inner:TableDual_11", "├─TableReader_10 10.00 root data:Selection_9", "│ └─Selection_9 10.00 cop eq(test.t1.a, 2)", "│ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableDual_11 8000.00 root rows:0", )) tk.MustQuery("explain select * from t1 left join t2 on t2.a = 1 and t2.a = 2;").Check(testkit.Rows( - "HashLeftJoin_6 0.00 root left outer join, inner:TableReader_11", + "HashLeftJoin_6 0.00 root CARTESIAN left outer join, inner:TableReader_11", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_11 0.00 root data:Selection_10", @@ -237,14 +237,14 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { )) // Constant propagation for DNF in outer join. tk.MustQuery("explain select * from t1 left join t2 on t1.a = 1 or (t1.a = 2 and t1.a = 3);").Check(testkit.Rows( - "HashLeftJoin_6 100000000.00 root left outer join, inner:TableReader_10, left cond:[or(eq(test.t1.a, 1), 0)]", + "HashLeftJoin_6 100000000.00 root CARTESIAN left outer join, inner:TableReader_10, left cond:[or(eq(test.t1.a, 1), 0)]", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_10 10000.00 root data:TableScan_9", " └─TableScan_9 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) tk.MustQuery("explain select * from t1 left join t2 on true where t1.a = 1 or (t1.a = 2 and t1.a = 3);").Check(testkit.Rows( - "HashLeftJoin_7 80000.00 root left outer join, inner:TableReader_12", + "HashLeftJoin_7 80000.00 root CARTESIAN left outer join, inner:TableReader_12", "├─TableReader_10 10.00 root data:Selection_9", "│ └─Selection_9 10.00 cop or(eq(test.t1.a, 1), 0)", "│ └─TableScan_8 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -255,7 +255,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { tk.MustQuery("explain select * from t1 where t1.b > 1 or t1.b in (select b from t2);").Check(testkit.Rows( "Projection_7 8000.00 root test.t1.id, test.t1.a, test.t1.b", "└─Selection_8 8000.00 root or(gt(test.t1.b, 1), 5_aux_0)", - " └─HashLeftJoin_9 10000.00 root left outer semi join, inner:TableReader_13, other cond:eq(test.t1.b, test.t2.b)", + " └─HashLeftJoin_9 10000.00 root CARTESIAN left outer semi join, inner:TableReader_13, other cond:eq(test.t1.b, test.t2.b)", " ├─TableReader_11 10000.00 root data:TableScan_10", " │ └─TableScan_10 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", " └─TableReader_13 10000.00 root data:TableScan_12", diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 222512c2419c0..d0e43567b556a 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -136,9 +136,9 @@ func (s *testAnalyzeSuite) TestStraightJoin(c *C) { } testKit.MustQuery("explain select straight_join * from t1, t2, t3, t4").Check(testkit.Rows( - "HashLeftJoin_10 10000000000000000.00 root inner join, inner:TableReader_23", - "├─HashLeftJoin_12 1000000000000.00 root inner join, inner:TableReader_21", - "│ ├─HashLeftJoin_14 100000000.00 root inner join, inner:TableReader_19", + "HashLeftJoin_10 10000000000000000.00 root CARTESIAN inner join, inner:TableReader_23", + "├─HashLeftJoin_12 1000000000000.00 root CARTESIAN inner join, inner:TableReader_21", + "│ ├─HashLeftJoin_14 100000000.00 root CARTESIAN inner join, inner:TableReader_19", "│ │ ├─TableReader_17 10000.00 root data:TableScan_16", "│ │ │ └─TableScan_16 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "│ │ └─TableReader_19 10000.00 root data:TableScan_18", @@ -150,9 +150,9 @@ func (s *testAnalyzeSuite) TestStraightJoin(c *C) { )) testKit.MustQuery("explain select * from t1 straight_join t2 straight_join t3 straight_join t4").Check(testkit.Rows( - "HashLeftJoin_10 10000000000000000.00 root inner join, inner:TableReader_23", - "├─HashLeftJoin_12 1000000000000.00 root inner join, inner:TableReader_21", - "│ ├─HashLeftJoin_14 100000000.00 root inner join, inner:TableReader_19", + "HashLeftJoin_10 10000000000000000.00 root CARTESIAN inner join, inner:TableReader_23", + "├─HashLeftJoin_12 1000000000000.00 root CARTESIAN inner join, inner:TableReader_21", + "│ ├─HashLeftJoin_14 100000000.00 root CARTESIAN inner join, inner:TableReader_19", "│ │ ├─TableReader_17 10000.00 root data:TableScan_16", "│ │ │ └─TableScan_16 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "│ │ └─TableReader_19 10000.00 root data:TableScan_18", @@ -165,8 +165,8 @@ func (s *testAnalyzeSuite) TestStraightJoin(c *C) { testKit.MustQuery("explain select straight_join * from t1, t2, t3, t4 where t1.a=t4.a;").Check(testkit.Rows( "HashLeftJoin_11 1248750000000.00 root inner join, inner:TableReader_26, equal:[eq(test.t1.a, test.t4.a)]", - "├─HashLeftJoin_13 999000000000.00 root inner join, inner:TableReader_23", - "│ ├─HashRightJoin_16 99900000.00 root inner join, inner:TableReader_19", + "├─HashLeftJoin_13 999000000000.00 root CARTESIAN inner join, inner:TableReader_23", + "│ ├─HashRightJoin_16 99900000.00 root CARTESIAN inner join, inner:TableReader_19", "│ │ ├─TableReader_19 9990.00 root data:Selection_18", "│ │ │ └─Selection_18 9990.00 cop not(isnull(test.t1.a))", "│ │ │ └─TableScan_17 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", @@ -688,7 +688,7 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { tk.MustQuery("explain select t.c in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t;"). Check(testkit.Rows( "Projection_11 10.00 root 9_aux_0", - "└─Apply_13 10.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0)", + "└─Apply_13 10.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.c, 7_col_0)", " ├─TableReader_15 10.00 root data:TableScan_14", " │ └─TableScan_14 10.00 cop table:t, range:[-inf,+inf], keep order:false", " └─StreamAgg_20 1.00 root funcs:count(1)", @@ -703,7 +703,7 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { tk.MustQuery("explain select (select concat(t1.a, \",\", t1.b) from t t1 where t1.a=t.a and t1.c=t.c) from t"). Check(testkit.Rows( "Projection_8 10.00 root concat(t1.a, \",\", t1.b)", - "└─Apply_10 10.00 root left outer join, inner:MaxOneRow_13", + "└─Apply_10 10.00 root CARTESIAN left outer join, inner:MaxOneRow_13", " ├─TableReader_12 10.00 root data:TableScan_11", " │ └─TableScan_11 10.00 cop table:t, range:[-inf,+inf], keep order:false", " └─MaxOneRow_13 1.00 root ", diff --git a/planner/core/explain.go b/planner/core/explain.go index 3b3a027fbe606..e3ed7f5a18e55 100644 --- a/planner/core/explain.go +++ b/planner/core/explain.go @@ -233,7 +233,13 @@ func (p *PhysicalIndexJoin) ExplainInfo() string { // ExplainInfo implements PhysicalPlan interface. func (p *PhysicalHashJoin) ExplainInfo() string { - buffer := bytes.NewBufferString(p.JoinType.String()) + buffer := new(bytes.Buffer) + + if len(p.EqualConditions) == 0 { + buffer.WriteString("CARTESIAN ") + } + + buffer.WriteString(p.JoinType.String()) fmt.Fprintf(buffer, ", inner:%s", p.Children()[p.InnerChildIdx].ExplainID()) if len(p.EqualConditions) > 0 { fmt.Fprintf(buffer, ", equal:%v", p.EqualConditions) diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 47a05cb9efb94..89daf91d4643f 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1047,7 +1047,7 @@ func (s *testRangerSuite) TestCompIndexInExprCorrCol(c *C) { testKit.MustExec("analyze table t") testKit.MustQuery("explain select t.e in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c in (1, 2) and s.d = t.a and s.a = t1.a) from t").Check(testkit.Rows( "Projection_11 2.00 root 9_aux_0", - "└─Apply_13 2.00 root left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.e, 7_col_0)", + "└─Apply_13 2.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.e, 7_col_0)", " ├─TableReader_15 2.00 root data:TableScan_14", " │ └─TableScan_14 2.00 cop table:t, range:[-inf,+inf], keep order:false", " └─StreamAgg_20 1.00 root funcs:count(1)", From abbca5375eea8129393a9ee1e517971893d163c1 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 24 Jul 2019 19:03:47 +0800 Subject: [PATCH 089/196] *: add trace support for subquery (#11182) Test pass, auto merge by Bot --- executor/adapter.go | 4 +- executor/compiler.go | 2 +- executor/executor.go | 8 +- executor/executor_test.go | 2 +- executor/grant.go | 2 +- executor/metrics_test.go | 3 +- executor/prepared.go | 16 +- executor/seqtest/prepared_test.go | 4 +- executor/show.go | 10 +- expression/builtin_miscellaneous.go | 1 + expression/expression.go | 2 +- expression/integration_test.go | 17 +- expression/typeinfer_test.go | 9 +- planner/cascades/optimize_test.go | 7 +- planner/core/cbo_test.go | 11 +- planner/core/common_plans.go | 19 +- planner/core/expression_rewriter.go | 203 ++++++++++--------- planner/core/indexmerge_test.go | 7 +- planner/core/logical_plan_builder.go | 162 +++++++-------- planner/core/logical_plan_test.go | 127 ++++++++---- planner/core/optimizer.go | 23 ++- planner/core/physical_plan_test.go | 35 ++-- planner/core/planbuilder.go | 103 +++++----- planner/core/planbuilder_test.go | 8 +- planner/core/prepare_test.go | 6 +- planner/core/rule_aggregation_elimination.go | 5 +- planner/core/rule_aggregation_push_down.go | 3 +- planner/core/rule_build_key_info.go | 4 +- planner/core/rule_column_pruning.go | 4 +- planner/core/rule_decorrelate.go | 17 +- planner/core/rule_eliminate_projection.go | 4 +- planner/core/rule_join_elimination.go | 4 +- planner/core/rule_join_reorder.go | 4 +- planner/core/rule_max_min_eliminate.go | 4 +- planner/core/rule_partition_processor.go | 4 +- planner/core/rule_predicate_push_down.go | 4 +- planner/core/rule_topn_push_down.go | 4 +- planner/optimize.go | 30 +-- session/session.go | 4 +- statistics/selectivity_test.go | 25 ++- util/ranger/ranger_test.go | 59 +++--- util/sqlexec/restricted_sql_executor.go | 2 +- 42 files changed, 533 insertions(+), 439 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index a08c26ab11abf..926d86a870dde 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -181,13 +181,13 @@ func (a *ExecStmt) IsReadOnly(vars *variable.SessionVars) bool { // RebuildPlan rebuilds current execute statement plan. // It returns the current information schema version that 'a' is using. -func (a *ExecStmt) RebuildPlan() (int64, error) { +func (a *ExecStmt) RebuildPlan(ctx context.Context) (int64, error) { is := GetInfoSchema(a.Ctx) a.InfoSchema = is if err := plannercore.Preprocess(a.Ctx, a.StmtNode, is, plannercore.InTxnRetry); err != nil { return 0, err } - p, err := planner.Optimize(a.Ctx, a.StmtNode, is) + p, err := planner.Optimize(ctx, a.Ctx, a.StmtNode, is) if err != nil { return 0, err } diff --git a/executor/compiler.go b/executor/compiler.go index 7c0c6a6a4cf97..7ac5f1eef3367 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -75,7 +75,7 @@ func (c *Compiler) compile(ctx context.Context, stmtNode ast.StmtNode, skipBind return nil, err } - finalPlan, err := planner.Optimize(c.Ctx, stmtNode, infoSchema) + finalPlan, err := planner.Optimize(ctx, c.Ctx, stmtNode, infoSchema) if err != nil { return nil, err } diff --git a/executor/executor.go b/executor/executor.go index 01a28bf0cde4b..1513bd6554f8a 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -810,13 +810,17 @@ func init() { // While doing optimization in the plan package, we need to execute uncorrelated subquery, // but the plan package cannot import the executor package because of the dependency cycle. // So we assign a function implemented in the executor package to the plan package to avoid the dependency cycle. - plannercore.EvalSubquery = func(p plannercore.PhysicalPlan, is infoschema.InfoSchema, sctx sessionctx.Context) (rows [][]types.Datum, err error) { + plannercore.EvalSubquery = func(ctx context.Context, p plannercore.PhysicalPlan, is infoschema.InfoSchema, sctx sessionctx.Context) (rows [][]types.Datum, err error) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("executor.EvalSubQuery", opentracing.ChildOf(span.Context())) + defer span1.Finish() + } + e := &executorBuilder{is: is, ctx: sctx} exec := e.build(p) if e.err != nil { return rows, err } - ctx := context.TODO() err = exec.Open(ctx) defer terror.Call(exec.Close) if err != nil { diff --git a/executor/executor_test.go b/executor/executor_test.go index 762d4fd044d90..762f31a7faee1 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2022,7 +2022,7 @@ func (s *testSuiteP1) TestIsPointGet(c *C) { c.Check(err, IsNil) err = plannercore.Preprocess(ctx, stmtNode, infoSchema) c.Check(err, IsNil) - p, err := planner.Optimize(ctx, stmtNode, infoSchema) + p, err := planner.Optimize(context.TODO(), ctx, stmtNode, infoSchema) c.Check(err, IsNil) ret, err := executor.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx, p) c.Assert(err, IsNil) diff --git a/executor/grant.go b/executor/grant.go index 3a952ab708f3c..8b99bb8ffdcbb 100644 --- a/executor/grant.go +++ b/executor/grant.go @@ -79,7 +79,7 @@ func (e *GrantExec) Next(ctx context.Context, req *chunk.Chunk) error { } user := fmt.Sprintf(`('%s', '%s', '%s')`, user.User.Hostname, user.User.Username, pwd) sql := fmt.Sprintf(`INSERT INTO %s.%s (Host, User, Password) VALUES %s;`, mysql.SystemDB, mysql.UserTable, user) - _, err := e.ctx.(sqlexec.SQLExecutor).Execute(context.TODO(), sql) + _, err := e.ctx.(sqlexec.SQLExecutor).Execute(ctx, sql) if err != nil { return err } diff --git a/executor/metrics_test.go b/executor/metrics_test.go index afb165038d8b6..662858fd9928c 100644 --- a/executor/metrics_test.go +++ b/executor/metrics_test.go @@ -14,6 +14,7 @@ package executor_test import ( + "context" "fmt" . "github.com/pingcap/check" @@ -63,7 +64,7 @@ func (s *testSuite4) TestStmtLabel(c *C) { is := executor.GetInfoSchema(tk.Se) err = plannercore.Preprocess(tk.Se.(sessionctx.Context), stmtNode, is) c.Assert(err, IsNil) - _, err = planner.Optimize(tk.Se, stmtNode, is) + _, err = planner.Optimize(context.TODO(), tk.Se, stmtNode, is) c.Assert(err, IsNil) c.Assert(executor.GetStmtLabel(stmtNode), Equals, tt.label) } diff --git a/executor/prepared.go b/executor/prepared.go index 01638d37cb786..03e039c83aef6 100644 --- a/executor/prepared.go +++ b/executor/prepared.go @@ -180,7 +180,7 @@ func (e *PrepareExec) Next(ctx context.Context, req *chunk.Chunk) error { param.InExecute = false } var p plannercore.Plan - p, err = plannercore.BuildLogicalPlan(e.ctx, stmt, e.is) + p, err = plannercore.BuildLogicalPlan(ctx, e.ctx, stmt, e.is) if err != nil { return err } @@ -266,14 +266,14 @@ func (e *DeallocateExec) Next(ctx context.Context, req *chunk.Chunk) error { } // CompileExecutePreparedStmt compiles a session Execute command to a stmt.Statement. -func CompileExecutePreparedStmt(ctx sessionctx.Context, ID uint32, args []types.Datum) (sqlexec.Statement, error) { +func CompileExecutePreparedStmt(ctx context.Context, sctx sessionctx.Context, ID uint32, args []types.Datum) (sqlexec.Statement, error) { execStmt := &ast.ExecuteStmt{ExecID: ID} - if err := ResetContextOfStmt(ctx, execStmt); err != nil { + if err := ResetContextOfStmt(sctx, execStmt); err != nil { return nil, err } execStmt.BinaryArgs = args - is := GetInfoSchema(ctx) - execPlan, err := planner.Optimize(ctx, execStmt, is) + is := GetInfoSchema(sctx) + execPlan, err := planner.Optimize(ctx, sctx, execStmt, is) if err != nil { return nil, err } @@ -282,11 +282,11 @@ func CompileExecutePreparedStmt(ctx sessionctx.Context, ID uint32, args []types. InfoSchema: is, Plan: execPlan, StmtNode: execStmt, - Ctx: ctx, + Ctx: sctx, } - if prepared, ok := ctx.GetSessionVars().PreparedStmts[ID]; ok { + if prepared, ok := sctx.GetSessionVars().PreparedStmts[ID]; ok { stmt.Text = prepared.Stmt.Text() - ctx.GetSessionVars().StmtCtx.OriginalSQL = stmt.Text + sctx.GetSessionVars().StmtCtx.OriginalSQL = stmt.Text } return stmt, nil } diff --git a/executor/seqtest/prepared_test.go b/executor/seqtest/prepared_test.go index 42a2fca138af3..59a4c0710fba0 100644 --- a/executor/seqtest/prepared_test.go +++ b/executor/seqtest/prepared_test.go @@ -131,13 +131,13 @@ func (s *seqTestSuite) TestPrepared(c *C) { tk.ResultSetToResult(rs, Commentf("%v", rs)).Check(testkit.Rows()) // Check that ast.Statement created by executor.CompileExecutePreparedStmt has query text. - stmt, err := executor.CompileExecutePreparedStmt(tk.Se, stmtID, []types.Datum{types.NewDatum(1)}) + stmt, err := executor.CompileExecutePreparedStmt(context.TODO(), tk.Se, stmtID, []types.Datum{types.NewDatum(1)}) c.Assert(err, IsNil) c.Assert(stmt.OriginText(), Equals, query) // Check that rebuild plan works. tk.Se.PrepareTxnCtx(ctx) - _, err = stmt.RebuildPlan() + _, err = stmt.RebuildPlan(ctx) c.Assert(err, IsNil) rs, err = stmt.Exec(ctx) c.Assert(err, IsNil) diff --git a/executor/show.go b/executor/show.go index 6c277687bdca2..c97e15e2b7a6d 100644 --- a/executor/show.go +++ b/executor/show.go @@ -83,7 +83,7 @@ func (e *ShowExec) Next(ctx context.Context, req *chunk.Chunk) error { req.GrowAndReset(e.maxChunkSize) if e.result == nil { e.result = newFirstChunk(e) - err := e.fetchAll() + err := e.fetchAll(ctx) if err != nil { return errors.Trace(err) } @@ -109,14 +109,14 @@ func (e *ShowExec) Next(ctx context.Context, req *chunk.Chunk) error { return nil } -func (e *ShowExec) fetchAll() error { +func (e *ShowExec) fetchAll(ctx context.Context) error { switch e.Tp { case ast.ShowCharset: return e.fetchShowCharset() case ast.ShowCollation: return e.fetchShowCollation() case ast.ShowColumns: - return e.fetchShowColumns() + return e.fetchShowColumns(ctx) case ast.ShowCreateTable: return e.fetchShowCreateTable() case ast.ShowCreateUser: @@ -367,7 +367,7 @@ func createOptions(tb *model.TableInfo) string { return "" } -func (e *ShowExec) fetchShowColumns() error { +func (e *ShowExec) fetchShowColumns(ctx context.Context) error { tb, err := e.getTable() if err != nil { @@ -384,7 +384,7 @@ func (e *ShowExec) fetchShowColumns() error { // Because view's undertable's column could change or recreate, so view's column type may change overtime. // To avoid this situation we need to generate a logical plan and extract current column types from Schema. planBuilder := plannercore.NewPlanBuilder(e.ctx, e.is) - viewLogicalPlan, err := planBuilder.BuildDataSourceFromView(e.DBName, tb.Meta()) + viewLogicalPlan, err := planBuilder.BuildDataSourceFromView(ctx, e.DBName, tb.Meta()) if err != nil { return err } diff --git a/expression/builtin_miscellaneous.go b/expression/builtin_miscellaneous.go index 239489acd2272..70c52503b52b0 100644 --- a/expression/builtin_miscellaneous.go +++ b/expression/builtin_miscellaneous.go @@ -114,6 +114,7 @@ func (b *builtinSleepSig) evalInt(row chunk.Row) (int64, bool, error) { if err != nil { return 0, isNull, err } + sessVars := b.ctx.GetSessionVars() if isNull { if sessVars.StrictSQLMode { diff --git a/expression/expression.go b/expression/expression.go index 162cacb015474..d1ea0e24d7730 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -36,7 +36,7 @@ const ( ) // EvalAstExpr evaluates ast expression directly. -var EvalAstExpr func(ctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) +var EvalAstExpr func(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) // Expression represents all scalar expression in SQL. type Expression interface { diff --git a/expression/integration_test.go b/expression/integration_test.go index 4b55bc284bc8e..a13edd67c2fb0 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3906,24 +3906,25 @@ func (s *testIntegrationSuite) TestFilterExtractFromDNF(c *C) { }, } + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprStr - ctx := tk.Se.(sessionctx.Context) - sc := ctx.GetSessionVars().StmtCtx - stmts, err := session.Parse(ctx, sql) + sctx := tk.Se.(sessionctx.Context) + sc := sctx.GetSessionVars().StmtCtx + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmts[0], is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) conds := make([]expression.Expression, 0, len(selection.Conditions)) for _, cond := range selection.Conditions { - conds = append(conds, expression.PushDownNot(ctx, cond, false)) + conds = append(conds, expression.PushDownNot(sctx, cond, false)) } - afterFunc := expression.ExtractFiltersFromDNFs(ctx, conds) + afterFunc := expression.ExtractFiltersFromDNFs(sctx, conds) sort.Slice(afterFunc, func(i, j int) bool { return bytes.Compare(afterFunc[i].HashCode(sc), afterFunc[j].HashCode(sc)) < 0 }) diff --git a/expression/typeinfer_test.go b/expression/typeinfer_test.go index 9ccaaffdb11ce..dcd7571813b97 100644 --- a/expression/typeinfer_test.go +++ b/expression/typeinfer_test.go @@ -127,8 +127,9 @@ func (s *testInferTypeSuite) TestInferType(c *C) { tests = append(tests, s.createTestCase4JSONFuncs()...) tests = append(tests, s.createTestCase4MiscellaneousFunc()...) + ctx := context.Background() for _, tt := range tests { - ctx := testKit.Se.(sessionctx.Context) + sctx := testKit.Se.(sessionctx.Context) sql := "select " + tt.sql + " from t" comment := Commentf("for %s", sql) stmt, err := s.ParseOneStmt(sql, "", "") @@ -137,10 +138,10 @@ func (s *testInferTypeSuite) TestInferType(c *C) { err = se.NewTxn(context.Background()) c.Assert(err, IsNil) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmt, is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmt, is) c.Assert(err, IsNil, comment) - p, err := plannercore.BuildLogicalPlan(ctx, stmt, is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmt, is) c.Assert(err, IsNil, comment) tp := p.Schema().Columns[0].RetType diff --git a/planner/cascades/optimize_test.go b/planner/cascades/optimize_test.go index 4841b32c271fc..3ea1cec66e580 100644 --- a/planner/cascades/optimize_test.go +++ b/planner/cascades/optimize_test.go @@ -14,6 +14,7 @@ package cascades import ( + "context" "math" "testing" @@ -54,7 +55,7 @@ func (s *testCascadesSuite) TearDownSuite(c *C) { func (s *testCascadesSuite) TestImplGroupZeroCost(c *C) { stmt, err := s.ParseOneStmt("select t1.a, t2.a from t as t1 left join t as t2 on t1.a = t2.a where t1.a < 1.0", "", "") c.Assert(err, IsNil) - p, err := plannercore.BuildLogicalPlan(s.sctx, stmt, s.is) + p, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt, s.is) c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) @@ -70,7 +71,7 @@ func (s *testCascadesSuite) TestImplGroupZeroCost(c *C) { func (s *testCascadesSuite) TestInitGroupSchema(c *C) { stmt, err := s.ParseOneStmt("select a from t", "", "") c.Assert(err, IsNil) - p, err := plannercore.BuildLogicalPlan(s.sctx, stmt, s.is) + p, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt, s.is) c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) @@ -84,7 +85,7 @@ func (s *testCascadesSuite) TestInitGroupSchema(c *C) { func (s *testCascadesSuite) TestFillGroupStats(c *C) { stmt, err := s.ParseOneStmt("select * from t t1 join t t2 on t1.a = t2.a", "", "") c.Assert(err, IsNil) - p, err := plannercore.BuildLogicalPlan(s.sctx, stmt, s.is) + p, err := plannercore.BuildLogicalPlan(context.Background(), s.sctx, stmt, s.is) c.Assert(err, IsNil) logic, ok := p.(plannercore.LogicalPlan) c.Assert(ok, IsTrue) diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index d0e43567b556a..69f8fbe284ece 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -14,6 +14,7 @@ package core_test import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -380,7 +381,7 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { is := domain.GetDomain(ctx).InfoSchema() err = core.Preprocess(ctx, stmt, is) c.Assert(err, IsNil) - p, err := planner.Optimize(ctx, stmt, is) + p, err := planner.Optimize(context.TODO(), ctx, stmt, is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, tt.best, Commentf("for %s", tt.sql)) } @@ -430,7 +431,7 @@ func (s *testAnalyzeSuite) TestEmptyTable(c *C) { is := domain.GetDomain(ctx).InfoSchema() err = core.Preprocess(ctx, stmt, is) c.Assert(err, IsNil) - p, err := planner.Optimize(ctx, stmt, is) + p, err := planner.Optimize(context.TODO(), ctx, stmt, is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, tt.best, Commentf("for %s", tt.sql)) } @@ -546,7 +547,7 @@ func (s *testAnalyzeSuite) TestAnalyze(c *C) { is := domain.GetDomain(ctx).InfoSchema() err = core.Preprocess(ctx, stmt, is) c.Assert(err, IsNil) - p, err := planner.Optimize(ctx, stmt, is) + p, err := planner.Optimize(context.TODO(), ctx, stmt, is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, tt.best, Commentf("for %s", tt.sql)) } @@ -623,7 +624,7 @@ func (s *testAnalyzeSuite) TestPreparedNullParam(c *C) { is := domain.GetDomain(ctx).InfoSchema() err = core.Preprocess(ctx, stmt, is, core.InPrepare) c.Assert(err, IsNil) - p, err := planner.Optimize(ctx, stmt, is) + p, err := planner.Optimize(context.TODO(), ctx, stmt, is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, best, Commentf("for %s", sql)) @@ -879,7 +880,7 @@ func BenchmarkOptimize(b *testing.B) { b.Run(tt.sql, func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := planner.Optimize(ctx, stmt, is) + _, err := planner.Optimize(context.TODO(), ctx, stmt, is) c.Assert(err, IsNil) } b.ReportAllocs() diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 8a458ffc90bc6..8cd2290e6682b 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -15,6 +15,7 @@ package core import ( "bytes" + "context" "fmt" "strconv" "strings" @@ -182,8 +183,8 @@ type Execute struct { } // OptimizePreparedPlan optimizes the prepared statement. -func (e *Execute) OptimizePreparedPlan(ctx sessionctx.Context, is infoschema.InfoSchema) error { - vars := ctx.GetSessionVars() +func (e *Execute) OptimizePreparedPlan(ctx context.Context, sctx sessionctx.Context, is infoschema.InfoSchema) error { + vars := sctx.GetSessionVars() if e.Name != "" { e.ExecID = vars.PreparedStmtNameToID[e.Name] } @@ -225,13 +226,13 @@ func (e *Execute) OptimizePreparedPlan(ctx sessionctx.Context, is infoschema.Inf if prepared.SchemaVersion != is.SchemaMetaVersion() { // If the schema version has changed we need to preprocess it again, // if this time it failed, the real reason for the error is schema changed. - err := Preprocess(ctx, prepared.Stmt, is, InPrepare) + err := Preprocess(sctx, prepared.Stmt, is, InPrepare) if err != nil { return ErrSchemaChanged.GenWithStack("Schema change caused error: %s", err.Error()) } prepared.SchemaVersion = is.SchemaMetaVersion() } - p, err := e.getPhysicalPlan(ctx, is, prepared) + p, err := e.getPhysicalPlan(ctx, sctx, is, prepared) if err != nil { return err } @@ -240,13 +241,13 @@ func (e *Execute) OptimizePreparedPlan(ctx sessionctx.Context, is infoschema.Inf return nil } -func (e *Execute) getPhysicalPlan(ctx sessionctx.Context, is infoschema.InfoSchema, prepared *ast.Prepared) (Plan, error) { +func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, is infoschema.InfoSchema, prepared *ast.Prepared) (Plan, error) { var cacheKey kvcache.Key - sessionVars := ctx.GetSessionVars() + sessionVars := sctx.GetSessionVars() sessionVars.StmtCtx.UseCache = prepared.UseCache if prepared.UseCache { cacheKey = NewPSTMTPlanCacheKey(sessionVars, e.ExecID, prepared.SchemaVersion) - if cacheValue, exists := ctx.PreparedPlanCache().Get(cacheKey); exists { + if cacheValue, exists := sctx.PreparedPlanCache().Get(cacheKey); exists { if metrics.ResettablePlanCacheCounterFortTest { metrics.PlanCacheCounter.WithLabelValues("prepare").Inc() } else { @@ -260,13 +261,13 @@ func (e *Execute) getPhysicalPlan(ctx sessionctx.Context, is infoschema.InfoSche return plan, nil } } - p, err := OptimizeAstNode(ctx, prepared.Stmt, is) + p, err := OptimizeAstNode(ctx, sctx, prepared.Stmt, is) if err != nil { return nil, err } _, isTableDual := p.(*PhysicalTableDual) if !isTableDual && prepared.UseCache { - ctx.PreparedPlanCache().Put(cacheKey, NewPSTMTPlanCacheValue(p)) + sctx.PreparedPlanCache().Put(cacheKey, NewPSTMTPlanCacheValue(p)) } return p, err } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 6d5dbbea7610b..4a79a68e3f7d9 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -14,6 +14,7 @@ package core import ( + "context" "strconv" "strings" @@ -35,31 +36,31 @@ import ( ) // EvalSubquery evaluates incorrelated subqueries once. -var EvalSubquery func(p PhysicalPlan, is infoschema.InfoSchema, ctx sessionctx.Context) ([][]types.Datum, error) +var EvalSubquery func(ctx context.Context, p PhysicalPlan, is infoschema.InfoSchema, sctx sessionctx.Context) ([][]types.Datum, error) // evalAstExpr evaluates ast expression directly. -func evalAstExpr(ctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) { +func evalAstExpr(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) { if val, ok := expr.(*driver.ValueExpr); ok { return val.Datum, nil } var is infoschema.InfoSchema - if ctx.GetSessionVars().TxnCtx.InfoSchema != nil { - is = ctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) + if sctx.GetSessionVars().TxnCtx.InfoSchema != nil { + is = sctx.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) } - b := NewPlanBuilder(ctx, is) - fakePlan := LogicalTableDual{}.Init(ctx) - newExpr, _, err := b.rewrite(expr, fakePlan, nil, true) + b := NewPlanBuilder(sctx, is) + fakePlan := LogicalTableDual{}.Init(sctx) + newExpr, _, err := b.rewrite(context.TODO(), expr, fakePlan, nil, true) if err != nil { return types.Datum{}, err } return newExpr.Eval(chunk.Row{}) } -func (b *PlanBuilder) rewriteInsertOnDuplicateUpdate(exprNode ast.ExprNode, mockPlan LogicalPlan, insertPlan *Insert) (expression.Expression, error) { +func (b *PlanBuilder) rewriteInsertOnDuplicateUpdate(ctx context.Context, exprNode ast.ExprNode, mockPlan LogicalPlan, insertPlan *Insert) (expression.Expression, error) { b.rewriterCounter++ defer func() { b.rewriterCounter-- }() - rewriter := b.getExpressionRewriter(mockPlan) + rewriter := b.getExpressionRewriter(ctx, mockPlan) // The rewriter maybe is obtained from "b.rewriterPool", "rewriter.err" is // not nil means certain previous procedure has not handled this error. // Here we give us one more chance to make a correct behavior by handling @@ -79,19 +80,19 @@ func (b *PlanBuilder) rewriteInsertOnDuplicateUpdate(exprNode ast.ExprNode, mock // aggMapper maps ast.AggregateFuncExpr to the columns offset in p's output schema. // asScalar means whether this expression must be treated as a scalar expression. // And this function returns a result expression, a new plan that may have apply or semi-join. -func (b *PlanBuilder) rewrite(exprNode ast.ExprNode, p LogicalPlan, aggMapper map[*ast.AggregateFuncExpr]int, asScalar bool) (expression.Expression, LogicalPlan, error) { - expr, resultPlan, err := b.rewriteWithPreprocess(exprNode, p, aggMapper, nil, asScalar, nil) +func (b *PlanBuilder) rewrite(ctx context.Context, exprNode ast.ExprNode, p LogicalPlan, aggMapper map[*ast.AggregateFuncExpr]int, asScalar bool) (expression.Expression, LogicalPlan, error) { + expr, resultPlan, err := b.rewriteWithPreprocess(ctx, exprNode, p, aggMapper, nil, asScalar, nil) return expr, resultPlan, err } // rewriteWithPreprocess is for handling the situation that we need to adjust the input ast tree // before really using its node in `expressionRewriter.Leave`. In that case, we first call // er.preprocess(expr), which returns a new expr. Then we use the new expr in `Leave`. -func (b *PlanBuilder) rewriteWithPreprocess(exprNode ast.ExprNode, p LogicalPlan, aggMapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int, asScalar bool, preprocess func(ast.Node) ast.Node) (expression.Expression, LogicalPlan, error) { +func (b *PlanBuilder) rewriteWithPreprocess(ctx context.Context, exprNode ast.ExprNode, p LogicalPlan, aggMapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int, asScalar bool, preprocess func(ast.Node) ast.Node) (expression.Expression, LogicalPlan, error) { b.rewriterCounter++ defer func() { b.rewriterCounter-- }() - rewriter := b.getExpressionRewriter(p) + rewriter := b.getExpressionRewriter(ctx, p) // The rewriter maybe is obtained from "b.rewriterPool", "rewriter.err" is // not nil means certain previous procedure has not handled this error. // Here we give us one more chance to make a correct behavior by handling @@ -109,7 +110,7 @@ func (b *PlanBuilder) rewriteWithPreprocess(exprNode ast.ExprNode, p LogicalPlan return expr, resultPlan, err } -func (b *PlanBuilder) getExpressionRewriter(p LogicalPlan) (rewriter *expressionRewriter) { +func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p LogicalPlan) (rewriter *expressionRewriter) { defer func() { if p != nil { rewriter.schema = p.Schema() @@ -117,7 +118,7 @@ func (b *PlanBuilder) getExpressionRewriter(p LogicalPlan) (rewriter *expression }() if len(b.rewriterPool) < b.rewriterCounter { - rewriter = &expressionRewriter{p: p, b: b, ctx: b.ctx} + rewriter = &expressionRewriter{p: p, b: b, sctx: b.ctx, ctx: ctx} b.rewriterPool = append(b.rewriterPool, rewriter) return } @@ -130,6 +131,7 @@ func (b *PlanBuilder) getExpressionRewriter(p LogicalPlan) (rewriter *expression rewriter.insertPlan = nil rewriter.disableFoldCounter = 0 rewriter.ctxStack = rewriter.ctxStack[:0] + rewriter.ctx = ctx return } @@ -159,7 +161,8 @@ type expressionRewriter struct { aggrMap map[*ast.AggregateFuncExpr]int windowMap map[*ast.WindowFuncExpr]int b *PlanBuilder - ctx sessionctx.Context + sctx sessionctx.Context + ctx context.Context // asScalar indicates the return value must be a scalar value. // NOTE: This value can be changed during expression rewritten. @@ -204,21 +207,21 @@ func (er *expressionRewriter) constructBinaryOpFunction(l expression.Expression, } } if op == ast.NE { - return expression.ComposeDNFCondition(er.ctx, funcs...), nil + return expression.ComposeDNFCondition(er.sctx, funcs...), nil } - return expression.ComposeCNFCondition(er.ctx, funcs...), nil + return expression.ComposeCNFCondition(er.sctx, funcs...), nil default: larg0, rarg0 := expression.GetFuncArg(l, 0), expression.GetFuncArg(r, 0) var expr1, expr2, expr3, expr4, expr5 expression.Expression - expr1 = expression.NewFunctionInternal(er.ctx, ast.NE, types.NewFieldType(mysql.TypeTiny), larg0, rarg0) - expr2 = expression.NewFunctionInternal(er.ctx, op, types.NewFieldType(mysql.TypeTiny), larg0, rarg0) - expr3 = expression.NewFunctionInternal(er.ctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), expr1) + expr1 = expression.NewFunctionInternal(er.sctx, ast.NE, types.NewFieldType(mysql.TypeTiny), larg0, rarg0) + expr2 = expression.NewFunctionInternal(er.sctx, op, types.NewFieldType(mysql.TypeTiny), larg0, rarg0) + expr3 = expression.NewFunctionInternal(er.sctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), expr1) var err error - l, err = expression.PopRowFirstArg(er.ctx, l) + l, err = expression.PopRowFirstArg(er.sctx, l) if err != nil { return nil, err } - r, err = expression.PopRowFirstArg(er.ctx, r) + r, err = expression.PopRowFirstArg(er.sctx, r) if err != nil { return nil, err } @@ -234,14 +237,14 @@ func (er *expressionRewriter) constructBinaryOpFunction(l expression.Expression, } } -func (er *expressionRewriter) buildSubquery(subq *ast.SubqueryExpr) (LogicalPlan, error) { +func (er *expressionRewriter) buildSubquery(ctx context.Context, subq *ast.SubqueryExpr) (LogicalPlan, error) { if er.schema != nil { outerSchema := er.schema.Clone() er.b.outerSchemas = append(er.b.outerSchemas, outerSchema) defer func() { er.b.outerSchemas = er.b.outerSchemas[0 : len(er.b.outerSchemas)-1] }() } - np, err := er.b.buildResultSetNode(subq.Query) + np, err := er.b.buildResultSetNode(ctx, subq.Query) if err != nil { return nil, err } @@ -268,12 +271,12 @@ func (er *expressionRewriter) Enter(inNode ast.Node) (ast.Node, bool) { return inNode, true } case *ast.CompareSubqueryExpr: - return er.handleCompareSubquery(v) + return er.handleCompareSubquery(er.ctx, v) case *ast.ExistsSubqueryExpr: - return er.handleExistSubquery(v) + return er.handleExistSubquery(er.ctx, v) case *ast.PatternInExpr: if v.Sel != nil { - return er.handleInSubquery(v) + return er.handleInSubquery(er.ctx, v) } if len(v.List) != 1 { break @@ -285,7 +288,7 @@ func (er *expressionRewriter) Enter(inNode ast.Node) (ast.Node, bool) { switch y := x.(type) { case *ast.SubqueryExpr: v.Sel = y - return er.handleInSubquery(v) + return er.handleInSubquery(er.ctx, v) case *ast.ParenthesesExpr: x = y.Expr default: @@ -293,7 +296,7 @@ func (er *expressionRewriter) Enter(inNode ast.Node) (ast.Node, bool) { } } case *ast.SubqueryExpr: - return er.handleScalarSubquery(v) + return er.handleScalarSubquery(er.ctx, v) case *ast.ParenthesesExpr: case *ast.ValuesExpr: schema := er.schema @@ -312,7 +315,7 @@ func (er *expressionRewriter) Enter(inNode ast.Node) (ast.Node, bool) { er.err = ErrUnknownColumn.GenWithStackByArgs(v.Column.Name.OrigColName(), "field list") return inNode, false } - er.ctxStack = append(er.ctxStack, expression.NewValuesFunc(er.ctx, col.Index, col.RetType)) + er.ctxStack = append(er.ctxStack, expression.NewValuesFunc(er.sctx, col.Index, col.RetType)) return inNode, true case *ast.WindowFuncExpr: index, ok := -1, false @@ -352,7 +355,7 @@ func (er *expressionRewriter) buildSemiApplyFromEqualSubq(np LogicalPlan, l, r e er.p, er.err = er.b.buildSemiApply(er.p, np, []expression.Expression{condition}, er.asScalar, not) } -func (er *expressionRewriter) handleCompareSubquery(v *ast.CompareSubqueryExpr) (ast.Node, bool) { +func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, v *ast.CompareSubqueryExpr) (ast.Node, bool) { v.L.Accept(er) if er.err != nil { return v, true @@ -363,7 +366,7 @@ func (er *expressionRewriter) handleCompareSubquery(v *ast.CompareSubqueryExpr) er.err = errors.Errorf("Unknown compare type %T.", v.R) return v, true } - np, err := er.buildSubquery(subq) + np, err := er.buildSubquery(ctx, subq) if err != nil { er.err = err return v, true @@ -435,7 +438,7 @@ func (er *expressionRewriter) handleCompareSubquery(v *ast.CompareSubqueryExpr) // handleOtherComparableSubq handles the queries like < any, < max, etc. For example, if the query is t.id < any (select s.id from s), // it will be rewrote to t.id < (select max(s.id) from s). func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression.Expression, np LogicalPlan, useMin bool, cmpFunc string, all bool) { - plan4Agg := LogicalAggregation{}.Init(er.ctx) + plan4Agg := LogicalAggregation{}.Init(er.sctx) plan4Agg.SetChildren(np) // Create a "max" or "min" aggregation. @@ -443,7 +446,7 @@ func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression. if useMin { funcName = ast.AggFuncMin } - funcMaxOrMin, err := aggregation.NewAggFuncDesc(er.ctx, funcName, []expression.Expression{rexpr}, false) + funcMaxOrMin, err := aggregation.NewAggFuncDesc(er.sctx, funcName, []expression.Expression{rexpr}, false) if err != nil { er.err = err return @@ -452,7 +455,7 @@ func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression. // Create a column and append it to the schema of that aggregation. colMaxOrMin := &expression.Column{ ColName: model.NewCIStr("agg_Col_0"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: funcMaxOrMin.RetTp, } schema := expression.NewSchema(colMaxOrMin) @@ -460,38 +463,38 @@ func (er *expressionRewriter) handleOtherComparableSubq(lexpr, rexpr expression. plan4Agg.SetSchema(schema) plan4Agg.AggFuncs = []*aggregation.AggFuncDesc{funcMaxOrMin} - cond := expression.NewFunctionInternal(er.ctx, cmpFunc, types.NewFieldType(mysql.TypeTiny), lexpr, colMaxOrMin) + cond := expression.NewFunctionInternal(er.sctx, cmpFunc, types.NewFieldType(mysql.TypeTiny), lexpr, colMaxOrMin) er.buildQuantifierPlan(plan4Agg, cond, lexpr, rexpr, all) } // buildQuantifierPlan adds extra condition for any / all subquery. func (er *expressionRewriter) buildQuantifierPlan(plan4Agg *LogicalAggregation, cond, lexpr, rexpr expression.Expression, all bool) { - innerIsNull := expression.NewFunctionInternal(er.ctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), rexpr) - outerIsNull := expression.NewFunctionInternal(er.ctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), lexpr) + innerIsNull := expression.NewFunctionInternal(er.sctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), rexpr) + outerIsNull := expression.NewFunctionInternal(er.sctx, ast.IsNull, types.NewFieldType(mysql.TypeTiny), lexpr) - funcSum, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncSum, []expression.Expression{innerIsNull}, false) + funcSum, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncSum, []expression.Expression{innerIsNull}, false) if err != nil { er.err = err return } colSum := &expression.Column{ ColName: model.NewCIStr("agg_col_sum"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: funcSum.RetTp, } plan4Agg.AggFuncs = append(plan4Agg.AggFuncs, funcSum) plan4Agg.schema.Append(colSum) - innerHasNull := expression.NewFunctionInternal(er.ctx, ast.NE, types.NewFieldType(mysql.TypeTiny), colSum, expression.Zero) + innerHasNull := expression.NewFunctionInternal(er.sctx, ast.NE, types.NewFieldType(mysql.TypeTiny), colSum, expression.Zero) // Build `count(1)` aggregation to check if subquery is empty. - funcCount, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncCount, []expression.Expression{expression.One}, false) + funcCount, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncCount, []expression.Expression{expression.One}, false) if err != nil { er.err = err return } colCount := &expression.Column{ ColName: model.NewCIStr("agg_col_cnt"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: funcCount.RetTp, } plan4Agg.AggFuncs = append(plan4Agg.AggFuncs, funcCount) @@ -500,23 +503,23 @@ func (er *expressionRewriter) buildQuantifierPlan(plan4Agg *LogicalAggregation, if all { // All of the inner record set should not contain null value. So for t.id < all(select s.id from s), it // should be rewrote to t.id < min(s.id) and if(sum(s.id is null) != 0, null, true). - innerNullChecker := expression.NewFunctionInternal(er.ctx, ast.If, types.NewFieldType(mysql.TypeTiny), innerHasNull, expression.Null, expression.One) - cond = expression.ComposeCNFCondition(er.ctx, cond, innerNullChecker) + innerNullChecker := expression.NewFunctionInternal(er.sctx, ast.If, types.NewFieldType(mysql.TypeTiny), innerHasNull, expression.Null, expression.One) + cond = expression.ComposeCNFCondition(er.sctx, cond, innerNullChecker) // If the subquery is empty, it should always return true. - emptyChecker := expression.NewFunctionInternal(er.ctx, ast.EQ, types.NewFieldType(mysql.TypeTiny), colCount, expression.Zero) + emptyChecker := expression.NewFunctionInternal(er.sctx, ast.EQ, types.NewFieldType(mysql.TypeTiny), colCount, expression.Zero) // If outer key is null, and subquery is not empty, it should always return null, even when it is `null = all (1, 2)`. - outerNullChecker := expression.NewFunctionInternal(er.ctx, ast.If, types.NewFieldType(mysql.TypeTiny), outerIsNull, expression.Null, expression.Zero) - cond = expression.ComposeDNFCondition(er.ctx, cond, emptyChecker, outerNullChecker) + outerNullChecker := expression.NewFunctionInternal(er.sctx, ast.If, types.NewFieldType(mysql.TypeTiny), outerIsNull, expression.Null, expression.Zero) + cond = expression.ComposeDNFCondition(er.sctx, cond, emptyChecker, outerNullChecker) } else { // For "any" expression, if the subquery has null and the cond returns false, the result should be NULL. // Specifically, `t.id < any (select s.id from s)` would be rewrote to `t.id < max(s.id) or if(sum(s.id is null) != 0, null, false)` - innerNullChecker := expression.NewFunctionInternal(er.ctx, ast.If, types.NewFieldType(mysql.TypeTiny), innerHasNull, expression.Null, expression.Zero) - cond = expression.ComposeDNFCondition(er.ctx, cond, innerNullChecker) + innerNullChecker := expression.NewFunctionInternal(er.sctx, ast.If, types.NewFieldType(mysql.TypeTiny), innerHasNull, expression.Null, expression.Zero) + cond = expression.ComposeDNFCondition(er.sctx, cond, innerNullChecker) // If the subquery is empty, it should always return false. - emptyChecker := expression.NewFunctionInternal(er.ctx, ast.NE, types.NewFieldType(mysql.TypeTiny), colCount, expression.Zero) + emptyChecker := expression.NewFunctionInternal(er.sctx, ast.NE, types.NewFieldType(mysql.TypeTiny), colCount, expression.Zero) // If outer key is null, and subquery is not empty, it should return null. - outerNullChecker := expression.NewFunctionInternal(er.ctx, ast.If, types.NewFieldType(mysql.TypeTiny), outerIsNull, expression.Null, expression.One) - cond = expression.ComposeCNFCondition(er.ctx, cond, emptyChecker, outerNullChecker) + outerNullChecker := expression.NewFunctionInternal(er.sctx, ast.If, types.NewFieldType(mysql.TypeTiny), outerIsNull, expression.Null, expression.One) + cond = expression.ComposeCNFCondition(er.sctx, cond, emptyChecker, outerNullChecker) } // TODO: Add a Projection if any argument of aggregate funcs or group by items are scalar functions. @@ -532,12 +535,12 @@ func (er *expressionRewriter) buildQuantifierPlan(plan4Agg *LogicalAggregation, joinSchema := er.p.Schema() proj := LogicalProjection{ Exprs: expression.Column2Exprs(joinSchema.Clone().Columns[:outerSchemaLen]), - }.Init(er.ctx) + }.Init(er.sctx) proj.SetSchema(expression.NewSchema(joinSchema.Clone().Columns[:outerSchemaLen]...)) proj.Exprs = append(proj.Exprs, cond) proj.schema.Append(&expression.Column{ ColName: model.NewCIStr("aux_col"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), IsReferenced: true, RetType: cond.GetType(), }) @@ -549,78 +552,78 @@ func (er *expressionRewriter) buildQuantifierPlan(plan4Agg *LogicalAggregation, // t.id != s.id or count(distinct s.id) > 1 or [any checker]. If there are two different values in s.id , // there must exist a s.id that doesn't equal to t.id. func (er *expressionRewriter) handleNEAny(lexpr, rexpr expression.Expression, np LogicalPlan) { - firstRowFunc, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncFirstRow, []expression.Expression{rexpr}, false) + firstRowFunc, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncFirstRow, []expression.Expression{rexpr}, false) if err != nil { er.err = err return } - countFunc, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncCount, []expression.Expression{rexpr}, true) + countFunc, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncCount, []expression.Expression{rexpr}, true) if err != nil { er.err = err return } plan4Agg := LogicalAggregation{ AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, - }.Init(er.ctx) + }.Init(er.sctx) plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ ColName: model.NewCIStr("col_firstRow"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: firstRowFunc.RetTp, } count := &expression.Column{ ColName: model.NewCIStr("col_count"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: countFunc.RetTp, } plan4Agg.SetSchema(expression.NewSchema(firstRowResultCol, count)) - gtFunc := expression.NewFunctionInternal(er.ctx, ast.GT, types.NewFieldType(mysql.TypeTiny), count, expression.One) - neCond := expression.NewFunctionInternal(er.ctx, ast.NE, types.NewFieldType(mysql.TypeTiny), lexpr, firstRowResultCol) - cond := expression.ComposeDNFCondition(er.ctx, gtFunc, neCond) + gtFunc := expression.NewFunctionInternal(er.sctx, ast.GT, types.NewFieldType(mysql.TypeTiny), count, expression.One) + neCond := expression.NewFunctionInternal(er.sctx, ast.NE, types.NewFieldType(mysql.TypeTiny), lexpr, firstRowResultCol) + cond := expression.ComposeDNFCondition(er.sctx, gtFunc, neCond) er.buildQuantifierPlan(plan4Agg, cond, lexpr, rexpr, false) } // handleEQAll handles the case of = all. For example, if the query is t.id = all (select s.id from s), it will be rewrote to // t.id = (select s.id from s having count(distinct s.id) <= 1 and [all checker]). func (er *expressionRewriter) handleEQAll(lexpr, rexpr expression.Expression, np LogicalPlan) { - firstRowFunc, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncFirstRow, []expression.Expression{rexpr}, false) + firstRowFunc, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncFirstRow, []expression.Expression{rexpr}, false) if err != nil { er.err = err return } - countFunc, err := aggregation.NewAggFuncDesc(er.ctx, ast.AggFuncCount, []expression.Expression{rexpr}, true) + countFunc, err := aggregation.NewAggFuncDesc(er.sctx, ast.AggFuncCount, []expression.Expression{rexpr}, true) if err != nil { er.err = err return } plan4Agg := LogicalAggregation{ AggFuncs: []*aggregation.AggFuncDesc{firstRowFunc, countFunc}, - }.Init(er.ctx) + }.Init(er.sctx) plan4Agg.SetChildren(np) firstRowResultCol := &expression.Column{ ColName: model.NewCIStr("col_firstRow"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: firstRowFunc.RetTp, } count := &expression.Column{ ColName: model.NewCIStr("col_count"), - UniqueID: er.ctx.GetSessionVars().AllocPlanColumnID(), + UniqueID: er.sctx.GetSessionVars().AllocPlanColumnID(), RetType: countFunc.RetTp, } plan4Agg.SetSchema(expression.NewSchema(firstRowResultCol, count)) - leFunc := expression.NewFunctionInternal(er.ctx, ast.LE, types.NewFieldType(mysql.TypeTiny), count, expression.One) - eqCond := expression.NewFunctionInternal(er.ctx, ast.EQ, types.NewFieldType(mysql.TypeTiny), lexpr, firstRowResultCol) - cond := expression.ComposeCNFCondition(er.ctx, leFunc, eqCond) + leFunc := expression.NewFunctionInternal(er.sctx, ast.LE, types.NewFieldType(mysql.TypeTiny), count, expression.One) + eqCond := expression.NewFunctionInternal(er.sctx, ast.EQ, types.NewFieldType(mysql.TypeTiny), lexpr, firstRowResultCol) + cond := expression.ComposeCNFCondition(er.sctx, leFunc, eqCond) er.buildQuantifierPlan(plan4Agg, cond, lexpr, rexpr, true) } -func (er *expressionRewriter) handleExistSubquery(v *ast.ExistsSubqueryExpr) (ast.Node, bool) { +func (er *expressionRewriter) handleExistSubquery(ctx context.Context, v *ast.ExistsSubqueryExpr) (ast.Node, bool) { subq, ok := v.Sel.(*ast.SubqueryExpr) if !ok { er.err = errors.Errorf("Unknown exists type %T.", v.Sel) return v, true } - np, err := er.buildSubquery(subq) + np, err := er.buildSubquery(ctx, subq) if err != nil { er.err = err return v, true @@ -633,12 +636,12 @@ func (er *expressionRewriter) handleExistSubquery(v *ast.ExistsSubqueryExpr) (as } er.ctxStack = append(er.ctxStack, er.p.Schema().Columns[er.p.Schema().Len()-1]) } else { - physicalPlan, err := DoOptimize(er.b.optFlag, np) + physicalPlan, err := DoOptimize(ctx, er.b.optFlag, np) if err != nil { er.err = err return v, true } - rows, err := EvalSubquery(physicalPlan, er.b.is, er.b.ctx) + rows, err := EvalSubquery(ctx, physicalPlan, er.b.is, er.b.ctx) if err != nil { er.err = err return v, true @@ -664,7 +667,7 @@ out: p = p.Children()[0] case *LogicalAggregation: if len(plan.GroupByItems) == 0 { - p = LogicalTableDual{RowCount: 1}.Init(er.ctx) + p = LogicalTableDual{RowCount: 1}.Init(er.sctx) break out } p = p.Children()[0] @@ -675,7 +678,7 @@ out: return p } -func (er *expressionRewriter) handleInSubquery(v *ast.PatternInExpr) (ast.Node, bool) { +func (er *expressionRewriter) handleInSubquery(ctx context.Context, v *ast.PatternInExpr) (ast.Node, bool) { asScalar := er.asScalar er.asScalar = true v.Expr.Accept(er) @@ -688,7 +691,7 @@ func (er *expressionRewriter) handleInSubquery(v *ast.PatternInExpr) (ast.Node, er.err = errors.Errorf("Unknown compare type %T.", v.Sel) return v, true } - np, err := er.buildSubquery(subq) + np, err := er.buildSubquery(ctx, subq) if err != nil { er.err = err return v, true @@ -732,7 +735,7 @@ func (er *expressionRewriter) handleInSubquery(v *ast.PatternInExpr) (ast.Node, // and has no correlated column from the current level plan(if the correlated column is from upper level, // we can treat it as constant, because the upper LogicalApply cannot be eliminated since current node is a join node), // and don't need to append a scalar value, we can rewrite it to inner join. - if er.ctx.GetSessionVars().AllowInSubqToJoinAndAgg && !v.Not && !asScalar && len(extractCorColumnsBySchema(np, er.p.Schema())) == 0 { + if er.sctx.GetSessionVars().AllowInSubqToJoinAndAgg && !v.Not && !asScalar && len(extractCorColumnsBySchema(np, er.p.Schema())) == 0 { // We need to try to eliminate the agg and the projection produced by this operation. er.b.optFlag |= flagEliminateAgg er.b.optFlag |= flagEliminateProjection @@ -747,7 +750,7 @@ func (er *expressionRewriter) handleInSubquery(v *ast.PatternInExpr) (ast.Node, col.IsReferenced = true } // Build inner join above the aggregation. - join := LogicalJoin{JoinType: InnerJoin}.Init(er.ctx) + join := LogicalJoin{JoinType: InnerJoin}.Init(er.sctx) join.SetChildren(er.p, agg) join.SetSchema(expression.MergeSchema(er.p.Schema(), agg.schema)) join.attachOnConds(expression.SplitCNFItems(checkCondition)) @@ -775,8 +778,8 @@ func (er *expressionRewriter) handleInSubquery(v *ast.PatternInExpr) (ast.Node, return v, true } -func (er *expressionRewriter) handleScalarSubquery(v *ast.SubqueryExpr) (ast.Node, bool) { - np, err := er.buildSubquery(v) +func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, v *ast.SubqueryExpr) (ast.Node, bool) { + np, err := er.buildSubquery(ctx, v) if err != nil { er.err = err return v, true @@ -800,12 +803,12 @@ func (er *expressionRewriter) handleScalarSubquery(v *ast.SubqueryExpr) (ast.Nod } return v, true } - physicalPlan, err := DoOptimize(er.b.optFlag, np) + physicalPlan, err := DoOptimize(ctx, er.b.optFlag, np) if err != nil { er.err = err return v, true } - rows, err := EvalSubquery(physicalPlan, er.b.is, er.b.ctx) + rows, err := EvalSubquery(ctx, physicalPlan, er.b.is, er.b.ctx) if err != nil { er.err = err return v, true @@ -849,7 +852,7 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok er.ctxStack = append(er.ctxStack, value) case *driver.ParamMarkerExpr: var value expression.Expression - value, er.err = expression.GetParamExpression(er.ctx, v) + value, er.err = expression.GetParamExpression(er.sctx, v) if er.err != nil { return retNode, false } @@ -884,7 +887,7 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok return retNode, false } - er.ctxStack[len(er.ctxStack)-1] = expression.BuildCastFunction(er.ctx, arg, v.Tp) + er.ctxStack[len(er.ctxStack)-1] = expression.BuildCastFunction(er.sctx, arg, v.Tp) case *ast.PatternLikeExpr: er.patternLikeToExpression(v) case *ast.PatternRegexpExpr: @@ -917,9 +920,9 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok // newFunction chooses which expression.NewFunctionImpl() will be used. func (er *expressionRewriter) newFunction(funcName string, retType *types.FieldType, args ...expression.Expression) (expression.Expression, error) { if er.disableFoldCounter > 0 { - return expression.NewFunctionBase(er.ctx, funcName, retType, args...) + return expression.NewFunctionBase(er.sctx, funcName, retType, args...) } - return expression.NewFunction(er.ctx, funcName, retType, args...) + return expression.NewFunction(er.sctx, funcName, retType, args...) } func (er *expressionRewriter) checkTimePrecision(ft *types.FieldType) error { @@ -930,7 +933,7 @@ func (er *expressionRewriter) checkTimePrecision(ft *types.FieldType) error { } func (er *expressionRewriter) useCache() bool { - return er.ctx.GetSessionVars().StmtCtx.UseCache + return er.sctx.GetSessionVars().StmtCtx.UseCache } func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { @@ -981,8 +984,8 @@ func (er *expressionRewriter) rewriteVariable(v *ast.VariableExpr) { return } e := expression.DatumToConstant(types.NewStringDatum(val), mysql.TypeVarString) - e.RetType.Charset, _ = er.ctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection) - e.RetType.Collate, _ = er.ctx.GetSessionVars().GetSystemVar(variable.CollationConnection) + e.RetType.Charset, _ = er.sctx.GetSessionVars().GetSystemVar(variable.CharacterSetConnection) + e.RetType.Collate, _ = er.sctx.GetSessionVars().GetSystemVar(variable.CollationConnection) er.ctxStack = append(er.ctxStack, e) } @@ -1069,7 +1072,7 @@ func (er *expressionRewriter) positionToScalarFunc(v *ast.PositionExpr) { if v.P != nil { stkLen := len(er.ctxStack) val := er.ctxStack[stkLen-1] - intNum, isNull, err := expression.GetIntFromConstant(er.ctx, val) + intNum, isNull, err := expression.GetIntFromConstant(er.sctx, val) str = "?" if err == nil { if isNull { @@ -1126,7 +1129,7 @@ func (er *expressionRewriter) inToExpression(lLen int, not bool, tp *types.Field for i := 1; i < len(args); i++ { if c, ok := args[i].(*expression.Constant); ok { var isExceptional bool - args[i], isExceptional = expression.RefineComparedConstant(er.ctx, *leftFt, c, opcode.EQ) + args[i], isExceptional = expression.RefineComparedConstant(er.sctx, *leftFt, c, opcode.EQ) if isExceptional { args[i] = c } @@ -1153,7 +1156,7 @@ func (er *expressionRewriter) inToExpression(lLen int, not bool, tp *types.Field } eqFunctions = append(eqFunctions, expr) } - function = expression.ComposeDNFCondition(er.ctx, eqFunctions...) + function = expression.ComposeDNFCondition(er.sctx, eqFunctions...) if not { var err error function, err = er.newFunction(ast.UnaryNot, tp, function) @@ -1297,9 +1300,9 @@ func (er *expressionRewriter) betweenToExpression(v *ast.BetweenExpr) { expr, lexp, rexp := er.ctxStack[stkLen-3], er.ctxStack[stkLen-2], er.ctxStack[stkLen-1] if expression.GetCmpTp4MinMax([]expression.Expression{expr, lexp, rexp}) == types.ETDatetime { - expr = expression.WrapWithCastAsTime(er.ctx, expr, types.NewFieldType(mysql.TypeDatetime)) - lexp = expression.WrapWithCastAsTime(er.ctx, lexp, types.NewFieldType(mysql.TypeDatetime)) - rexp = expression.WrapWithCastAsTime(er.ctx, rexp, types.NewFieldType(mysql.TypeDatetime)) + expr = expression.WrapWithCastAsTime(er.sctx, expr, types.NewFieldType(mysql.TypeDatetime)) + lexp = expression.WrapWithCastAsTime(er.sctx, lexp, types.NewFieldType(mysql.TypeDatetime)) + rexp = expression.WrapWithCastAsTime(er.sctx, rexp, types.NewFieldType(mysql.TypeDatetime)) } var op string @@ -1403,7 +1406,7 @@ func (er *expressionRewriter) funcCallToExpression(v *ast.FuncCallExpr) { var function expression.Expression er.ctxStack = er.ctxStack[:stackLen-len(v.Args)] if _, ok := expression.DeferredFunctions[v.FnName.L]; er.useCache() && ok { - function, er.err = expression.NewFunctionBase(er.ctx, v.FnName.L, &v.Type, args...) + function, er.err = expression.NewFunctionBase(er.sctx, v.FnName.L, &v.Type, args...) c := &expression.Constant{Value: types.NewDatum(nil), RetType: function.GetType().Clone(), DeferredExpr: function} er.ctxStack = append(er.ctxStack, c) } else { @@ -1476,7 +1479,7 @@ func (er *expressionRewriter) evalDefaultExpr(v *ast.DefaultExpr) { dbName := colExpr.DBName if dbName.O == "" { // if database name is not specified, use current database name - dbName = model.NewCIStr(er.ctx.GetSessionVars().CurrentDB) + dbName = model.NewCIStr(er.sctx.GetSessionVars().CurrentDB) } if colExpr.OrigTblName.O == "" { // column is evaluated by some expressions, for example: diff --git a/planner/core/indexmerge_test.go b/planner/core/indexmerge_test.go index a3600667d2458..d6ac524c28fa5 100644 --- a/planner/core/indexmerge_test.go +++ b/planner/core/indexmerge_test.go @@ -14,6 +14,8 @@ package core import ( + "context" + . "github.com/pingcap/check" "github.com/pingcap/parser" "github.com/pingcap/parser/model" @@ -102,19 +104,20 @@ func (s *testIndexMergeSuite) TestIndexMergePathGenerateion(c *C) { "{Idxs:[c_d_e,f_g],TbFilters:[or(lt(test.t.c, 1), gt(test.t.f, 2)),or(lt(test.t.e, 1), gt(test.t.f, 2))]}]", }, } + ctx := context.TODO() for i, tc := range tests { comment := Commentf("case:%v sql:%s", i, tc.sql) stmt, err := s.ParseOneStmt(tc.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) if err != nil { c.Assert(err.Error(), Equals, tc.idxMergeDigest, comment) continue } c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) lp := p.(LogicalPlan) c.Assert(err, IsNil) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 99a68742f3d63..94c36b75ac343 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -14,6 +14,7 @@ package core import ( + "context" "fmt" "math" "math/bits" @@ -71,7 +72,7 @@ func (la *LogicalAggregation) collectGroupByColumns() { } } -func (b *PlanBuilder) buildAggregation(p LogicalPlan, aggFuncList []*ast.AggregateFuncExpr, gbyItems []expression.Expression) (LogicalPlan, map[int]int, error) { +func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFuncList []*ast.AggregateFuncExpr, gbyItems []expression.Expression) (LogicalPlan, map[int]int, error) { b.optFlag = b.optFlag | flagBuildKeyInfo b.optFlag = b.optFlag | flagPushDownAgg // We may apply aggregation eliminate optimization. @@ -91,7 +92,7 @@ func (b *PlanBuilder) buildAggregation(p LogicalPlan, aggFuncList []*ast.Aggrega for i, aggFunc := range aggFuncList { newArgList := make([]expression.Expression, 0, len(aggFunc.Args)) for _, arg := range aggFunc.Args { - newArg, np, err := b.rewrite(arg, p, nil, true) + newArg, np, err := b.rewrite(ctx, arg, p, nil, true) if err != nil { return nil, nil, err } @@ -139,18 +140,18 @@ func (b *PlanBuilder) buildAggregation(p LogicalPlan, aggFuncList []*ast.Aggrega return plan4Agg, aggIndexMap, nil } -func (b *PlanBuilder) buildResultSetNode(node ast.ResultSetNode) (p LogicalPlan, err error) { +func (b *PlanBuilder) buildResultSetNode(ctx context.Context, node ast.ResultSetNode) (p LogicalPlan, err error) { switch x := node.(type) { case *ast.Join: - return b.buildJoin(x) + return b.buildJoin(ctx, x) case *ast.TableSource: switch v := x.Source.(type) { case *ast.SelectStmt: - p, err = b.buildSelect(v) + p, err = b.buildSelect(ctx, v) case *ast.UnionStmt: - p, err = b.buildUnion(v) + p, err = b.buildUnion(ctx, v) case *ast.TableName: - p, err = b.buildDataSource(v) + p, err = b.buildDataSource(ctx, v) default: err = ErrUnsupportedType.GenWithStackByArgs(v) } @@ -179,9 +180,9 @@ func (b *PlanBuilder) buildResultSetNode(node ast.ResultSetNode) (p LogicalPlan, } return p, nil case *ast.SelectStmt: - return b.buildSelect(x) + return b.buildSelect(ctx, x) case *ast.UnionStmt: - return b.buildUnion(x) + return b.buildUnion(ctx, x) default: return nil, ErrUnsupportedType.GenWithStack("Unsupported ast.ResultSetNode(%T) for buildResultSetNode()", x) } @@ -369,22 +370,22 @@ func resetNotNullFlag(schema *expression.Schema, start, end int) { } } -func (b *PlanBuilder) buildJoin(joinNode *ast.Join) (LogicalPlan, error) { +func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (LogicalPlan, error) { // We will construct a "Join" node for some statements like "INSERT", // "DELETE", "UPDATE", "REPLACE". For this scenario "joinNode.Right" is nil // and we only build the left "ResultSetNode". if joinNode.Right == nil { - return b.buildResultSetNode(joinNode.Left) + return b.buildResultSetNode(ctx, joinNode.Left) } b.optFlag = b.optFlag | flagPredicatePushDown - leftPlan, err := b.buildResultSetNode(joinNode.Left) + leftPlan, err := b.buildResultSetNode(ctx, joinNode.Left) if err != nil { return nil, err } - rightPlan, err := b.buildResultSetNode(joinNode.Right) + rightPlan, err := b.buildResultSetNode(ctx, joinNode.Right) if err != nil { return nil, err } @@ -447,7 +448,7 @@ func (b *PlanBuilder) buildJoin(joinNode *ast.Join) (LogicalPlan, error) { } } else if joinNode.On != nil { b.curClause = onClause - onExpr, newPlan, err := b.rewrite(joinNode.On.Expr, joinPlan, nil, false) + onExpr, newPlan, err := b.rewrite(ctx, joinNode.On.Expr, joinPlan, nil, false) if err != nil { return nil, err } @@ -558,7 +559,7 @@ func (b *PlanBuilder) coalesceCommonColumns(p *LogicalJoin, leftPlan, rightPlan return nil } -func (b *PlanBuilder) buildSelection(p LogicalPlan, where ast.ExprNode, AggMapper map[*ast.AggregateFuncExpr]int) (LogicalPlan, error) { +func (b *PlanBuilder) buildSelection(ctx context.Context, p LogicalPlan, where ast.ExprNode, AggMapper map[*ast.AggregateFuncExpr]int) (LogicalPlan, error) { b.optFlag = b.optFlag | flagPredicatePushDown if b.curClause != havingClause { b.curClause = whereClause @@ -568,7 +569,7 @@ func (b *PlanBuilder) buildSelection(p LogicalPlan, where ast.ExprNode, AggMappe expressions := make([]expression.Expression, 0, len(conditions)) selection := LogicalSelection{}.Init(b.ctx) for _, cond := range conditions { - expr, np, err := b.rewrite(cond, p, AggMapper, false) + expr, np, err := b.rewrite(ctx, cond, p, AggMapper, false) if err != nil { return nil, err } @@ -617,7 +618,7 @@ func (b *PlanBuilder) buildProjectionFieldNameFromColumns(origField *ast.SelectF } // buildProjectionFieldNameFromExpressions builds the field name when field expression is a normal expression. -func (b *PlanBuilder) buildProjectionFieldNameFromExpressions(field *ast.SelectField) (model.CIStr, error) { +func (b *PlanBuilder) buildProjectionFieldNameFromExpressions(ctx context.Context, field *ast.SelectField) (model.CIStr, error) { if agg, ok := field.Expr.(*ast.AggregateFuncExpr); ok && agg.F == ast.AggFuncFirstRow { // When the query is select t.a from t group by a; The Column Name should be a but not t.a; return agg.Args[0].(*ast.ColumnNameExpr).Name.Name, nil @@ -682,7 +683,7 @@ func (b *PlanBuilder) buildProjectionFieldNameFromExpressions(field *ast.SelectF } // buildProjectionField builds the field object according to SelectField in projection. -func (b *PlanBuilder) buildProjectionField(id, position int, field *ast.SelectField, expr expression.Expression) (*expression.Column, error) { +func (b *PlanBuilder) buildProjectionField(ctx context.Context, id, position int, field *ast.SelectField, expr expression.Expression) (*expression.Column, error) { var origTblName, tblName, origColName, colName, dbName model.CIStr innerNode := getInnerFromParenthesesAndUnaryPlus(field.Expr) col, isCol := expr.(*expression.Column) @@ -697,7 +698,7 @@ func (b *PlanBuilder) buildProjectionField(id, position int, field *ast.SelectFi } else { // Other: field is an expression. var err error - if colName, err = b.buildProjectionFieldNameFromExpressions(field); err != nil { + if colName, err = b.buildProjectionFieldNameFromExpressions(ctx, field); err != nil { return nil, err } } @@ -713,7 +714,7 @@ func (b *PlanBuilder) buildProjectionField(id, position int, field *ast.SelectFi } // buildProjection returns a Projection plan and non-aux columns length. -func (b *PlanBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, mapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int, considerWindow bool) (LogicalPlan, int, error) { +func (b *PlanBuilder) buildProjection(ctx context.Context, p LogicalPlan, fields []*ast.SelectField, mapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int, considerWindow bool) (LogicalPlan, int, error) { b.optFlag |= flagEliminateProjection b.curClause = fieldList proj := LogicalProjection{Exprs: make([]expression.Expression, 0, len(fields))}.Init(b.ctx) @@ -738,14 +739,14 @@ func (b *PlanBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, } else if !considerWindow && isWindowFuncField { expr := expression.Zero proj.Exprs = append(proj.Exprs, expr) - col, err := b.buildProjectionField(proj.id, schema.Len()+1, field, expr) + col, err := b.buildProjectionField(ctx, proj.id, schema.Len()+1, field, expr) if err != nil { return nil, 0, err } schema.Append(col) continue } - newExpr, np, err := b.rewriteWithPreprocess(field.Expr, p, mapper, windowMapper, true, nil) + newExpr, np, err := b.rewriteWithPreprocess(ctx, field.Expr, p, mapper, windowMapper, true, nil) if err != nil { return nil, 0, err } @@ -761,7 +762,7 @@ func (b *PlanBuilder) buildProjection(p LogicalPlan, fields []*ast.SelectField, p = np proj.Exprs = append(proj.Exprs, newExpr) - col, err := b.buildProjectionField(proj.id, schema.Len()+1, field, newExpr) + col, err := b.buildProjectionField(ctx, proj.id, schema.Len()+1, field, newExpr) if err != nil { return nil, 0, err } @@ -820,7 +821,7 @@ func unionJoinFieldType(a, b *types.FieldType) *types.FieldType { return resultTp } -func (b *PlanBuilder) buildProjection4Union(u *LogicalUnionAll) { +func (b *PlanBuilder) buildProjection4Union(ctx context.Context, u *LogicalUnionAll) { unionCols := make([]*expression.Column, 0, u.children[0].Schema().Len()) // Infer union result types by its children's schema. @@ -858,13 +859,13 @@ func (b *PlanBuilder) buildProjection4Union(u *LogicalUnionAll) { } } -func (b *PlanBuilder) buildUnion(union *ast.UnionStmt) (LogicalPlan, error) { - distinctSelectPlans, allSelectPlans, err := b.divideUnionSelectPlans(union.SelectList.Selects) +func (b *PlanBuilder) buildUnion(ctx context.Context, union *ast.UnionStmt) (LogicalPlan, error) { + distinctSelectPlans, allSelectPlans, err := b.divideUnionSelectPlans(ctx, union.SelectList.Selects) if err != nil { return nil, err } - unionDistinctPlan := b.buildUnionAll(distinctSelectPlans) + unionDistinctPlan := b.buildUnionAll(ctx, distinctSelectPlans) if unionDistinctPlan != nil { unionDistinctPlan, err = b.buildDistinct(unionDistinctPlan, unionDistinctPlan.Schema().Len()) if err != nil { @@ -876,7 +877,7 @@ func (b *PlanBuilder) buildUnion(union *ast.UnionStmt) (LogicalPlan, error) { } } - unionAllPlan := b.buildUnionAll(allSelectPlans) + unionAllPlan := b.buildUnionAll(ctx, allSelectPlans) unionPlan := unionDistinctPlan if unionAllPlan != nil { unionPlan = unionAllPlan @@ -885,7 +886,7 @@ func (b *PlanBuilder) buildUnion(union *ast.UnionStmt) (LogicalPlan, error) { oldLen := unionPlan.Schema().Len() if union.OrderBy != nil { - unionPlan, err = b.buildSort(unionPlan, union.OrderBy.Items, nil, nil) + unionPlan, err = b.buildSort(ctx, unionPlan, union.OrderBy.Items, nil, nil) if err != nil { return nil, err } @@ -918,7 +919,7 @@ func (b *PlanBuilder) buildUnion(union *ast.UnionStmt) (LogicalPlan, error) { // and divide result plans into "union-distinct" and "union-all" parts. // divide rule ref: https://dev.mysql.com/doc/refman/5.7/en/union.html // "Mixed UNION types are treated such that a DISTINCT union overrides any ALL union to its left." -func (b *PlanBuilder) divideUnionSelectPlans(selects []*ast.SelectStmt) (distinctSelects []LogicalPlan, allSelects []LogicalPlan, err error) { +func (b *PlanBuilder) divideUnionSelectPlans(ctx context.Context, selects []*ast.SelectStmt) (distinctSelects []LogicalPlan, allSelects []LogicalPlan, err error) { firstUnionAllIdx, columnNums := 0, -1 // The last slot is reserved for appending distinct union outside this function. children := make([]LogicalPlan, len(selects), len(selects)+1) @@ -928,7 +929,7 @@ func (b *PlanBuilder) divideUnionSelectPlans(selects []*ast.SelectStmt) (distinc firstUnionAllIdx = i + 1 } - selectPlan, err := b.buildSelect(stmt) + selectPlan, err := b.buildSelect(ctx, stmt) if err != nil { return nil, nil, err } @@ -944,13 +945,13 @@ func (b *PlanBuilder) divideUnionSelectPlans(selects []*ast.SelectStmt) (distinc return children[:firstUnionAllIdx], children[firstUnionAllIdx:], nil } -func (b *PlanBuilder) buildUnionAll(subPlan []LogicalPlan) LogicalPlan { +func (b *PlanBuilder) buildUnionAll(ctx context.Context, subPlan []LogicalPlan) LogicalPlan { if len(subPlan) == 0 { return nil } u := LogicalUnionAll{}.Init(b.ctx) u.children = subPlan - b.buildProjection4Union(u) + b.buildProjection4Union(ctx, u) return u } @@ -990,7 +991,7 @@ func (t *itemTransformer) Leave(inNode ast.Node) (ast.Node, bool) { return inNode, false } -func (b *PlanBuilder) buildSort(p LogicalPlan, byItems []*ast.ByItem, aggMapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int) (*LogicalSort, error) { +func (b *PlanBuilder) buildSort(ctx context.Context, p LogicalPlan, byItems []*ast.ByItem, aggMapper map[*ast.AggregateFuncExpr]int, windowMapper map[*ast.WindowFuncExpr]int) (*LogicalSort, error) { if _, isUnion := p.(*LogicalUnionAll); isUnion { b.curClause = globalOrderByClause } else { @@ -1002,7 +1003,7 @@ func (b *PlanBuilder) buildSort(p LogicalPlan, byItems []*ast.ByItem, aggMapper for _, item := range byItems { newExpr, _ := item.Expr.Accept(transformer) item.Expr = newExpr.(ast.ExprNode) - it, np, err := b.rewriteWithPreprocess(item.Expr, p, aggMapper, windowMapper, true, nil) + it, np, err := b.rewriteWithPreprocess(ctx, item.Expr, p, aggMapper, windowMapper, true, nil) if err != nil { return nil, err } @@ -1864,7 +1865,7 @@ func allColFromExprNode(p LogicalPlan, n ast.Node, cols map[*expression.Column]s n.Accept(extractor) } -func (b *PlanBuilder) resolveGbyExprs(p LogicalPlan, gby *ast.GroupByClause, fields []*ast.SelectField) (LogicalPlan, []expression.Expression, error) { +func (b *PlanBuilder) resolveGbyExprs(ctx context.Context, p LogicalPlan, gby *ast.GroupByClause, fields []*ast.SelectField) (LogicalPlan, []expression.Expression, error) { b.curClause = groupByClause exprs := make([]expression.Expression, 0, len(gby.Items)) resolver := &gbyResolver{ @@ -1883,7 +1884,7 @@ func (b *PlanBuilder) resolveGbyExprs(p LogicalPlan, gby *ast.GroupByClause, fie } itemExpr := retExpr.(ast.ExprNode) - expr, np, err := b.rewrite(itemExpr, p, nil, true) + expr, np, err := b.rewrite(ctx, itemExpr, p, nil, true) if err != nil { return nil, nil, err } @@ -1981,7 +1982,7 @@ func (b *PlanBuilder) TableHints() *tableHintInfo { return &(b.tableHintInfo[len(b.tableHintInfo)-1]) } -func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error) { +func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p LogicalPlan, err error) { if b.pushTableHints(sel.TableHints) { // table hints are only visible in the current SELECT statement. defer b.popTableHints() @@ -2000,7 +2001,7 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error ) if sel.From != nil { - p, err = b.buildResultSetNode(sel.From.TableRefs) + p, err = b.buildResultSetNode(ctx, sel.From.TableRefs) if err != nil { return nil, err } @@ -2015,7 +2016,7 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error } if sel.GroupBy != nil { - p, gbyCols, err = b.resolveGbyExprs(p, sel.GroupBy, sel.Fields.Fields) + p, gbyCols, err = b.resolveGbyExprs(ctx, p, sel.GroupBy, sel.Fields.Fields) if err != nil { return nil, err } @@ -2044,7 +2045,7 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error } if sel.Where != nil { - p, err = b.buildSelection(p, sel.Where, nil) + p, err = b.buildSelection(ctx, p, sel.Where, nil) if err != nil { return nil, err } @@ -2058,7 +2059,7 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error if hasAgg { aggFuncs, totalMap = b.extractAggFuncs(sel.Fields.Fields) var aggIndexMap map[int]int - p, aggIndexMap, err = b.buildAggregation(p, aggFuncs, gbyCols) + p, aggIndexMap, err = b.buildAggregation(ctx, p, aggFuncs, gbyCols) if err != nil { return nil, err } @@ -2070,14 +2071,14 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error var oldLen int // According to https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html, // we can only process window functions after having clause, so `considerWindow` is false now. - p, oldLen, err = b.buildProjection(p, sel.Fields.Fields, totalMap, nil, false) + p, oldLen, err = b.buildProjection(ctx, p, sel.Fields.Fields, totalMap, nil, false) if err != nil { return nil, err } if sel.Having != nil { b.curClause = havingClause - p, err = b.buildSelection(p, sel.Having.Expr, havingMap) + p, err = b.buildSelection(ctx, p, sel.Having.Expr, havingMap) if err != nil { return nil, err } @@ -2095,12 +2096,12 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error if err != nil { return nil, err } - p, windowMapper, err = b.buildWindowFunctions(p, groupedFuncs, windowAggMap) + p, windowMapper, err = b.buildWindowFunctions(ctx, p, groupedFuncs, windowAggMap) if err != nil { return nil, err } // Now we build the window function fields. - p, oldLen, err = b.buildProjection(p, sel.Fields.Fields, windowAggMap, windowMapper, true) + p, oldLen, err = b.buildProjection(ctx, p, sel.Fields.Fields, windowAggMap, windowMapper, true) if err != nil { return nil, err } @@ -2114,7 +2115,7 @@ func (b *PlanBuilder) buildSelect(sel *ast.SelectStmt) (p LogicalPlan, err error } if sel.OrderBy != nil { - p, err = b.buildSort(p, sel.OrderBy.Items, orderMap, windowMapper) + p, err = b.buildSort(ctx, p, sel.OrderBy.Items, orderMap, windowMapper) if err != nil { return nil, err } @@ -2192,7 +2193,7 @@ func getStatsTable(ctx sessionctx.Context, tblInfo *model.TableInfo, pid int64) return statsTbl } -func (b *PlanBuilder) buildDataSource(tn *ast.TableName) (LogicalPlan, error) { +func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (LogicalPlan, error) { dbName := tn.Schema if dbName.L == "" { dbName = model.NewCIStr(b.ctx.GetSessionVars().CurrentDB) @@ -2211,7 +2212,7 @@ func (b *PlanBuilder) buildDataSource(tn *ast.TableName) (LogicalPlan, error) { b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, dbName.L, tableInfo.Name.L, "", authErr) if tableInfo.IsView() { - return b.BuildDataSourceFromView(dbName, tableInfo) + return b.BuildDataSourceFromView(ctx, dbName, tableInfo) } if tableInfo.GetPartitionInfo() != nil { @@ -2312,7 +2313,7 @@ func (b *PlanBuilder) buildDataSource(tn *ast.TableName) (LogicalPlan, error) { // If this table contains any virtual generated columns, we need a // "Projection" to calculate these columns. - proj, err := b.projectVirtualColumns(ds, columns) + proj, err := b.projectVirtualColumns(ctx, ds, columns) if err != nil { return nil, err } @@ -2325,7 +2326,7 @@ func (b *PlanBuilder) buildDataSource(tn *ast.TableName) (LogicalPlan, error) { } // BuildDataSourceFromView is used to build LogicalPlan from view -func (b *PlanBuilder) BuildDataSourceFromView(dbName model.CIStr, tableInfo *model.TableInfo) (LogicalPlan, error) { +func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName model.CIStr, tableInfo *model.TableInfo) (LogicalPlan, error) { charset, collation := b.ctx.GetSessionVars().GetCharsetInfo() viewParser := parser.New() viewParser.EnableWindowFunc(b.ctx.GetSessionVars().EnableWindowFunction) @@ -2335,7 +2336,7 @@ func (b *PlanBuilder) BuildDataSourceFromView(dbName model.CIStr, tableInfo *mod } originalVisitInfo := b.visitInfo b.visitInfo = make([]visitInfo, 0) - selectLogicalPlan, err := b.Build(selectNode) + selectLogicalPlan, err := b.Build(ctx, selectNode) if err != nil { return nil, err } @@ -2384,7 +2385,7 @@ func (b *PlanBuilder) BuildDataSourceFromView(dbName model.CIStr, tableInfo *mod // projectVirtualColumns is only for DataSource. If some table has virtual generated columns, // we add a projection on the original DataSource, and calculate those columns in the projection // so that plans above it can reference generated columns by their name. -func (b *PlanBuilder) projectVirtualColumns(ds *DataSource, columns []*table.Column) (*LogicalProjection, error) { +func (b *PlanBuilder) projectVirtualColumns(ctx context.Context, ds *DataSource, columns []*table.Column) (*LogicalProjection, error) { hasVirtualGeneratedColumn := false for _, column := range columns { if column.IsGenerated() && !column.GeneratedStored { @@ -2406,7 +2407,7 @@ func (b *PlanBuilder) projectVirtualColumns(ds *DataSource, columns []*table.Col if i < len(columns) { if columns[i].IsGenerated() && !columns[i].GeneratedStored { var err error - expr, _, err = b.rewrite(columns[i].GeneratedExpr, ds, nil, true) + expr, _, err = b.rewrite(ctx, columns[i].GeneratedExpr, ds, nil, true) if err != nil { return nil, err } @@ -2521,7 +2522,7 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio return joinPlan, nil } -func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { +func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { if b.pushTableHints(update.TableHints) { // table hints are only visible in the current UPDATE statement. defer b.popTableHints() @@ -2540,7 +2541,7 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { b.inUpdateStmt = true - p, err := b.buildResultSetNode(update.TableRefs.TableRefs) + p, err := b.buildResultSetNode(ctx, update.TableRefs.TableRefs) if err != nil { return nil, err } @@ -2559,13 +2560,13 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { } if update.Where != nil { - p, err = b.buildSelection(p, update.Where, nil) + p, err = b.buildSelection(ctx, p, update.Where, nil) if err != nil { return nil, err } } if update.Order != nil { - p, err = b.buildSort(p, update.Order.Items, nil, nil) + p, err = b.buildSort(ctx, p, update.Order.Items, nil, nil) if err != nil { return nil, err } @@ -2579,7 +2580,7 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { var updateTableList []*ast.TableName updateTableList = extractTableList(update.TableRefs.TableRefs, updateTableList, true) - orderedList, np, err := b.buildUpdateLists(updateTableList, update.List, p) + orderedList, np, err := b.buildUpdateLists(ctx, updateTableList, update.List, p) if err != nil { return nil, err } @@ -2589,7 +2590,7 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { updt.SetSchema(p.Schema()) // We cannot apply projection elimination when building the subplan, because // columns in orderedList cannot be resolved. - updt.SelectPlan, err = DoOptimize(b.optFlag&^flagEliminateProjection, p) + updt.SelectPlan, err = DoOptimize(ctx, b.optFlag&^flagEliminateProjection, p) if err != nil { return nil, err } @@ -2597,7 +2598,7 @@ func (b *PlanBuilder) buildUpdate(update *ast.UpdateStmt) (Plan, error) { return updt, err } -func (b *PlanBuilder) buildUpdateLists(tableList []*ast.TableName, list []*ast.Assignment, p LogicalPlan) ([]*expression.Assignment, LogicalPlan, error) { +func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.TableName, list []*ast.Assignment, p LogicalPlan) ([]*expression.Assignment, LogicalPlan, error) { b.curClause = fieldList modifyColumns := make(map[string]struct{}, p.Schema().Len()) // Which columns are in set list. for _, assign := range list { @@ -2648,7 +2649,7 @@ func (b *PlanBuilder) buildUpdateLists(tableList []*ast.TableName, list []*ast.A var newExpr expression.Expression var np LogicalPlan if i < len(list) { - newExpr, np, err = b.rewrite(assign.Expr, p, nil, false) + newExpr, np, err = b.rewrite(ctx, assign.Expr, p, nil, false) } else { // rewrite with generation expression rewritePreprocess := func(expr ast.Node) ast.Node { @@ -2663,7 +2664,7 @@ func (b *PlanBuilder) buildUpdateLists(tableList []*ast.TableName, list []*ast.A return expr } } - newExpr, np, err = b.rewriteWithPreprocess(assign.Expr, p, nil, nil, false, rewritePreprocess) + newExpr, np, err = b.rewriteWithPreprocess(ctx, assign.Expr, p, nil, nil, false, rewritePreprocess) } if err != nil { return nil, nil, err @@ -2734,13 +2735,13 @@ func extractTableAsNameForUpdate(p LogicalPlan, asNames map[*model.TableInfo][]* } } -func (b *PlanBuilder) buildDelete(delete *ast.DeleteStmt) (Plan, error) { +func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) (Plan, error) { if b.pushTableHints(delete.TableHints) { // table hints are only visible in the current DELETE statement. defer b.popTableHints() } - p, err := b.buildResultSetNode(delete.TableRefs.TableRefs) + p, err := b.buildResultSetNode(ctx, delete.TableRefs.TableRefs) if err != nil { return nil, err } @@ -2748,14 +2749,14 @@ func (b *PlanBuilder) buildDelete(delete *ast.DeleteStmt) (Plan, error) { oldLen := oldSchema.Len() if delete.Where != nil { - p, err = b.buildSelection(p, delete.Where, nil) + p, err = b.buildSelection(ctx, p, delete.Where, nil) if err != nil { return nil, err } } if delete.Order != nil { - p, err = b.buildSort(p, delete.Order.Items, nil, nil) + p, err = b.buildSort(ctx, p, delete.Order.Items, nil, nil) if err != nil { return nil, err } @@ -2787,7 +2788,7 @@ func (b *PlanBuilder) buildDelete(delete *ast.DeleteStmt) (Plan, error) { IsMultiTable: delete.IsMultiTable, }.Init(b.ctx) - del.SelectPlan, err = DoOptimize(b.optFlag, p) + del.SelectPlan, err = DoOptimize(ctx, b.optFlag, p) if err != nil { return nil, err } @@ -2862,7 +2863,7 @@ func getWindowName(name string) string { // buildProjectionForWindow builds the projection for expressions in the window specification that is not an column, // so after the projection, window functions only needs to deal with columns. -func (b *PlanBuilder) buildProjectionForWindow(p LogicalPlan, spec *ast.WindowSpec, args []ast.ExprNode, aggMap map[*ast.AggregateFuncExpr]int) (LogicalPlan, []property.Item, []property.Item, []expression.Expression, error) { +func (b *PlanBuilder) buildProjectionForWindow(ctx context.Context, p LogicalPlan, spec *ast.WindowSpec, args []ast.ExprNode, aggMap map[*ast.AggregateFuncExpr]int) (LogicalPlan, []property.Item, []property.Item, []expression.Expression, error) { b.optFlag |= flagEliminateProjection var partitionItems, orderItems []*ast.ByItem @@ -2883,19 +2884,19 @@ func (b *PlanBuilder) buildProjectionForWindow(p LogicalPlan, spec *ast.WindowSp propertyItems := make([]property.Item, 0, len(partitionItems)+len(orderItems)) var err error - p, propertyItems, err = b.buildByItemsForWindow(p, proj, partitionItems, propertyItems, aggMap) + p, propertyItems, err = b.buildByItemsForWindow(ctx, p, proj, partitionItems, propertyItems, aggMap) if err != nil { return nil, nil, nil, nil, err } lenPartition := len(propertyItems) - p, propertyItems, err = b.buildByItemsForWindow(p, proj, orderItems, propertyItems, aggMap) + p, propertyItems, err = b.buildByItemsForWindow(ctx, p, proj, orderItems, propertyItems, aggMap) if err != nil { return nil, nil, nil, nil, err } newArgList := make([]expression.Expression, 0, len(args)) for _, arg := range args { - newArg, np, err := b.rewrite(arg, p, aggMap, true) + newArg, np, err := b.rewrite(ctx, arg, p, aggMap, true) if err != nil { return nil, nil, nil, nil, err } @@ -2920,6 +2921,7 @@ func (b *PlanBuilder) buildProjectionForWindow(p LogicalPlan, spec *ast.WindowSp } func (b *PlanBuilder) buildByItemsForWindow( + ctx context.Context, p LogicalPlan, proj *LogicalProjection, items []*ast.ByItem, @@ -2930,7 +2932,7 @@ func (b *PlanBuilder) buildByItemsForWindow( for _, item := range items { newExpr, _ := item.Expr.Accept(transformer) item.Expr = newExpr.(ast.ExprNode) - it, np, err := b.rewrite(item.Expr, p, aggMap, true) + it, np, err := b.rewrite(ctx, item.Expr, p, aggMap, true) if err != nil { return nil, nil, err } @@ -2957,7 +2959,7 @@ func (b *PlanBuilder) buildByItemsForWindow( // buildWindowFunctionFrameBound builds the bounds of window function frames. // For type `Rows`, the bound expr must be an unsigned integer. // For type `Range`, the bound expr must be temporal or numeric types. -func (b *PlanBuilder) buildWindowFunctionFrameBound(spec *ast.WindowSpec, orderByItems []property.Item, boundClause *ast.FrameBound) (*FrameBound, error) { +func (b *PlanBuilder) buildWindowFunctionFrameBound(ctx context.Context, spec *ast.WindowSpec, orderByItems []property.Item, boundClause *ast.FrameBound) (*FrameBound, error) { frameType := spec.Frame.Type bound := &FrameBound{Type: boundClause.Type, UnBounded: boundClause.UnBounded} if bound.UnBounded { @@ -3087,7 +3089,7 @@ func (pc *paramMarkerInPrepareChecker) Leave(in ast.Node) (out ast.Node, ok bool // buildWindowFunctionFrame builds the window function frames. // See https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html -func (b *PlanBuilder) buildWindowFunctionFrame(spec *ast.WindowSpec, orderByItems []property.Item) (*WindowFrame, error) { +func (b *PlanBuilder) buildWindowFunctionFrame(ctx context.Context, spec *ast.WindowSpec, orderByItems []property.Item) (*WindowFrame, error) { frameClause := spec.Frame if frameClause == nil { return nil, nil @@ -3101,7 +3103,7 @@ func (b *PlanBuilder) buildWindowFunctionFrame(spec *ast.WindowSpec, orderByItem return nil, ErrWindowFrameStartIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) } var err error - frame.Start, err = b.buildWindowFunctionFrameBound(spec, orderByItems, &start) + frame.Start, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &start) if err != nil { return nil, err } @@ -3110,7 +3112,7 @@ func (b *PlanBuilder) buildWindowFunctionFrame(spec *ast.WindowSpec, orderByItem if end.Type == ast.Preceding && end.UnBounded { return nil, ErrWindowFrameEndIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) } - frame.End, err = b.buildWindowFunctionFrameBound(spec, orderByItems, &end) + frame.End, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &end) return frame, err } @@ -3172,7 +3174,7 @@ func sortWindowSpecs(groupedFuncs map[*ast.WindowSpec][]*ast.WindowFuncExpr) []w return windows } -func (b *PlanBuilder) buildWindowFunctions(p LogicalPlan, groupedFuncs map[*ast.WindowSpec][]*ast.WindowFuncExpr, aggMap map[*ast.AggregateFuncExpr]int) (LogicalPlan, map[*ast.WindowFuncExpr]int, error) { +func (b *PlanBuilder) buildWindowFunctions(ctx context.Context, p LogicalPlan, groupedFuncs map[*ast.WindowSpec][]*ast.WindowFuncExpr, aggMap map[*ast.AggregateFuncExpr]int) (LogicalPlan, map[*ast.WindowFuncExpr]int, error) { args := make([]ast.ExprNode, 0, 4) windowMap := make(map[*ast.WindowFuncExpr]int) for _, window := range sortWindowSpecs(groupedFuncs) { @@ -3181,11 +3183,11 @@ func (b *PlanBuilder) buildWindowFunctions(p LogicalPlan, groupedFuncs map[*ast. for _, windowFunc := range funcs { args = append(args, windowFunc.Args...) } - np, partitionBy, orderBy, args, err := b.buildProjectionForWindow(p, spec, args, aggMap) + np, partitionBy, orderBy, args, err := b.buildProjectionForWindow(ctx, p, spec, args, aggMap) if err != nil { return nil, nil, err } - frame, err := b.buildWindowFunctionFrame(spec, orderBy) + frame, err := b.buildWindowFunctionFrame(ctx, spec, orderBy) if err != nil { return nil, nil, err } diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index a9839e100d6e5..7761ea3f2f4cf 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -14,6 +14,7 @@ package core import ( + "context" "fmt" "sort" "strings" @@ -199,13 +200,15 @@ func (s *testPlanSuite) TestPredicatePushDown(c *C) { best: "Dual->Projection", }, } + + ctx := context.Background() for ith, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - p, err = logicalOptimize(flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, ca.best, Commentf("for %s %d", ca.sql, ith)) } @@ -292,13 +295,15 @@ func (s *testPlanSuite) TestJoinPredicatePushDown(c *C) { right: "[]", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil, comment) proj, ok := p.(*LogicalProjection) c.Assert(ok, IsTrue, comment) @@ -343,13 +348,15 @@ func (s *testPlanSuite) TestOuterWherePredicatePushDown(c *C) { right: "[]", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagDecorrelate|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil, comment) proj, ok := p.(*LogicalProjection) c.Assert(ok, IsTrue, comment) @@ -408,13 +415,15 @@ func (s *testPlanSuite) TestSimplifyOuterJoin(c *C) { joinType: "left outer join", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil, comment) c.Assert(ToString(p), Equals, ca.best, comment) join, ok := p.(LogicalPlan).Children()[0].(*LogicalJoin) @@ -439,13 +448,15 @@ func (s *testPlanSuite) TestAntiSemiJoinConstFalse(c *C) { joinType: "anti semi join", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagDecorrelate|flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagDecorrelate|flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil, comment) c.Assert(ToString(p), Equals, ca.best, comment) join, _ := p.(LogicalPlan).Children()[0].(*LogicalJoin) @@ -542,13 +553,15 @@ func (s *testPlanSuite) TestDeriveNotNullConds(c *C) { right: "[]", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagPredicatePushDown|flagPrunColumns|flagDecorrelate, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagPrunColumns|flagDecorrelate, p.(LogicalPlan)) c.Assert(err, IsNil, comment) c.Assert(ToString(p), Equals, ca.plan, comment) join := p.(LogicalPlan).Children()[0].(*LogicalJoin) @@ -566,9 +579,9 @@ func (s *testPlanSuite) TestDupRandJoinCondsPushDown(c *C) { comment := Commentf("for %s", sql) stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(context.Background(), s.ctx, stmt, s.is) c.Assert(err, IsNil, comment) - p, err = logicalOptimize(flagPredicatePushDown, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown, p.(LogicalPlan)) c.Assert(err, IsNil, comment) proj, ok := p.(*LogicalProjection) c.Assert(ok, IsTrue, comment) @@ -669,13 +682,15 @@ func (s *testPlanSuite) TestTablePartition(c *C) { is: is1, }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, ca.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, ca.is) c.Assert(err, IsNil) - p, err = logicalOptimize(flagDecorrelate|flagPrunColumns|flagPredicatePushDown|flagPartitionProcessor, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagDecorrelate|flagPrunColumns|flagPredicatePushDown|flagPartitionProcessor, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, ca.best, Commentf("for %s", ca.sql)) } @@ -755,16 +770,17 @@ func (s *testPlanSuite) TestSubquery(c *C) { }, } + ctx := context.Background() for ith, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) if lp, ok := p.(LogicalPlan); ok { - p, err = logicalOptimize(flagBuildKeyInfo|flagDecorrelate|flagPrunColumns, lp) + p, err = logicalOptimize(context.TODO(), flagBuildKeyInfo|flagDecorrelate|flagPrunColumns, lp) c.Assert(err, IsNil) } c.Assert(ToString(p), Equals, ca.best, Commentf("for %s %d", ca.sql, ith)) @@ -859,6 +875,8 @@ func (s *testPlanSuite) TestPlanBuilder(c *C) { plan: "LeftHashJoin{LeftHashJoin{TableReader(Table(t))->IndexLookUp(Index(t.c_d_e)[[666,666]], Table(t))}(test.t.a,test.t.b)->IndexReader(Index(t.c_d_e)[[42,42]])}(test.t.b,test.t.a)->Sel([or(6_aux_0, 10_aux_0)])->Projection->Delete", }, } + + ctx := context.Background() for _, ca := range tests { comment := Commentf("for %s", ca.sql) stmt, err := s.ParseOneStmt(ca.sql, "", "") @@ -866,10 +884,10 @@ func (s *testPlanSuite) TestPlanBuilder(c *C) { s.ctx.GetSessionVars().HashJoinConcurrency = 1 Preprocess(s.ctx, stmt, s.is) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) if lp, ok := p.(LogicalPlan); ok { - p, err = logicalOptimize(flagPrunColumns, lp) + p, err = logicalOptimize(context.TODO(), flagPrunColumns, lp) c.Assert(err, IsNil) } c.Assert(ToString(p), Equals, ca.plan, Commentf("for %s", ca.sql)) @@ -907,14 +925,16 @@ func (s *testPlanSuite) TestJoinReOrder(c *C) { best: "Apply{DataScan(o)->Join{Join{DataScan(t1)->DataScan(t2)}->DataScan(t3)}->Projection}->Projection", }, } + + ctx := context.Background() for _, tt := range tests { comment := Commentf("for %s", tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - p, err = logicalOptimize(flagPredicatePushDown|flagJoinReOrder, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagJoinReOrder, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, Commentf("for %s", tt.sql)) } @@ -1007,15 +1027,17 @@ func (s *testPlanSuite) TestEagerAggregation(c *C) { best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.a,test.t2.a)->Projection->Projection", }, } + + ctx := context.Background() s.ctx.GetSessionVars().AllowAggPushDown = true for ith, tt := range tests { comment := Commentf("for %s", tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - p, err = logicalOptimize(flagBuildKeyInfo|flagPredicatePushDown|flagPrunColumns|flagPushDownAgg, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagBuildKeyInfo|flagPredicatePushDown|flagPrunColumns|flagPushDownAgg, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, Commentf("for %s %d", tt.sql, ith)) } @@ -1166,14 +1188,16 @@ func (s *testPlanSuite) TestColumnPruning(c *C) { }, }, } + + ctx := context.Background() for _, tt := range tests { comment := Commentf("for %s", tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - lp, err := logicalOptimize(flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) + lp, err := logicalOptimize(ctx, flagPredicatePushDown|flagPrunColumns, p.(LogicalPlan)) c.Assert(err, IsNil) checkDataSourceCols(lp, c, tt.ans, comment) } @@ -1359,13 +1383,15 @@ func (s *testPlanSuite) TestValidate(c *C) { err: ErrUnknownColumn, }, } + + ctx := context.Background() for _, tt := range tests { sql := tt.sql comment := Commentf("for %s", sql) stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) - _, err = BuildLogicalPlan(s.ctx, stmt, s.is) + _, err = BuildLogicalPlan(ctx, s.ctx, stmt, s.is) if tt.err == nil { c.Assert(err, IsNil, comment) } else { @@ -1456,14 +1482,16 @@ func (s *testPlanSuite) TestUniqueKeyInfo(c *C) { }, }, } + + ctx := context.Background() for ith, tt := range tests { comment := Commentf("for %s %d", tt.sql, ith) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - lp, err := logicalOptimize(flagPredicatePushDown|flagPrunColumns|flagBuildKeyInfo, p.(LogicalPlan)) + lp, err := logicalOptimize(context.TODO(), flagPredicatePushDown|flagPrunColumns|flagBuildKeyInfo, p.(LogicalPlan)) c.Assert(err, IsNil) checkUniqueKeys(lp, c, tt.ans, tt.sql) } @@ -1500,15 +1528,17 @@ func (s *testPlanSuite) TestAggPrune(c *C) { best: "DataScan(t)->Projection->Projection", }, } + + ctx := context.Background() for _, tt := range tests { comment := Commentf("for %s", tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) - p, err := BuildLogicalPlan(s.ctx, stmt, s.is) + p, err := BuildLogicalPlan(ctx, s.ctx, stmt, s.is) c.Assert(err, IsNil) - p, err = logicalOptimize(flagPredicatePushDown|flagPrunColumns|flagBuildKeyInfo|flagEliminateAgg|flagEliminateProjection, p.(LogicalPlan)) + p, err = logicalOptimize(context.TODO(), flagPredicatePushDown|flagPrunColumns|flagBuildKeyInfo|flagEliminateAgg|flagEliminateProjection, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, comment) } @@ -1715,7 +1745,7 @@ func (s *testPlanSuite) TestVisitInfo(c *C) { Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) builder.ctx.GetSessionVars().HashJoinConcurrency = 1 - _, err = builder.Build(stmt) + _, err = builder.Build(context.TODO(), stmt) c.Assert(err, IsNil, comment) checkVisitInfo(c, builder.visitInfo, tt.ans, comment) @@ -1822,20 +1852,21 @@ func (s *testPlanSuite) TestUnion(c *C) { err: false, }, } + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - plan, err := builder.Build(stmt) + plan, err := builder.Build(ctx, stmt) if tt.err { c.Assert(err, NotNil) continue } c.Assert(err, IsNil) p := plan.(LogicalPlan) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, comment) } @@ -1950,15 +1981,16 @@ func (s *testPlanSuite) TestTopNPushDown(c *C) { best: "Join{DataScan(t1)->DataScan(t2)}(test.t1.e,test.t2.e)->TopN([ifnull(test.t1.h, test.t2.b)],0,5)->Projection->Projection", }, } + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, comment) } @@ -1996,13 +2028,14 @@ func (s *testPlanSuite) TestNameResolver(c *C) { {"update t, (select * from t) as b set b.a = t.a", "[planner:1288]The target table b of the UPDATE is not updatable"}, } + ctx := context.Background() for _, t := range tests { comment := Commentf("for %s", t.sql) stmt, err := s.ParseOneStmt(t.sql, "", "") c.Assert(err, IsNil, comment) s.ctx.GetSessionVars().HashJoinConcurrency = 1 - _, err = BuildLogicalPlan(s.ctx, stmt, s.is) + _, err = BuildLogicalPlan(ctx, s.ctx, stmt, s.is) if t.err == "" { c.Check(err, IsNil) } else { @@ -2071,15 +2104,16 @@ func (s *testPlanSuite) TestOuterJoinEliminator(c *C) { }, } + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, comment) } @@ -2098,15 +2132,16 @@ func (s *testPlanSuite) TestSelectView(c *C) { best: "DataScan(t)->Projection", }, } + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) c.Assert(ToString(p), Equals, tt.best, comment) } @@ -2321,19 +2356,20 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { defer func() { s.Parser.EnableWindowFunc(false) }() + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) if err != nil { c.Assert(err.Error(), Equals, tt.result, comment) continue } c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) lp, ok := p.(LogicalPlan) c.Assert(ok, IsTrue) @@ -2402,19 +2438,20 @@ func (s *testPlanSuite) TestSkylinePruning(c *C) { result: "PRIMARY_KEY,f,g,f_g", }, } + ctx := context.TODO() for i, tt := range tests { comment := Commentf("case:%v sql:%s", i, tt.sql) stmt, err := s.ParseOneStmt(tt.sql, "", "") c.Assert(err, IsNil, comment) Preprocess(s.ctx, stmt, s.is) builder := NewPlanBuilder(MockContext(), s.is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) if err != nil { c.Assert(err.Error(), Equals, tt.result, comment) continue } c.Assert(err, IsNil) - p, err = logicalOptimize(builder.optFlag, p.(LogicalPlan)) + p, err = logicalOptimize(ctx, builder.optFlag, p.(LogicalPlan)) c.Assert(err, IsNil) lp := p.(LogicalPlan) _, err = lp.recursiveDeriveStats() diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index f164dbb5a92a4..142e1b7203045 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -14,6 +14,7 @@ package core import ( + "context" "math" "github.com/pingcap/errors" @@ -31,7 +32,7 @@ import ( ) // OptimizeAstNode optimizes the query to a physical plan directly. -var OptimizeAstNode func(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (Plan, error) +var OptimizeAstNode func(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (Plan, error) // AllowCartesianProduct means whether tidb allows cartesian join without equal conditions. var AllowCartesianProduct = atomic.NewBool(true) @@ -68,16 +69,16 @@ var optRuleList = []logicalOptRule{ // logicalOptRule means a logical optimizing rule, which contains decorrelate, ppd, column pruning, etc. type logicalOptRule interface { - optimize(LogicalPlan) (LogicalPlan, error) + optimize(context.Context, LogicalPlan) (LogicalPlan, error) name() string } // BuildLogicalPlan used to build logical plan from ast.Node. -func BuildLogicalPlan(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (Plan, error) { - ctx.GetSessionVars().PlanID = 0 - ctx.GetSessionVars().PlanColumnID = 0 - builder := NewPlanBuilder(ctx, is) - p, err := builder.Build(node) +func BuildLogicalPlan(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (Plan, error) { + sctx.GetSessionVars().PlanID = 0 + sctx.GetSessionVars().PlanColumnID = 0 + builder := NewPlanBuilder(sctx, is) + p, err := builder.Build(ctx, node) if err != nil { return nil, err } @@ -113,8 +114,8 @@ func CheckTableLock(ctx sessionctx.Context, is infoschema.InfoSchema, vs []visit } // DoOptimize optimizes a logical plan to a physical plan. -func DoOptimize(flag uint64, logic LogicalPlan) (PhysicalPlan, error) { - logic, err := logicalOptimize(flag, logic) +func DoOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (PhysicalPlan, error) { + logic, err := logicalOptimize(ctx, flag, logic) if err != nil { return nil, err } @@ -135,7 +136,7 @@ func postOptimize(plan PhysicalPlan) PhysicalPlan { return plan } -func logicalOptimize(flag uint64, logic LogicalPlan) (LogicalPlan, error) { +func logicalOptimize(ctx context.Context, flag uint64, logic LogicalPlan) (LogicalPlan, error) { var err error for i, rule := range optRuleList { // The order of flags is same as the order of optRule in the list. @@ -144,7 +145,7 @@ func logicalOptimize(flag uint64, logic LogicalPlan) (LogicalPlan, error) { if flag&(1<TableReader(Table(t))}") warnings := se.GetSessionVars().StmtCtx.GetWarnings() @@ -1462,7 +1462,7 @@ func (s *testPlanSuite) TestSemiJoinToInner(c *C) { sql := "select t1.a, (select count(t2.a) from t t2 where t2.g in (select t3.d from t t3 where t3.c = t1.a)) as agg_col from t t1;" stmt, err := s.ParseOneStmt(sql, "", "") c.Assert(err, IsNil) - p, err := planner.Optimize(se, stmt, s.is) + p, err := planner.Optimize(context.TODO(), se, stmt, s.is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, "Apply{TableReader(Table(t))->IndexJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]]->HashAgg)->HashAgg->IndexReader(Index(t.g)[[NULL,+inf]])}(test.t3.d,test.t2.g)}->StreamAgg") } @@ -1508,7 +1508,7 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { se.GetSessionVars().StmtCtx.SetWarnings(nil) stmt, err := s.ParseOneStmt(test.sql, "", "") c.Assert(err, IsNil) - _, err = planner.Optimize(se, stmt, s.is) + _, err = planner.Optimize(context.TODO(), se, stmt, s.is) c.Assert(err, IsNil) warnings := se.GetSessionVars().StmtCtx.GetWarnings() if test.warning == "" { @@ -1550,12 +1550,13 @@ func (s *testPlanSuite) TestIndexJoinHint(c *C) { warning: "[planner:1815]Optimizer Hint /*+ TIDB_INLJ(t1) */ is inapplicable", }, } + ctx := context.Background() for i, test := range tests { comment := Commentf("case:%v sql:%s", i, test) stmt, err := s.ParseOneStmt(test.sql, "", "") c.Assert(err, IsNil, comment) - p, err := planner.Optimize(se, stmt, s.is) + p, err := planner.Optimize(ctx, se, stmt, s.is) c.Assert(err, IsNil) c.Assert(core.ToString(p), Equals, test.best) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 8ff371db64cd3..b4ae31c8bae6d 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -15,6 +15,7 @@ package core import ( "bytes" + "context" "fmt" "strings" @@ -228,43 +229,43 @@ func NewPlanBuilder(sctx sessionctx.Context, is infoschema.InfoSchema) *PlanBuil } // Build builds the ast node to a Plan. -func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { +func (b *PlanBuilder) Build(ctx context.Context, node ast.Node) (Plan, error) { b.optFlag = flagPrunColumns switch x := node.(type) { case *ast.AdminStmt: - return b.buildAdmin(x) + return b.buildAdmin(ctx, x) case *ast.DeallocateStmt: return &Deallocate{Name: x.Name}, nil case *ast.DeleteStmt: - return b.buildDelete(x) + return b.buildDelete(ctx, x) case *ast.ExecuteStmt: - return b.buildExecute(x) + return b.buildExecute(ctx, x) case *ast.ExplainStmt: - return b.buildExplain(x) + return b.buildExplain(ctx, x) case *ast.ExplainForStmt: return b.buildExplainFor(x) case *ast.TraceStmt: return b.buildTrace(x) case *ast.InsertStmt: - return b.buildInsert(x) + return b.buildInsert(ctx, x) case *ast.LoadDataStmt: - return b.buildLoadData(x) + return b.buildLoadData(ctx, x) case *ast.LoadStatsStmt: return b.buildLoadStats(x), nil case *ast.PrepareStmt: return b.buildPrepare(x), nil case *ast.SelectStmt: - return b.buildSelect(x) + return b.buildSelect(ctx, x) case *ast.UnionStmt: - return b.buildUnion(x) + return b.buildUnion(ctx, x) case *ast.UpdateStmt: - return b.buildUpdate(x) + return b.buildUpdate(ctx, x) case *ast.ShowStmt: - return b.buildShow(x) + return b.buildShow(ctx, x) case *ast.DoStmt: - return b.buildDo(x) + return b.buildDo(ctx, x) case *ast.SetStmt: - return b.buildSet(x) + return b.buildSet(ctx, x) case *ast.AnalyzeTableStmt: return b.buildAnalyze(x) case *ast.BinlogStmt, *ast.FlushStmt, *ast.UseStmt, @@ -273,7 +274,7 @@ func (b *PlanBuilder) Build(node ast.Node) (Plan, error) { *ast.GrantRoleStmt, *ast.RevokeRoleStmt, *ast.SetRoleStmt, *ast.SetDefaultRoleStmt: return b.buildSimple(node.(ast.StmtNode)) case ast.DDLNode: - return b.buildDDL(x) + return b.buildDDL(ctx, x) case *ast.CreateBindingStmt: return b.buildCreateBindPlan(x) case *ast.DropBindingStmt: @@ -293,10 +294,10 @@ func (b *PlanBuilder) buildChange(v *ast.ChangeStmt) (Plan, error) { return exe, nil } -func (b *PlanBuilder) buildExecute(v *ast.ExecuteStmt) (Plan, error) { +func (b *PlanBuilder) buildExecute(ctx context.Context, v *ast.ExecuteStmt) (Plan, error) { vars := make([]expression.Expression, 0, len(v.UsingVars)) for _, expr := range v.UsingVars { - newExpr, _, err := b.rewrite(expr, nil, nil, true) + newExpr, _, err := b.rewrite(ctx, expr, nil, nil, true) if err != nil { return nil, err } @@ -309,7 +310,7 @@ func (b *PlanBuilder) buildExecute(v *ast.ExecuteStmt) (Plan, error) { return exe, nil } -func (b *PlanBuilder) buildDo(v *ast.DoStmt) (Plan, error) { +func (b *PlanBuilder) buildDo(ctx context.Context, v *ast.DoStmt) (Plan, error) { var p LogicalPlan dual := LogicalTableDual{RowCount: 1}.Init(b.ctx) dual.SetSchema(expression.NewSchema()) @@ -317,7 +318,7 @@ func (b *PlanBuilder) buildDo(v *ast.DoStmt) (Plan, error) { proj := LogicalProjection{Exprs: make([]expression.Expression, 0, len(v.Exprs))}.Init(b.ctx) schema := expression.NewSchema(make([]*expression.Column, 0, len(v.Exprs))...) for _, astExpr := range v.Exprs { - expr, np, err := b.rewrite(astExpr, p, nil, true) + expr, np, err := b.rewrite(ctx, astExpr, p, nil, true) if err != nil { return nil, err } @@ -335,7 +336,7 @@ func (b *PlanBuilder) buildDo(v *ast.DoStmt) (Plan, error) { return proj, nil } -func (b *PlanBuilder) buildSet(v *ast.SetStmt) (Plan, error) { +func (b *PlanBuilder) buildSet(ctx context.Context, v *ast.SetStmt) (Plan, error) { p := &Set{} for _, vars := range v.Variables { if vars.IsGlobal { @@ -354,7 +355,7 @@ func (b *PlanBuilder) buildSet(v *ast.SetStmt) (Plan, error) { } mockTablePlan := LogicalTableDual{}.Init(b.ctx) var err error - assign.Expr, _, err = b.rewrite(vars.Value, mockTablePlan, nil, true) + assign.Expr, _, err = b.rewrite(ctx, vars.Value, mockTablePlan, nil, true) if err != nil { return nil, err } @@ -605,12 +606,12 @@ func (b *PlanBuilder) buildCheckIndex(dbName model.CIStr, as *ast.AdminStmt) (Pl return rootT.p, nil } -func (b *PlanBuilder) buildAdmin(as *ast.AdminStmt) (Plan, error) { +func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (Plan, error) { var ret Plan var err error switch as.Tp { case ast.AdminCheckTable: - ret, err = b.buildAdminCheckTable(as) + ret, err = b.buildAdminCheckTable(ctx, as) if err != nil { return ret, err } @@ -688,7 +689,7 @@ func (b *PlanBuilder) buildAdmin(as *ast.AdminStmt) (Plan, error) { return ret, nil } -func (b *PlanBuilder) buildAdminCheckTable(as *ast.AdminStmt) (*CheckTable, error) { +func (b *PlanBuilder) buildAdminCheckTable(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { p := &CheckTable{Tables: as.Tables} p.GenExprs = make(map[model.TableColumnID]expression.Expression, len(p.Tables)) @@ -717,7 +718,7 @@ func (b *PlanBuilder) buildAdminCheckTable(as *ast.AdminStmt) (*CheckTable, erro return nil, err } - expr, _, err := b.rewrite(column.GeneratedExpr, mockTablePlan, nil, true) + expr, _, err := b.rewrite(ctx, column.GeneratedExpr, mockTablePlan, nil, true) if err != nil { return nil, err } @@ -1092,7 +1093,7 @@ func splitWhere(where ast.ExprNode) []ast.ExprNode { return conditions } -func (b *PlanBuilder) buildShow(show *ast.ShowStmt) (Plan, error) { +func (b *PlanBuilder) buildShow(ctx context.Context, show *ast.ShowStmt) (Plan, error) { p := Show{ Tp: show.Tp, DBName: show.DBName, @@ -1139,13 +1140,13 @@ func (b *PlanBuilder) buildShow(show *ast.ShowStmt) (Plan, error) { show.Pattern.Expr = &ast.ColumnNameExpr{ Name: &ast.ColumnName{Name: p.Schema().Columns[0].ColName}, } - np, err = b.buildSelection(np, show.Pattern, nil) + np, err = b.buildSelection(ctx, np, show.Pattern, nil) if err != nil { return nil, err } } if show.Where != nil { - np, err = b.buildSelection(np, show.Where, nil) + np, err = b.buildSelection(ctx, np, show.Where, nil) if err != nil { return nil, err } @@ -1162,7 +1163,7 @@ func (b *PlanBuilder) buildShow(show *ast.ShowStmt) (Plan, error) { } proj.SetSchema(schema) proj.SetChildren(np) - physical, err := DoOptimize(b.optFlag|flagEliminateProjection, proj) + physical, err := DoOptimize(ctx, b.optFlag|flagEliminateProjection, proj) if err != nil { return nil, err } @@ -1286,7 +1287,7 @@ func (b *PlanBuilder) findDefaultValue(cols []*table.Column, name *ast.ColumnNam // resolveGeneratedColumns resolves generated columns with their generation // expressions respectively. onDups indicates which columns are in on-duplicate list. -func (b *PlanBuilder) resolveGeneratedColumns(columns []*table.Column, onDups map[string]struct{}, mockPlan LogicalPlan) (igc InsertGeneratedColumns, err error) { +func (b *PlanBuilder) resolveGeneratedColumns(ctx context.Context, columns []*table.Column, onDups map[string]struct{}, mockPlan LogicalPlan) (igc InsertGeneratedColumns, err error) { for _, column := range columns { if !column.IsGenerated() { continue @@ -1299,7 +1300,7 @@ func (b *PlanBuilder) resolveGeneratedColumns(columns []*table.Column, onDups ma return igc, err } - expr, _, err := b.rewrite(column.GeneratedExpr, mockPlan, nil, true) + expr, _, err := b.rewrite(ctx, column.GeneratedExpr, mockPlan, nil, true) if err != nil { return igc, err } @@ -1321,7 +1322,7 @@ func (b *PlanBuilder) resolveGeneratedColumns(columns []*table.Column, onDups ma return igc, nil } -func (b *PlanBuilder) buildInsert(insert *ast.InsertStmt) (Plan, error) { +func (b *PlanBuilder) buildInsert(ctx context.Context, insert *ast.InsertStmt) (Plan, error) { ts, ok := insert.Table.TableRefs.Left.(*ast.TableSource) if !ok { return nil, infoschema.ErrTableNotExists.GenWithStackByArgs() @@ -1377,19 +1378,19 @@ func (b *PlanBuilder) buildInsert(insert *ast.InsertStmt) (Plan, error) { if len(insert.Setlist) > 0 { // Branch for `INSERT ... SET ...`. - err := b.buildSetValuesOfInsert(insert, insertPlan, mockTablePlan, checkRefColumn) + err := b.buildSetValuesOfInsert(ctx, insert, insertPlan, mockTablePlan, checkRefColumn) if err != nil { return nil, err } } else if len(insert.Lists) > 0 { // Branch for `INSERT ... VALUES ...`. - err := b.buildValuesListOfInsert(insert, insertPlan, mockTablePlan, checkRefColumn) + err := b.buildValuesListOfInsert(ctx, insert, insertPlan, mockTablePlan, checkRefColumn) if err != nil { return nil, err } } else { // Branch for `INSERT ... SELECT ...`. - err := b.buildSelectPlanOfInsert(insert, insertPlan) + err := b.buildSelectPlanOfInsert(ctx, insert, insertPlan) if err != nil { return nil, err } @@ -1406,7 +1407,7 @@ func (b *PlanBuilder) buildInsert(insert *ast.InsertStmt) (Plan, error) { } for i, assign := range insert.OnDuplicate { // Construct the function which calculates the assign value of the column. - expr, err1 := b.rewriteInsertOnDuplicateUpdate(assign.Expr, mockTablePlan, insertPlan) + expr, err1 := b.rewriteInsertOnDuplicateUpdate(ctx, assign.Expr, mockTablePlan, insertPlan) if err1 != nil { return nil, err1 } @@ -1419,7 +1420,7 @@ func (b *PlanBuilder) buildInsert(insert *ast.InsertStmt) (Plan, error) { // Calculate generated columns. mockTablePlan.schema = insertPlan.tableSchema - insertPlan.GenCols, err = b.resolveGeneratedColumns(insertPlan.Table.Cols(), onDupColSet, mockTablePlan) + insertPlan.GenCols, err = b.resolveGeneratedColumns(ctx, insertPlan.Table.Cols(), onDupColSet, mockTablePlan) if err != nil { return nil, err } @@ -1474,7 +1475,7 @@ func (b *PlanBuilder) getAffectCols(insertStmt *ast.InsertStmt, insertPlan *Inse return affectedValuesCols, nil } -func (b *PlanBuilder) buildSetValuesOfInsert(insert *ast.InsertStmt, insertPlan *Insert, mockTablePlan *LogicalTableDual, checkRefColumn func(n ast.Node) ast.Node) error { +func (b *PlanBuilder) buildSetValuesOfInsert(ctx context.Context, insert *ast.InsertStmt, insertPlan *Insert, mockTablePlan *LogicalTableDual, checkRefColumn func(n ast.Node) ast.Node) error { tableInfo := insertPlan.Table.Meta() colNames := make([]string, 0, len(insert.Setlist)) exprCols := make([]*expression.Column, 0, len(insert.Setlist)) @@ -1502,7 +1503,7 @@ func (b *PlanBuilder) buildSetValuesOfInsert(insert *ast.InsertStmt, insertPlan } for i, assign := range insert.Setlist { - expr, _, err := b.rewriteWithPreprocess(assign.Expr, mockTablePlan, nil, nil, true, checkRefColumn) + expr, _, err := b.rewriteWithPreprocess(ctx, assign.Expr, mockTablePlan, nil, nil, true, checkRefColumn) if err != nil { return err } @@ -1515,7 +1516,7 @@ func (b *PlanBuilder) buildSetValuesOfInsert(insert *ast.InsertStmt, insertPlan return nil } -func (b *PlanBuilder) buildValuesListOfInsert(insert *ast.InsertStmt, insertPlan *Insert, mockTablePlan *LogicalTableDual, checkRefColumn func(n ast.Node) ast.Node) error { +func (b *PlanBuilder) buildValuesListOfInsert(ctx context.Context, insert *ast.InsertStmt, insertPlan *Insert, mockTablePlan *LogicalTableDual, checkRefColumn func(n ast.Node) ast.Node) error { affectedValuesCols, err := b.getAffectCols(insert, insertPlan) if err != nil { return err @@ -1563,7 +1564,7 @@ func (b *PlanBuilder) buildValuesListOfInsert(insert *ast.InsertStmt, insertPlan RetType: &x.Type, } default: - expr, _, err = b.rewriteWithPreprocess(valueItem, mockTablePlan, nil, nil, true, checkRefColumn) + expr, _, err = b.rewriteWithPreprocess(ctx, valueItem, mockTablePlan, nil, nil, true, checkRefColumn) } if err != nil { return err @@ -1576,12 +1577,12 @@ func (b *PlanBuilder) buildValuesListOfInsert(insert *ast.InsertStmt, insertPlan return nil } -func (b *PlanBuilder) buildSelectPlanOfInsert(insert *ast.InsertStmt, insertPlan *Insert) error { +func (b *PlanBuilder) buildSelectPlanOfInsert(ctx context.Context, insert *ast.InsertStmt, insertPlan *Insert) error { affectedValuesCols, err := b.getAffectCols(insert, insertPlan) if err != nil { return err } - selectPlan, err := b.Build(insert.Select) + selectPlan, err := b.Build(ctx, insert.Select) if err != nil { return err } @@ -1604,7 +1605,7 @@ func (b *PlanBuilder) buildSelectPlanOfInsert(insert *ast.InsertStmt, insertPlan } } - insertPlan.SelectPlan, err = DoOptimize(b.optFlag, selectPlan.(LogicalPlan)) + insertPlan.SelectPlan, err = DoOptimize(ctx, b.optFlag, selectPlan.(LogicalPlan)) if err != nil { return err } @@ -1629,7 +1630,7 @@ func (b *PlanBuilder) buildSelectPlanOfInsert(insert *ast.InsertStmt, insertPlan return nil } -func (b *PlanBuilder) buildLoadData(ld *ast.LoadDataStmt) (Plan, error) { +func (b *PlanBuilder) buildLoadData(ctx context.Context, ld *ast.LoadDataStmt) (Plan, error) { p := &LoadData{ IsLocal: ld.IsLocal, OnDuplicate: ld.OnDuplicate, @@ -1651,7 +1652,7 @@ func (b *PlanBuilder) buildLoadData(ld *ast.LoadDataStmt) (Plan, error) { mockTablePlan.SetSchema(schema) var err error - p.GenCols, err = b.resolveGeneratedColumns(tableInPlan.Cols(), nil, mockTablePlan) + p.GenCols, err = b.resolveGeneratedColumns(ctx, tableInPlan.Cols(), nil, mockTablePlan) if err != nil { return nil, err } @@ -1756,7 +1757,7 @@ func (b *PlanBuilder) convertValue(valueItem ast.ExprNode, mockTablePlan Logical RetType: &x.Type, } default: - expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) + expr, _, err = b.rewrite(context.TODO(), valueItem, mockTablePlan, nil, true) if err != nil { return d, err } @@ -1844,7 +1845,7 @@ func (b *PlanBuilder) buildSplitTableRegion(node *ast.SplitRegionStmt) (Plan, er return p, nil } -func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { +func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, error) { var authErr error switch v := node.(type) { case *ast.AlterDatabaseStmt: @@ -1927,7 +1928,7 @@ func (b *PlanBuilder) buildDDL(node ast.DDLNode) (Plan, error) { v.ReferTable.Name.L, "", authErr) } case *ast.CreateViewStmt: - plan, err := b.Build(v.Select) + plan, err := b.Build(ctx, v.Select) if err != nil { return nil, err } @@ -2100,11 +2101,11 @@ func (b *PlanBuilder) buildExplainFor(explainFor *ast.ExplainForStmt) (Plan, err return b.buildExplainPlan(targetPlan, explainFor.Format, false, nil) } -func (b *PlanBuilder) buildExplain(explain *ast.ExplainStmt) (Plan, error) { +func (b *PlanBuilder) buildExplain(ctx context.Context, explain *ast.ExplainStmt) (Plan, error) { if show, ok := explain.Stmt.(*ast.ShowStmt); ok { - return b.buildShow(show) + return b.buildShow(ctx, show) } - targetPlan, err := OptimizeAstNode(b.ctx, explain.Stmt, b.is) + targetPlan, err := OptimizeAstNode(ctx, b.ctx, explain.Stmt, b.is) if err != nil { return nil, err } diff --git a/planner/core/planbuilder_test.go b/planner/core/planbuilder_test.go index 282e95a2cd9ff..9c6a90ea6addb 100644 --- a/planner/core/planbuilder_test.go +++ b/planner/core/planbuilder_test.go @@ -14,6 +14,8 @@ package core import ( + "context" + . "github.com/pingcap/check" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -99,7 +101,7 @@ func (s *testPlanBuilderSuite) TestRewriterPool(c *C) { // Make sure PlanBuilder.getExpressionRewriter() provides clean rewriter from pool. // First, pick one rewriter from the pool and make it dirty. builder.rewriterCounter++ - dirtyRewriter := builder.getExpressionRewriter(nil) + dirtyRewriter := builder.getExpressionRewriter(context.TODO(), nil) dirtyRewriter.asScalar = true dirtyRewriter.aggrMap = make(map[*ast.AggregateFuncExpr]int) dirtyRewriter.preprocess = func(ast.Node) ast.Node { return nil } @@ -109,7 +111,7 @@ func (s *testPlanBuilderSuite) TestRewriterPool(c *C) { builder.rewriterCounter-- // Then, pick again and check if it's cleaned up. builder.rewriterCounter++ - cleanRewriter := builder.getExpressionRewriter(nil) + cleanRewriter := builder.getExpressionRewriter(context.TODO(), nil) c.Assert(cleanRewriter, Equals, dirtyRewriter) // Rewriter should be reused. c.Assert(cleanRewriter.asScalar, Equals, false) c.Assert(cleanRewriter.aggrMap, IsNil) @@ -149,7 +151,7 @@ func (s *testPlanBuilderSuite) TestDisableFold(c *C) { builder := NewPlanBuilder(ctx, nil) builder.rewriterCounter++ - rewriter := builder.getExpressionRewriter(nil) + rewriter := builder.getExpressionRewriter(context.TODO(), nil) c.Assert(rewriter, NotNil) c.Assert(rewriter.disableFoldCounter, Equals, 0) rewritenExpression, _, err := builder.rewriteExprNode(rewriter, expr, true) diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index 3c6db4b271ea9..6b53eb65a06a6 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -14,6 +14,7 @@ package core_test import ( + "context" "math" "strconv" "time" @@ -157,17 +158,18 @@ func (s *testPlanSuite) TestPrepareCacheDeferredFunction(c *C) { metrics.ResettablePlanCacheCounterFortTest = true metrics.PlanCacheCounter.Reset() counter := metrics.PlanCacheCounter.WithLabelValues("prepare") + ctx := context.TODO() for i := 0; i < 2; i++ { stmt, err := s.ParseOneStmt(sql1, "", "") c.Check(err, IsNil) is := tk.Se.GetSessionVars().TxnCtx.InfoSchema.(infoschema.InfoSchema) builder := core.NewPlanBuilder(tk.Se, is) - p, err := builder.Build(stmt) + p, err := builder.Build(ctx, stmt) c.Check(err, IsNil) execPlan, ok := p.(*core.Execute) c.Check(ok, IsTrue) executor.ResetContextOfStmt(tk.Se, stmt) - err = execPlan.OptimizePreparedPlan(tk.Se, is) + err = execPlan.OptimizePreparedPlan(ctx, tk.Se, is) c.Check(err, IsNil) planStr[i] = core.ToString(execPlan.Plan) c.Check(planStr[i], Matches, expectedPattern, Commentf("for %s", sql1)) diff --git a/planner/core/rule_aggregation_elimination.go b/planner/core/rule_aggregation_elimination.go index 84249a61419e3..1b3a91ae1f937 100644 --- a/planner/core/rule_aggregation_elimination.go +++ b/planner/core/rule_aggregation_elimination.go @@ -14,6 +14,7 @@ package core import ( + "context" "math" "github.com/pingcap/parser/ast" @@ -136,10 +137,10 @@ func (a *aggregationEliminateChecker) wrapCastFunction(ctx sessionctx.Context, a return expression.BuildCastFunction(ctx, arg, targetTp) } -func (a *aggregationEliminator) optimize(p LogicalPlan) (LogicalPlan, error) { +func (a *aggregationEliminator) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { newChildren := make([]LogicalPlan, 0, len(p.Children())) for _, child := range p.Children() { - newChild, err := a.optimize(child) + newChild, err := a.optimize(ctx, child) if err != nil { return nil, err } diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index 4cc3556084cb2..6f7488667b0ac 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -13,6 +13,7 @@ package core import ( + "context" "fmt" "github.com/pingcap/parser/ast" @@ -314,7 +315,7 @@ func (a *aggregationPushDownSolver) pushAggCrossUnion(agg *LogicalAggregation, u return newAgg } -func (a *aggregationPushDownSolver) optimize(p LogicalPlan) (LogicalPlan, error) { +func (a *aggregationPushDownSolver) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { if !p.context().GetSessionVars().AllowAggPushDown { return p, nil } diff --git a/planner/core/rule_build_key_info.go b/planner/core/rule_build_key_info.go index dca0da8094d15..fefba82f10e9b 100644 --- a/planner/core/rule_build_key_info.go +++ b/planner/core/rule_build_key_info.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" @@ -21,7 +23,7 @@ import ( type buildKeySolver struct{} -func (s *buildKeySolver) optimize(lp LogicalPlan) (LogicalPlan, error) { +func (s *buildKeySolver) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { lp.buildKeyInfo() return lp, nil } diff --git a/planner/core/rule_column_pruning.go b/planner/core/rule_column_pruning.go index b2442a566c1f7..ba84c17b80e01 100644 --- a/planner/core/rule_column_pruning.go +++ b/planner/core/rule_column_pruning.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" @@ -28,7 +30,7 @@ import ( type columnPruner struct { } -func (s *columnPruner) optimize(lp LogicalPlan) (LogicalPlan, error) { +func (s *columnPruner) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { err := lp.PruneColumns(lp.Schema().Columns) return lp, err } diff --git a/planner/core/rule_decorrelate.go b/planner/core/rule_decorrelate.go index e7c2f734de649..2d3eeb6650fb8 100644 --- a/planner/core/rule_decorrelate.go +++ b/planner/core/rule_decorrelate.go @@ -14,6 +14,7 @@ package core import ( + "context" "math" "github.com/pingcap/parser/ast" @@ -97,7 +98,7 @@ func (s *decorrelateSolver) aggDefaultValueMap(agg *LogicalAggregation) map[int] } // optimize implements logicalOptRule interface. -func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { +func (s *decorrelateSolver) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { if apply, ok := p.(*LogicalApply); ok { outerPlan := apply.children[0] innerPlan := apply.children[1] @@ -117,12 +118,12 @@ func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { apply.attachOnConds(newConds) innerPlan = sel.children[0] apply.SetChildren(outerPlan, innerPlan) - return s.optimize(p) + return s.optimize(ctx, p) } else if m, ok := innerPlan.(*LogicalMaxOneRow); ok { if m.children[0].MaxOneRow() { innerPlan = m.children[0] apply.SetChildren(outerPlan, innerPlan) - return s.optimize(p) + return s.optimize(ctx, p) } } else if proj, ok := innerPlan.(*LogicalProjection); ok { for i, expr := range proj.Exprs { @@ -135,14 +136,14 @@ func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { proj.SetSchema(apply.Schema()) proj.Exprs = append(expression.Column2Exprs(outerPlan.Schema().Clone().Columns), proj.Exprs...) apply.SetSchema(expression.MergeSchema(outerPlan.Schema(), innerPlan.Schema())) - np, err := s.optimize(p) + np, err := s.optimize(ctx, p) if err != nil { return nil, err } proj.SetChildren(np) return proj, nil } - return s.optimize(p) + return s.optimize(ctx, p) } else if agg, ok := innerPlan.(*LogicalAggregation); ok { if apply.canPullUpAgg() && agg.canPullUp() { innerPlan = agg.children[0] @@ -167,7 +168,7 @@ func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { newAggFuncs = append(newAggFuncs, agg.AggFuncs...) agg.AggFuncs = newAggFuncs apply.SetSchema(expression.MergeSchema(expression.NewSchema(outerColsInSchema...), innerPlan.Schema())) - np, err := s.optimize(p) + np, err := s.optimize(ctx, p) if err != nil { return nil, err } @@ -236,7 +237,7 @@ func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { proj.SetChildren(apply) p = proj } - return s.optimize(p) + return s.optimize(ctx, p) } sel.Conditions = originalExpr apply.corCols = extractCorColumnsBySchema(apply.children[1], apply.children[0].Schema()) @@ -246,7 +247,7 @@ func (s *decorrelateSolver) optimize(p LogicalPlan) (LogicalPlan, error) { } newChildren := make([]LogicalPlan, 0, len(p.Children())) for _, child := range p.Children() { - np, err := s.optimize(child) + np, err := s.optimize(ctx, child) if err != nil { return nil, err } diff --git a/planner/core/rule_eliminate_projection.go b/planner/core/rule_eliminate_projection.go index 05ff34c100fc8..5e0f07c758d10 100644 --- a/planner/core/rule_eliminate_projection.go +++ b/planner/core/rule_eliminate_projection.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/pingcap/tidb/expression" ) @@ -104,7 +106,7 @@ type projectionEliminater struct { } // optimize implements the logicalOptRule interface. -func (pe *projectionEliminater) optimize(lp LogicalPlan) (LogicalPlan, error) { +func (pe *projectionEliminater) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { root := pe.eliminate(lp, make(map[string]*expression.Column), false) return root, nil } diff --git a/planner/core/rule_join_elimination.go b/planner/core/rule_join_elimination.go index 59eaf3363ecb2..21807e9b101e6 100644 --- a/planner/core/rule_join_elimination.go +++ b/planner/core/rule_join_elimination.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/util/set" @@ -232,7 +234,7 @@ func (o *outerJoinEliminator) doOptimize(p LogicalPlan, aggCols []*expression.Co return p, nil } -func (o *outerJoinEliminator) optimize(p LogicalPlan) (LogicalPlan, error) { +func (o *outerJoinEliminator) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { return o.doOptimize(p, nil, nil) } diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 4885b887af41c..562ab9bf5a438 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" ) @@ -52,7 +54,7 @@ type jrNode struct { cumCost float64 } -func (s *joinReOrderSolver) optimize(p LogicalPlan) (LogicalPlan, error) { +func (s *joinReOrderSolver) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { return s.optimizeRecursive(p.context(), p) } diff --git a/planner/core/rule_max_min_eliminate.go b/planner/core/rule_max_min_eliminate.go index 48f4a7055a010..fddedf0440b70 100644 --- a/planner/core/rule_max_min_eliminate.go +++ b/planner/core/rule_max_min_eliminate.go @@ -13,6 +13,8 @@ package core import ( + "context" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" @@ -25,7 +27,7 @@ import ( type maxMinEliminator struct { } -func (a *maxMinEliminator) optimize(p LogicalPlan) (LogicalPlan, error) { +func (a *maxMinEliminator) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { a.eliminateMaxMin(p) return p, nil } diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index cdd882c38d73a..38335b9ce9804 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -13,6 +13,8 @@ package core import ( + "context" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" @@ -39,7 +41,7 @@ import ( // partitionProcessor is here because it's easier to prune partition after predicate push down. type partitionProcessor struct{} -func (s *partitionProcessor) optimize(lp LogicalPlan) (LogicalPlan, error) { +func (s *partitionProcessor) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { return s.rewriteDataSource(lp) } diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index 68eb0352417f7..aa909538abadc 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -13,6 +13,8 @@ package core import ( + "context" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -23,7 +25,7 @@ import ( type ppdSolver struct{} -func (s *ppdSolver) optimize(lp LogicalPlan) (LogicalPlan, error) { +func (s *ppdSolver) optimize(ctx context.Context, lp LogicalPlan) (LogicalPlan, error) { _, p := lp.PredicatePushDown(nil) return p, nil } diff --git a/planner/core/rule_topn_push_down.go b/planner/core/rule_topn_push_down.go index 7bdb2de0a728c..cfa3cc73c6319 100644 --- a/planner/core/rule_topn_push_down.go +++ b/planner/core/rule_topn_push_down.go @@ -14,6 +14,8 @@ package core import ( + "context" + "github.com/cznic/mathutil" "github.com/pingcap/tidb/expression" ) @@ -22,7 +24,7 @@ import ( type pushDownTopNOptimizer struct { } -func (s *pushDownTopNOptimizer) optimize(p LogicalPlan) (LogicalPlan, error) { +func (s *pushDownTopNOptimizer) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { return p.pushDownTopN(nil), nil } diff --git a/planner/optimize.go b/planner/optimize.go index e658899b0efaa..4045e2f10ac0d 100644 --- a/planner/optimize.go +++ b/planner/optimize.go @@ -14,6 +14,8 @@ package planner import ( + "context" + "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/planner/cascades" @@ -24,39 +26,39 @@ import ( // Optimize does optimization and creates a Plan. // The node must be prepared first. -func Optimize(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (plannercore.Plan, error) { - fp := plannercore.TryFastPlan(ctx, node) +func Optimize(ctx context.Context, sctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) (plannercore.Plan, error) { + fp := plannercore.TryFastPlan(sctx, node) if fp != nil { return fp, nil } // build logical plan - ctx.GetSessionVars().PlanID = 0 - ctx.GetSessionVars().PlanColumnID = 0 - builder := plannercore.NewPlanBuilder(ctx, is) - p, err := builder.Build(node) + sctx.GetSessionVars().PlanID = 0 + sctx.GetSessionVars().PlanColumnID = 0 + builder := plannercore.NewPlanBuilder(sctx, is) + p, err := builder.Build(ctx, node) if err != nil { return nil, err } - ctx.GetSessionVars().StmtCtx.Tables = builder.GetDBTableInfo() - activeRoles := ctx.GetSessionVars().ActiveRoles + sctx.GetSessionVars().StmtCtx.Tables = builder.GetDBTableInfo() + activeRoles := sctx.GetSessionVars().ActiveRoles // Check privilege. Maybe it's better to move this to the Preprocess, but // we need the table information to check privilege, which is collected // into the visitInfo in the logical plan builder. - if pm := privilege.GetPrivilegeManager(ctx); pm != nil { + if pm := privilege.GetPrivilegeManager(sctx); pm != nil { if err := plannercore.CheckPrivilege(activeRoles, pm, builder.GetVisitInfo()); err != nil { return nil, err } } - if err := plannercore.CheckTableLock(ctx, is, builder.GetVisitInfo()); err != nil { + if err := plannercore.CheckTableLock(sctx, is, builder.GetVisitInfo()); err != nil { return nil, err } // Handle the execute statement. if execPlan, ok := p.(*plannercore.Execute); ok { - err := execPlan.OptimizePreparedPlan(ctx, is) + err := execPlan.OptimizePreparedPlan(ctx, sctx, is) return p, err } @@ -67,10 +69,10 @@ func Optimize(ctx sessionctx.Context, node ast.Node, is infoschema.InfoSchema) ( } // Handle the logical plan statement, use cascades planner if enabled. - if ctx.GetSessionVars().EnableCascadesPlanner { - return cascades.FindBestPlan(ctx, logic) + if sctx.GetSessionVars().EnableCascadesPlanner { + return cascades.FindBestPlan(sctx, logic) } - return plannercore.DoOptimize(builder.GetOptFlag(), logic) + return plannercore.DoOptimize(ctx, builder.GetOptFlag(), logic) } func init() { diff --git a/session/session.go b/session/session.go index 7564dd1ec7750..2d58ac5f2258a 100644 --- a/session/session.go +++ b/session/session.go @@ -642,7 +642,7 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) { s.sessionVars.StmtCtx = sr.stmtCtx s.sessionVars.StmtCtx.ResetForRetry() s.sessionVars.PreparedParams = s.sessionVars.PreparedParams[:0] - schemaVersion, err = st.RebuildPlan() + schemaVersion, err = st.RebuildPlan(ctx) if err != nil { return err } @@ -1217,7 +1217,7 @@ func (s *session) PrepareStmt(sql string) (stmtID uint32, paramCount int, fields // ExecutePreparedStmt executes a prepared statement. func (s *session) ExecutePreparedStmt(ctx context.Context, stmtID uint32, args []types.Datum) (sqlexec.RecordSet, error) { s.PrepareTxnCtx(ctx) - st, err := executor.CompileExecutePreparedStmt(s, stmtID, args) + st, err := executor.CompileExecutePreparedStmt(ctx, s, stmtID, args) if err != nil { return nil, err } diff --git a/statistics/selectivity_test.go b/statistics/selectivity_test.go index a6912a41b02f3..41b03aaaafa6a 100644 --- a/statistics/selectivity_test.go +++ b/statistics/selectivity_test.go @@ -14,6 +14,7 @@ package statistics_test import ( + "context" "fmt" "math" "os" @@ -275,17 +276,19 @@ func (s *testStatsSuite) TestSelectivity(c *C) { selectivity: 0.001, }, } + + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprs comment := Commentf("for %s", tt.exprs) - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprs)) c.Assert(stmts, HasLen, 1) - err = plannercore.Preprocess(ctx, stmts[0], is) + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, comment) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for building plan, expr %s", err, tt.exprs)) sel := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) @@ -293,12 +296,12 @@ func (s *testStatsSuite) TestSelectivity(c *C) { histColl := statsTbl.GenerateHistCollFromColumnInfo(ds.Columns, ds.Schema().Columns) - ratio, _, err := histColl.Selectivity(ctx, sel.Conditions) + ratio, _, err := histColl.Selectivity(sctx, sel.Conditions) c.Assert(err, IsNil, comment) c.Assert(math.Abs(ratio-tt.selectivity) < eps, IsTrue, Commentf("for %s, needed: %v, got: %v", tt.exprs, tt.selectivity, ratio)) histColl.Count *= 10 - ratio, _, err = histColl.Selectivity(ctx, sel.Conditions) + ratio, _, err = histColl.Selectivity(sctx, sel.Conditions) c.Assert(err, IsNil, comment) c.Assert(math.Abs(ratio-tt.selectivity) < eps, IsTrue, Commentf("for %s, needed: %v, got: %v", tt.exprs, tt.selectivity, ratio)) } @@ -451,13 +454,13 @@ func BenchmarkSelectivity(b *testing.B) { exprs := "a > 1 and b < 2 and c > 3 and d < 4 and e > 5" sql := "select * from t where " + exprs comment := Commentf("for %s", exprs) - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, exprs)) c.Assert(stmts, HasLen, 1) - err = plannercore.Preprocess(ctx, stmts[0], is) + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, comment) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(context.Background(), sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for building plan, expr %s", err, exprs)) file, err := os.Create("cpu.profile") @@ -468,7 +471,7 @@ func BenchmarkSelectivity(b *testing.B) { b.Run("Selectivity", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, err := statsTbl.Selectivity(ctx, p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection).Conditions) + _, _, err := statsTbl.Selectivity(sctx, p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection).Conditions) c.Assert(err, IsNil) } b.ReportAllocs() diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 89daf91d4643f..6fc8d102aff25 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -14,6 +14,7 @@ package ranger_test import ( + "context" "fmt" "testing" @@ -292,27 +293,28 @@ func (s *testRangerSuite) TestTableRange(c *C) { }, } + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprStr - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmts[0], is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) conds := make([]expression.Expression, 0, len(selection.Conditions)) for _, cond := range selection.Conditions { - conds = append(conds, expression.PushDownNot(ctx, cond, false)) + conds = append(conds, expression.PushDownNot(sctx, cond, false)) } tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() col := expression.ColInfo2Col(selection.Schema().Columns, tbl.Columns[0]) c.Assert(col, NotNil) var filter []expression.Expression - conds, filter = ranger.DetachCondsForColumn(ctx, conds, col) + conds, filter = ranger.DetachCondsForColumn(sctx, conds, col) c.Assert(fmt.Sprintf("%s", conds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr)) c.Assert(fmt.Sprintf("%s", filter), Equals, tt.filterConds, Commentf("wrong filter conditions for expr: %s", tt.exprStr)) result, err := ranger.BuildTableRange(conds, new(stmtctx.StatementContext), col.RetType) @@ -575,27 +577,28 @@ func (s *testRangerSuite) TestIndexRange(c *C) { }, } + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprStr - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmts[0], is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() c.Assert(selection, NotNil, Commentf("expr:%v", tt.exprStr)) conds := make([]expression.Expression, 0, len(selection.Conditions)) for _, cond := range selection.Conditions { - conds = append(conds, expression.PushDownNot(ctx, cond, false)) + conds = append(conds, expression.PushDownNot(sctx, cond, false)) } cols, lengths := expression.IndexInfo2Cols(selection.Schema().Columns, tbl.Indices[tt.indexPos]) c.Assert(cols, NotNil) - res, err := ranger.DetachCondAndBuildRangeForIndex(ctx, conds, cols, lengths) + res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths) c.Assert(err, IsNil) c.Assert(fmt.Sprintf("%s", res.AccessConds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr)) c.Assert(fmt.Sprintf("%s", res.RemainedConds), Equals, tt.filterConds, Commentf("wrong filter conditions for expr: %s", tt.exprStr)) @@ -695,27 +698,28 @@ func (s *testRangerSuite) TestIndexRangeForUnsignedInt(c *C) { }, } + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprStr - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmts[0], is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() c.Assert(selection, NotNil, Commentf("expr:%v", tt.exprStr)) conds := make([]expression.Expression, 0, len(selection.Conditions)) for _, cond := range selection.Conditions { - conds = append(conds, expression.PushDownNot(ctx, cond, false)) + conds = append(conds, expression.PushDownNot(sctx, cond, false)) } cols, lengths := expression.IndexInfo2Cols(selection.Schema().Columns, tbl.Indices[tt.indexPos]) c.Assert(cols, NotNil) - res, err := ranger.DetachCondAndBuildRangeForIndex(ctx, conds, cols, lengths) + res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths) c.Assert(err, IsNil) c.Assert(fmt.Sprintf("%s", res.AccessConds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr)) c.Assert(fmt.Sprintf("%s", res.RemainedConds), Equals, tt.filterConds, Commentf("wrong filter conditions for expr: %s", tt.exprStr)) @@ -975,23 +979,24 @@ func (s *testRangerSuite) TestColumnRange(c *C) { }, } + ctx := context.Background() for _, tt := range tests { sql := "select * from t where " + tt.exprStr - ctx := testKit.Se.(sessionctx.Context) - stmts, err := session.Parse(ctx, sql) + sctx := testKit.Se.(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr)) c.Assert(stmts, HasLen, 1) - is := domain.GetDomain(ctx).InfoSchema() - err = plannercore.Preprocess(ctx, stmts[0], is) + is := domain.GetDomain(sctx).InfoSchema() + err = plannercore.Preprocess(sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr)) - p, err := plannercore.BuildLogicalPlan(ctx, stmts[0], is) + p, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is) c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr)) sel := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) ds, ok := sel.Children()[0].(*plannercore.DataSource) c.Assert(ok, IsTrue, Commentf("expr:%v", tt.exprStr)) conds := make([]expression.Expression, 0, len(sel.Conditions)) for _, cond := range sel.Conditions { - conds = append(conds, expression.PushDownNot(ctx, cond, false)) + conds = append(conds, expression.PushDownNot(sctx, cond, false)) } col := expression.ColInfo2Col(sel.Schema().Columns, ds.TableInfo().Columns[tt.colPos]) c.Assert(col, NotNil) diff --git a/util/sqlexec/restricted_sql_executor.go b/util/sqlexec/restricted_sql_executor.go index ec42336087752..cfbe37e4603cb 100644 --- a/util/sqlexec/restricted_sql_executor.go +++ b/util/sqlexec/restricted_sql_executor.go @@ -78,7 +78,7 @@ type Statement interface { IsReadOnly(vars *variable.SessionVars) bool // RebuildPlan rebuilds the plan of the statement. - RebuildPlan() (schemaVersion int64, err error) + RebuildPlan(ctx context.Context) (schemaVersion int64, err error) } // RecordSet is an abstract result set interface to help get data from Plan. From 86d8f9a4bb098c125f7e89a53526a6d3148d4a78 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 24 Jul 2019 19:20:18 +0800 Subject: [PATCH 090/196] *: add succ filed to slow log and fix shallow copy problem when parse slow log file. (#11417) Test pass, auto merge by Bot --- executor/adapter.go | 4 +- infoschema/slow_log.go | 144 ++++++++++------------------ infoschema/slow_log_test.go | 29 +++++- infoschema/tables_test.go | 24 ++++- sessionctx/variable/session.go | 6 +- sessionctx/variable/session_test.go | 3 +- 6 files changed, 108 insertions(+), 102 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index 926d86a870dde..a7c63c854d081 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -650,10 +650,10 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool) { memMax := sessVars.StmtCtx.MemTracker.MaxConsumed() if costTime < threshold { _, digest := sessVars.StmtCtx.SQLDigest() - logutil.SlowQueryLogger.Debug(sessVars.SlowLogFormat(txnTS, costTime, execDetail, indexIDs, digest, statsInfos, copTaskInfo, memMax, sql)) + logutil.SlowQueryLogger.Debug(sessVars.SlowLogFormat(txnTS, costTime, execDetail, indexIDs, digest, statsInfos, copTaskInfo, memMax, succ, sql)) } else { _, digest := sessVars.StmtCtx.SQLDigest() - logutil.SlowQueryLogger.Warn(sessVars.SlowLogFormat(txnTS, costTime, execDetail, indexIDs, digest, statsInfos, copTaskInfo, memMax, sql)) + logutil.SlowQueryLogger.Warn(sessVars.SlowLogFormat(txnTS, costTime, execDetail, indexIDs, digest, statsInfos, copTaskInfo, memMax, succ, sql)) metrics.TotalQueryProcHistogram.Observe(costTime.Seconds()) metrics.TotalCopProcHistogram.Observe(execDetail.ProcessTime.Seconds()) metrics.TotalCopWaitHistogram.Observe(execDetail.WaitTime.Seconds()) diff --git a/infoschema/slow_log.go b/infoschema/slow_log.go index bd35fb16b3791..7c85f30fa3c36 100644 --- a/infoschema/slow_log.go +++ b/infoschema/slow_log.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/tidb/util/execdetails" "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/logutil" - "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" ) @@ -60,7 +59,8 @@ var slowQueryCols = []columnInfo{ {variable.SlowLogCopWaitMax, mysql.TypeDouble, 22, 0, nil, nil}, {variable.SlowLogCopWaitAddr, mysql.TypeVarchar, 64, 0, nil, nil}, {variable.SlowLogMemMax, mysql.TypeLonglong, 20, 0, nil, nil}, - {variable.SlowLogQuerySQLStr, mysql.TypeVarchar, 4096, 0, nil, nil}, + {variable.SlowLogSucc, mysql.TypeTiny, 1, 0, nil, nil}, + {variable.SlowLogQuerySQLStr, mysql.TypeLongBlob, types.UnspecifiedLength, 0, nil, nil}, } func dataForSlowLog(ctx sessionctx.Context) ([][]types.Datum, error) { @@ -139,24 +139,34 @@ func ParseSlowLog(tz *time.Location, reader *bufio.Reader) ([][]types.Datum, err } func getOneLine(reader *bufio.Reader) ([]byte, error) { + var resByte []byte lineByte, isPrefix, err := reader.ReadLine() + if isPrefix { + // Need to read more data. + resByte = make([]byte, len(lineByte), len(lineByte)*2) + } else { + resByte = make([]byte, len(lineByte)) + } + // Use copy here to avoid shallow copy problem. + copy(resByte, lineByte) if err != nil { - return lineByte, err + return resByte, err } + var tempLine []byte for isPrefix { tempLine, isPrefix, err = reader.ReadLine() - lineByte = append(lineByte, tempLine...) + resByte = append(resByte, tempLine...) // Use the max value of max_allowed_packet to check the single line length. - if len(lineByte) > int(variable.MaxOfMaxAllowedPacket) { - return lineByte, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket) + if len(resByte) > int(variable.MaxOfMaxAllowedPacket) { + return resByte, errors.Errorf("single line length exceeds limit: %v", variable.MaxOfMaxAllowedPacket) } if err != nil { - return lineByte, err + return resByte, err } } - return lineByte, err + return resByte, err } type slowQueryTuple struct { @@ -186,27 +196,23 @@ type slowQueryTuple struct { maxWaitTime float64 maxWaitAddress string memMax int64 + succ bool sql string } func (st *slowQueryTuple) setFieldValue(tz *time.Location, field, value string) error { - value = stringutil.Copy(value) + var err error switch field { case variable.SlowLogTimeStr: - t, err := ParseTime(value) + st.time, err = ParseTime(value) if err != nil { - return err + break } - if t.Location() != tz { - t = t.In(tz) + if st.time.Location() != tz { + st.time = st.time.In(tz) } - st.time = t case variable.SlowLogTxnStartTSStr: - num, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.txnStartTs = num + st.txnStartTs, err = strconv.ParseUint(value, 10, 64) case variable.SlowLogUserStr: fields := strings.SplitN(value, "@", 2) if len(field) > 0 { @@ -216,53 +222,21 @@ func (st *slowQueryTuple) setFieldValue(tz *time.Location, field, value string) st.host = fields[1] } case variable.SlowLogConnIDStr: - num, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.connID = num + st.connID, err = strconv.ParseUint(value, 10, 64) case variable.SlowLogQueryTimeStr: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.queryTime = num + st.queryTime, err = strconv.ParseFloat(value, 64) case execdetails.ProcessTimeStr: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.processTime = num + st.processTime, err = strconv.ParseFloat(value, 64) case execdetails.WaitTimeStr: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.waitTime = num + st.waitTime, err = strconv.ParseFloat(value, 64) case execdetails.BackoffTimeStr: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.backOffTime = num + st.backOffTime, err = strconv.ParseFloat(value, 64) case execdetails.RequestCountStr: - num, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.requestCount = num + st.requestCount, err = strconv.ParseUint(value, 10, 64) case execdetails.TotalKeysStr: - num, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.totalKeys = num + st.totalKeys, err = strconv.ParseUint(value, 10, 64) case execdetails.ProcessKeysStr: - num, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.processKeys = num + st.processKeys, err = strconv.ParseUint(value, 10, 64) case variable.SlowLogDBStr: st.db = value case variable.SlowLogIndexIDsStr: @@ -274,54 +248,31 @@ func (st *slowQueryTuple) setFieldValue(tz *time.Location, field, value string) case variable.SlowLogStatsInfoStr: st.statsInfo = value case variable.SlowLogCopProcAvg: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.avgProcessTime = num + st.avgProcessTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopProcP90: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.p90ProcessTime = num + st.p90ProcessTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopProcMax: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.maxProcessTime = num + st.maxProcessTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopProcAddr: st.maxProcessAddress = value case variable.SlowLogCopWaitAvg: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.avgWaitTime = num + st.avgWaitTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopWaitP90: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.p90WaitTime = num + st.p90WaitTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopWaitMax: - num, err := strconv.ParseFloat(value, 64) - if err != nil { - return errors.AddStack(err) - } - st.maxWaitTime = num + st.maxWaitTime, err = strconv.ParseFloat(value, 64) case variable.SlowLogCopWaitAddr: st.maxWaitAddress = value case variable.SlowLogMemMax: - num, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return errors.AddStack(err) - } - st.memMax = num + st.memMax, err = strconv.ParseInt(value, 10, 64) + case variable.SlowLogSucc: + st.succ, err = strconv.ParseBool(value) case variable.SlowLogQuerySQLStr: st.sql = value } + if err != nil { + return errors.Wrap(err, "parse slow log failed `"+field+"` error") + } return nil } @@ -357,6 +308,11 @@ func (st *slowQueryTuple) convertToDatumRow() []types.Datum { record = append(record, types.NewFloat64Datum(st.maxWaitTime)) record = append(record, types.NewStringDatum(st.maxWaitAddress)) record = append(record, types.NewIntDatum(st.memMax)) + if st.succ { + record = append(record, types.NewIntDatum(1)) + } else { + record = append(record, types.NewIntDatum(0)) + } record = append(record, types.NewStringDatum(st.sql)) return record } diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index 1dc076af79948..a0f9e7b7a1039 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -37,6 +37,7 @@ func (s *testSuite) TestParseSlowLogFile(c *C) { # Cop_proc_avg: 0.1 Cop_proc_p90: 0.2 Cop_proc_max: 0.03 Cop_proc_addr: 127.0.0.1:20160 # Cop_wait_avg: 0.05 Cop_wait_p90: 0.6 Cop_wait_max: 0.8 Cop_wait_addr: 0.0.0.0:20160 # Mem_max: 70724 +# Succ: false select * from t;`) reader := bufio.NewReader(slowLog) loc, err := time.LoadLocation("Asia/Shanghai") @@ -53,7 +54,7 @@ select * from t;`) } recordString += str } - expectRecordString := "2019-04-28 15:24:04.309074,405888132465033227,,,0,0.216905,0.021,0,0,1,637,0,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,select * from t;" + expectRecordString := "2019-04-28 15:24:04.309074,405888132465033227,,,0,0.216905,0.021,0,0,1,637,0,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,0,select * from t;" c.Assert(expectRecordString, Equals, recordString) // fix sql contain '# ' bug @@ -67,6 +68,7 @@ select a# from t; # Is_internal: true # Digest: 42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 # Stats: t1:1,t2:2 +# Succ: false select * from t; `) reader = bufio.NewReader(slowLog) @@ -110,6 +112,17 @@ select * from t; reader = bufio.NewReader(slowLog) _, err = infoschema.ParseSlowLog(loc, reader) c.Assert(err, IsNil) + + // Add parse error check. + slowLog = bytes.NewBufferString( + `# Time: 2019-04-28T15:24:04.309074+08:00 +# Succ: abc +select * from t; +`) + reader = bufio.NewReader(slowLog) + _, err = infoschema.ParseSlowLog(loc, reader) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "parse slow log failed `Succ` error: strconv.ParseBool: parsing \"abc\": invalid syntax") } func (s *testSuite) TestSlowLogParseTime(c *C) { @@ -159,3 +172,17 @@ select * from t;`) _, err = infoschema.ParseSlowLog(loc, scanner) c.Assert(err, IsNil) } + +func (s *testSuite) TestSlowLogLongQuery(c *C) { + t1Str := "2019-01-24T22:32:29.313255+08:00" + t2Str := "2019-01-24T22:32:29.313255" + t1, err := infoschema.ParseTime(t1Str) + c.Assert(err, IsNil) + loc, err := time.LoadLocation("Asia/Shanghai") + c.Assert(err, IsNil) + t2, err := time.ParseInLocation("2006-01-02T15:04:05.999999999", t2Str, loc) + c.Assert(err, IsNil) + c.Assert(t1.Unix(), Equals, t2.Unix()) + t1Format := t1.In(loc).Format(logutil.SlowLogTimeFormat) + c.Assert(t1Format, Equals, t1Str) +} diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 9e9220d015095..13d0999243083 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -458,18 +458,36 @@ func (s *testTableSuite) TestSlowQuery(c *C) { # Cop_proc_avg: 0.1 Cop_proc_p90: 0.2 Cop_proc_max: 0.03 Cop_proc_addr: 127.0.0.1:20160 # Cop_wait_avg: 0.05 Cop_wait_p90: 0.6 Cop_wait_max: 0.8 Cop_wait_addr: 0.0.0.0:20160 # Mem_max: 70724 +# Succ: true select * from t_slim;`)) - c.Assert(f.Close(), IsNil) + c.Assert(f.Sync(), IsNil) c.Assert(err, IsNil) tk.MustExec(fmt.Sprintf("set @@tidb_slow_query_file='%v'", slowLogFileName)) tk.MustExec("set time_zone = '+08:00';") re := tk.MustQuery("select * from information_schema.slow_query") re.Check(testutil.RowsWithSep("|", - "2019-02-12 19:33:56.571953|406315658548871171|root|127.0.0.1|6|4.895492|0.161|0.101|0.092|1|100001|100000|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|select * from t_slim;")) + "2019-02-12 19:33:56.571953|406315658548871171|root|127.0.0.1|6|4.895492|0.161|0.101|0.092|1|100001|100000|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|1|select * from t_slim;")) tk.MustExec("set time_zone = '+00:00';") re = tk.MustQuery("select * from information_schema.slow_query") - re.Check(testutil.RowsWithSep("|", "2019-02-12 11:33:56.571953|406315658548871171|root|127.0.0.1|6|4.895492|0.161|0.101|0.092|1|100001|100000|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|select * from t_slim;")) + re.Check(testutil.RowsWithSep("|", "2019-02-12 11:33:56.571953|406315658548871171|root|127.0.0.1|6|4.895492|0.161|0.101|0.092|1|100001|100000|test||0|42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772|t1:1,t2:2|0.1|0.2|0.03|127.0.0.1:20160|0.05|0.6|0.8|0.0.0.0:20160|70724|1|select * from t_slim;")) + + // Test for long query. + _, err = f.Write([]byte(` +# Time: 2019-02-13T19:33:56.571953+08:00 +`)) + c.Assert(err, IsNil) + sql := "select * from " + for len(sql) < 5000 { + sql += "abcdefghijklmnopqrstuvwxyz_1234567890_qwertyuiopasdfghjklzxcvbnm" + } + sql += ";" + _, err = f.Write([]byte(sql)) + c.Assert(err, IsNil) + c.Assert(f.Close(), IsNil) + re = tk.MustQuery("select query from information_schema.slow_query order by time desc limit 1") + rows := re.Rows() + c.Assert(rows[0][0], Equals, sql) } func (s *testTableSuite) TestForAnalyzeStatus(c *C) { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 940c20a6e7863..715cede8bc46c 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1019,6 +1019,8 @@ const ( SlowLogCopWaitAddr = "Cop_wait_addr" // SlowLogMemMax is the max number bytes of memory used in this statement. SlowLogMemMax = "Mem_max" + // SlowLogSucc is used to indicate whether this sql execute successfully. + SlowLogSucc = "Succ" ) // SlowLogFormat uses for formatting slow log. @@ -1038,9 +1040,10 @@ const ( // # Cop_process: Avg_time: 1s P90_time: 2s Max_time: 3s Max_addr: 10.6.131.78 // # Cop_wait: Avg_time: 10ms P90_time: 20ms Max_time: 30ms Max_Addr: 10.6.131.79 // # Memory_max: 4096 +// # Succ: true // select * from t_slim; func (s *SessionVars) SlowLogFormat(txnTS uint64, costTime time.Duration, execDetail execdetails.ExecDetails, indexIDs string, digest string, - statsInfos map[string]uint64, copTasks *stmtctx.CopTasksDetails, memMax int64, sql string) string { + statsInfos map[string]uint64, copTasks *stmtctx.CopTasksDetails, memMax int64, succ bool, sql string) string { var buf bytes.Buffer execDetailStr := execDetail.String() buf.WriteString(SlowLogRowPrefixStr + SlowLogTxnStartTSStr + SlowLogSpaceMarkStr + strconv.FormatUint(txnTS, 10) + "\n") @@ -1100,6 +1103,7 @@ func (s *SessionVars) SlowLogFormat(txnTS uint64, costTime time.Duration, execDe if memMax > 0 { buf.WriteString(SlowLogRowPrefixStr + SlowLogMemMax + SlowLogSpaceMarkStr + strconv.FormatInt(memMax, 10) + "\n") } + buf.WriteString(SlowLogRowPrefixStr + SlowLogSucc + SlowLogSpaceMarkStr + strconv.FormatBool(succ) + "\n") if len(sql) == 0 { sql = ";" } diff --git a/sessionctx/variable/session_test.go b/sessionctx/variable/session_test.go index 87a874930ad33..179579ce8b5e9 100644 --- a/sessionctx/variable/session_test.go +++ b/sessionctx/variable/session_test.go @@ -166,9 +166,10 @@ func (*testSessionSuite) TestSlowLogFormat(c *C) { # Cop_proc_avg: 1 Cop_proc_p90: 2 Cop_proc_max: 3 Cop_proc_addr: 10.6.131.78 # Cop_wait_avg: 0.01 Cop_wait_p90: 0.02 Cop_wait_max: 0.03 Cop_wait_addr: 10.6.131.79 # Mem_max: 2333 +# Succ: true select * from t;` sql := "select * from t" digest := parser.DigestHash(sql) - logString := seVar.SlowLogFormat(txnTS, costTime, execDetail, "[1,2]", digest, statsInfos, copTasks, memMax, sql) + logString := seVar.SlowLogFormat(txnTS, costTime, execDetail, "[1,2]", digest, statsInfos, copTasks, memMax, true, sql) c.Assert(logString, Equals, resultString) } From c95d42a5371e353a04febb931238a8f380ffd99d Mon Sep 17 00:00:00 2001 From: Wang Xiaoyong Date: Wed, 24 Jul 2019 19:41:17 +0800 Subject: [PATCH 091/196] expression, session: handle `CASE WHEN` specially when folding constant during outerJoin Simplification (#11105) Test pass, auto merge by Bot --- expression/constant_fold.go | 54 ++++++++++++++++++++++++++++++++++ expression/integration_test.go | 12 ++++++++ 2 files changed, 66 insertions(+) diff --git a/expression/constant_fold.go b/expression/constant_fold.go index 4c0c906744b83..b421bcd30f173 100644 --- a/expression/constant_fold.go +++ b/expression/constant_fold.go @@ -27,6 +27,7 @@ func init() { specialFoldHandler = map[string]func(*ScalarFunction) (Expression, bool){ ast.If: ifFoldHandler, ast.Ifnull: ifNullFoldHandler, + ast.Case: caseWhenHandler, } } @@ -78,6 +79,59 @@ func ifNullFoldHandler(expr *ScalarFunction) (Expression, bool) { return expr, isDeferredConst } +func caseWhenHandler(expr *ScalarFunction) (Expression, bool) { + args, l := expr.GetArgs(), len(expr.GetArgs()) + var isDeferred, isDeferredConst, hasNonConstCondition bool + for i := 0; i < l-1; i += 2 { + expr.GetArgs()[i], isDeferred = foldConstant(args[i]) + isDeferredConst = isDeferredConst || isDeferred + if _, isConst := expr.GetArgs()[i].(*Constant); isConst && !hasNonConstCondition { + // If the condition is const and true, and the previous conditions + // has no expr, then the folded execution body is returned, otherwise + // the arguments of the casewhen are folded and replaced. + val, isNull, err := args[i].EvalInt(expr.GetCtx(), chunk.Row{}) + if err != nil { + return expr, false + } + if val != 0 && !isNull { + foldedExpr, isDeferred := foldConstant(args[i+1]) + isDeferredConst = isDeferredConst || isDeferred + if _, isConst := foldedExpr.(*Constant); isConst { + foldedExpr.GetType().Decimal = expr.GetType().Decimal + return foldedExpr, isDeferredConst + } + return BuildCastFunction(expr.GetCtx(), foldedExpr, foldedExpr.GetType()), isDeferredConst + } + } else { + hasNonConstCondition = true + } + expr.GetArgs()[i+1], isDeferred = foldConstant(args[i+1]) + isDeferredConst = isDeferredConst || isDeferred + } + + if l%2 == 0 { + return expr, isDeferredConst + } + + // If the number of arguments in casewhen is odd, and the previous conditions + // is const and false, then the folded else execution body is returned. otherwise + // the execution body of the else are folded and replaced. + if !hasNonConstCondition { + foldedExpr, isDeferred := foldConstant(args[l-1]) + isDeferredConst = isDeferredConst || isDeferred + if _, isConst := foldedExpr.(*Constant); isConst { + foldedExpr.GetType().Decimal = expr.GetType().Decimal + return foldedExpr, isDeferredConst + } + return BuildCastFunction(expr.GetCtx(), foldedExpr, foldedExpr.GetType()), isDeferredConst + } + + expr.GetArgs()[l-1], isDeferred = foldConstant(args[l-1]) + isDeferredConst = isDeferredConst || isDeferred + + return expr, isDeferredConst +} + func foldConstant(expr Expression) (Expression, bool) { switch x := expr.(type) { case *ScalarFunction: diff --git a/expression/integration_test.go b/expression/integration_test.go index a13edd67c2fb0..bbec68f771067 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4648,5 +4648,17 @@ func (s *testIntegrationSuite) TestDatetimeMicrosecond(c *C) { testkit.Rows("2007-03-28 22:06:28")) tk.MustQuery(`select DATE_ADD('2007-03-28 22:08:28',INTERVAL "-2.-2" MICROSECOND);`).Check( testkit.Rows("2007-03-28 22:08:27.999998")) +} + +func (s *testIntegrationSuite) TestFuncCaseWithLeftJoin(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + + tk.MustExec("create table kankan1(id int, name text)") + tk.MustExec("insert into kankan1 values(1, 'a')") + tk.MustExec("insert into kankan1 values(2, 'a')") + + tk.MustExec("create table kankan2(id int, h1 text)") + tk.MustExec("insert into kankan2 values(2, 'z')") + tk.MustQuery("select t1.id from kankan1 t1 left join kankan2 t2 on t1.id = t2.id where (case when t1.name='b' then 'case2' when t1.name='a' then 'case1' else NULL end) = 'case1' order by t1.id").Check(testkit.Rows("1", "2")) } From 9e9a5649215c12261c2b4abb11805bcbe92eea19 Mon Sep 17 00:00:00 2001 From: crazycs Date: Thu, 25 Jul 2019 11:21:37 +0800 Subject: [PATCH 092/196] =?UTF-8?q?infoschema:=20remove=20redundant=20test?= =?UTF-8?q?=20introduce=20by=20=EF=BC=88#11417)=20(#11422)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infoschema/slow_log_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/infoschema/slow_log_test.go b/infoschema/slow_log_test.go index a0f9e7b7a1039..ff790ee42b475 100644 --- a/infoschema/slow_log_test.go +++ b/infoschema/slow_log_test.go @@ -172,17 +172,3 @@ select * from t;`) _, err = infoschema.ParseSlowLog(loc, scanner) c.Assert(err, IsNil) } - -func (s *testSuite) TestSlowLogLongQuery(c *C) { - t1Str := "2019-01-24T22:32:29.313255+08:00" - t2Str := "2019-01-24T22:32:29.313255" - t1, err := infoschema.ParseTime(t1Str) - c.Assert(err, IsNil) - loc, err := time.LoadLocation("Asia/Shanghai") - c.Assert(err, IsNil) - t2, err := time.ParseInLocation("2006-01-02T15:04:05.999999999", t2Str, loc) - c.Assert(err, IsNil) - c.Assert(t1.Unix(), Equals, t2.Unix()) - t1Format := t1.In(loc).Format(logutil.SlowLogTimeFormat) - c.Assert(t1Format, Equals, t1Str) -} From 67221062fd5da1ea08285b9a5c8c5b0814d9556a Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 25 Jul 2019 11:28:55 +0800 Subject: [PATCH 093/196] executor: add `CopyReconstruct()` and `CopyConstruct()` methods to `Column` (#11408) --- util/chunk/chunk.go | 8 +-- util/chunk/column.go | 64 +++++++++++++++++++-- util/chunk/column_test.go | 118 +++++++++++++++++++++++++++++++++++++- 3 files changed, 179 insertions(+), 11 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 8f6d166787905..fc8ff8a5a6152 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -102,11 +102,7 @@ func Renew(chk *Chunk, maxChunkSize int) *Chunk { func renewColumns(oldCol []*Column, cap int) []*Column { columns := make([]*Column, 0, len(oldCol)) for _, col := range oldCol { - if col.isFixed() { - columns = append(columns, newFixedLenColumn(len(col.elemBuf), cap)) - } else { - columns = append(columns, newVarLenColumn(cap, col)) - } + columns = append(columns, newColumn(col.typeSize(), cap)) } return columns } @@ -257,7 +253,7 @@ func (c *Chunk) Reset() { func (c *Chunk) CopyConstruct() *Chunk { newChk := &Chunk{numVirtualRows: c.numVirtualRows, capacity: c.capacity, columns: make([]*Column, len(c.columns))} for i := range c.columns { - newChk.columns[i] = c.columns[i].copyConstruct() + newChk.columns[i] = c.columns[i].CopyConstruct(nil) } if c.sel != nil { newChk.sel = make([]int, len(c.sel)) diff --git a/util/chunk/column.go b/util/chunk/column.go index bd8df6eeb2726..d6695ba0142dd 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -67,11 +67,24 @@ type Column struct { // NewColumn creates a new column with the specific length and capacity. func NewColumn(ft *types.FieldType, cap int) *Column { - typeSize := getFixedLen(ft) + return newColumn(getFixedLen(ft), cap) +} + +func newColumn(typeSize, cap int) *Column { + var col *Column if typeSize == varElemLen { - return newVarLenColumn(cap, nil) + col = newVarLenColumn(cap, nil) + } else { + col = newFixedLenColumn(typeSize, cap) + } + return col +} + +func (c *Column) typeSize() int { + if len(c.elemBuf) > 0 { + return len(c.elemBuf) } - return newFixedLenColumn(typeSize, cap) + return varElemLen } func (c *Column) isFixed() bool { @@ -96,7 +109,18 @@ func (c *Column) IsNull(rowIdx int) bool { return nullByte&(1<<(uint(rowIdx)&7)) == 0 } -func (c *Column) copyConstruct() *Column { +// CopyConstruct copies this Column to dst. +// If dst is nil, it creates a new Column and returns it. +func (c *Column) CopyConstruct(dst *Column) *Column { + if dst != nil { + dst.length = c.length + dst.nullCount = c.nullCount + dst.nullBitmap = append(dst.nullBitmap[:0], c.nullBitmap...) + dst.offsets = append(dst.offsets[:0], c.offsets...) + dst.data = append(dst.data[:0], c.data...) + dst.elemBuf = append(dst.elemBuf[:0], c.elemBuf...) + return dst + } newCol := &Column{length: c.length, nullCount: c.nullCount} newCol.nullBitmap = append(newCol.nullBitmap, c.nullBitmap...) newCol.offsets = append(newCol.offsets, c.offsets...) @@ -386,3 +410,35 @@ func (c *Column) reconstruct(sel []int) { c.nullBitmap[idx] &= byte((1 << pos) - 1) } } + +// CopyReconstruct copies this Column to dst and removes unselected rows. +// If dst is nil, it creates a new Column and returns it. +func (c *Column) CopyReconstruct(sel []int, dst *Column) *Column { + if sel == nil { + return c.CopyConstruct(dst) + } + + if dst == nil { + dst = newColumn(c.typeSize(), len(sel)) + } else { + dst.Reset() + } + + if c.isFixed() { + elemLen := len(c.elemBuf) + for _, i := range sel { + dst.appendNullBitmap(!c.IsNull(i)) + dst.data = append(dst.data, c.data[i*elemLen:i*elemLen+elemLen]...) + dst.length++ + } + } else { + for _, i := range sel { + dst.appendNullBitmap(!c.IsNull(i)) + start, end := c.offsets[i], c.offsets[i+1] + dst.data = append(dst.data, c.data[start:end]...) + dst.offsets = append(dst.offsets, int64(len(dst.data))) + dst.length++ + } + } + return dst +} diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 39e6c1cca4fbf..b40a91bfac7e6 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -64,8 +64,124 @@ func (s *testChunkSuite) TestColumnCopy(c *check.C) { col.AppendInt64(int64(i)) } - c1 := col.copyConstruct() + c1 := col.CopyConstruct(nil) c.Check(equalColumn(col, c1), check.IsTrue) + + c2 := newFixedLenColumn(8, 10) + c2 = col.CopyConstruct(c2) + c.Check(equalColumn(col, c2), check.IsTrue) +} + +func (s *testChunkSuite) TestColumnCopyReconstructFixedLen(c *check.C) { + col := NewColumn(types.NewFieldType(mysql.TypeLonglong), 1024) + results := make([]int64, 0, 1024) + nulls := make([]bool, 0, 1024) + sel := make([]int, 0, 1024) + for i := 0; i < 1024; i++ { + if rand.Intn(10) < 6 { + sel = append(sel, i) + } + + if rand.Intn(10) < 2 { + col.AppendNull() + nulls = append(nulls, true) + results = append(results, 0) + continue + } + + v := rand.Int63() + col.AppendInt64(v) + results = append(results, v) + nulls = append(nulls, false) + } + + col = col.CopyReconstruct(sel, nil) + nullCnt := 0 + for n, i := range sel { + if nulls[i] { + nullCnt++ + c.Assert(col.IsNull(n), check.Equals, true) + } else { + c.Assert(col.GetInt64(n), check.Equals, results[i]) + } + } + c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(col.length, check.Equals, len(sel)) + + for i := 0; i < 128; i++ { + if i%2 == 0 { + col.AppendNull() + } else { + col.AppendInt64(int64(i * i * i)) + } + } + + c.Assert(col.length, check.Equals, len(sel)+128) + c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + for i := 0; i < 128; i++ { + if i%2 == 0 { + c.Assert(col.IsNull(len(sel)+i), check.Equals, true) + } else { + c.Assert(col.GetInt64(len(sel)+i), check.Equals, int64(i*i*i)) + c.Assert(col.IsNull(len(sel)+i), check.Equals, false) + } + } +} + +func (s *testChunkSuite) TestColumnCopyReconstructVarLen(c *check.C) { + col := NewColumn(types.NewFieldType(mysql.TypeVarString), 1024) + results := make([]string, 0, 1024) + nulls := make([]bool, 0, 1024) + sel := make([]int, 0, 1024) + for i := 0; i < 1024; i++ { + if rand.Intn(10) < 6 { + sel = append(sel, i) + } + + if rand.Intn(10) < 2 { + col.AppendNull() + nulls = append(nulls, true) + results = append(results, "") + continue + } + + v := fmt.Sprintf("%v", rand.Int63()) + col.AppendString(v) + results = append(results, v) + nulls = append(nulls, false) + } + + col = col.CopyReconstruct(sel, nil) + nullCnt := 0 + for n, i := range sel { + if nulls[i] { + nullCnt++ + c.Assert(col.IsNull(n), check.Equals, true) + } else { + c.Assert(col.GetString(n), check.Equals, results[i]) + } + } + c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(col.length, check.Equals, len(sel)) + + for i := 0; i < 128; i++ { + if i%2 == 0 { + col.AppendNull() + } else { + col.AppendString(fmt.Sprintf("%v", i*i*i)) + } + } + + c.Assert(col.length, check.Equals, len(sel)+128) + c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + for i := 0; i < 128; i++ { + if i%2 == 0 { + c.Assert(col.IsNull(len(sel)+i), check.Equals, true) + } else { + c.Assert(col.GetString(len(sel)+i), check.Equals, fmt.Sprintf("%v", i*i*i)) + c.Assert(col.IsNull(len(sel)+i), check.Equals, false) + } + } } func (s *testChunkSuite) TestLargeStringColumnOffset(c *check.C) { From dcadc0acfb054372689f21355521c8dc46ea1381 Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Thu, 25 Jul 2019 12:52:40 +0800 Subject: [PATCH 094/196] types: fix uint64 overflow bug in `ConvertJSONToFloat` (#11355) All tests passed, auto merged by Bot --- expression/integration_test.go | 6 +++++- types/convert.go | 3 +-- types/convert_test.go | 30 ++++++++++++++++-------------- types/datum_test.go | 1 + 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index bbec68f771067..403d77286c3a1 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2172,10 +2172,14 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result.Check(testkit.Rows("")) result = tk.MustQuery(`select cast(cast('2017-01-01 01:01:11.12' as date) as datetime(2));`) result.Check(testkit.Rows("2017-01-01 00:00:00.00")) - result = tk.MustQuery(`select cast(20170118.999 as datetime);`) result.Check(testkit.Rows("2017-01-18 00:00:00")) + tk.MustExec(`create table tb5(a bigint(64) unsigned, b double);`) + tk.MustExec(`insert into tb5 (a, b) values (9223372036854776000, 9223372036854776000);`) + tk.MustExec(`insert into tb5 (a, b) select * from (select cast(a as json) as a1, b from tb5) as t where t.a1 = t.b;`) + tk.MustExec(`drop table tb5;`) + // Test corner cases of cast string as datetime result = tk.MustQuery(`select cast("170102034" as datetime);`) result.Check(testkit.Rows("2017-01-02 03:04:00")) diff --git a/types/convert.go b/types/convert.go index 7ddfa5e1dd2b1..6e330abee0f69 100644 --- a/types/convert.go +++ b/types/convert.go @@ -553,8 +553,7 @@ func ConvertJSONToFloat(sc *stmtctx.StatementContext, j json.BinaryJSON) (float6 case json.TypeCodeInt64: return float64(j.GetInt64()), nil case json.TypeCodeUint64: - u, err := ConvertIntToUint(sc, j.GetInt64(), IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) - return float64(u), errors.Trace(err) + return float64(j.GetUint64()), nil case json.TypeCodeFloat64: return j.GetFloat64(), nil case json.TypeCodeString: diff --git a/types/convert_test.go b/types/convert_test.go index 52e1b58832683..3f6ba218589dc 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -828,24 +828,26 @@ func (s *testTypeConvertSuite) TestConvertJSONToInt(c *C) { func (s *testTypeConvertSuite) TestConvertJSONToFloat(c *C) { var tests = []struct { - In string + In interface{} Out float64 + ty json.TypeCode }{ - {`{}`, 0}, - {`[]`, 0}, - {`3`, 3}, - {`-3`, -3}, - {`4.5`, 4.5}, - {`true`, 1}, - {`false`, 0}, - {`null`, 0}, - {`"hello"`, 0}, - {`"123.456hello"`, 123.456}, - {`"1234"`, 1234}, + {make(map[string]interface{}, 0), 0, json.TypeCodeObject}, + {make([]interface{}, 0), 0, json.TypeCodeArray}, + {int64(3), 3, json.TypeCodeInt64}, + {int64(-3), -3, json.TypeCodeInt64}, + {uint64(1 << 63), 1 << 63, json.TypeCodeUint64}, + {float64(4.5), 4.5, json.TypeCodeFloat64}, + {true, 1, json.TypeCodeLiteral}, + {false, 0, json.TypeCodeLiteral}, + {nil, 0, json.TypeCodeLiteral}, + {"hello", 0, json.TypeCodeString}, + {"123.456hello", 123.456, json.TypeCodeString}, + {"1234", 1234, json.TypeCodeString}, } for _, tt := range tests { - j, err := json.ParseBinaryFromString(tt.In) - c.Assert(err, IsNil) + j := json.CreateBinary(tt.In) + c.Assert(j.TypeCode, Equals, tt.ty) casted, _ := ConvertJSONToFloat(new(stmtctx.StatementContext), j) c.Assert(casted, Equals, tt.Out) } diff --git a/types/datum_test.go b/types/datum_test.go index dc389f3f16a71..a1e8eb468983b 100644 --- a/types/datum_test.go +++ b/types/datum_test.go @@ -250,6 +250,7 @@ func (ts *testDatumSuite) TestToJSON(c *C) { {NewStringDatum("[1, 2, 3]"), `[1, 2, 3]`, true}, {NewStringDatum("{}"), `{}`, true}, {mustParseTimeIntoDatum("2011-11-10 11:11:11.111111", mysql.TypeTimestamp, 6), `"2011-11-10 11:11:11.111111"`, true}, + {NewStringDatum(`{"a": "9223372036854775809"}`), `{"a": "9223372036854775809"}`, true}, // can not parse JSON from this string, so error occurs. {NewStringDatum("hello, 世界"), "", false}, From 268cbf1a5d2491b4242f26109e05fbdf684e16fa Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Thu, 25 Jul 2019 13:25:24 +0800 Subject: [PATCH 095/196] stats: support more analyze options (#11278) --- executor/analyze.go | 39 ++++++++++---------- executor/analyze_test.go | 21 ++++++++--- executor/builder.go | 48 ++++++++++++------------- executor/executor_test.go | 1 + go.mod | 2 +- go.sum | 4 +-- planner/core/common_plans.go | 6 ++-- planner/core/planbuilder.go | 70 ++++++++++++++++++++++++++---------- 8 files changed, 119 insertions(+), 72 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index c159b82313206..05ae1c5ffc3e9 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/debugpb" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/distsql" @@ -70,10 +71,8 @@ var ( ) const ( - maxRegionSampleSize = 1000 - maxSketchSize = 10000 - defaultCMSketchDepth = 5 - defaultCMSketchWidth = 2048 + maxRegionSampleSize = 1000 + maxSketchSize = 10000 ) // Next implements the Executor Next interface. @@ -252,7 +251,7 @@ type AnalyzeIndexExec struct { analyzePB *tipb.AnalyzeReq result distsql.SelectResult countNullRes distsql.SelectResult - maxNumBuckets uint64 + opts map[ast.AnalyzeOptionType]uint64 job *statistics.AnalyzeJob } @@ -307,7 +306,7 @@ func (e *AnalyzeIndexExec) buildStatsFromResult(result distsql.SelectResult, nee hist := &statistics.Histogram{} var cms *statistics.CMSketch if needCMS { - cms = statistics.NewCMSketch(defaultCMSketchDepth, defaultCMSketchWidth) + cms = statistics.NewCMSketch(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth])) } for { data, err := result.NextRaw(context.TODO()) @@ -324,7 +323,7 @@ func (e *AnalyzeIndexExec) buildStatsFromResult(result distsql.SelectResult, nee } respHist := statistics.HistogramFromProto(resp.Hist) e.job.Update(int64(respHist.TotalRowCount())) - hist, err = statistics.MergeHistograms(e.ctx.GetSessionVars().StmtCtx, hist, respHist, int(e.maxNumBuckets)) + hist, err = statistics.MergeHistograms(e.ctx.GetSessionVars().StmtCtx, hist, respHist, int(e.opts[ast.AnalyzeOptNumBuckets])) if err != nil { return nil, nil, err } @@ -401,7 +400,7 @@ type AnalyzeColumnsExec struct { priority int analyzePB *tipb.AnalyzeReq resultHandler *tableResultHandler - maxNumBuckets uint64 + opts map[ast.AnalyzeOptionType]uint64 job *statistics.AnalyzeJob } @@ -465,7 +464,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis IsMerger: true, FMSketch: statistics.NewFMSketch(maxSketchSize), MaxSampleSize: int64(MaxSampleSize), - CMSketch: statistics.NewCMSketch(defaultCMSketchDepth, defaultCMSketchWidth), + CMSketch: statistics.NewCMSketch(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth])), } } for { @@ -486,7 +485,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis if e.pkInfo != nil { respHist := statistics.HistogramFromProto(resp.PkHist) rowCount = int64(respHist.TotalRowCount()) - pkHist, err = statistics.MergeHistograms(sc, pkHist, respHist, int(e.maxNumBuckets)) + pkHist, err = statistics.MergeHistograms(sc, pkHist, respHist, int(e.opts[ast.AnalyzeOptNumBuckets])) if err != nil { return nil, nil, err } @@ -516,7 +515,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis return nil, nil, err } } - hg, err := statistics.BuildColumn(e.ctx, int64(e.maxNumBuckets), col.ID, collectors[i], &col.FieldType) + hg, err := statistics.BuildColumn(e.ctx, int64(e.opts[ast.AnalyzeOptNumBuckets]), col.ID, collectors[i], &col.FieldType) if err != nil { return nil, nil, err } @@ -591,7 +590,7 @@ type AnalyzeFastExec struct { colsInfo []*model.ColumnInfo idxsInfo []*model.IndexInfo concurrency int - maxNumBuckets uint64 + opts map[ast.AnalyzeOptionType]uint64 tblInfo *model.TableInfo cache *tikv.RegionCache wg *sync.WaitGroup @@ -1006,9 +1005,9 @@ func (e *AnalyzeFastExec) buildColumnStats(ID int64, collector *statistics.Sampl data = append(data, bytes) } // Build CMSketch. - cmSketch, ndv, scaleRatio := statistics.NewCMSketchWithTopN(defaultCMSketchDepth, defaultCMSketchWidth, data, 20, uint64(rowCount)) + cmSketch, ndv, scaleRatio := statistics.NewCMSketchWithTopN(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth]), data, uint32(e.opts[ast.AnalyzeOptNumTopN]), uint64(rowCount)) // Build Histogram. - hist, err := statistics.BuildColumnHist(e.ctx, int64(e.maxNumBuckets), ID, collector, tp, rowCount, int64(ndv), collector.NullCount*int64(scaleRatio)) + hist, err := statistics.BuildColumnHist(e.ctx, int64(e.opts[ast.AnalyzeOptNumBuckets]), ID, collector, tp, rowCount, int64(ndv), collector.NullCount*int64(scaleRatio)) return hist, cmSketch, err } @@ -1029,20 +1028,20 @@ func (e *AnalyzeFastExec) buildIndexStats(idxInfo *model.IndexInfo, collector *s data[i] = append(data[i], sample.Value.GetBytes()[:preLen]) } } - numTop := uint32(20) - cmSketch, ndv, scaleRatio := statistics.NewCMSketchWithTopN(defaultCMSketchDepth, defaultCMSketchWidth, data[0], numTop, uint64(rowCount)) + numTop := uint32(e.opts[ast.AnalyzeOptNumTopN]) + cmSketch, ndv, scaleRatio := statistics.NewCMSketchWithTopN(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth]), data[0], numTop, uint64(rowCount)) // Build CM Sketch for each prefix and merge them into one. for i := 1; i < len(idxInfo.Columns); i++ { var curCMSketch *statistics.CMSketch // `ndv` should be the ndv of full index, so just rewrite it here. - curCMSketch, ndv, scaleRatio = statistics.NewCMSketchWithTopN(defaultCMSketchDepth, defaultCMSketchWidth, data[i], numTop, uint64(rowCount)) + curCMSketch, ndv, scaleRatio = statistics.NewCMSketchWithTopN(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth]), data[i], numTop, uint64(rowCount)) err := cmSketch.MergeCMSketch(curCMSketch, numTop) if err != nil { return nil, nil, err } } // Build Histogram. - hist, err := statistics.BuildColumnHist(e.ctx, int64(e.maxNumBuckets), idxInfo.ID, collector, types.NewFieldType(mysql.TypeBlob), rowCount, int64(ndv), collector.NullCount*int64(scaleRatio)) + hist, err := statistics.BuildColumnHist(e.ctx, int64(e.opts[ast.AnalyzeOptNumBuckets]), idxInfo.ID, collector, types.NewFieldType(mysql.TypeBlob), rowCount, int64(ndv), collector.NullCount*int64(scaleRatio)) return hist, cmSketch, err } @@ -1209,7 +1208,7 @@ func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult if err != nil { return analyzeResult{Err: err, job: idxExec.job} } - hist, err = statistics.MergeHistograms(idxExec.ctx.GetSessionVars().StmtCtx, idxExec.oldHist, hist, int(idxExec.maxNumBuckets)) + hist, err = statistics.MergeHistograms(idxExec.ctx.GetSessionVars().StmtCtx, idxExec.oldHist, hist, int(idxExec.opts[ast.AnalyzeOptNumBuckets])) if err != nil { return analyzeResult{Err: err, job: idxExec.job} } @@ -1252,7 +1251,7 @@ func analyzePKIncremental(colExec *analyzePKIncrementalExec) analyzeResult { return analyzeResult{Err: err, job: colExec.job} } hist := hists[0] - hist, err = statistics.MergeHistograms(colExec.ctx.GetSessionVars().StmtCtx, colExec.oldHist, hist, int(colExec.maxNumBuckets)) + hist, err = statistics.MergeHistograms(colExec.ctx.GetSessionVars().StmtCtx, colExec.oldHist, hist, int(colExec.opts[ast.AnalyzeOptNumBuckets])) if err != nil { return analyzeResult{Err: err, job: colExec.job} } diff --git a/executor/analyze_test.go b/executor/analyze_test.go index ce0f579e56757..e7bc61e02f89b 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -123,18 +123,31 @@ func (s *testSuite1) TestAnalyzeParameters(c *C) { for i := 0; i < 20; i++ { tk.MustExec(fmt.Sprintf("insert into t values (%d)", i)) } + tk.MustExec("insert into t values (19), (19), (19)") + tk.MustExec("set @@tidb_enable_fast_analyze = 1") + executor.MaxSampleSize = 30 tk.MustExec("analyze table t") is := executor.GetInfoSchema(tk.Se.(sessionctx.Context)) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tableInfo := table.Meta() tbl := s.dom.StatsHandle().GetTableStats(tableInfo) - c.Assert(tbl.Columns[1].Len(), Equals, 20) - - tk.MustExec("analyze table t with 4 buckets") + col := tbl.Columns[1] + c.Assert(col.Len(), Equals, 20) + c.Assert(len(col.CMSketch.TopN()), Equals, 20) + width, depth := col.CMSketch.GetWidthAndDepth() + c.Assert(depth, Equals, int32(5)) + c.Assert(width, Equals, int32(2048)) + + tk.MustExec("analyze table t with 4 buckets, 1 topn, 4 cmsketch width, 4 cmsketch depth") tbl = s.dom.StatsHandle().GetTableStats(tableInfo) - c.Assert(tbl.Columns[1].Len(), Equals, 4) + col = tbl.Columns[1] + c.Assert(col.Len(), Equals, 4) + c.Assert(len(col.CMSketch.TopN()), Equals, 1) + width, depth = col.CMSketch.GetWidthAndDepth() + c.Assert(depth, Equals, int32(4)) + c.Assert(width, Equals, int32(4)) } func (s *testSuite1) TestAnalyzeTooLongColumns(c *C) { diff --git a/executor/builder.go b/executor/builder.go index d2d048616bd93..cb77e13ed179e 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1408,7 +1408,7 @@ func (b *executorBuilder) buildDelete(v *plannercore.Delete) Executor { return deleteExec } -func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeIndexTask, maxNumBuckets uint64, autoAnalyze string) *analyzeTask { +func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeIndexTask, opts map[ast.AnalyzeOptionType]uint64, autoAnalyze string) *analyzeTask { _, offset := timeutil.Zone(b.ctx.GetSessionVars().Location()) sc := b.ctx.GetSessionVars().StmtCtx e := &AnalyzeIndexExec{ @@ -1422,24 +1422,24 @@ func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeInde Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, - maxNumBuckets: maxNumBuckets, + opts: opts, } e.analyzePB.IdxReq = &tipb.AnalyzeIndexReq{ - BucketSize: int64(maxNumBuckets), + BucketSize: int64(opts[ast.AnalyzeOptNumBuckets]), NumColumns: int32(len(task.IndexInfo.Columns)), } - depth := int32(defaultCMSketchDepth) - width := int32(defaultCMSketchWidth) + depth := int32(opts[ast.AnalyzeOptCMSketchDepth]) + width := int32(opts[ast.AnalyzeOptCMSketchWidth]) e.analyzePB.IdxReq.CmsketchDepth = &depth e.analyzePB.IdxReq.CmsketchWidth = &width job := &statistics.AnalyzeJob{DBName: task.DBName, TableName: task.TableName, PartitionName: task.PartitionName, JobInfo: autoAnalyze + "analyze index " + task.IndexInfo.Name.O} return &analyzeTask{taskType: idxTask, idxExec: e, job: job} } -func (b *executorBuilder) buildAnalyzeIndexIncremental(task plannercore.AnalyzeIndexTask, maxNumBuckets uint64) *analyzeTask { +func (b *executorBuilder) buildAnalyzeIndexIncremental(task plannercore.AnalyzeIndexTask, opts map[ast.AnalyzeOptionType]uint64) *analyzeTask { h := domain.GetDomain(b.ctx).StatsHandle() statsTbl := h.GetPartitionStats(&model.TableInfo{}, task.PhysicalTableID) - analyzeTask := b.buildAnalyzeIndexPushdown(task, maxNumBuckets, "") + analyzeTask := b.buildAnalyzeIndexPushdown(task, opts, "") if statsTbl.Pseudo { return analyzeTask } @@ -1470,7 +1470,7 @@ func (b *executorBuilder) buildAnalyzeIndexIncremental(task plannercore.AnalyzeI return analyzeTask } -func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeColumnsTask, maxNumBuckets uint64, autoAnalyze string) *analyzeTask { +func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64, autoAnalyze string) *analyzeTask { cols := task.ColsInfo if task.PKInfo != nil { cols = append([]*model.ColumnInfo{task.PKInfo}, cols...) @@ -1490,12 +1490,12 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo Flags: sc.PushDownFlags(), TimeZoneOffset: offset, }, - maxNumBuckets: maxNumBuckets, + opts: opts, } - depth := int32(defaultCMSketchDepth) - width := int32(defaultCMSketchWidth) + depth := int32(opts[ast.AnalyzeOptCMSketchDepth]) + width := int32(opts[ast.AnalyzeOptCMSketchWidth]) e.analyzePB.ColReq = &tipb.AnalyzeColumnsReq{ - BucketSize: int64(maxNumBuckets), + BucketSize: int64(opts[ast.AnalyzeOptNumBuckets]), SampleSize: maxRegionSampleSize, SketchSize: maxSketchSize, ColumnsInfo: model.ColumnsToProto(cols, task.PKInfo != nil), @@ -1507,10 +1507,10 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(task plannercore.AnalyzeCo return &analyzeTask{taskType: colTask, colExec: e, job: job} } -func (b *executorBuilder) buildAnalyzePKIncremental(task plannercore.AnalyzeColumnsTask, maxNumBuckets uint64) *analyzeTask { +func (b *executorBuilder) buildAnalyzePKIncremental(task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64) *analyzeTask { h := domain.GetDomain(b.ctx).StatsHandle() statsTbl := h.GetPartitionStats(&model.TableInfo{}, task.PhysicalTableID) - analyzeTask := b.buildAnalyzeColumnsPushdown(task, maxNumBuckets, "") + analyzeTask := b.buildAnalyzeColumnsPushdown(task, opts, "") if statsTbl.Pseudo { return analyzeTask } @@ -1541,7 +1541,7 @@ func (b *executorBuilder) buildAnalyzePKIncremental(task plannercore.AnalyzeColu return analyzeTask } -func (b *executorBuilder) buildAnalyzeFastColumn(e *AnalyzeExec, task plannercore.AnalyzeColumnsTask, maxNumBuckets uint64) { +func (b *executorBuilder) buildAnalyzeFastColumn(e *AnalyzeExec, task plannercore.AnalyzeColumnsTask, opts map[ast.AnalyzeOptionType]uint64) { findTask := false for _, eTask := range e.tasks { if eTask.fastExec.physicalTableID == task.PhysicalTableID { @@ -1563,7 +1563,7 @@ func (b *executorBuilder) buildAnalyzeFastColumn(e *AnalyzeExec, task plannercor physicalTableID: task.PhysicalTableID, colsInfo: task.ColsInfo, pkInfo: task.PKInfo, - maxNumBuckets: maxNumBuckets, + opts: opts, tblInfo: task.TblInfo, concurrency: concurrency, wg: &sync.WaitGroup{}, @@ -1573,7 +1573,7 @@ func (b *executorBuilder) buildAnalyzeFastColumn(e *AnalyzeExec, task plannercor } } -func (b *executorBuilder) buildAnalyzeFastIndex(e *AnalyzeExec, task plannercore.AnalyzeIndexTask, maxNumBuckets uint64) { +func (b *executorBuilder) buildAnalyzeFastIndex(e *AnalyzeExec, task plannercore.AnalyzeIndexTask, opts map[ast.AnalyzeOptionType]uint64) { findTask := false for _, eTask := range e.tasks { if eTask.fastExec.physicalTableID == task.PhysicalTableID { @@ -1594,7 +1594,7 @@ func (b *executorBuilder) buildAnalyzeFastIndex(e *AnalyzeExec, task plannercore ctx: b.ctx, physicalTableID: task.PhysicalTableID, idxsInfo: []*model.IndexInfo{task.IndexInfo}, - maxNumBuckets: maxNumBuckets, + opts: opts, tblInfo: task.TblInfo, concurrency: concurrency, wg: &sync.WaitGroup{}, @@ -1617,12 +1617,12 @@ func (b *executorBuilder) buildAnalyze(v *plannercore.Analyze) Executor { } for _, task := range v.ColTasks { if task.Incremental { - e.tasks = append(e.tasks, b.buildAnalyzePKIncremental(task, v.MaxNumBuckets)) + e.tasks = append(e.tasks, b.buildAnalyzePKIncremental(task, v.Opts)) } else { if enableFastAnalyze { - b.buildAnalyzeFastColumn(e, task, v.MaxNumBuckets) + b.buildAnalyzeFastColumn(e, task, v.Opts) } else { - e.tasks = append(e.tasks, b.buildAnalyzeColumnsPushdown(task, v.MaxNumBuckets, autoAnalyze)) + e.tasks = append(e.tasks, b.buildAnalyzeColumnsPushdown(task, v.Opts, autoAnalyze)) } } if b.err != nil { @@ -1631,12 +1631,12 @@ func (b *executorBuilder) buildAnalyze(v *plannercore.Analyze) Executor { } for _, task := range v.IdxTasks { if task.Incremental { - e.tasks = append(e.tasks, b.buildAnalyzeIndexIncremental(task, v.MaxNumBuckets)) + e.tasks = append(e.tasks, b.buildAnalyzeIndexIncremental(task, v.Opts)) } else { if enableFastAnalyze { - b.buildAnalyzeFastIndex(e, task, v.MaxNumBuckets) + b.buildAnalyzeFastIndex(e, task, v.Opts) } else { - e.tasks = append(e.tasks, b.buildAnalyzeIndexPushdown(task, v.MaxNumBuckets, autoAnalyze)) + e.tasks = append(e.tasks, b.buildAnalyzeIndexPushdown(task, v.Opts, autoAnalyze)) } } if b.err != nil { diff --git a/executor/executor_test.go b/executor/executor_test.go index 762f31a7faee1..139bb7f8b2629 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2714,6 +2714,7 @@ func (s *testSuite1) SetUpSuite(c *C) { mockstore.WithHijackClient(hijackClient), ) c.Assert(err, IsNil) + session.SetStatsLease(0) s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) s.dom.SetStatsUpdating(true) diff --git a/go.mod b/go.mod index ba19c42dd4d89..70745f85587d5 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903 + github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 6d6d387061618..ca2f017ad302e 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903 h1:mRZH1M//ZhlpJ9ByB6TyEFErVO5vsfeWyA8a0SklkF0= -github.com/pingcap/parser v0.0.0-20190719041739-ff945b25f903/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1 h1:/L2n0wamoKiRlXOn7xCNk8ejgXJbjmC3X54pGYSgPvQ= +github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 8cd2290e6682b..0cf8a357a5716 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -495,9 +495,9 @@ type AnalyzeIndexTask struct { type Analyze struct { baseSchemaProducer - ColTasks []AnalyzeColumnsTask - IdxTasks []AnalyzeIndexTask - MaxNumBuckets uint64 + ColTasks []AnalyzeColumnsTask + IdxTasks []AnalyzeIndexTask + Opts map[ast.AnalyzeOptionType]uint64 } // LoadData represents a loaddata plan. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index b4ae31c8bae6d..48b9a76669577 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -16,10 +16,10 @@ package core import ( "bytes" "context" + "encoding/binary" "fmt" "strings" - "github.com/cznic/mathutil" "github.com/pingcap/errors" "github.com/pingcap/parser" "github.com/pingcap/parser/ast" @@ -30,6 +30,7 @@ import ( "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" + "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -821,8 +822,8 @@ func getPhysicalIDsAndPartitionNames(tblInfo *model.TableInfo, partitionNames [] return ids, names, nil } -func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt) (Plan, error) { - p := &Analyze{MaxNumBuckets: as.MaxNumBuckets} +func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64) (Plan, error) { + p := &Analyze{Opts: opts} for _, tbl := range as.TableNames { if tbl.TableInfo.IsView() { return nil, errors.Errorf("analyze %s is not supported now.", tbl.Name.O) @@ -857,8 +858,8 @@ func (b *PlanBuilder) buildAnalyzeTable(as *ast.AnalyzeTableStmt) (Plan, error) return p, nil } -func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt) (Plan, error) { - p := &Analyze{MaxNumBuckets: as.MaxNumBuckets} +func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64) (Plan, error) { + p := &Analyze{Opts: opts} tblInfo := as.TableNames[0].TableInfo physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames) if err != nil { @@ -885,8 +886,8 @@ func (b *PlanBuilder) buildAnalyzeIndex(as *ast.AnalyzeTableStmt) (Plan, error) return p, nil } -func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt) (Plan, error) { - p := &Analyze{MaxNumBuckets: as.MaxNumBuckets} +func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt, opts map[ast.AnalyzeOptionType]uint64) (Plan, error) { + p := &Analyze{Opts: opts} tblInfo := as.TableNames[0].TableInfo physicalIDs, names, err := getPhysicalIDsAndPartitionNames(tblInfo, as.PartitionNames) if err != nil { @@ -910,10 +911,44 @@ func (b *PlanBuilder) buildAnalyzeAllIndex(as *ast.AnalyzeTableStmt) (Plan, erro return p, nil } -const ( - defaultMaxNumBuckets = 256 - numBucketsLimit = 1024 -) +var cmSketchSizeLimit = kv.TxnEntrySizeLimit / binary.MaxVarintLen32 + +var analyzeOptionLimit = map[ast.AnalyzeOptionType]uint64{ + ast.AnalyzeOptNumBuckets: 1024, + ast.AnalyzeOptNumTopN: 1024, + ast.AnalyzeOptCMSketchWidth: uint64(cmSketchSizeLimit), + ast.AnalyzeOptCMSketchDepth: uint64(cmSketchSizeLimit), +} + +var analyzeOptionDefault = map[ast.AnalyzeOptionType]uint64{ + ast.AnalyzeOptNumBuckets: 256, + ast.AnalyzeOptNumTopN: 20, + ast.AnalyzeOptCMSketchWidth: 2048, + ast.AnalyzeOptCMSketchDepth: 5, +} + +func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint64, error) { + optMap := make(map[ast.AnalyzeOptionType]uint64, len(analyzeOptionDefault)) + for key, val := range analyzeOptionDefault { + optMap[key] = val + } + for _, opt := range opts { + if opt.Type == ast.AnalyzeOptNumTopN { + if opt.Value > analyzeOptionLimit[opt.Type] { + return nil, errors.Errorf("value of analyze option %s should not larger than %d", ast.AnalyzeOptionString[opt.Type], analyzeOptionLimit[opt.Type]) + } + } else { + if opt.Value == 0 || opt.Value > analyzeOptionLimit[opt.Type] { + return nil, errors.Errorf("value of analyze option %s should be positive and not larger than %d", ast.AnalyzeOptionString[opt.Type], analyzeOptionLimit[opt.Type]) + } + } + optMap[opt.Type] = opt.Value + } + if optMap[ast.AnalyzeOptCMSketchWidth]*optMap[ast.AnalyzeOptCMSketchDepth] > uint64(cmSketchSizeLimit) { + return nil, errors.Errorf("cm sketch size(depth * width) should not larger than %d", cmSketchSizeLimit) + } + return optMap, nil +} func (b *PlanBuilder) buildAnalyze(as *ast.AnalyzeTableStmt) (Plan, error) { // If enable fast analyze, the storage must be tikv.Storage. @@ -930,18 +965,17 @@ func (b *PlanBuilder) buildAnalyze(as *ast.AnalyzeTableStmt) (Plan, error) { b.visitInfo = appendVisitInfo(b.visitInfo, mysql.InsertPriv, tbl.Schema.O, tbl.Name.O, "", insertErr) b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, tbl.Schema.O, tbl.Name.O, "", selectErr) } - if as.MaxNumBuckets == 0 { - as.MaxNumBuckets = defaultMaxNumBuckets - } else { - as.MaxNumBuckets = mathutil.MinUint64(as.MaxNumBuckets, numBucketsLimit) + opts, err := handleAnalyzeOptions(as.AnalyzeOpts) + if err != nil { + return nil, err } if as.IndexFlag { if len(as.IndexNames) == 0 { - return b.buildAnalyzeAllIndex(as) + return b.buildAnalyzeAllIndex(as, opts) } - return b.buildAnalyzeIndex(as) + return b.buildAnalyzeIndex(as, opts) } - return b.buildAnalyzeTable(as) + return b.buildAnalyzeTable(as, opts) } func buildShowNextRowID() *expression.Schema { From 566c3278e3cc4a036384fef644b0f2279ab4a3ff Mon Sep 17 00:00:00 2001 From: ono-yoko <53291476+ono-yoko@users.noreply.github.com> Date: Thu, 25 Jul 2019 18:12:06 +0800 Subject: [PATCH 096/196] expression: improve the compatibility with mysql when datatime is invalid. (#11445) All tests passed, auto merged by Bot --- expression/integration_test.go | 5 +++++ types/time.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 403d77286c3a1..e07ffe91fc2ac 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -3562,6 +3562,11 @@ func (s *testIntegrationSuite) TestTimeLiteral(c *C) { _, err = tk.Exec("select time '20171231235959.999999';") c.Assert(err, NotNil) c.Assert(terror.ErrorEqual(err, types.ErrIncorrectDatetimeValue.GenWithStackByArgs("20171231235959.999999")), IsTrue) + + _, err = tk.Exec("select ADDDATE('2008-01-34', -1);") + c.Assert(err, IsNil) + tk.MustQuery("Show warnings;").Check(testutil.RowsWithSep("|", + "Warning|1292|Incorrect datetime value: '2008-1-34'")) } func (s *testIntegrationSuite) TestTimestampLiteral(c *C) { diff --git a/types/time.go b/types/time.go index cccb63c95306f..b44c261597521 100644 --- a/types/time.go +++ b/types/time.go @@ -1469,7 +1469,7 @@ func checkDateRange(t MysqlTime) error { func checkMonthDay(year, month, day int, allowInvalidDate bool) error { if month < 0 || month > 12 { - return errors.Trace(ErrIncorrectDatetimeValue.GenWithStackByArgs(month)) + return errors.Trace(ErrIncorrectDatetimeValue.GenWithStackByArgs(fmt.Sprintf("%d-%d-%d", year, month, day))) } maxDay := 31 @@ -1483,7 +1483,7 @@ func checkMonthDay(year, month, day int, allowInvalidDate bool) error { } if day < 0 || day > maxDay { - return errors.Trace(ErrIncorrectDatetimeValue.GenWithStackByArgs(day)) + return errors.Trace(ErrIncorrectDatetimeValue.GenWithStackByArgs(fmt.Sprintf("%d-%d-%d", year, month, day))) } return nil } From 47ed7ec170cce1d60361886bfcdebbdc97c85284 Mon Sep 17 00:00:00 2001 From: gaoxingliang Date: Fri, 26 Jul 2019 10:21:01 +0800 Subject: [PATCH 097/196] =?UTF-8?q?planner/core:=20give=20the=20full=20sch?= =?UTF-8?q?ema=20information=20when=20group=20by=20f=E2=80=A6=20(#11406)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plan_test.go | 66 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 94c36b75ac343..77325fccafc51 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1769,7 +1769,7 @@ func (b *PlanBuilder) checkOnlyFullGroupByWithGroupClause(p LogicalPlan, sel *as } switch errExprLoc.Loc { case ErrExprInSelect: - return ErrFieldNotInGroupBy.GenWithStackByArgs(errExprLoc.Offset+1, errExprLoc.Loc, sel.Fields.Fields[errExprLoc.Offset].Text()) + return ErrFieldNotInGroupBy.GenWithStackByArgs(errExprLoc.Offset+1, errExprLoc.Loc, col.DBName.O+"."+col.TblName.O+"."+col.OrigColName.O) case ErrExprInOrderBy: return ErrFieldNotInGroupBy.GenWithStackByArgs(errExprLoc.Offset+1, errExprLoc.Loc, sel.OrderBy.Items[errExprLoc.Offset].Expr.Text()) } diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 7761ea3f2f4cf..b2770092a5f9c 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -22,6 +22,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -574,6 +575,71 @@ func (s *testPlanSuite) TestDeriveNotNullConds(c *C) { } } +func buildLogicPlan4GroupBy(s *testPlanSuite, c *C, sql string) (Plan, error) { + sqlMode := s.ctx.GetSessionVars().SQLMode + mockedTableInfo := MockSignedTable() + // mock the table info here for later use + // enable only full group by + s.ctx.GetSessionVars().SQLMode = sqlMode | mysql.ModeOnlyFullGroupBy + defer func() { s.ctx.GetSessionVars().SQLMode = sqlMode }() // restore it + comment := Commentf("for %s", sql) + stmt, err := s.ParseOneStmt(sql, "", "") + c.Assert(err, IsNil, comment) + + stmt.(*ast.SelectStmt).From.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName).TableInfo = mockedTableInfo + + return BuildLogicalPlan(context.Background(), s.ctx, stmt, s.is) +} + +func (s *testPlanSuite) TestGroupByWhenNotExistCols(c *C) { + sqlTests := []struct { + sql string + expectedErrMatch string + }{ + { + sql: "select a from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + { + // has an as column alias + sql: "select a as tempField from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + { + // has as table alias + sql: "select tempTable.a from t as tempTable group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.tempTable\\.a'.*", + }, + { + // has a func call + sql: "select length(a) from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + { + // has a func call with two cols + sql: "select length(b + a) from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + { + // has a func call with two cols + sql: "select length(a + b) from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + { + // has a func call with two cols + sql: "select length(a + b) as tempField from t group by b", + expectedErrMatch: ".*contains nonaggregated column 'test\\.t\\.a'.*", + }, + } + for _, test := range sqlTests { + sql := test.sql + p, err := buildLogicPlan4GroupBy(s, c, sql) + c.Assert(err, NotNil) + c.Assert(p, IsNil) + c.Assert(err, ErrorMatches, test.expectedErrMatch) + } +} + func (s *testPlanSuite) TestDupRandJoinCondsPushDown(c *C) { sql := "select * from t as t1 join t t2 on t1.a > rand() and t1.a > rand()" comment := Commentf("for %s", sql) From 884371a580d2d8463eda3871679ea5e3f96ddfe5 Mon Sep 17 00:00:00 2001 From: sdojjy Date: Fri, 26 Jul 2019 11:02:27 +0800 Subject: [PATCH 098/196] planner: add table info into context in fast plan case (#11446) --- planner/core/logical_plan_test.go | 42 +++++++++++++++++++++++++++++++ planner/core/point_get_plan.go | 2 ++ 2 files changed, 44 insertions(+) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index b2770092a5f9c..fac77b8e6c31a 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2539,3 +2539,45 @@ func (s *testPlanSuite) TestSkylinePruning(c *C) { c.Assert(pathsName(paths), Equals, tt.result) } } + +func (s *testPlanSuite) TestFastPlanContextTables(c *C) { + defer testleak.AfterTest(c)() + tests := []struct { + sql string + fastPlan bool + }{ + { + "select * from t where a=1", + true, + }, + { + + "update t set f=0 where a=43215", + true, + }, + { + "delete from t where a =43215", + true, + }, + { + "select * from t where a>1", + false, + }, + } + for _, tt := range tests { + stmt, err := s.ParseOneStmt(tt.sql, "", "") + c.Assert(err, IsNil) + Preprocess(s.ctx, stmt, s.is) + s.ctx.GetSessionVars().StmtCtx.Tables = nil + p := TryFastPlan(s.ctx, stmt) + if tt.fastPlan { + c.Assert(p, NotNil) + c.Assert(len(s.ctx.GetSessionVars().StmtCtx.Tables), Equals, 1) + c.Assert(s.ctx.GetSessionVars().StmtCtx.Tables[0].Table, Equals, "t") + c.Assert(s.ctx.GetSessionVars().StmtCtx.Tables[0].DB, Equals, "test") + } else { + c.Assert(p, IsNil) + c.Assert(len(s.ctx.GetSessionVars().StmtCtx.Tables), Equals, 0) + } + } +} diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 7be3ba1e49b07..00c1665f405ce 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/privilege" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tipb/go-tipb" @@ -264,6 +265,7 @@ func newPointGetPlan(ctx sessionctx.Context, schema *expression.Schema, tbl *mod schema: schema, TblInfo: tbl, } + ctx.GetSessionVars().StmtCtx.Tables = []stmtctx.TableEntry{{DB: ctx.GetSessionVars().CurrentDB, Table: tbl.Name.L}} return p } From ff0132ff34f8d1574f427c3ab8d8e7cf4f295333 Mon Sep 17 00:00:00 2001 From: gaoxingliang Date: Fri, 26 Jul 2019 14:21:15 +0800 Subject: [PATCH 099/196] expression, parser: add function octet_length support (#11451) --- expression/builtin.go | 1 + expression/builtin_string_test.go | 25 ++++++++++++++----------- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/expression/builtin.go b/expression/builtin.go index 9c95f7468027a..98049ba148ba5 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -466,6 +466,7 @@ var funcs = map[string]functionClass{ ast.Mid: &substringFunctionClass{baseFunctionClass{ast.Mid, 3, 3}}, ast.MakeSet: &makeSetFunctionClass{baseFunctionClass{ast.MakeSet, 2, -1}}, ast.Oct: &octFunctionClass{baseFunctionClass{ast.Oct, 1, 1}}, + ast.OctetLength: &lengthFunctionClass{baseFunctionClass{ast.OctetLength, 1, 1}}, ast.Ord: &ordFunctionClass{baseFunctionClass{ast.Ord, 1, 1}}, ast.Position: &locateFunctionClass{baseFunctionClass{ast.Position, 2, 2}}, ast.Quote: "eFunctionClass{baseFunctionClass{ast.Quote, 1, 1}}, diff --git a/expression/builtin_string_test.go b/expression/builtin_string_test.go index b07ebd7d11f13..a9bcaf581ae15 100644 --- a/expression/builtin_string_test.go +++ b/expression/builtin_string_test.go @@ -33,7 +33,7 @@ import ( "github.com/pingcap/tidb/util/testutil" ) -func (s *testEvaluatorSuite) TestLength(c *C) { +func (s *testEvaluatorSuite) TestLengthAndOctetLength(c *C) { defer testleak.AfterTest(c)() cases := []struct { args interface{} @@ -54,18 +54,21 @@ func (s *testEvaluatorSuite) TestLength(c *C) { {errors.New("must error"), 0, false, true}, } - for _, t := range cases { - f, err := newFunctionForTest(s.ctx, ast.Length, s.primitiveValsToConstants([]interface{}{t.args})...) - c.Assert(err, IsNil) - d, err := f.Eval(chunk.Row{}) - if t.getErr { - c.Assert(err, NotNil) - } else { + lengthMethods := []string{ast.Length, ast.OctetLength} + for _, lengthMethod := range lengthMethods { + for _, t := range cases { + f, err := newFunctionForTest(s.ctx, lengthMethod, s.primitiveValsToConstants([]interface{}{t.args})...) c.Assert(err, IsNil) - if t.isNil { - c.Assert(d.Kind(), Equals, types.KindNull) + d, err := f.Eval(chunk.Row{}) + if t.getErr { + c.Assert(err, NotNil) } else { - c.Assert(d.GetInt64(), Equals, t.expected) + c.Assert(err, IsNil) + if t.isNil { + c.Assert(d.Kind(), Equals, types.KindNull) + } else { + c.Assert(d.GetInt64(), Equals, t.expected) + } } } } diff --git a/go.mod b/go.mod index 70745f85587d5..6f5060725a50f 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1 + github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index ca2f017ad302e..368f4dd6e0d9a 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1 h1:/L2n0wamoKiRlXOn7xCNk8ejgXJbjmC3X54pGYSgPvQ= -github.com/pingcap/parser v0.0.0-20190723083556-57e1f3b7a1c1/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd h1:vFLYlcLyvWrIN+3Y9uPExiisDjeRCPh1Vic0vnkxBMs= +github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From bd419c8fdb0c5e4bc0be1f64503fbac38bd97399 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Fri, 26 Jul 2019 14:43:20 +0800 Subject: [PATCH 100/196] stats: fix panic when dump pseudo columns (#11430) --- statistics/handle/dump_test.go | 17 +++++++++++++++++ statistics/table.go | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index 628652c6ae7ca..934518aa44a1c 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -158,3 +158,20 @@ func (s *testStatsSuite) TestDumpCMSketchWithTopN(c *C) { cmsFromJSON := stat.Columns[tableInfo.Columns[0].ID].CMSketch.Copy() c.Check(cms.Equal(cmsFromJSON), IsTrue) } + +func (s *testStatsSuite) TestDumpPseudoColumns(c *C) { + defer cleanEnv(c, s.store, s.do) + testKit := testkit.NewTestKit(c, s.store) + testKit.MustExec("use test") + testKit.MustExec("create table t(a int, b int, index idx(a))") + // Force adding an pseudo tables in stats cache. + testKit.MustQuery("select * from t") + testKit.MustExec("analyze table t index idx") + + is := s.do.InfoSchema() + tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + h := s.do.StatsHandle() + _, err = h.DumpStatsToJSON("test", tbl.Meta(), nil) + c.Assert(err, IsNil) +} diff --git a/statistics/table.go b/statistics/table.go index c95979e81dbde..5805b177e347b 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -454,12 +454,15 @@ func PseudoTable(tblInfo *model.TableInfo) *Table { PhysicalID: fakePhysicalID, Info: col, IsHandle: tblInfo.PKIsHandle && mysql.HasPriKeyFlag(col.Flag), + Histogram: *NewHistogram(col.ID, 0, 0, 0, &col.FieldType, 0, 0), } } } for _, idx := range tblInfo.Indices { if idx.State == model.StatePublic { - t.Indices[idx.ID] = &Index{Info: idx} + t.Indices[idx.ID] = &Index{ + Info: idx, + Histogram: *NewHistogram(idx.ID, 0, 0, 0, types.NewFieldType(mysql.TypeBlob), 0, 0)} } } return t From 1e1cc1f3259315ad01eb8b8c6c45346ac8440894 Mon Sep 17 00:00:00 2001 From: amyangfei Date: Fri, 26 Jul 2019 19:10:30 +0800 Subject: [PATCH 101/196] types: fix string to integer cast (#11295) --- executor/executor.go | 2 + executor/point_get_test.go | 2 + expression/builtin_cast_test.go | 4 ++ planner/core/point_get_plan.go | 5 +- sessionctx/stmtctx/stmtctx.go | 5 ++ types/convert.go | 38 +++++++++-- types/convert_test.go | 117 ++++++++++++++++++++++++++++---- types/datum.go | 6 +- 8 files changed, 154 insertions(+), 25 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 1513bd6554f8a..35a6254e49553 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1350,8 +1350,10 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) { sc.NotFillCache = !opts.SQLCache } sc.PadCharToFullLength = ctx.GetSessionVars().SQLMode.HasPadCharToFullLengthMode() + sc.CastStrToIntStrict = true case *ast.ExplainStmt: sc.InExplainStmt = true + sc.CastStrToIntStrict = true case *ast.ShowStmt: sc.IgnoreTruncate = true sc.IgnoreZeroInDate = true diff --git a/executor/point_get_test.go b/executor/point_get_test.go index b14a7de0cadaf..e4df3c7a2c1bc 100644 --- a/executor/point_get_test.go +++ b/executor/point_get_test.go @@ -399,4 +399,6 @@ func (s *testPointGetSuite) TestIssue10677(c *C) { tk.MustQuery("select * from t where pk = 1").Check(testkit.Rows("1")) tk.MustQuery("desc select * from t where pk = '1'").Check(testkit.Rows("Point_Get_1 1.00 root table:t, handle:1")) tk.MustQuery("select * from t where pk = '1'").Check(testkit.Rows("1")) + tk.MustQuery("desc select * from t where pk = '1.0'").Check(testkit.Rows("Point_Get_1 1.00 root table:t, handle:1")) + tk.MustQuery("select * from t where pk = '1.0'").Check(testkit.Rows("1")) } diff --git a/expression/builtin_cast_test.go b/expression/builtin_cast_test.go index 05207acfde41a..c0636e4b34a04 100644 --- a/expression/builtin_cast_test.go +++ b/expression/builtin_cast_test.go @@ -88,7 +88,11 @@ func (s *testEvaluatorSuite) TestCast(c *C) { c.Assert(terror.ErrorEqual(errWarnAllowedPacketOverflowed, lastWarn.Err), IsTrue, Commentf("err %v", lastWarn.Err)) origSc := sc + oldInSelectStmt := sc.InSelectStmt sc.InSelectStmt = true + defer func() { + sc.InSelectStmt = oldInSelectStmt + }() sc.OverflowAsWarning = true // cast('18446744073709551616' as unsigned); diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 00c1665f405ce..83810ce96a10c 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -220,7 +220,10 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt) *PointGetP p.IsTableDual = true return p } - return nil + // some scenarios cast to int with error, but we may use this value in point get + if !terror.ErrorEqual(types.ErrTruncatedWrongVal, err) { + return nil + } } cmp, err := intDatum.CompareDatum(ctx.GetSessionVars().StmtCtx, &handlePair.value) if err != nil { diff --git a/sessionctx/stmtctx/stmtctx.go b/sessionctx/stmtctx/stmtctx.go index e19345d469ffd..1fd16287633ef 100644 --- a/sessionctx/stmtctx/stmtctx.go +++ b/sessionctx/stmtctx/stmtctx.go @@ -70,6 +70,11 @@ type StatementContext struct { BatchCheck bool InNullRejectCheck bool AllowInvalidDate bool + // CastStrToIntStrict is used to control the way we cast float format string to int. + // If ConvertStrToIntStrict is false, we convert it to a valid float string first, + // then cast the float string to int string. Otherwise, we cast string to integer + // prefix in a strict way, only extract 0-9 and (+ or - in first bit). + CastStrToIntStrict bool // mu struct holds variables that change during execution. mu struct { diff --git a/types/convert.go b/types/convert.go index 6e330abee0f69..9a52701cabf2e 100644 --- a/types/convert.go +++ b/types/convert.go @@ -362,11 +362,37 @@ func NumberToDuration(number int64, fsp int) (Duration, error) { // getValidIntPrefix gets prefix of the string which can be successfully parsed as int. func getValidIntPrefix(sc *stmtctx.StatementContext, str string) (string, error) { - floatPrefix, err := getValidFloatPrefix(sc, str) - if err != nil { - return floatPrefix, errors.Trace(err) + if !sc.CastStrToIntStrict { + floatPrefix, err := getValidFloatPrefix(sc, str) + if err != nil { + return floatPrefix, errors.Trace(err) + } + return floatStrToIntStr(sc, floatPrefix, str) + } + + validLen := 0 + + for i := 0; i < len(str); i++ { + c := str[i] + if (c == '+' || c == '-') && i == 0 { + continue + } + + if c >= '0' && c <= '9' { + validLen = i + 1 + continue + } + + break + } + valid := str[:validLen] + if valid == "" { + valid = "0" + } + if validLen == 0 || validLen != len(str) { + return valid, errors.Trace(handleTruncateError(sc, ErrTruncatedWrongVal.GenWithStackByArgs("INTEGER", str))) } - return floatStrToIntStr(sc, floatPrefix, str) + return valid, nil } // roundIntStr is to round int string base on the number following dot. @@ -580,7 +606,7 @@ func ConvertJSONToDecimal(sc *stmtctx.StatementContext, j json.BinaryJSON) (*MyD // getValidFloatPrefix gets prefix of string which can be successfully parsed as float. func getValidFloatPrefix(sc *stmtctx.StatementContext, s string) (valid string, err error) { - if sc.InDeleteStmt && s == "" { + if (sc.InDeleteStmt || sc.InSelectStmt || sc.InUpdateStmt) && s == "" { return "0", nil } @@ -624,7 +650,7 @@ func getValidFloatPrefix(sc *stmtctx.StatementContext, s string) (valid string, valid = "0" } if validLen == 0 || validLen != len(s) { - err = errors.Trace(handleTruncateError(sc)) + err = errors.Trace(handleTruncateError(sc, ErrTruncated)) } return valid, err } diff --git a/types/convert_test.go b/types/convert_test.go index 3f6ba218589dc..d2bcb372a0269 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -462,28 +462,41 @@ func (s *testTypeConvertSuite) TestStrToNum(c *C) { testStrToFloat(c, "-1e649", -math.MaxFloat64, true, ErrTruncatedWrongVal) testStrToFloat(c, "-1e649", -math.MaxFloat64, false, nil) - // for issue #10806 - testDeleteEmptyStringError(c) + // for issue #10806, #11179 + testSelectUpdateDeleteEmptyStringError(c) } -func testDeleteEmptyStringError(c *C) { +func testSelectUpdateDeleteEmptyStringError(c *C) { + testCases := []struct { + inSelect bool + inUpdate bool + inDelete bool + }{ + {true, false, false}, + {false, true, false}, + {false, false, true}, + } sc := new(stmtctx.StatementContext) - sc.InDeleteStmt = true + for _, tc := range testCases { + sc.InSelectStmt = tc.inSelect + sc.InUpdateStmt = tc.inUpdate + sc.InDeleteStmt = tc.inDelete - str := "" - expect := 0 + str := "" + expect := 0 - val, err := StrToInt(sc, str) - c.Assert(err, IsNil) - c.Assert(val, Equals, int64(expect)) + val, err := StrToInt(sc, str) + c.Assert(err, IsNil) + c.Assert(val, Equals, int64(expect)) - val1, err := StrToUint(sc, str) - c.Assert(err, IsNil) - c.Assert(val1, Equals, uint64(expect)) + val1, err := StrToUint(sc, str) + c.Assert(err, IsNil) + c.Assert(val1, Equals, uint64(expect)) - val2, err := StrToFloat(sc, str) - c.Assert(err, IsNil) - c.Assert(val2, Equals, float64(expect)) + val2, err := StrToFloat(sc, str) + c.Assert(err, IsNil) + c.Assert(val2, Equals, float64(expect)) + } } func (s *testTypeConvertSuite) TestFieldTypeToStr(c *C) { @@ -689,6 +702,80 @@ func (s *testTypeConvertSuite) TestConvert(c *C) { signedAccept(c, mysql.TypeNewDecimal, dec, "-0.00123") } +func (s *testTypeConvertSuite) TestGetValidInt(c *C) { + tests := []struct { + origin string + valid string + warning bool + }{ + {"100", "100", false}, + {"-100", "-100", false}, + {"1abc", "1", true}, + {"-1-1", "-1", true}, + {"+1+1", "+1", true}, + {"123..34", "123", true}, + {"123.23E-10", "123", true}, + {"1.1e1.3", "1", true}, + {"11e1.3", "11", true}, + {"1.", "1", true}, + {".1", "0", true}, + {"", "0", true}, + {"123e+", "123", true}, + {"123de", "123", true}, + } + sc := new(stmtctx.StatementContext) + sc.TruncateAsWarning = true + sc.CastStrToIntStrict = true + warningCount := 0 + for _, tt := range tests { + prefix, err := getValidIntPrefix(sc, tt.origin) + c.Assert(err, IsNil) + c.Assert(prefix, Equals, tt.valid) + _, err = strconv.ParseInt(prefix, 10, 64) + c.Assert(err, IsNil) + warnings := sc.GetWarnings() + if tt.warning { + c.Assert(warnings, HasLen, warningCount+1) + c.Assert(terror.ErrorEqual(warnings[len(warnings)-1].Err, ErrTruncatedWrongVal), IsTrue) + warningCount += 1 + } else { + c.Assert(warnings, HasLen, warningCount) + } + } + + tests2 := []struct { + origin string + valid string + warning bool + }{ + {"100", "100", false}, + {"-100", "-100", false}, + {"1abc", "1", true}, + {"-1-1", "-1", true}, + {"+1+1", "+1", true}, + {"123..34", "123.", true}, + {"123.23E-10", "0", false}, + {"1.1e1.3", "1.1e1", true}, + {"11e1.3", "11e1", true}, + {"1.", "1", false}, + {".1", "0", false}, + {"", "0", true}, + {"123e+", "123", true}, + {"123de", "123", true}, + } + sc.TruncateAsWarning = false + sc.CastStrToIntStrict = false + for _, tt := range tests2 { + prefix, err := getValidIntPrefix(sc, tt.origin) + if tt.warning { + c.Assert(terror.ErrorEqual(err, ErrTruncated), IsTrue) + } else { + c.Assert(err, IsNil) + } + c.Assert(prefix, Equals, tt.valid) + } +} + func (s *testTypeConvertSuite) TestGetValidFloat(c *C) { tests := []struct { origin string diff --git a/types/datum.go b/types/datum.go index 92dd3dc40ba86..c4f7edb276467 100644 --- a/types/datum.go +++ b/types/datum.go @@ -1780,14 +1780,14 @@ func (ds *datumsSorter) Swap(i, j int) { ds.datums[i], ds.datums[j] = ds.datums[j], ds.datums[i] } -func handleTruncateError(sc *stmtctx.StatementContext) error { +func handleTruncateError(sc *stmtctx.StatementContext, err error) error { if sc.IgnoreTruncate { return nil } if !sc.TruncateAsWarning { - return ErrTruncated + return err } - sc.AppendWarning(ErrTruncated) + sc.AppendWarning(err) return nil } From 85c790ec97df7a04e9ca4a016acb253d316f93aa Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 26 Jul 2019 19:31:06 +0800 Subject: [PATCH 102/196] planner, executor: tiny refactor (#11420) --- executor/builder.go | 72 +------------- executor/delete.go | 83 +++------------- executor/update.go | 114 ++++++++++----------- executor/write.go | 9 -- planner/core/common_plans.go | 5 +- planner/core/logical_plan_builder.go | 143 +++++++++++++++++++++++++-- planner/core/point_get_plan.go | 24 +++++ 7 files changed, 236 insertions(+), 214 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index cb77e13ed179e..ebd8295088404 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -1314,78 +1314,17 @@ func (b *executorBuilder) buildUpdate(v *plannercore.Update) Executor { if b.err != nil { return nil } - columns2Handle := buildColumns2Handle(v.SelectPlan.Schema(), tblID2table) base := newBaseExecutor(b.ctx, nil, v.ExplainID(), selExec) base.initCap = chunk.ZeroCapacity updateExec := &UpdateExec{ baseExecutor: base, - SelectExec: selExec, OrderedList: v.OrderedList, tblID2table: tblID2table, - columns2Handle: columns2Handle, + tblColPosInfos: v.TblColPosInfos, } return updateExec } -// cols2Handle represents an mapper from column index to handle index. -type cols2Handle struct { - // start and end represent the ordinal range [start, end) of the consecutive columns. - start, end int32 - // handleOrdinal represents the ordinal of the handle column. - handleOrdinal int32 -} - -// cols2HandleSlice attaches the methods of sort.Interface to []cols2Handle sorting in increasing order. -type cols2HandleSlice []cols2Handle - -// Len implements sort.Interface#Len. -func (c cols2HandleSlice) Len() int { - return len(c) -} - -// Swap implements sort.Interface#Swap. -func (c cols2HandleSlice) Swap(i, j int) { - c[i], c[j] = c[j], c[i] -} - -// Less implements sort.Interface#Less. -func (c cols2HandleSlice) Less(i, j int) bool { - return c[i].start < c[j].start -} - -// findHandle finds the ordinal of the corresponding handle column. -func (c cols2HandleSlice) findHandle(ordinal int32) (int32, bool) { - if c == nil || len(c) == 0 { - return 0, false - } - // find the smallest index of the range that its start great than ordinal. - // @see https://godoc.org/sort#Search - rangeBehindOrdinal := sort.Search(len(c), func(i int) bool { return c[i].start > ordinal }) - if rangeBehindOrdinal == 0 { - return 0, false - } - return c[rangeBehindOrdinal-1].handleOrdinal, true -} - -// buildColumns2Handle builds columns to handle mapping. -func buildColumns2Handle(schema *expression.Schema, tblID2Table map[int64]table.Table) cols2HandleSlice { - if len(schema.TblID2Handle) < 2 { - // skip buildColumns2Handle mapping if there are only single table. - return nil - } - var cols2Handles cols2HandleSlice - for tblID, handleCols := range schema.TblID2Handle { - tbl := tblID2Table[tblID] - for _, handleCol := range handleCols { - offset := getTableOffset(schema, handleCol) - end := offset + len(tbl.WritableCols()) - cols2Handles = append(cols2Handles, cols2Handle{int32(offset), int32(end), int32(handleCol.Index)}) - } - } - sort.Sort(cols2Handles) - return cols2Handles -} - func (b *executorBuilder) buildDelete(v *plannercore.Delete) Executor { tblID2table := make(map[int64]table.Table) for id := range v.SelectPlan.Schema().TblID2Handle { @@ -1399,11 +1338,10 @@ func (b *executorBuilder) buildDelete(v *plannercore.Delete) Executor { base := newBaseExecutor(b.ctx, nil, v.ExplainID(), selExec) base.initCap = chunk.ZeroCapacity deleteExec := &DeleteExec{ - baseExecutor: base, - SelectExec: selExec, - Tables: v.Tables, - IsMultiTable: v.IsMultiTable, - tblID2Table: tblID2table, + baseExecutor: base, + IsMultiTable: v.IsMultiTable, + tblID2Table: tblID2table, + tblColPosInfos: v.TblColPosInfos, } return deleteExec } diff --git a/executor/delete.go b/executor/delete.go index 7004dcf989edb..6170c1ea17101 100644 --- a/executor/delete.go +++ b/executor/delete.go @@ -16,8 +16,8 @@ package executor import ( "context" - "github.com/pingcap/parser/ast" "github.com/pingcap/tidb/expression" + plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" @@ -29,16 +29,12 @@ import ( type DeleteExec struct { baseExecutor - SelectExec Executor - - Tables []*ast.TableName IsMultiTable bool tblID2Table map[int64]table.Table - // tblMap is the table map value is an array which contains table aliases. - // Table ID may not be unique for deleting multiple tables, for statements like - // `delete from t as t1, t as t2`, the same table has two alias, we have to identify a table - // by its alias instead of ID. - tblMap map[int64][]*ast.TableName + + // tblColPosInfos stores relationship between column ordinal to its table handle. + // the columns ordinals is present in ordinal range format, @see plannercore.TblColPosInfos + tblColPosInfos plannercore.TblColPosInfoSlice } // Next implements the Executor Next interface. @@ -50,20 +46,6 @@ func (e *DeleteExec) Next(ctx context.Context, req *chunk.Chunk) error { return e.deleteSingleTableByChunk(ctx) } -// matchingDeletingTable checks whether this column is from the table which is in the deleting list. -func (e *DeleteExec) matchingDeletingTable(tableID int64, col *expression.Column) bool { - names, ok := e.tblMap[tableID] - if !ok { - return false - } - for _, n := range names { - if (col.DBName.L == "" || col.DBName.L == n.Schema.L) && col.TblName.L == n.Name.L { - return true - } - } - return false -} - func (e *DeleteExec) deleteOneRow(tbl table.Table, handleCol *expression.Column, row []types.Datum) error { end := len(row) if handleIsExtra(handleCol) { @@ -132,50 +114,20 @@ func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error { return nil } -func (e *DeleteExec) initialMultiTableTblMap() { - e.tblMap = make(map[int64][]*ast.TableName, len(e.Tables)) - for _, t := range e.Tables { - e.tblMap[t.TableInfo.ID] = append(e.tblMap[t.TableInfo.ID], t) - } -} - -func (e *DeleteExec) getColPosInfos(schema *expression.Schema) []tblColPosInfo { - var colPosInfos []tblColPosInfo - // Extract the columns' position information of this table in the delete's schema, together with the table id - // and its handle's position in the schema. - for id, cols := range schema.TblID2Handle { - tbl := e.tblID2Table[id] - for _, col := range cols { - if !e.matchingDeletingTable(id, col) { - continue - } - offset := getTableOffset(schema, col) - end := offset + len(tbl.Cols()) - colPosInfos = append(colPosInfos, tblColPosInfo{tblID: id, colBeginIndex: offset, colEndIndex: end, handleIndex: col.Index}) - } - } - return colPosInfos -} - -func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []tblColPosInfo, joinedRow []types.Datum) { +func (e *DeleteExec) composeTblRowMap(tblRowMap tableRowMapType, colPosInfos []plannercore.TblColPosInfo, joinedRow []types.Datum) { // iterate all the joined tables, and got the copresonding rows in joinedRow. for _, info := range colPosInfos { - if tblRowMap[info.tblID] == nil { - tblRowMap[info.tblID] = make(map[int64][]types.Datum) + if tblRowMap[info.TblID] == nil { + tblRowMap[info.TblID] = make(map[int64][]types.Datum) } - handle := joinedRow[info.handleIndex].GetInt64() - // tblRowMap[info.tblID][handle] hold the row datas binding to this table and this handle. - tblRowMap[info.tblID][handle] = joinedRow[info.colBeginIndex:info.colEndIndex] + handle := joinedRow[info.HandleOrdinal].GetInt64() + // tblRowMap[info.TblID][handle] hold the row datas binding to this table and this handle. + tblRowMap[info.TblID][handle] = joinedRow[info.Start:info.End] } } func (e *DeleteExec) deleteMultiTablesByChunk(ctx context.Context) error { - if len(e.Tables) == 0 { - return nil - } - - e.initialMultiTableTblMap() - colPosInfos := e.getColPosInfos(e.children[0].Schema()) + colPosInfos := e.tblColPosInfos tblRowMap := make(tableRowMapType) fields := retTypes(e.children[0]) chk := newFirstChunk(e.children[0]) @@ -223,19 +175,12 @@ func (e *DeleteExec) removeRow(ctx sessionctx.Context, t table.Table, h int64, d // Close implements the Executor Close interface. func (e *DeleteExec) Close() error { - return e.SelectExec.Close() + return e.children[0].Close() } // Open implements the Executor Open interface. func (e *DeleteExec) Open(ctx context.Context) error { - return e.SelectExec.Open(ctx) -} - -type tblColPosInfo struct { - tblID int64 - colBeginIndex int - colEndIndex int - handleIndex int + return e.children[0].Open(ctx) } // tableRowMapType is a map for unique (Table, Row) pair. key is the tableID. diff --git a/executor/update.go b/executor/update.go index e32b46170b812..2ac64ad2ed07a 100644 --- a/executor/update.go +++ b/executor/update.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" + plannercore "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" @@ -33,7 +34,6 @@ import ( type UpdateExec struct { baseExecutor - SelectExec Executor OrderedList []*expression.Assignment // updatedRowKeys is a map for unique (Table, handle) pair. @@ -46,9 +46,9 @@ type UpdateExec struct { fetched bool cursor int matched uint64 // a counter of matched rows during update - // columns2Handle stores relationship between column ordinal to its table handle. - // the columns ordinals is present in ordinal range format, @see executor.cols2Handle - columns2Handle cols2HandleSlice + // tblColPosInfos stores relationship between column ordinal to its table handle. + // the columns ordinals is present in ordinal range format, @see plannercore.TblColPosInfos + tblColPosInfos plannercore.TblColPosInfoSlice evalBuffer chunk.MutRow } @@ -65,57 +65,53 @@ func (e *UpdateExec) exec(schema *expression.Schema) ([]types.Datum, error) { } row := e.rows[e.cursor] newData := e.newRowsData[e.cursor] - for id, cols := range schema.TblID2Handle { - tbl := e.tblID2table[id] - if e.updatedRowKeys[id] == nil { - e.updatedRowKeys[id] = make(map[int64]bool) + for _, content := range e.tblColPosInfos { + tbl := e.tblID2table[content.TblID] + if e.updatedRowKeys[content.TblID] == nil { + e.updatedRowKeys[content.TblID] = make(map[int64]bool) } - for _, col := range cols { - offset := getTableOffset(schema, col) - end := offset + len(tbl.WritableCols()) - handleDatum := row[col.Index] - if e.canNotUpdate(handleDatum) { - continue - } - handle := row[col.Index].GetInt64() - oldData := row[offset:end] - newTableData := newData[offset:end] - updatable := false - flags := assignFlag[offset:end] - for _, flag := range flags { - if flag { - updatable = true - break - } - } - if !updatable { - // If there's nothing to update, we can just skip current row - continue - } - changed, ok := e.updatedRowKeys[id][handle] - if !ok { - // Row is matched for the first time, increment `matched` counter - e.matched++ - } - if changed { - // Each matched row is updated once, even if it matches the conditions multiple times. - continue + handleDatum := row[content.HandleOrdinal] + if e.canNotUpdate(handleDatum) { + continue + } + handle := row[content.HandleOrdinal].GetInt64() + oldData := row[content.Start:content.End] + newTableData := newData[content.Start:content.End] + updatable := false + flags := assignFlag[content.Start:content.End] + for _, flag := range flags { + if flag { + updatable = true + break } + } + if !updatable { + // If there's nothing to update, we can just skip current row + continue + } + changed, ok := e.updatedRowKeys[content.TblID][handle] + if !ok { + // Row is matched for the first time, increment `matched` counter + e.matched++ + } + if changed { + // Each matched row is updated once, even if it matches the conditions multiple times. + continue + } - // Update row - changed, _, _, err1 := updateRecord(e.ctx, handle, oldData, newTableData, flags, tbl, false) - if err1 == nil { - e.updatedRowKeys[id][handle] = changed - continue - } + // Update row + changed, _, _, err1 := updateRecord(e.ctx, handle, oldData, newTableData, flags, tbl, false) + if err1 == nil { + e.updatedRowKeys[content.TblID][handle] = changed + continue + } - sc := e.ctx.GetSessionVars().StmtCtx - if kv.ErrKeyExists.Equal(err1) && sc.DupKeyAsWarning { - sc.AppendWarning(err1) - continue - } - return nil, err1 + sc := e.ctx.GetSessionVars().StmtCtx + if kv.ErrKeyExists.Equal(err1) && sc.DupKeyAsWarning { + sc.AppendWarning(err1) + continue } + return nil, err1 } e.cursor++ return []types.Datum{}, nil @@ -166,15 +162,11 @@ func (e *UpdateExec) Next(ctx context.Context, req *chunk.Chunk) error { func (e *UpdateExec) fetchChunkRows(ctx context.Context) error { fields := retTypes(e.children[0]) - schema := e.children[0].Schema() colsInfo := make([]*table.Column, len(fields)) - for id, cols := range schema.TblID2Handle { - tbl := e.tblID2table[id] - for _, col := range cols { - offset := getTableOffset(schema, col) - for i, c := range tbl.WritableCols() { - colsInfo[offset+i] = c - } + for _, content := range e.tblColPosInfos { + tbl := e.tblID2table[content.TblID] + for i, c := range tbl.WritableCols() { + colsInfo[content.Start+i] = c } } globalRowIdx := 0 @@ -226,7 +218,7 @@ func (e *UpdateExec) composeNewRow(rowIdx int, oldRow []types.Datum, cols []*tab newRowData := types.CloneRow(oldRow) e.evalBuffer.SetDatums(newRowData...) for _, assign := range e.OrderedList { - handleIdx, handleFound := e.columns2Handle.findHandle(int32(assign.Col.Index)) + handleIdx, handleFound := e.tblColPosInfos.FindHandle(assign.Col.Index) if handleFound && e.canNotUpdate(oldRow[handleIdx]) { continue } @@ -253,12 +245,12 @@ func (e *UpdateExec) composeNewRow(rowIdx int, oldRow []types.Datum, cols []*tab // Close implements the Executor Close interface. func (e *UpdateExec) Close() error { e.setMessage() - return e.SelectExec.Close() + return e.children[0].Close() } // Open implements the Executor Open interface. func (e *UpdateExec) Open(ctx context.Context) error { - return e.SelectExec.Open(ctx) + return e.children[0].Open(ctx) } func (e *UpdateExec) getUpdateColumns(ctx sessionctx.Context, schemaLen int) ([]bool, error) { diff --git a/executor/write.go b/executor/write.go index 62574621b19de..e551ad558dc38 100644 --- a/executor/write.go +++ b/executor/write.go @@ -182,12 +182,3 @@ func resetErrDataTooLong(colName string, rowIdx int, err error) error { logutil.BgLogger().Error("data too long for column", zap.String("colName", colName), zap.Int("rowIndex", rowIdx)) return newErr } - -func getTableOffset(schema *expression.Schema, handleCol *expression.Column) int { - for i, col := range schema.Columns { - if col.DBName.L == handleCol.DBName.L && col.TblName.L == handleCol.TblName.L { - return i - } - } - panic("Couldn't get column information when do update/delete") -} diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 0cf8a357a5716..71e720f3c3b18 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -454,16 +454,19 @@ type Update struct { OrderedList []*expression.Assignment SelectPlan PhysicalPlan + + TblColPosInfos TblColPosInfoSlice } // Delete represents a delete plan. type Delete struct { baseSchemaProducer - Tables []*ast.TableName IsMultiTable bool SelectPlan PhysicalPlan + + TblColPosInfos TblColPosInfoSlice } // analyzeInfo is used to store the database name, table name and partition name of analyze task. diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 77325fccafc51..bc2087d1190b3 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2522,6 +2522,85 @@ func (b *PlanBuilder) buildSemiJoin(outerPlan, innerPlan LogicalPlan, onConditio return joinPlan, nil } +func getTableOffset(schema *expression.Schema, handleCol *expression.Column) (int, error) { + for i, col := range schema.Columns { + if col.DBName.L == handleCol.DBName.L && col.TblName.L == handleCol.TblName.L { + return i, nil + } + } + return -1, errors.Errorf("Couldn't get column information when do update/delete") +} + +// TblColPosInfo represents an mapper from column index to handle index. +type TblColPosInfo struct { + TblID int64 + // Start and End represent the ordinal range [Start, End) of the consecutive columns. + Start, End int + // HandleOrdinal represents the ordinal of the handle column. + HandleOrdinal int +} + +// TblColPosInfoSlice attaches the methods of sort.Interface to []TblColPosInfos sorting in increasing order. +type TblColPosInfoSlice []TblColPosInfo + +// Len implements sort.Interface#Len. +func (c TblColPosInfoSlice) Len() int { + return len(c) +} + +// Swap implements sort.Interface#Swap. +func (c TblColPosInfoSlice) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} + +// Less implements sort.Interface#Less. +func (c TblColPosInfoSlice) Less(i, j int) bool { + return c[i].Start < c[j].Start +} + +// FindHandle finds the ordinal of the corresponding handle column. +func (c TblColPosInfoSlice) FindHandle(colOrdinal int) (int, bool) { + if len(c) == 0 { + return 0, false + } + // find the smallest index of the range that its start great than colOrdinal. + // @see https://godoc.org/sort#Search + rangeBehindOrdinal := sort.Search(len(c), func(i int) bool { return c[i].Start > colOrdinal }) + if rangeBehindOrdinal == 0 { + return 0, false + } + return c[rangeBehindOrdinal-1].HandleOrdinal, true +} + +// buildColumns2Handle builds columns to handle mapping. +func buildColumns2Handle( + schema *expression.Schema, + tblID2Handle map[int64][]*expression.Column, + tblID2Table map[int64]table.Table, + onlyWritableCol bool, +) (TblColPosInfoSlice, error) { + var cols2Handles TblColPosInfoSlice + for tblID, handleCols := range tblID2Handle { + tbl := tblID2Table[tblID] + var tblLen int + if onlyWritableCol { + tblLen = len(tbl.WritableCols()) + } else { + tblLen = len(tbl.Cols()) + } + for _, handleCol := range handleCols { + offset, err := getTableOffset(schema, handleCol) + if err != nil { + return nil, err + } + end := offset + tblLen + cols2Handles = append(cols2Handles, TblColPosInfo{tblID, offset, end, handleCol.Index}) + } + } + sort.Sort(cols2Handles) + return cols2Handles, nil +} + func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { if b.pushTableHints(update.TableHints) { // table hints are only visible in the current UPDATE statement. @@ -2595,6 +2674,14 @@ func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) ( return nil, err } err = updt.ResolveIndices() + if err != nil { + return nil, err + } + tblID2table := make(map[int64]table.Table) + for id := range updt.SelectPlan.Schema().TblID2Handle { + tblID2table[id], _ = b.is.TableByID(id) + } + updt.TblColPosInfos, err = buildColumns2Handle(updt.SelectPlan.Schema(), updt.SelectPlan.Schema().TblID2Handle, tblID2table, true) return updt, err } @@ -2778,13 +2865,7 @@ func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) ( p = proj } - var tables []*ast.TableName - if delete.Tables != nil { - tables = delete.Tables.Tables - } - del := Delete{ - Tables: tables, IsMultiTable: delete.IsMultiTable, }.Init(b.ctx) @@ -2851,7 +2932,55 @@ func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) ( } } - return del, nil + tblID2Handles := del.SelectPlan.Schema().TblID2Handle + if del.IsMultiTable { + // tblID2TableName is the table map value is an array which contains table aliases. + // Table ID may not be unique for deleting multiple tables, for statements like + // `delete from t as t1, t as t2`, the same table has two alias, we have to identify a table + // by its alias instead of ID. + tblID2TableName := make(map[int64][]*ast.TableName, len(delete.Tables.Tables)) + for _, tn := range delete.Tables.Tables { + tblID2TableName[tn.TableInfo.ID] = append(tblID2TableName[tn.TableInfo.ID], tn) + } + tblID2Handles = del.cleanTblID2HandleMap(tblID2TableName, tblID2Handles) + } + tblID2table := make(map[int64]table.Table) + for id := range tblID2Handles { + tblID2table[id], _ = b.is.TableByID(id) + } + del.TblColPosInfos, err = buildColumns2Handle(del.SelectPlan.Schema(), tblID2Handles, tblID2table, false) + return del, err +} + +func (p *Delete) cleanTblID2HandleMap(tablesToDelete map[int64][]*ast.TableName, tblID2Handle map[int64][]*expression.Column) map[int64][]*expression.Column { + for id, cols := range tblID2Handle { + names, ok := tablesToDelete[id] + if !ok { + delete(tblID2Handle, id) + continue + } + for i := len(cols) - 1; i >= 0; i-- { + if !p.matchingDeletingTable(names, cols[i]) { + cols = append(cols[:i], cols[i+1:]...) + } + } + if len(cols) == 0 { + delete(tblID2Handle, id) + continue + } + tblID2Handle[id] = cols + } + return tblID2Handle +} + +// matchingDeletingTable checks whether this column is from the table which is in the deleting list. +func (p *Delete) matchingDeletingTable(names []*ast.TableName, col *expression.Column) bool { + for _, n := range names { + if (col.DBName.L == "" || col.DBName.L == n.Schema.L) && col.TblName.L == n.Name.L { + return true + } + } + return false } func getWindowName(name string) string { diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index 83810ce96a10c..b7243ac4c9a05 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -473,9 +473,21 @@ func tryUpdatePointPlan(ctx sessionctx.Context, updateStmt *ast.UpdateStmt) Plan if orderedList == nil { return nil } + var handleCol *expression.Column + for _, handles := range fastSelect.schema.TblID2Handle { + handleCol = handles[0] + } updatePlan := Update{ SelectPlan: fastSelect, OrderedList: orderedList, + TblColPosInfos: TblColPosInfoSlice{ + TblColPosInfo{ + TblID: fastSelect.TblInfo.ID, + Start: 0, + End: fastSelect.schema.Len(), + HandleOrdinal: handleCol.Index, + }, + }, }.Init(ctx) updatePlan.SetSchema(fastSelect.schema) return updatePlan @@ -532,8 +544,20 @@ func tryDeletePointPlan(ctx sessionctx.Context, delStmt *ast.DeleteStmt) Plan { if ctx.GetSessionVars().TxnCtx.IsPessimistic { fastSelect.Lock = true } + var handleCol *expression.Column + for _, handles := range fastSelect.schema.TblID2Handle { + handleCol = handles[0] + } delPlan := Delete{ SelectPlan: fastSelect, + TblColPosInfos: TblColPosInfoSlice{ + TblColPosInfo{ + TblID: fastSelect.TblInfo.ID, + Start: 0, + End: fastSelect.schema.Len(), + HandleOrdinal: handleCol.Index, + }, + }, }.Init(ctx) delPlan.SetSchema(fastSelect.schema) return delPlan From e220499086197e4ace6c09c7a9582fff8bbc0318 Mon Sep 17 00:00:00 2001 From: goroutine Date: Fri, 26 Jul 2019 19:52:14 +0800 Subject: [PATCH 103/196] util: clean up MemoizeStr function (#11450) --- util/stringutil/string_util.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/util/stringutil/string_util.go b/util/stringutil/string_util.go index 6e69f61d2bd52..f69fb03165e64 100644 --- a/util/stringutil/string_util.go +++ b/util/stringutil/string_util.go @@ -262,13 +262,8 @@ func (l stringerFunc) String() string { // MemoizeStr returns memoized version of stringFunc. func MemoizeStr(l func() string) fmt.Stringer { - var result string return stringerFunc(func() string { - if result != "" { - return result - } - result = l() - return result + return l() }) } From d135ff161fc8ed2b6156e26c91d7946c7917c14b Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Fri, 26 Jul 2019 20:07:43 +0800 Subject: [PATCH 104/196] server: handle partitioned table in some http APIs (#11463) --- server/http_handler.go | 83 +++++++++++++++++++++++++++---------- server/http_handler_test.go | 22 +++++++++- 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index 296031ef393d6..ece6aba284eea 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -479,14 +479,32 @@ type RegionDetail struct { func (rt *RegionDetail) addTableInRange(dbName string, curTable *model.TableInfo, r *helper.RegionFrameRange) { tName := curTable.Name.String() tID := curTable.ID - + pi := curTable.GetPartitionInfo() for _, index := range curTable.Indices { - if f := r.GetIndexFrame(tID, index.ID, dbName, tName, index.Name.String()); f != nil { - rt.Frames = append(rt.Frames, f) + if pi != nil { + for _, def := range pi.Definitions { + if f := r.GetIndexFrame(def.ID, index.ID, dbName, fmt.Sprintf("%s(%s)", tName, def.Name.O), index.Name.String()); f != nil { + rt.Frames = append(rt.Frames, f) + } + } + } else { + if f := r.GetIndexFrame(tID, index.ID, dbName, tName, index.Name.String()); f != nil { + rt.Frames = append(rt.Frames, f) + } } + } - if f := r.GetRecordFrame(tID, dbName, tName); f != nil { - rt.Frames = append(rt.Frames, f) + + if pi != nil { + for _, def := range pi.Definitions { + if f := r.GetRecordFrame(def.ID, dbName, fmt.Sprintf("%s(%s)", tName, def.Name.O)); f != nil { + rt.Frames = append(rt.Frames, f) + } + } + } else { + if f := r.GetRecordFrame(tID, dbName, tName); f != nil { + rt.Frames = append(rt.Frames, f) + } } } @@ -913,18 +931,43 @@ func (h tableHandler) handleStopScatterTableRequest(schema infoschema.InfoSchema } func (h tableHandler) handleRegionRequest(schema infoschema.InfoSchema, tbl table.Table, w http.ResponseWriter, req *http.Request) { - tableID := tbl.Meta().ID - // for record - startKey, endKey := tablecodec.GetTableHandleKeyRange(tableID) - recordRegionIDs, err := h.RegionCache.ListRegionIDsInKeyRange(tikv.NewBackoffer(context.Background(), 500), startKey, endKey) + pi := tbl.Meta().GetPartitionInfo() + if pi != nil { + // Partitioned table. + var data []*TableRegions + for _, def := range pi.Definitions { + tableRegions, err := h.getRegionsByID(tbl, def.ID, def.Name.O) + if err != nil { + writeError(w, err) + return + } + + data = append(data, tableRegions) + } + writeData(w, data) + return + } + + meta := tbl.Meta() + tableRegions, err := h.getRegionsByID(tbl, meta.ID, meta.Name.O) if err != nil { writeError(w, err) return } + + writeData(w, tableRegions) +} + +func (h tableHandler) getRegionsByID(tbl table.Table, id int64, name string) (*TableRegions, error) { + // for record + startKey, endKey := tablecodec.GetTableHandleKeyRange(id) + recordRegionIDs, err := h.RegionCache.ListRegionIDsInKeyRange(tikv.NewBackoffer(context.Background(), 500), startKey, endKey) + if err != nil { + return nil, err + } recordRegions, err := h.getRegionsMeta(recordRegionIDs) if err != nil { - writeError(w, err) - return + return nil, err } // for indices @@ -933,27 +976,23 @@ func (h tableHandler) handleRegionRequest(schema infoschema.InfoSchema, tbl tabl indexID := index.Meta().ID indices[i].Name = index.Meta().Name.String() indices[i].ID = indexID - startKey, endKey := tablecodec.GetTableIndexKeyRange(tableID, indexID) + startKey, endKey := tablecodec.GetTableIndexKeyRange(id, indexID) rIDs, err := h.RegionCache.ListRegionIDsInKeyRange(tikv.NewBackoffer(context.Background(), 500), startKey, endKey) if err != nil { - writeError(w, err) - return + return nil, err } indices[i].Regions, err = h.getRegionsMeta(rIDs) if err != nil { - writeError(w, err) - return + return nil, err } } - tableRegions := &TableRegions{ - TableName: tbl.Meta().Name.O, - TableID: tableID, + return &TableRegions{ + TableName: name, + TableID: id, Indices: indices, RecordRegions: recordRegions, - } - - writeData(w, tableRegions) + }, nil } // pdRegionStats is the json response from PD. diff --git a/server/http_handler_test.go b/server/http_handler_test.go index c9c51a77576f7..00616c157e23d 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -204,13 +204,28 @@ func regionContainsTable(c *C, regionID uint64, tableID int64) bool { return false } -func (ts *HTTPHandlerTestSuite) TestListTableRegionsWithError(c *C) { +func (ts *HTTPHandlerTestSuite) TestListTableRegions(c *C) { ts.startServer(c) defer ts.stopServer(c) + ts.prepareData(c) + // Test list table regions with error resp, err := http.Get("http://127.0.0.1:10090/tables/fdsfds/aaa/regions") c.Assert(err, IsNil) defer resp.Body.Close() c.Assert(resp.StatusCode, Equals, http.StatusBadRequest) + + resp, err = http.Get("http://127.0.0.1:10090/tables/tidb/pt/regions") + c.Assert(err, IsNil) + defer resp.Body.Close() + + var data []*TableRegions + dec := json.NewDecoder(resp.Body) + err = dec.Decode(&data) + c.Assert(err, IsNil) + + region := data[1] + resp, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/regions/%d", region.TableID)) + c.Assert(err, IsNil) } func (ts *HTTPHandlerTestSuite) TestGetRegionByIDWithError(c *C) { @@ -305,6 +320,11 @@ func (ts *HTTPHandlerTestSuite) prepareData(c *C) { c.Assert(err, IsNil) dbt.mustExec("alter table tidb.test add index idx1 (a, b);") dbt.mustExec("alter table tidb.test add unique index idx2 (a, b);") + + dbt.mustExec(`create table tidb.pt (a int) partition by range (a) +(partition p0 values less than (256), + partition p1 values less than (512), + partition p2 values less than (1024))`) } func decodeKeyMvcc(closer io.ReadCloser, c *C, valid bool) { From e09745bd02cd20b497861f0e587f01541ccac8b2 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Fri, 26 Jul 2019 20:19:03 +0800 Subject: [PATCH 105/196] metrics: add metrics for bind info (#10921) All tests passed, auto merged by Bot --- bindinfo/bind_test.go | 38 ++++++++++++++++++++++++++++++--- bindinfo/cache.go | 19 +++++++++++++++++ bindinfo/handle.go | 20 +++++++++++------- bindinfo/session_handle.go | 15 +++++++++++-- executor/compiler.go | 2 ++ metrics/bindinfo.go | 43 ++++++++++++++++++++++++++++++++++++++ metrics/metrics.go | 7 +++++++ session/session.go | 4 ++++ 8 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 metrics/bindinfo.go diff --git a/bindinfo/bind_test.go b/bindinfo/bind_test.go index b768ef4edd394..aab1a82f1b6be 100644 --- a/bindinfo/bind_test.go +++ b/bindinfo/bind_test.go @@ -19,7 +19,6 @@ import ( "fmt" "os" "testing" - "time" . "github.com/pingcap/check" "github.com/pingcap/parser" @@ -27,6 +26,7 @@ import ( "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/store/mockstore" "github.com/pingcap/tidb/store/mockstore/mocktikv" @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/util/mock" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" + dto "github.com/prometheus/client_model/go" ) func TestT(t *testing.T) { @@ -146,13 +147,21 @@ func (s *testSuite) TestGlobalBinding(c *C) { tk.MustExec("create table t1(i int, s varchar(20))") tk.MustExec("create index index_t on t(i,s)") + metrics.BindTotalGauge.Reset() + metrics.BindMemoryUsage.Reset() + _, err := tk.Exec("create global binding for select * from t where i>100 using select * from t use index(index_t) where i>100") c.Assert(err, IsNil, Commentf("err %v", err)) - time.Sleep(time.Second * 1) _, err = tk.Exec("create global binding for select * from t where i>99 using select * from t use index(index_t) where i>99") c.Assert(err, IsNil) + pb := &dto.Metric{} + metrics.BindTotalGauge.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(1)) + metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(161)) + sql, hash := parser.NormalizeDigest("select * from t where i > ?") bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test") @@ -203,6 +212,12 @@ func (s *testSuite) TestGlobalBinding(c *C) { bindData = s.domain.BindHandle().GetBindRecord(hash, sql, "test") c.Check(bindData, IsNil) + metrics.BindTotalGauge.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(0)) + metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, bindinfo.Using).Write(pb) + // From newly created global bind handle. + c.Assert(pb.GetGauge().GetValue(), Equals, float64(161)) + bindHandle = bindinfo.NewBindHandle(tk.Se) err = bindHandle.Update(true) c.Check(err, IsNil) @@ -235,13 +250,21 @@ func (s *testSuite) TestSessionBinding(c *C) { tk.MustExec("create table t1(i int, s varchar(20))") tk.MustExec("create index index_t on t(i,s)") + metrics.BindTotalGauge.Reset() + metrics.BindMemoryUsage.Reset() + _, err := tk.Exec("create session binding for select * from t where i>100 using select * from t use index(index_t) where i>100") c.Assert(err, IsNil, Commentf("err %v", err)) - time.Sleep(time.Second * 1) _, err = tk.Exec("create session binding for select * from t where i>99 using select * from t use index(index_t) where i>99") c.Assert(err, IsNil) + pb := &dto.Metric{} + metrics.BindTotalGauge.WithLabelValues(metrics.ScopeSession, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(1)) + metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeSession, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(161)) + handle := tk.Se.Value(bindinfo.SessionBindInfoKeyType).(*bindinfo.SessionHandle) bindData := handle.GetBindRecord("select * from t where i > ?", "test") c.Check(bindData, NotNil) @@ -283,6 +306,11 @@ func (s *testSuite) TestSessionBinding(c *C) { c.Check(bindData, NotNil) c.Check(bindData.OriginalSQL, Equals, "select * from t where i > ?") c.Check(bindData.Status, Equals, "deleted") + + metrics.BindTotalGauge.WithLabelValues(metrics.ScopeSession, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(0)) + metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeSession, bindinfo.Using).Write(pb) + c.Assert(pb.GetGauge().GetValue(), Equals, float64(0)) } func (s *testSuite) TestGlobalAndSessionBindingBothExist(c *C) { @@ -318,6 +346,7 @@ func (s *testSuite) TestGlobalAndSessionBindingBothExist(c *C) { tk.MustExec("create global binding for SELECT * from t1,t2 where t1.id = t2.id using SELECT /*+ TIDB_SMJ(t1, t2) */ * from t1,t2 where t1.id = t2.id") + metrics.BindUsageCounter.Reset() tk.MustQuery("explain SELECT * from t1,t2 where t1.id = t2.id").Check(testkit.Rows( "MergeJoin_7 12487.50 root inner join, left key:test.t1.id, right key:test.t2.id", "├─Sort_11 9990.00 root test.t1.id:asc", @@ -329,6 +358,9 @@ func (s *testSuite) TestGlobalAndSessionBindingBothExist(c *C) { " └─Selection_13 9990.00 cop not(isnull(test.t2.id))", " └─TableScan_12 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo", )) + pb := &dto.Metric{} + metrics.BindUsageCounter.WithLabelValues(metrics.ScopeGlobal).Write(pb) + c.Assert(pb.GetCounter().GetValue(), Equals, float64(1)) tk.MustExec("drop global binding for SELECT * from t1,t2 where t1.id = t2.id") diff --git a/bindinfo/cache.go b/bindinfo/cache.go index a4c2785eb9c64..5b74a8c316832 100644 --- a/bindinfo/cache.go +++ b/bindinfo/cache.go @@ -14,7 +14,10 @@ package bindinfo import ( + "unsafe" + "github.com/pingcap/parser/ast" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" ) @@ -64,3 +67,19 @@ func newBindRecord(row chunk.Row) *BindRecord { Collation: row.GetString(7), } } + +// size calculates the memory size of a bind meta. +func (m *BindRecord) size() float64 { + res := len(m.OriginalSQL) + len(m.BindSQL) + len(m.Db) + len(m.Status) + 2*int(unsafe.Sizeof(m.CreateTime)) + len(m.Charset) + len(m.Collation) + return float64(res) +} + +func (m *BindRecord) updateMetrics(scope string, inc bool) { + if inc { + metrics.BindMemoryUsage.WithLabelValues(scope, m.Status).Add(float64(m.size())) + metrics.BindTotalGauge.WithLabelValues(scope, m.Status).Inc() + } else { + metrics.BindMemoryUsage.WithLabelValues(scope, m.Status).Sub(float64(m.size())) + metrics.BindTotalGauge.WithLabelValues(scope, m.Status).Dec() + } +} diff --git a/bindinfo/handle.go b/bindinfo/handle.go index 3e3da3b90ea09..2e5124b95f990 100644 --- a/bindinfo/handle.go +++ b/bindinfo/handle.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/parser" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/tikv/oracle" "github.com/pingcap/tidb/types" @@ -123,9 +124,10 @@ func (h *BindHandle) Update(fullLoad bool) (err error) { continue } - newCache.removeStaleBindMetas(hash, meta) + newCache.removeStaleBindMetas(hash, meta, metrics.ScopeGlobal) if meta.Status == Using { newCache[hash] = append(newCache[hash], meta) + metrics.BindMemoryUsage.WithLabelValues(metrics.ScopeGlobal, meta.Status).Add(meta.size()) } } return nil @@ -254,6 +256,7 @@ func (h *BindHandle) DropInvalidBindRecord() { if time.Since(invalidBindRecord.droppedTime) > 6*time.Second { delete(invalidBindRecordMap, key) + invalidBindRecord.bindRecord.updateMetrics(metrics.ScopeGlobal, false) } } h.invalidBindRecordMap.Store(invalidBindRecordMap) @@ -275,6 +278,7 @@ func (h *BindHandle) AddDropInvalidBindTask(invalidBindRecord *BindRecord) { bindRecord: invalidBindRecord, } h.invalidBindRecordMap.Store(newMap) + invalidBindRecord.updateMetrics(metrics.ScopeGlobal, true) } // Size return the size of bind info cache. @@ -320,8 +324,9 @@ func newBindMetaWithoutAst(record *BindRecord) (hash string, meta *BindMeta) { // removed from the cache after this operation. func (h *BindHandle) appendBindMeta(hash string, meta *BindMeta) { newCache := h.bindInfo.Value.Load().(cache).copy() - newCache.removeStaleBindMetas(hash, meta) + newCache.removeStaleBindMetas(hash, meta, metrics.ScopeGlobal) newCache[hash] = append(newCache[hash], meta) + meta.updateMetrics(metrics.ScopeGlobal, true) h.bindInfo.Value.Store(newCache) } @@ -334,18 +339,19 @@ func (h *BindHandle) removeBindMeta(hash string, meta *BindMeta) { h.bindInfo.Unlock() }() - newCache.removeDeletedBindMeta(hash, meta) + newCache.removeDeletedBindMeta(hash, meta, metrics.ScopeGlobal) } // removeDeletedBindMeta removes all the BindMeta which originSQL and db are the same with the parameter's meta. -func (c cache) removeDeletedBindMeta(hash string, meta *BindMeta) { +func (c cache) removeDeletedBindMeta(hash string, meta *BindMeta, scope string) { metas, ok := c[hash] if !ok { return } for i := len(metas) - 1; i >= 0; i-- { - if meta.isSame(meta) { + if metas[i].isSame(meta) { + metas[i].updateMetrics(scope, false) metas = append(metas[:i], metas[i+1:]...) if len(metas) == 0 { delete(c, hash) @@ -356,15 +362,15 @@ func (c cache) removeDeletedBindMeta(hash string, meta *BindMeta) { } // removeStaleBindMetas removes all the stale BindMeta in the cache. -func (c cache) removeStaleBindMetas(hash string, meta *BindMeta) { +func (c cache) removeStaleBindMetas(hash string, meta *BindMeta, scope string) { metas, ok := c[hash] if !ok { return } - // remove stale bindMetas. for i := len(metas) - 1; i >= 0; i-- { if metas[i].isStale(meta) { + metas[i].updateMetrics(scope, false) metas = append(metas[:i], metas[i+1:]...) if len(metas) == 0 { delete(c, hash) diff --git a/bindinfo/session_handle.go b/bindinfo/session_handle.go index f343b3ca8e24d..f52c7d0f92e22 100644 --- a/bindinfo/session_handle.go +++ b/bindinfo/session_handle.go @@ -18,6 +18,7 @@ import ( "github.com/pingcap/parser" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/types" ) @@ -38,8 +39,9 @@ func NewSessionBindHandle(parser *parser.Parser) *SessionHandle { // removed from the cache after this operation. func (h *SessionHandle) appendBindMeta(hash string, meta *BindMeta) { // Make sure there is only one goroutine writes the cache. - h.ch.removeStaleBindMetas(hash, meta) + h.ch.removeStaleBindMetas(hash, meta, metrics.ScopeSession) h.ch[hash] = append(h.ch[hash], meta) + meta.updateMetrics(metrics.ScopeSession, true) } func (h *SessionHandle) newBindMeta(record *BindRecord) (hash string, meta *BindMeta, err error) { @@ -74,7 +76,7 @@ func (h *SessionHandle) DropBindRecord(record *BindRecord) { meta := &BindMeta{BindRecord: record} meta.Status = deleted hash := parser.DigestHash(record.OriginalSQL) - h.ch.removeDeletedBindMeta(hash, meta) + h.ch.removeDeletedBindMeta(hash, meta, metrics.ScopeSession) h.appendBindMeta(hash, meta) } @@ -100,6 +102,15 @@ func (h *SessionHandle) GetAllBindRecord() (bindRecords []*BindMeta) { return bindRecords } +// Close closes the session handle. +func (h *SessionHandle) Close() { + for _, bindRecords := range h.ch { + for _, bindRecord := range bindRecords { + bindRecord.updateMetrics(metrics.ScopeSession, false) + } + } +} + // sessionBindInfoKeyType is a dummy type to avoid naming collision in context. type sessionBindInfoKeyType int diff --git a/executor/compiler.go b/executor/compiler.go index 7ac5f1eef3367..9190763fe68fd 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -404,6 +404,7 @@ func addHintForSelect(hash, normdOrigSQL string, ctx sessionctx.Context, stmt as return stmt } if bindRecord.Status == bindinfo.Using { + metrics.BindUsageCounter.WithLabelValues(metrics.ScopeSession).Inc() return bindinfo.BindHint(stmt, bindRecord.Ast) } } @@ -413,6 +414,7 @@ func addHintForSelect(hash, normdOrigSQL string, ctx sessionctx.Context, stmt as bindRecord = globalHandle.GetBindRecord(hash, normdOrigSQL, "") } if bindRecord != nil { + metrics.BindUsageCounter.WithLabelValues(metrics.ScopeGlobal).Inc() return bindinfo.BindHint(stmt, bindRecord.Ast) } return stmt diff --git a/metrics/bindinfo.go b/metrics/bindinfo.go new file mode 100644 index 0000000000000..958bd110c2b23 --- /dev/null +++ b/metrics/bindinfo.go @@ -0,0 +1,43 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +// bindinfo metrics. +var ( + BindUsageCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "bindinfo", + Name: "bind_usage_counter", + Help: "Counter of query using sql bind", + }, []string{LableScope}) + + BindTotalGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "bindinfo", + Name: "bind_total_gauge", + Help: "Total number of sql bind", + }, []string{LableScope, LblType}) + + BindMemoryUsage = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "tidb", + Subsystem: "bindinfo", + Name: "bind_memory_usage", + Help: "Memory usage of sql bind", + }, []string{LableScope, LblType}) +) diff --git a/metrics/metrics.go b/metrics/metrics.go index ba6a3ce73736d..a31d5ec3837c6 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -41,6 +41,10 @@ const ( opSucc = "ok" opFailed = "err" + + LableScope = "scope" + ScopeGlobal = "global" + ScopeSession = "session" ) // RetLabel returns "ok" when err == nil and "err" when err != nil. @@ -58,6 +62,9 @@ func RegisterMetrics() { prometheus.MustRegister(AutoAnalyzeHistogram) prometheus.MustRegister(AutoIDHistogram) prometheus.MustRegister(BatchAddIdxHistogram) + prometheus.MustRegister(BindUsageCounter) + prometheus.MustRegister(BindTotalGauge) + prometheus.MustRegister(BindMemoryUsage) prometheus.MustRegister(CampaignOwnerCounter) prometheus.MustRegister(ConnGauge) prometheus.MustRegister(PreparedStmtGauge) diff --git a/session/session.go b/session/session.go index 2d58ac5f2258a..bca00fdf4e7fa 100644 --- a/session/session.go +++ b/session/session.go @@ -1364,6 +1364,10 @@ func (s *session) Close() { if s.statsCollector != nil { s.statsCollector.Delete() } + bindValue := s.Value(bindinfo.SessionBindInfoKeyType) + if bindValue != nil { + bindValue.(*bindinfo.SessionHandle).Close() + } ctx := context.TODO() s.RollbackTxn(ctx) if s.sessionVars != nil { From 5611acd3782dd1577e3f0486564d07b203e8591e Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Fri, 26 Jul 2019 21:53:22 +0800 Subject: [PATCH 106/196] =?UTF-8?q?types:=20floatStrToIntStr=20will=20fail?= =?UTF-8?q?ed=20in=20some=20case=20such=20as=20=20the=E2=80=A6=20(#11376)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- types/convert.go | 26 +++++++++++++++++--------- types/convert_test.go | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/types/convert.go b/types/convert.go index 9a52701cabf2e..4c427c5522a4f 100644 --- a/types/convert.go +++ b/types/convert.go @@ -395,23 +395,30 @@ func getValidIntPrefix(sc *stmtctx.StatementContext, str string) (string, error) return valid, nil } -// roundIntStr is to round int string base on the number following dot. +// roundIntStr is to round a **valid int string** base on the number following dot. func roundIntStr(numNextDot byte, intStr string) string { if numNextDot < '5' { return intStr } retStr := []byte(intStr) - for i := len(intStr) - 1; i >= 0; i-- { - if retStr[i] != '9' { - retStr[i]++ + idx := len(intStr) - 1 + for ; idx >= 1; idx-- { + if retStr[idx] != '9' { + retStr[idx]++ break } - if i == 0 { - retStr[i] = '1' + retStr[idx] = '0' + } + if idx == 0 { + if intStr[0] == '9' { + retStr[0] = '1' + retStr = append(retStr, '0') + } else if isDigit(intStr[0]) { + retStr[0]++ + } else { + retStr[1] = '1' retStr = append(retStr, '0') - break } - retStr[i] = '0' } return string(retStr) } @@ -458,6 +465,7 @@ func floatStrToIntStr(sc *stmtctx.StatementContext, validFloat string, oriStr st } return intStr, nil } + // intCnt and digits contain the prefix `+/-` if validFloat[0] is `+/-` var intCnt int digits := make([]byte, 0, len(validFloat)) if dotIdx == -1 { @@ -483,7 +491,7 @@ func floatStrToIntStr(sc *stmtctx.StatementContext, validFloat string, oriStr st } if intCnt <= 0 { intStr = "0" - if intCnt == 0 && len(digits) > 0 { + if intCnt == 0 && len(digits) > 0 && isDigit(digits[0]) { intStr = roundIntStr(digits[0], intStr) } return intStr, nil diff --git a/types/convert_test.go b/types/convert_test.go index d2bcb372a0269..2d5c97cc2a215 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -702,6 +702,21 @@ func (s *testTypeConvertSuite) TestConvert(c *C) { signedAccept(c, mysql.TypeNewDecimal, dec, "-0.00123") } +func (s *testTypeConvertSuite) TestRoundIntStr(c *C) { + cases := []struct { + a string + b byte + c string + }{ + {"+999", '5', "+1000"}, + {"999", '5', "1000"}, + {"-999", '5', "-1000"}, + } + for _, cc := range cases { + c.Assert(roundIntStr(cc.b, cc.a), Equals, cc.c) + } +} + func (s *testTypeConvertSuite) TestGetValidInt(c *C) { tests := []struct { origin string @@ -821,6 +836,7 @@ func (s *testTypeConvertSuite) TestGetValidFloat(c *C) { {".5", "1"}, {"123.456789e5", "12345679"}, {"123.456784e5", "12345678"}, + {"+999.9999e2", "+100000"}, } for _, t := range tests2 { str, err := floatStrToIntStr(sc, t.origin, t.origin) From 5c35082f6af637f13f5d4a936a1e91f94e520259 Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 27 Jul 2019 18:22:21 +0800 Subject: [PATCH 107/196] expression: handle builtin add_date/sub_date func overflow (#11472) --- expression/builtin_time.go | 8 ++++++++ expression/builtin_time_test.go | 29 +++++++++++++++++++++++++++++ expression/integration_test.go | 6 ++++++ 3 files changed, 43 insertions(+) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 7a085453fdb06..a0ac0de54d51a 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -2744,6 +2744,10 @@ func (du *baseDateArithmitical) add(ctx sessionctx.Context, date types.Time, int date.Fsp = 6 } + if goTime.Year() < 0 || goTime.Year() > (1<<16-1) { + return types.Time{}, true, handleInvalidTimeError(ctx, types.ErrDatetimeFunctionOverflow.GenWithStackByArgs("datetime")) + } + date.Time = types.FromGoTime(goTime) overflow, err := types.DateTimeIsOverflow(ctx.GetSessionVars().StmtCtx, date) if err := handleInvalidTimeError(ctx, err); err != nil { @@ -2801,6 +2805,10 @@ func (du *baseDateArithmitical) sub(ctx sessionctx.Context, date types.Time, int date.Fsp = 6 } + if goTime.Year() < 0 || goTime.Year() > (1<<16-1) { + return types.Time{}, true, handleInvalidTimeError(ctx, types.ErrDatetimeFunctionOverflow.GenWithStackByArgs("datetime")) + } + date.Time = types.FromGoTime(goTime) overflow, err := types.DateTimeIsOverflow(ctx.GetSessionVars().StmtCtx, date) if err := handleInvalidTimeError(ctx, err); err != nil { diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index 88566b9fbd584..14e7e70094fc4 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -1826,6 +1826,35 @@ func (s *testEvaluatorSuite) TestDateArithFuncs(c *C) { c.Assert(err, IsNil) c.Assert(v.GetMysqlTime().String(), Equals, test.expected) } + + testOverflowYears := []struct { + input string + year int + }{ + {"2008-11-23", -1465647104}, + {"2008-11-23", 1465647104}, + } + + for _, test := range testOverflowYears { + args = types.MakeDatums(test.input, test.year, "YEAR") + f, err = fcAdd.getFunction(s.ctx, s.datumsToConstants(args)) + c.Assert(err, IsNil) + c.Assert(f, NotNil) + v, err = evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + c.Assert(v.IsNull(), IsTrue) + } + + for _, test := range testOverflowYears { + args = types.MakeDatums(test.input, test.year, "YEAR") + f, err = fcSub.getFunction(s.ctx, s.datumsToConstants(args)) + c.Assert(err, IsNil) + c.Assert(f, NotNil) + v, err = evalBuiltinFunc(f, chunk.Row{}) + c.Assert(err, IsNil) + c.Assert(v.IsNull(), IsTrue) + } + testDurations := []struct { fc functionClass dur string diff --git a/expression/integration_test.go b/expression/integration_test.go index e07ffe91fc2ac..5a5b949dba916 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2103,6 +2103,12 @@ func (s *testIntegrationSuite) TestDatetimeOverflow(c *C) { rows = append(rows, "") } tk.MustQuery("select * from t1").Check(testkit.Rows(rows...)) + + //Fix ISSUE 11256 + tk.MustQuery(`select DATE_ADD('2000-04-13 07:17:02',INTERVAL -1465647104 YEAR);`).Check(testkit.Rows("")) + tk.MustQuery(`select DATE_ADD('2008-11-23 22:47:31',INTERVAL 266076160 QUARTER);`).Check(testkit.Rows("")) + tk.MustQuery(`select DATE_SUB('2000-04-13 07:17:02',INTERVAL 1465647104 YEAR);`).Check(testkit.Rows("")) + tk.MustQuery(`select DATE_SUB('2008-11-23 22:47:31',INTERVAL -266076160 QUARTER);`).Check(testkit.Rows("")) } func (s *testIntegrationSuite) TestBuiltin(c *C) { From 1ae5e4ce983bccd99a08d60a93b3509182e4ae09 Mon Sep 17 00:00:00 2001 From: wshwsh12 <793703860@qq.com> Date: Mon, 29 Jul 2019 10:28:12 +0800 Subject: [PATCH 108/196] parser: support cast as double (#11443) --- expression/builtin_cast.go | 2 +- expression/integration_test.go | 28 ++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 1b8205c009118..366c48def1b80 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -458,7 +458,7 @@ func (b *builtinCastIntAsRealSig) evalReal(row chunk.Row) (res float64, isNull b if isNull || err != nil { return res, isNull, err } - if !mysql.HasUnsignedFlag(b.tp.Flag) { + if !mysql.HasUnsignedFlag(b.tp.Flag) && !mysql.HasUnsignedFlag(b.args[0].GetType().Flag) { res = float64(val) } else if b.inUnion && val < 0 { res = 0 diff --git a/expression/integration_test.go b/expression/integration_test.go index 5a5b949dba916..ed4e73872fc2d 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2336,6 +2336,34 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result = tk.MustQuery("select cast(1 as signed int)") result.Check(testkit.Rows("1")) + // test cast as double + result = tk.MustQuery("select cast(1 as double)") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(12345 as unsigned) as double)") + result.Check(testkit.Rows("12345")) + result = tk.MustQuery("select cast(1.1 as double)") + result.Check(testkit.Rows("1.1")) + result = tk.MustQuery("select cast(-1.1 as double)") + result.Check(testkit.Rows("-1.1")) + result = tk.MustQuery("select cast('123.321' as double)") + result.Check(testkit.Rows("123.321")) + result = tk.MustQuery("select cast('12345678901234567890' as double) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(-1 as double)") + result.Check(testkit.Rows("-1")) + result = tk.MustQuery("select cast(null as double)") + result.Check(testkit.Rows("")) + result = tk.MustQuery("select cast(12345678901234567890 as double) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(-1 as unsigned) as double) = 1.8446744073709552e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(1e100 as double) = 1e100") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(123456789012345678901234567890 as double) = 1.2345678901234568e29") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(0x12345678 as double)") + result.Check(testkit.Rows("305419896")) + // test cast time as decimal overflow tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(s1 time);") diff --git a/go.mod b/go.mod index 6f5060725a50f..a6b13ca525c92 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd + github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 368f4dd6e0d9a..da1a87c8df089 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd h1:vFLYlcLyvWrIN+3Y9uPExiisDjeRCPh1Vic0vnkxBMs= -github.com/pingcap/parser v0.0.0-20190726023712-ca2f45b420fd/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2 h1:uPLL91Kf8Sh86ht/ijNj6iUWw7JBArTcJtyn8aR+uDs= +github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From 0f6a47102972bf0990e331e874b52b933b8ec193 Mon Sep 17 00:00:00 2001 From: qupeng Date: Mon, 29 Jul 2019 12:43:57 +0800 Subject: [PATCH 109/196] stabilize a test case (#11465) --- store/tikv/client_fail_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/tikv/client_fail_test.go b/store/tikv/client_fail_test.go index 74c67ab25b0a5..198bd7e889906 100644 --- a/store/tikv/client_fail_test.go +++ b/store/tikv/client_fail_test.go @@ -48,8 +48,8 @@ func (s *testClientSuite) TestPanicInRecvLoop(c *C) { c.Assert(err, IsNil) time.Sleep(time.Second) - c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/panicInFailPendingRequests"), IsNil) c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop"), IsNil) + c.Assert(failpoint.Disable("github.com/pingcap/tidb/store/tikv/panicInFailPendingRequests"), IsNil) time.Sleep(time.Second) req := tikvrpc.NewRequest(tikvrpc.CmdEmpty, &tikvpb.BatchCommandsEmptyRequest{}) From 6ba79d4d4405006ecf6fe28d9c53496131da306e Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Mon, 29 Jul 2019 13:33:54 +0800 Subject: [PATCH 110/196] fix http handler ut (#11494) Signed-off-by: Shuaipeng Yu --- server/http_handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/http_handler_test.go b/server/http_handler_test.go index 00616c157e23d..d10e60903cde1 100644 --- a/server/http_handler_test.go +++ b/server/http_handler_test.go @@ -586,7 +586,7 @@ func (ts *HTTPHandlerTestSuite) TestGetSchema(c *C) { decoder = json.NewDecoder(resp.Body) err = decoder.Decode(<) c.Assert(err, IsNil) - c.Assert(lt[0].Name.L, Equals, "test") + c.Assert(len(lt), Greater, 0) _, err = http.Get(fmt.Sprintf("http://127.0.0.1:10090/schema/abc")) c.Assert(err, IsNil) From 05b4e2299d0ed6656114deb13550de7f1a1ad8b7 Mon Sep 17 00:00:00 2001 From: lysu Date: Mon, 29 Jul 2019 16:11:09 +0800 Subject: [PATCH 111/196] infoschema: add current txn start time in processlist table (#11491) --- infoschema/tables.go | 3 ++- infoschema/tables_test.go | 31 ++++++++++++++++--------------- util/misc_test.go | 2 +- util/processinfo.go | 13 +++++++++++-- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index 3a447bb3264ae..e31db88b9af68 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -552,6 +552,7 @@ var tableProcesslistCols = []columnInfo{ {"STATE", mysql.TypeVarchar, 7, 0, nil, nil}, {"INFO", mysql.TypeString, 512, 0, nil, nil}, {"MEM", mysql.TypeLonglong, 21, 0, nil, nil}, + {"TxnStart", mysql.TypeVarchar, 64, mysql.NotNullFlag, "", nil}, } var tableTiDBIndexesCols = []columnInfo{ @@ -871,7 +872,7 @@ func dataForProcesslist(ctx sessionctx.Context) [][]types.Datum { continue } - rows := pi.ToRow() + rows := pi.ToRow(ctx.GetSessionVars().StmtCtx.TimeZone) record := types.MakeDatums(rows...) records = append(records, record) } diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 13d0999243083..c2c7c84cb1ea5 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -18,6 +18,7 @@ import ( "os" "strconv" "strings" + "time" . "github.com/pingcap/check" "github.com/pingcap/parser/auth" @@ -343,8 +344,8 @@ func (s *testTableSuite) TestSomeTables(c *C) { tk.Se.SetSessionManager(sm) tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Sort().Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", "do something"), - fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0 ", "do something"), + fmt.Sprintf("2 user-2 localhost test Init DB 9223372036 2 %s 0 ", strings.Repeat("x", 101)), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( testkit.Rows( @@ -365,24 +366,24 @@ func (s *testTableSuite) TestSomeTables(c *C) { DB: "information_schema", Command: byte(1), State: 1, - Info: nil, StmtCtx: tk.Se.GetSessionVars().StmtCtx, } sm.processInfoMap[2] = &util.ProcessInfo{ - ID: 2, - User: "user-2", - Host: "localhost", - DB: nil, - Command: byte(2), - State: 2, - Info: strings.Repeat("x", 101), - StmtCtx: tk.Se.GetSessionVars().StmtCtx, + ID: 2, + User: "user-2", + Host: "localhost", + Command: byte(2), + State: 2, + Info: strings.Repeat("x", 101), + StmtCtx: tk.Se.GetSessionVars().StmtCtx, + CurTxnStartTS: 410090409861578752, } tk.Se.SetSessionManager(sm) + tk.Se.GetSessionVars().TimeZone = time.UTC tk.MustQuery("select * from information_schema.PROCESSLIST order by ID;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", ""), - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0 ", ""), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 2 %s 0 07-29 03:26:05.158(410090409861578752)", strings.Repeat("x", 101)), )) tk.MustQuery("SHOW PROCESSLIST;").Sort().Check( testkit.Rows( @@ -396,11 +397,11 @@ func (s *testTableSuite) TestSomeTables(c *C) { )) tk.MustQuery("select * from information_schema.PROCESSLIST where db is null;").Check( testkit.Rows( - fmt.Sprintf("2 user-2 localhost Init DB 9223372036 2 %s 0", strings.Repeat("x", 101)), + fmt.Sprintf("2 user-2 localhost Init DB 9223372036 2 %s 0 07-29 03:26:05.158(410090409861578752)", strings.Repeat("x", 101)), )) tk.MustQuery("select * from information_schema.PROCESSLIST where Info is null;").Check( testkit.Rows( - fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0", ""), + fmt.Sprintf("1 user-1 localhost information_schema Quit 9223372036 1 %s 0 ", ""), )) } diff --git a/util/misc_test.go b/util/misc_test.go index 7c365a98fdb5a..dba9ccd4eabf1 100644 --- a/util/misc_test.go +++ b/util/misc_test.go @@ -172,7 +172,7 @@ func (s *testMiscSuite) TestBasicFunc(c *C) { c.Assert(row[6], Equals, "1") c.Assert(row[7], Equals, "test") - row3 := pi.ToRow() + row3 := pi.ToRow(time.UTC) c.Assert(row3[:8], DeepEquals, row) c.Assert(row3[8], Equals, int64(0)) diff --git a/util/processinfo.go b/util/processinfo.go index b09edf810b184..9d95ff72b56b9 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/store/tikv/oracle" ) // ProcessInfo is a struct used for show processlist statement. @@ -64,10 +65,18 @@ func (pi *ProcessInfo) ToRowForShow(full bool) []interface{} { } } +func (pi *ProcessInfo) txnStartTs(tz *time.Location) (txnStart string) { + if pi.CurTxnStartTS > 0 { + physicalTime := oracle.GetTimeFromTS(pi.CurTxnStartTS) + txnStart = fmt.Sprintf("%s(%d)", physicalTime.In(tz).Format("01-02 15:04:05.000"), pi.CurTxnStartTS) + } + return +} + // ToRow returns []interface{} for the row data of // "SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST". -func (pi *ProcessInfo) ToRow() []interface{} { - return append(pi.ToRowForShow(true), pi.StmtCtx.MemTracker.BytesConsumed()) +func (pi *ProcessInfo) ToRow(tz *time.Location) []interface{} { + return append(pi.ToRowForShow(true), pi.StmtCtx.MemTracker.BytesConsumed(), pi.txnStartTs(tz)) } // SessionManager is an interface for session manage. Show processlist and From 5c07ccec3a46428fc6a7dbdd98f3cc5ad335c82f Mon Sep 17 00:00:00 2001 From: lysu Date: Mon, 29 Jul 2019 16:56:59 +0800 Subject: [PATCH 112/196] expression: make `regex binary` and `rlike binary` be case sensitive (#11502) --- expression/builtin_like.go | 2 +- expression/integration_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/expression/builtin_like.go b/expression/builtin_like.go index f64df725d6129..093dfaf2abd97 100644 --- a/expression/builtin_like.go +++ b/expression/builtin_like.go @@ -95,7 +95,7 @@ func (c *regexpFunctionClass) getFunction(ctx sessionctx.Context, args []Express bf := newBaseBuiltinFuncWithTp(ctx, args, types.ETInt, types.ETString, types.ETString) bf.tp.Flen = 1 var sig builtinFunc - if types.IsBinaryStr(args[0].GetType()) { + if types.IsBinaryStr(args[0].GetType()) || types.IsBinaryStr(args[1].GetType()) { sig = &builtinRegexpBinarySig{bf} } else { sig = &builtinRegexpSig{bf} diff --git a/expression/integration_test.go b/expression/integration_test.go index ed4e73872fc2d..e53c37b7729ff 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2545,18 +2545,24 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result.Check(testkit.Rows("1")) result = tk.MustQuery(`select b regexp 'Xt' from t;`) result.Check(testkit.Rows("1")) + result = tk.MustQuery(`select b regexp binary 'Xt' from t;`) + result.Check(testkit.Rows("0")) result = tk.MustQuery(`select c regexp 'Xt' from t;`) result.Check(testkit.Rows("0")) result = tk.MustQuery(`select d regexp 'Xt' from t;`) result.Check(testkit.Rows("0")) result = tk.MustQuery(`select a rlike 'Xt' from t;`) result.Check(testkit.Rows("1")) + result = tk.MustQuery(`select a rlike binary 'Xt' from t;`) + result.Check(testkit.Rows("0")) result = tk.MustQuery(`select b rlike 'Xt' from t;`) result.Check(testkit.Rows("1")) result = tk.MustQuery(`select c rlike 'Xt' from t;`) result.Check(testkit.Rows("0")) result = tk.MustQuery(`select d rlike 'Xt' from t;`) result.Check(testkit.Rows("0")) + result = tk.MustQuery(`select 'a' regexp 'A', 'a' regexp binary 'A'`) + result.Check(testkit.Rows("1 0")) // testCase is for like and regexp type testCase struct { From af15f725b5558651627b94faacdb525f2d60920f Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 29 Jul 2019 17:21:08 +0800 Subject: [PATCH 113/196] stats: handle feedback updates for topn items (#11434) --- statistics/feedback.go | 24 +++++++++++++------ statistics/handle/update_test.go | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/statistics/feedback.go b/statistics/feedback.go index a32003492336c..013b9c2d6f9b2 100644 --- a/statistics/feedback.go +++ b/statistics/feedback.go @@ -34,7 +34,6 @@ import ( "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" - "github.com/spaolacci/murmur3" "go.uber.org/atomic" "go.uber.org/zap" ) @@ -687,8 +686,11 @@ func buildNewHistogram(h *Histogram, buckets []bucket) *Histogram { type queryFeedback struct { IntRanges []int64 // HashValues is the murmur hash values for each index point. + // Note that index points will be stored in `IndexPoints`, we keep it here only for compatibility. HashValues []uint64 IndexRanges [][]byte + // IndexPoints stores the value of each equal condition. + IndexPoints [][]byte // Counts is the number of scan keys in each range. It first stores the count for `IntRanges`, `IndexRanges` or `ColumnRanges`. // After that, it stores the Ranges for `HashValues`. Counts []int64 @@ -721,8 +723,7 @@ func encodeIndexFeedback(q *QueryFeedback) *queryFeedback { var pointCounts []int64 for _, fb := range q.Feedback { if bytes.Compare(kv.Key(fb.Lower.GetBytes()).PrefixNext(), fb.Upper.GetBytes()) >= 0 { - h1, h2 := murmur3.Sum128(fb.Lower.GetBytes()) - pb.HashValues = append(pb.HashValues, h1, h2) + pb.IndexPoints = append(pb.IndexPoints, fb.Lower.GetBytes()) pointCounts = append(pointCounts, fb.Count) } else { pb.IndexRanges = append(pb.IndexRanges, fb.Lower.GetBytes(), fb.Upper.GetBytes()) @@ -782,9 +783,18 @@ func decodeFeedbackForIndex(q *QueryFeedback, pb *queryFeedback, c *CMSketch) { if c != nil { // decode the index point feedback, just set value count in CM Sketch start := len(pb.IndexRanges) / 2 - for i := 0; i < len(pb.HashValues); i += 2 { - // TODO: update using raw bytes instead of hash values. - c.setValue(pb.HashValues[i], pb.HashValues[i+1], uint64(pb.Counts[start+i/2])) + if len(pb.HashValues) > 0 { + // It needs raw values to update the top n, so just skip it here. + if len(c.topN) > 0 { + return + } + for i := 0; i < len(pb.HashValues); i += 2 { + c.setValue(pb.HashValues[i], pb.HashValues[i+1], uint64(pb.Counts[start+i/2])) + } + return + } + for i := 0; i < len(pb.IndexPoints); i++ { + c.updateValueBytes(pb.IndexPoints[i], uint64(pb.Counts[start+i])) } } } @@ -854,7 +864,7 @@ func DecodeFeedback(val []byte, q *QueryFeedback, c *CMSketch, ft *types.FieldTy if err != nil { return errors.Trace(err) } - if len(pb.IndexRanges) > 0 || len(pb.HashValues) > 0 { + if len(pb.IndexRanges) > 0 || len(pb.HashValues) > 0 || len(pb.IndexPoints) > 0 { decodeFeedbackForIndex(q, pb, c) } else if len(pb.IntRanges) > 0 { decodeFeedbackForPK(q, pb, mysql.HasUnsignedFlag(ft.Flag)) diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index 84e428ee22f5a..f4eea552d570a 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -1390,6 +1390,47 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { } } +func (s *testStatsSuite) TestIndexQueryFeedback4TopN(c *C) { + defer cleanEnv(c, s.store, s.do) + testKit := testkit.NewTestKit(c, s.store) + + oriProbability := statistics.FeedbackProbability + defer func() { + statistics.FeedbackProbability = oriProbability + }() + statistics.FeedbackProbability.Store(1) + + testKit.MustExec("use test") + testKit.MustExec("create table t (a bigint(64), index idx(a))") + for i := 0; i < 20; i++ { + testKit.MustExec(`insert into t values (1)`) + } + h := s.do.StatsHandle() + h.HandleDDLEvent(<-h.DDLEventCh()) + c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) + testKit.MustExec("set @@tidb_enable_fast_analyze = 1") + testKit.MustExec("analyze table t with 3 buckets") + for i := 0; i < 20; i++ { + testKit.MustExec(`insert into t values (1)`) + } + c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) + is := s.do.InfoSchema() + c.Assert(h.Update(is), IsNil) + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tblInfo := table.Meta() + + testKit.MustQuery("select * from t use index(idx) where a = 1") + c.Assert(h.DumpStatsDeltaToKV(handle.DumpAll), IsNil) + c.Assert(h.DumpStatsFeedbackToKV(), IsNil) + c.Assert(h.HandleUpdateStats(s.do.InfoSchema()), IsNil) + c.Assert(h.Update(is), IsNil) + tbl := h.GetTableStats(tblInfo) + val, err := codec.EncodeKey(testKit.Se.GetSessionVars().StmtCtx, nil, types.NewIntDatum(1)) + c.Assert(err, IsNil) + c.Assert(tbl.Indices[1].CMSketch.QueryBytes(val), Equals, uint64(40)) +} + func (s *testStatsSuite) TestAbnormalIndexFeedback(c *C) { defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) From 0668cae6651ae3930006151c2af06cee3436ddab Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 29 Jul 2019 19:05:19 +0800 Subject: [PATCH 114/196] executor, stats: extract topn from cm sketch (#11409) --- executor/analyze.go | 6 ++- executor/analyze_test.go | 33 ++++++++++++++-- statistics/cmsketch.go | 37 ++++++++++++++---- statistics/cmsketch_test.go | 2 +- statistics/handle/update_test.go | 1 + statistics/histogram.go | 66 ++++++++++++++++++++++++++++++++ statistics/sample.go | 31 +++++++++++++++ 7 files changed, 162 insertions(+), 14 deletions(-) mode change 100644 => 100755 executor/analyze.go diff --git a/executor/analyze.go b/executor/analyze.go old mode 100644 new mode 100755 index 05ae1c5ffc3e9..ee4a91a49fce8 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -335,7 +335,8 @@ func (e *AnalyzeIndexExec) buildStatsFromResult(result distsql.SelectResult, nee } } } - return hist, cms, nil + err := hist.ExtractTopN(cms, len(e.idxInfo.Columns), uint32(e.opts[ast.AnalyzeOptNumTopN])) + return hist, cms, err } func (e *AnalyzeIndexExec) buildStats(ranges []*ranger.Range, considerNull bool) (hist *statistics.Histogram, cms *statistics.CMSketch, err error) { @@ -508,6 +509,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis cms = append(cms, nil) } for i, col := range e.colsInfo { + collectors[i].ExtractTopN(uint32(e.opts[ast.AnalyzeOptNumTopN])) for j, s := range collectors[i].Samples { collectors[i].Samples[j].Ordinal = j collectors[i].Samples[j].Value, err = tablecodec.DecodeColumnValue(s.Value.GetBytes(), &col.FieldType, timeZone) @@ -1213,7 +1215,7 @@ func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult return analyzeResult{Err: err, job: idxExec.job} } if idxExec.oldCMS != nil && cms != nil { - err = cms.MergeCMSketch4IncrementalAnalyze(idxExec.oldCMS) + err = cms.MergeCMSketch4IncrementalAnalyze(idxExec.oldCMS, uint32(idxExec.opts[ast.AnalyzeOptNumTopN])) if err != nil { return analyzeResult{Err: err, job: idxExec.job} } diff --git a/executor/analyze_test.go b/executor/analyze_test.go index e7bc61e02f89b..26affd809d1b2 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -135,16 +135,16 @@ func (s *testSuite1) TestAnalyzeParameters(c *C) { tbl := s.dom.StatsHandle().GetTableStats(tableInfo) col := tbl.Columns[1] c.Assert(col.Len(), Equals, 20) - c.Assert(len(col.CMSketch.TopN()), Equals, 20) + c.Assert(len(col.CMSketch.TopN()), Equals, 1) width, depth := col.CMSketch.GetWidthAndDepth() c.Assert(depth, Equals, int32(5)) c.Assert(width, Equals, int32(2048)) - tk.MustExec("analyze table t with 4 buckets, 1 topn, 4 cmsketch width, 4 cmsketch depth") + tk.MustExec("analyze table t with 4 buckets, 0 topn, 4 cmsketch width, 4 cmsketch depth") tbl = s.dom.StatsHandle().GetTableStats(tableInfo) col = tbl.Columns[1] c.Assert(col.Len(), Equals, 4) - c.Assert(len(col.CMSketch.TopN()), Equals, 1) + c.Assert(len(col.CMSketch.TopN()), Equals, 0) width, depth = col.CMSketch.GetWidthAndDepth() c.Assert(depth, Equals, int32(4)) c.Assert(width, Equals, int32(4)) @@ -462,3 +462,30 @@ func (s *testSuite1) TestFailedAnalyzeRequest(c *C) { c.Assert(err.Error(), Equals, "mock buildStatsFromResult error") c.Assert(failpoint.Disable("github.com/pingcap/tidb/executor/buildStatsFromResult"), IsNil) } + +func (s *testSuite1) TestExtractTopN(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int primary key, b int, index index_b(b))") + for i := 0; i < 10; i++ { + tk.MustExec(fmt.Sprintf("insert into t values (%d, %d)", i, i)) + } + for i := 0; i < 10; i++ { + tk.MustExec(fmt.Sprintf("insert into t values (%d, 0)", i+10)) + } + tk.MustExec("analyze table t") + is := s.dom.InfoSchema() + table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tblInfo := table.Meta() + tblStats := s.dom.StatsHandle().GetTableStats(tblInfo) + colStats := tblStats.Columns[tblInfo.Columns[1].ID] + c.Assert(len(colStats.CMSketch.TopN()), Equals, 1) + item := colStats.CMSketch.TopN()[0] + c.Assert(item.Count, Equals, uint64(11)) + idxStats := tblStats.Indices[tblInfo.Indices[0].ID] + c.Assert(len(idxStats.CMSketch.TopN()), Equals, 1) + item = idxStats.CMSketch.TopN()[0] + c.Assert(item.Count, Equals, uint64(11)) +} diff --git a/statistics/cmsketch.go b/statistics/cmsketch.go index 86ea0338a4530..a543ae41ad786 100644 --- a/statistics/cmsketch.go +++ b/statistics/cmsketch.go @@ -103,6 +103,9 @@ func newTopNHelper(sample [][]byte, numTop uint32) *topNHelper { if i >= numTop && sorted[i]*3 < sorted[numTop-1]*2 && last != sorted[i] { break } + if sorted[i] == 1 { + break + } last = sorted[i] sumTopN += sorted[i] } @@ -244,6 +247,14 @@ func (c *CMSketch) setValue(h1, h2 uint64, count uint64) { } } +func (c *CMSketch) subValue(h1, h2 uint64, count uint64) { + c.count -= count + for i := range c.table { + j := (h1 + h2*uint64(i)) % uint64(c.width) + c.table[i][j] = c.table[i][j] - uint32(count) + } +} + func (c *CMSketch) queryValue(sc *stmtctx.StatementContext, val types.Datum) (uint64, error) { bytes, err := codec.EncodeValue(sc, nil, val) if err != nil { @@ -287,7 +298,7 @@ func (c *CMSketch) queryHashValue(h1, h2 uint64) uint64 { return uint64(res) } -func (c *CMSketch) mergeTopN(lTopN map[uint64][]*TopNMeta, rTopN map[uint64][]*TopNMeta, numTop uint32) { +func (c *CMSketch) mergeTopN(lTopN map[uint64][]*TopNMeta, rTopN map[uint64][]*TopNMeta, numTop uint32, usingMax bool) { counter := make(map[hack.MutableString]uint64) for _, metas := range lTopN { for _, meta := range metas { @@ -296,7 +307,11 @@ func (c *CMSketch) mergeTopN(lTopN map[uint64][]*TopNMeta, rTopN map[uint64][]*T } for _, metas := range rTopN { for _, meta := range metas { - counter[hack.String(meta.Data)] += meta.Count + if usingMax { + counter[hack.String(meta.Data)] = mathutil.MaxUint64(counter[hack.String(meta.Data)], meta.Count) + } else { + counter[hack.String(meta.Data)] += meta.Count + } } } sorted := make([]uint64, len(counter)) @@ -326,7 +341,7 @@ func (c *CMSketch) MergeCMSketch(rc *CMSketch, numTopN uint32) error { return errors.New("Dimensions of Count-Min Sketch should be the same") } if c.topN != nil || rc.topN != nil { - c.mergeTopN(c.topN, rc.topN, numTopN) + c.mergeTopN(c.topN, rc.topN, numTopN, false) } c.count += rc.count for i := range c.table { @@ -345,12 +360,12 @@ func (c *CMSketch) MergeCMSketch(rc *CMSketch, numTopN uint32) error { // (3): For values that appears both in `c` and `rc`, if they do not appear partially in `c` and `rc`, for example, // if `v` appears 5 times in the table, it can appears 5 times in `c` and 3 times in `rc`, then `max` also gives the correct answer. // So in fact, if we can know the number of appearances of each value in the first place, it is better to use `max` to construct the CM sketch rather than `sum`. -func (c *CMSketch) MergeCMSketch4IncrementalAnalyze(rc *CMSketch) error { +func (c *CMSketch) MergeCMSketch4IncrementalAnalyze(rc *CMSketch, numTopN uint32) error { if c.depth != rc.depth || c.width != rc.width { return errors.New("Dimensions of Count-Min Sketch should be the same") } if c.topN != nil || rc.topN != nil { - return errors.New("CMSketch with Top-N does not support merge") + c.mergeTopN(c.topN, rc.topN, numTopN, true) } for i := range c.table { c.count = 0 @@ -393,10 +408,10 @@ func CMSketchFromProto(protoSketch *tipb.CMSketch) *CMSketch { c.count = c.count + uint64(counter) } } + c.defaultValue = protoSketch.DefaultValue if len(protoSketch.TopN) == 0 { return c } - c.defaultValue = protoSketch.DefaultValue c.topN = make(map[uint64][]*TopNMeta) for _, e := range protoSketch.TopN { h1, h2 := murmur3.Sum128(e.Data) @@ -450,9 +465,15 @@ func LoadCMSketchWithTopN(exec sqlexec.RestrictedSQLExecutor, tableID, isIndex, return decodeCMSketch(cms, topN) } -// TotalCount returns the count, it is only used for test. +// TotalCount returns the total count in the sketch, it is only used for test. func (c *CMSketch) TotalCount() uint64 { - return c.count + res := c.count + for _, metas := range c.topN { + for _, meta := range metas { + res += meta.Count + } + } + return res } // Equal tests if two CM Sketch equal, it is only used for test. diff --git a/statistics/cmsketch_test.go b/statistics/cmsketch_test.go index 0c6508f4524a5..15decc68384c2 100644 --- a/statistics/cmsketch_test.go +++ b/statistics/cmsketch_test.go @@ -240,7 +240,7 @@ func (s *testStatisticsSuite) TestMergeCMSketch4IncrementalAnalyze(c *C) { for key, val := range rMap { lMap[key] += val } - c.Assert(lSketch.MergeCMSketch4IncrementalAnalyze(rSketch), IsNil) + c.Assert(lSketch.MergeCMSketch4IncrementalAnalyze(rSketch, 0), IsNil) avg, err = averageAbsoluteError(lSketch, lMap) c.Assert(err, IsNil) c.Check(avg, LessEqual, t.avgError) diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index f4eea552d570a..33206cabe556f 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -1257,6 +1257,7 @@ func (s *testStatsSuite) TestNeedAnalyzeTable(c *C) { } func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { + c.Skip("support update the topn of index equal conditions") defer cleanEnv(c, s.store, s.do) testKit := testkit.NewTestKit(c, s.store) diff --git a/statistics/histogram.go b/statistics/histogram.go index 2ffbd933909a0..8bdb2b25e00b0 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -17,6 +17,7 @@ import ( "bytes" "fmt" "math" + "sort" "strings" "time" @@ -33,6 +34,7 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" "github.com/pingcap/tipb/go-tipb" + "github.com/spaolacci/murmur3" "go.uber.org/zap" ) @@ -1054,3 +1056,67 @@ func matchPrefix(row chunk.Row, colIdx int, ad *types.Datum) bool { } return false } + +type dataCnt struct { + data []byte + cnt uint64 +} + +func getIndexPrefixLens(data []byte, numCols int) (prefixLens []int, err error) { + prefixLens = make([]int, 0, numCols) + var colData []byte + prefixLen := 0 + for len(data) > 0 { + colData, data, err = codec.CutOne(data) + if err != nil { + return nil, err + } + prefixLen += len(colData) + prefixLens = append(prefixLens, prefixLen) + } + return prefixLens, nil +} + +// ExtractTopN extracts topn from histogram. +func (hg *Histogram) ExtractTopN(cms *CMSketch, numCols int, numTopN uint32) error { + if hg.Len() == 0 || cms == nil || numTopN == 0 { + return nil + } + dataSet := make(map[string]struct{}, hg.Bounds.NumRows()) + dataCnts := make([]dataCnt, 0, hg.Bounds.NumRows()) + hg.PreCalculateScalar() + // Set a limit on the frequency of boundary values to avoid extract values with low frequency. + limit := hg.notNullCount() / float64(hg.Len()) + // Since our histogram are equal depth, they must occurs on the boundaries of buckets. + for i := 0; i < hg.Bounds.NumRows(); i++ { + data := hg.Bounds.GetRow(i).GetBytes(0) + prefixLens, err := getIndexPrefixLens(data, numCols) + if err != nil { + return err + } + for _, prefixLen := range prefixLens { + prefixColData := data[:prefixLen] + _, ok := dataSet[string(prefixColData)] + if ok { + continue + } + dataSet[string(prefixColData)] = struct{}{} + res := hg.BetweenRowCount(types.NewBytesDatum(prefixColData), types.NewBytesDatum(kv.Key(prefixColData).PrefixNext())) + if res >= limit { + dataCnts = append(dataCnts, dataCnt{prefixColData, uint64(res)}) + } + } + } + sort.SliceStable(dataCnts, func(i, j int) bool { return dataCnts[i].cnt >= dataCnts[j].cnt }) + cms.topN = make(map[uint64][]*TopNMeta) + if len(dataCnts) > int(numTopN) { + dataCnts = dataCnts[:numTopN] + } + for _, dataCnt := range dataCnts { + h1, h2 := murmur3.Sum128(dataCnt.data) + realCnt := cms.queryHashValue(h1, h2) + cms.subValue(h1, h2, realCnt) + cms.topN[h1] = append(cms.topN[h1], &TopNMeta{h2, dataCnt.data, realCnt}) + } + return nil +} diff --git a/statistics/sample.go b/statistics/sample.go index 8dbf0c4676125..96cc22dead8a8 100644 --- a/statistics/sample.go +++ b/statistics/sample.go @@ -25,8 +25,10 @@ import ( "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tipb/go-tipb" + "github.com/spaolacci/murmur3" ) // SampleItem is an item of sampled column value. @@ -257,3 +259,32 @@ func RowToDatums(row chunk.Row, fields []*ast.ResultField) []types.Datum { } return datums } + +// ExtractTopN extracts the topn from the CM Sketch. +func (c *SampleCollector) ExtractTopN(numTop uint32) { + if numTop == 0 { + return + } + values := make([][]byte, 0, len(c.Samples)) + for _, sample := range c.Samples { + values = append(values, sample.Value.GetBytes()) + } + helper := newTopNHelper(values, numTop) + cms := c.CMSketch + cms.topN = make(map[uint64][]*TopNMeta) + dataCnts := make([]dataCnt, 0, len(helper.counter)) + for key, cnt := range helper.counter { + if cnt >= helper.lastVal { + dataCnts = append(dataCnts, dataCnt{hack.Slice(string(key)), cnt}) + } + } + // Sort them decreasingly so we can handle most frequent values first and reduce the probability of hash collision + // by small values. + sort.SliceStable(dataCnts, func(i, j int) bool { return dataCnts[i].cnt >= dataCnts[j].cnt }) + for _, dc := range dataCnts { + h1, h2 := murmur3.Sum128(dc.data) + realCnt := cms.queryHashValue(h1, h2) + cms.subValue(h1, h2, realCnt) + cms.topN[h1] = append(cms.topN[h1], &TopNMeta{h2, dc.data, realCnt}) + } +} From b17848b8805bb99898ce9abdefa73c62d15b7317 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 29 Jul 2019 19:56:50 +0800 Subject: [PATCH 115/196] stats: fix estimation for time equal conditions (#11511) --- .../r/explain_complex_stats.result | 12 +++--- cmd/explaintest/r/explain_easy.result | 14 +++++++ cmd/explaintest/r/tpch.result | 38 +++++++++---------- cmd/explaintest/t/explain_easy.test | 7 ++++ statistics/cmsketch.go | 4 +- statistics/handle/update_test.go | 4 +- 6 files changed, 50 insertions(+), 29 deletions(-) diff --git a/cmd/explaintest/r/explain_complex_stats.result b/cmd/explaintest/r/explain_complex_stats.result index 4e08eb2d4eddf..419d1ba3834d8 100644 --- a/cmd/explaintest/r/explain_complex_stats.result +++ b/cmd/explaintest/r/explain_complex_stats.result @@ -115,13 +115,13 @@ PRIMARY KEY (aid,dic) load stats 's/explain_complex_stats_rr.json'; explain SELECT ds, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(dic) as install_device FROM dt use index (cm) WHERE (ds >= '2016-09-01') AND (ds <= '2016-11-03') AND (cm IN ('1062', '1086', '1423', '1424', '1425', '1426', '1427', '1428', '1429', '1430', '1431', '1432', '1433', '1434', '1435', '1436', '1437', '1438', '1439', '1440', '1441', '1442', '1443', '1444', '1445', '1446', '1447', '1448', '1449', '1450', '1451', '1452', '1488', '1489', '1490', '1491', '1492', '1493', '1494', '1495', '1496', '1497', '1550', '1551', '1552', '1553', '1554', '1555', '1556', '1557', '1558', '1559', '1597', '1598', '1599', '1600', '1601', '1602', '1603', '1604', '1605', '1606', '1607', '1608', '1609', '1610', '1611', '1612', '1613', '1614', '1615', '1616', '1623', '1624', '1625', '1626', '1627', '1628', '1629', '1630', '1631', '1632', '1709', '1719', '1720', '1843', '2813', '2814', '2815', '2816', '2817', '2818', '2819', '2820', '2821', '2822', '2823', '2824', '2825', '2826', '2827', '2828', '2829', '2830', '2831', '2832', '2833', '2834', '2835', '2836', '2837', '2838', '2839', '2840', '2841', '2842', '2843', '2844', '2845', '2846', '2847', '2848', '2849', '2850', '2851', '2852', '2853', '2854', '2855', '2856', '2857', '2858', '2859', '2860', '2861', '2862', '2863', '2864', '2865', '2866', '2867', '2868', '2869', '2870', '2871', '2872', '3139', '3140', '3141', '3142', '3143', '3144', '3145', '3146', '3147', '3148', '3149', '3150', '3151', '3152', '3153', '3154', '3155', '3156', '3157', '3158', '3386', '3387', '3388', '3389', '3390', '3391', '3392', '3393', '3394', '3395', '3664', '3665', '3666', '3667', '3668', '3670', '3671', '3672', '3673', '3674', '3676', '3677', '3678', '3679', '3680', '3681', '3682', '3683', '3684', '3685', '3686', '3687', '3688', '3689', '3690', '3691', '3692', '3693', '3694', '3695', '3696', '3697', '3698', '3699', '3700', '3701', '3702', '3703', '3704', '3705', '3706', '3707', '3708', '3709', '3710', '3711', '3712', '3713', '3714', '3715', '3960', '3961', '3962', '3963', '3964', '3965', '3966', '3967', '3968', '3978', '3979', '3980', '3981', '3982', '3983', '3984', '3985', '3986', '3987', '4208', '4209', '4210', '4211', '4212', '4304', '4305', '4306', '4307', '4308', '4866', '4867', '4868', '4869', '4870', '4871', '4872', '4873', '4874', '4875')) GROUP BY ds, p1, p2, p3, p4, p5, p6_md5, p7_md5 ORDER BY ds2 DESC; id count task operator info -Projection_7 21.40 root test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, install_device -└─Sort_8 21.40 root test.dt.ds2:desc - └─HashAgg_16 21.40 root group by:col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, funcs:count(col_0), firstrow(col_1), firstrow(col_2), firstrow(col_3), firstrow(col_4), firstrow(col_5), firstrow(col_6), firstrow(col_7), firstrow(col_8), firstrow(col_9) - └─IndexLookUp_17 21.40 root +Projection_7 21.53 root test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, install_device +└─Sort_8 21.53 root test.dt.ds2:desc + └─HashAgg_16 21.53 root group by:col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, funcs:count(col_0), firstrow(col_1), firstrow(col_2), firstrow(col_3), firstrow(col_4), firstrow(col_5), firstrow(col_6), firstrow(col_7), firstrow(col_8), firstrow(col_9) + └─IndexLookUp_17 21.53 root ├─IndexScan_13 128.32 cop table:dt, index:cm, range:[1062,1062], [1086,1086], [1423,1423], [1424,1424], [1425,1425], [1426,1426], [1427,1427], [1428,1428], [1429,1429], [1430,1430], [1431,1431], [1432,1432], [1433,1433], [1434,1434], [1435,1435], [1436,1436], [1437,1437], [1438,1438], [1439,1439], [1440,1440], [1441,1441], [1442,1442], [1443,1443], [1444,1444], [1445,1445], [1446,1446], [1447,1447], [1448,1448], [1449,1449], [1450,1450], [1451,1451], [1452,1452], [1488,1488], [1489,1489], [1490,1490], [1491,1491], [1492,1492], [1493,1493], [1494,1494], [1495,1495], [1496,1496], [1497,1497], [1550,1550], [1551,1551], [1552,1552], [1553,1553], [1554,1554], [1555,1555], [1556,1556], [1557,1557], [1558,1558], [1559,1559], [1597,1597], [1598,1598], [1599,1599], [1600,1600], [1601,1601], [1602,1602], [1603,1603], [1604,1604], [1605,1605], [1606,1606], [1607,1607], [1608,1608], [1609,1609], [1610,1610], [1611,1611], [1612,1612], [1613,1613], [1614,1614], [1615,1615], [1616,1616], [1623,1623], [1624,1624], [1625,1625], [1626,1626], [1627,1627], [1628,1628], [1629,1629], [1630,1630], [1631,1631], [1632,1632], [1709,1709], [1719,1719], [1720,1720], [1843,1843], [2813,2813], [2814,2814], [2815,2815], [2816,2816], [2817,2817], [2818,2818], [2819,2819], [2820,2820], [2821,2821], [2822,2822], [2823,2823], [2824,2824], [2825,2825], [2826,2826], [2827,2827], [2828,2828], [2829,2829], [2830,2830], [2831,2831], [2832,2832], [2833,2833], [2834,2834], [2835,2835], [2836,2836], [2837,2837], [2838,2838], [2839,2839], [2840,2840], [2841,2841], [2842,2842], [2843,2843], [2844,2844], [2845,2845], [2846,2846], [2847,2847], [2848,2848], [2849,2849], [2850,2850], [2851,2851], [2852,2852], [2853,2853], [2854,2854], [2855,2855], [2856,2856], [2857,2857], [2858,2858], [2859,2859], [2860,2860], [2861,2861], [2862,2862], [2863,2863], [2864,2864], [2865,2865], [2866,2866], [2867,2867], [2868,2868], [2869,2869], [2870,2870], [2871,2871], [2872,2872], [3139,3139], [3140,3140], [3141,3141], [3142,3142], [3143,3143], [3144,3144], [3145,3145], [3146,3146], [3147,3147], [3148,3148], [3149,3149], [3150,3150], [3151,3151], [3152,3152], [3153,3153], [3154,3154], [3155,3155], [3156,3156], [3157,3157], [3158,3158], [3386,3386], [3387,3387], [3388,3388], [3389,3389], [3390,3390], [3391,3391], [3392,3392], [3393,3393], [3394,3394], [3395,3395], [3664,3664], [3665,3665], [3666,3666], [3667,3667], [3668,3668], [3670,3670], [3671,3671], [3672,3672], [3673,3673], [3674,3674], [3676,3676], [3677,3677], [3678,3678], [3679,3679], [3680,3680], [3681,3681], [3682,3682], [3683,3683], [3684,3684], [3685,3685], [3686,3686], [3687,3687], [3688,3688], [3689,3689], [3690,3690], [3691,3691], [3692,3692], [3693,3693], [3694,3694], [3695,3695], [3696,3696], [3697,3697], [3698,3698], [3699,3699], [3700,3700], [3701,3701], [3702,3702], [3703,3703], [3704,3704], [3705,3705], [3706,3706], [3707,3707], [3708,3708], [3709,3709], [3710,3710], [3711,3711], [3712,3712], [3713,3713], [3714,3714], [3715,3715], [3960,3960], [3961,3961], [3962,3962], [3963,3963], [3964,3964], [3965,3965], [3966,3966], [3967,3967], [3968,3968], [3978,3978], [3979,3979], [3980,3980], [3981,3981], [3982,3982], [3983,3983], [3984,3984], [3985,3985], [3986,3986], [3987,3987], [4208,4208], [4209,4209], [4210,4210], [4211,4211], [4212,4212], [4304,4304], [4305,4305], [4306,4306], [4307,4307], [4308,4308], [4866,4866], [4867,4867], [4868,4868], [4869,4869], [4870,4870], [4871,4871], [4872,4872], [4873,4873], [4874,4874], [4875,4875], keep order:false - └─HashAgg_11 21.40 cop group by:test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, funcs:count(test.dt.dic), firstrow(test.dt.ds), firstrow(test.dt.ds2), firstrow(test.dt.p1), firstrow(test.dt.p2), firstrow(test.dt.p3), firstrow(test.dt.p4), firstrow(test.dt.p5), firstrow(test.dt.p6_md5), firstrow(test.dt.p7_md5) - └─Selection_15 21.43 cop ge(test.dt.ds, 2016-09-01 00:00:00.000000), le(test.dt.ds, 2016-11-03 00:00:00.000000) + └─HashAgg_11 21.53 cop group by:test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, funcs:count(test.dt.dic), firstrow(test.dt.ds), firstrow(test.dt.ds2), firstrow(test.dt.p1), firstrow(test.dt.p2), firstrow(test.dt.p3), firstrow(test.dt.p4), firstrow(test.dt.p5), firstrow(test.dt.p6_md5), firstrow(test.dt.p7_md5) + └─Selection_15 21.56 cop ge(test.dt.ds, 2016-09-01 00:00:00.000000), le(test.dt.ds, 2016-11-03 00:00:00.000000) └─TableScan_14 128.32 cop table:dt, keep order:false explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext, gad.t as gtime from st gad join (select id, aid, pt, dic, ip, t from dd where pt = 'android' and bm = 0 and t > 1478143908) sdk on gad.aid = sdk.aid and gad.ip = sdk.ip and sdk.t > gad.t where gad.t > 1478143908 and gad.bm = 0 and gad.pt = 'android' group by gad.aid, sdk.dic limit 2500; id count task operator info diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 9699f6c7a5c95..aa9b6ace34e26 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -689,3 +689,17 @@ Projection_8 8320.83 root test.t.a, test.t1.a └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo rollback; drop table if exists t; +create table t(a time, b date); +insert into t values (1, "1000-01-01"), (2, "1000-01-02"), (3, "1000-01-03"); +analyze table t; +explain select * from t where a = 1; +id count task operator info +TableReader_7 1.00 root data:Selection_6 +└─Selection_6 1.00 cop eq(test.t.a, 00:00:01.000000) + └─TableScan_5 3.00 cop table:t, range:[-inf,+inf], keep order:false +explain select * from t where b = "1000-01-01"; +id count task operator info +TableReader_7 1.00 root data:Selection_6 +└─Selection_6 1.00 cop eq(test.t.b, 1000-01-01 00:00:00.000000) + └─TableScan_5 3.00 cop table:t, range:[-inf,+inf], keep order:false +drop table t; diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 48daaa9b7d8b9..eefcc37ffaf98 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -124,7 +124,7 @@ Sort_6 2.94 root tpch.lineitem.l_returnflag:asc, tpch.lineitem.l_linestatus:asc └─HashAgg_14 2.94 root group by:col_13, col_14, funcs:sum(col_0), sum(col_1), sum(col_2), sum(col_3), avg(col_4, col_5), avg(col_6, col_7), avg(col_8, col_9), count(col_10), firstrow(col_11), firstrow(col_12) └─TableReader_15 2.94 root data:HashAgg_9 └─HashAgg_9 2.94 cop group by:tpch.lineitem.l_linestatus, tpch.lineitem.l_returnflag, funcs:sum(tpch.lineitem.l_quantity), sum(tpch.lineitem.l_extendedprice), sum(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))), sum(mul(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), plus(1, tpch.lineitem.l_tax))), avg(tpch.lineitem.l_quantity), avg(tpch.lineitem.l_extendedprice), avg(tpch.lineitem.l_discount), count(1), firstrow(tpch.lineitem.l_returnflag), firstrow(tpch.lineitem.l_linestatus) - └─Selection_13 293683189.00 cop le(tpch.lineitem.l_shipdate, 1998-08-15) + └─Selection_13 293795345.00 cop le(tpch.lineitem.l_shipdate, 1998-08-15) └─TableScan_12 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false /* Q2 Minimum Cost Supplier Query @@ -250,7 +250,7 @@ limit 10; id count task operator info Projection_14 10.00 root tpch.lineitem.l_orderkey, 7_col_0, tpch.orders.o_orderdate, tpch.orders.o_shippriority └─TopN_17 10.00 root 7_col_0:desc, tpch.orders.o_orderdate:asc, offset:0, count:10 - └─HashAgg_23 40227041.09 root group by:col_4, col_5, col_6, funcs:sum(col_0), firstrow(col_1), firstrow(col_2), firstrow(col_3) + └─HashAgg_23 40252367.98 root group by:col_4, col_5, col_6, funcs:sum(col_0), firstrow(col_1), firstrow(col_2), firstrow(col_3) └─Projection_59 91515927.49 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), tpch.orders.o_orderdate, tpch.orders.o_shippriority, tpch.lineitem.l_orderkey, tpch.lineitem.l_orderkey, tpch.orders.o_orderdate, tpch.orders.o_shippriority └─IndexJoin_29 91515927.49 root inner join, inner:IndexLookUp_28, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey ├─HashRightJoin_49 22592975.51 root inner join, inner:TableReader_55, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] @@ -443,9 +443,9 @@ supp_nation, cust_nation, l_year; id count task operator info -Sort_22 768.91 root tpch.shipping.supp_nation:asc, tpch.shipping.cust_nation:asc, shipping.l_year:asc -└─Projection_24 768.91 root tpch.shipping.supp_nation, tpch.shipping.cust_nation, shipping.l_year, 14_col_0 - └─HashAgg_27 768.91 root group by:shipping.l_year, tpch.shipping.cust_nation, tpch.shipping.supp_nation, funcs:sum(shipping.volume), firstrow(tpch.shipping.supp_nation), firstrow(tpch.shipping.cust_nation), firstrow(shipping.l_year) +Sort_22 769.96 root tpch.shipping.supp_nation:asc, tpch.shipping.cust_nation:asc, shipping.l_year:asc +└─Projection_24 769.96 root tpch.shipping.supp_nation, tpch.shipping.cust_nation, shipping.l_year, 14_col_0 + └─HashAgg_27 769.96 root group by:shipping.l_year, tpch.shipping.cust_nation, tpch.shipping.supp_nation, funcs:sum(shipping.volume), firstrow(tpch.shipping.supp_nation), firstrow(tpch.shipping.cust_nation), firstrow(shipping.l_year) └─Projection_28 1957240.42 root tpch.n1.n_name, tpch.n2.n_name, extract("YEAR", tpch.lineitem.l_shipdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)) └─HashLeftJoin_33 1957240.42 root inner join, inner:TableReader_68, equal:[eq(tpch.customer.c_nationkey, tpch.n2.n_nationkey)], other cond:or(and(eq(tpch.n1.n_name, "JAPAN"), eq(tpch.n2.n_name, "INDIA")), and(eq(tpch.n1.n_name, "INDIA"), eq(tpch.n2.n_name, "JAPAN"))) ├─IndexJoin_37 24465505.20 root inner join, inner:TableReader_36, outer key:tpch.orders.o_custkey, inner key:tpch.customer.c_custkey @@ -457,8 +457,8 @@ Sort_22 768.91 root tpch.shipping.supp_nation:asc, tpch.shipping.cust_nation:asc │ │ │ │ │ └─TableScan_56 25.00 cop table:n1, range:[-inf,+inf], keep order:false │ │ │ │ └─TableReader_55 500000.00 root data:TableScan_54 │ │ │ │ └─TableScan_54 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false - │ │ │ └─TableReader_61 91321768.29 root data:Selection_60 - │ │ │ └─Selection_60 91321768.29 cop ge(tpch.lineitem.l_shipdate, 1995-01-01 00:00:00.000000), le(tpch.lineitem.l_shipdate, 1996-12-31 00:00:00.000000) + │ │ │ └─TableReader_61 91446230.29 root data:Selection_60 + │ │ │ └─Selection_60 91446230.29 cop ge(tpch.lineitem.l_shipdate, 1995-01-01 00:00:00.000000), le(tpch.lineitem.l_shipdate, 1996-12-31 00:00:00.000000) │ │ │ └─TableScan_59 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false │ │ └─TableReader_42 1.00 root data:TableScan_41 │ │ └─TableScan_41 1.00 cop table:orders, range: decided by [tpch.lineitem.l_orderkey], keep order:false @@ -515,16 +515,16 @@ o_year order by o_year; id count task operator info -Sort_29 718.01 root all_nations.o_year:asc -└─Projection_31 718.01 root all_nations.o_year, div(18_col_0, 18_col_1) - └─HashAgg_34 718.01 root group by:col_3, funcs:sum(col_0), sum(col_1), firstrow(col_2) - └─Projection_89 562348.12 root case(eq(tpch.all_nations.nation, "INDIA"), all_nations.volume, 0), all_nations.volume, all_nations.o_year, all_nations.o_year - └─Projection_35 562348.12 root extract("YEAR", tpch.orders.o_orderdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), tpch.n2.n_name - └─HashLeftJoin_39 562348.12 root inner join, inner:TableReader_87, equal:[eq(tpch.supplier.s_nationkey, tpch.n2.n_nationkey)] - ├─IndexJoin_43 562348.12 root inner join, inner:TableReader_42, outer key:tpch.lineitem.l_suppkey, inner key:tpch.supplier.s_suppkey - │ ├─HashLeftJoin_50 562348.12 root inner join, inner:TableReader_83, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)] - │ │ ├─IndexJoin_56 90661378.61 root inner join, inner:IndexLookUp_55, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey - │ │ │ ├─HashRightJoin_60 22382008.93 root inner join, inner:HashRightJoin_62, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] +Sort_29 719.02 root all_nations.o_year:asc +└─Projection_31 719.02 root all_nations.o_year, div(18_col_0, 18_col_1) + └─HashAgg_34 719.02 root group by:col_3, funcs:sum(col_0), sum(col_1), firstrow(col_2) + └─Projection_89 563136.02 root case(eq(tpch.all_nations.nation, "INDIA"), all_nations.volume, 0), all_nations.volume, all_nations.o_year, all_nations.o_year + └─Projection_35 563136.02 root extract("YEAR", tpch.orders.o_orderdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), tpch.n2.n_name + └─HashLeftJoin_39 563136.02 root inner join, inner:TableReader_87, equal:[eq(tpch.supplier.s_nationkey, tpch.n2.n_nationkey)] + ├─IndexJoin_43 563136.02 root inner join, inner:TableReader_42, outer key:tpch.lineitem.l_suppkey, inner key:tpch.supplier.s_suppkey + │ ├─HashLeftJoin_50 563136.02 root inner join, inner:TableReader_83, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)] + │ │ ├─IndexJoin_56 90788402.51 root inner join, inner:IndexLookUp_55, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey + │ │ │ ├─HashRightJoin_60 22413367.93 root inner join, inner:HashRightJoin_62, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] │ │ │ │ ├─HashRightJoin_62 1500000.00 root inner join, inner:HashRightJoin_68, equal:[eq(tpch.n1.n_nationkey, tpch.customer.c_nationkey)] │ │ │ │ │ ├─HashRightJoin_68 5.00 root inner join, inner:TableReader_73, equal:[eq(tpch.region.r_regionkey, tpch.n1.n_regionkey)] │ │ │ │ │ │ ├─TableReader_73 1.00 root data:Selection_72 @@ -534,8 +534,8 @@ Sort_29 718.01 root all_nations.o_year:asc │ │ │ │ │ │ └─TableScan_69 25.00 cop table:n1, range:[-inf,+inf], keep order:false │ │ │ │ │ └─TableReader_75 7500000.00 root data:TableScan_74 │ │ │ │ │ └─TableScan_74 7500000.00 cop table:customer, range:[-inf,+inf], keep order:false - │ │ │ │ └─TableReader_78 22382008.93 root data:Selection_77 - │ │ │ │ └─Selection_77 22382008.93 cop ge(tpch.orders.o_orderdate, 1995-01-01 00:00:00.000000), le(tpch.orders.o_orderdate, 1996-12-31 00:00:00.000000) + │ │ │ │ └─TableReader_78 22413367.93 root data:Selection_77 + │ │ │ │ └─Selection_77 22413367.93 cop ge(tpch.orders.o_orderdate, 1995-01-01 00:00:00.000000), le(tpch.orders.o_orderdate, 1996-12-31 00:00:00.000000) │ │ │ │ └─TableScan_76 75000000.00 cop table:orders, range:[-inf,+inf], keep order:false │ │ │ └─IndexLookUp_55 1.00 root │ │ │ ├─IndexScan_53 1.00 cop table:lineitem, index:L_ORDERKEY, L_LINENUMBER, range: decided by [eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)], keep order:false diff --git a/cmd/explaintest/t/explain_easy.test b/cmd/explaintest/t/explain_easy.test index fe056d84d0208..3510b4fabb8f6 100644 --- a/cmd/explaintest/t/explain_easy.test +++ b/cmd/explaintest/t/explain_easy.test @@ -152,3 +152,10 @@ insert into t values (1); explain select * from t left outer join t t1 on t.a = t1.a where t.a not between 1 and 2; rollback; drop table if exists t; + +create table t(a time, b date); +insert into t values (1, "1000-01-01"), (2, "1000-01-02"), (3, "1000-01-03"); +analyze table t; +explain select * from t where a = 1; +explain select * from t where b = "1000-01-01"; +drop table t; diff --git a/statistics/cmsketch.go b/statistics/cmsketch.go index a543ae41ad786..00935d0c45638 100644 --- a/statistics/cmsketch.go +++ b/statistics/cmsketch.go @@ -24,8 +24,8 @@ import ( "github.com/cznic/sortutil" "github.com/pingcap/errors" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/hack" "github.com/pingcap/tidb/util/sqlexec" "github.com/pingcap/tipb/go-tipb" @@ -256,7 +256,7 @@ func (c *CMSketch) subValue(h1, h2 uint64, count uint64) { } func (c *CMSketch) queryValue(sc *stmtctx.StatementContext, val types.Datum) (uint64, error) { - bytes, err := codec.EncodeValue(sc, nil, val) + bytes, err := tablecodec.EncodeValue(sc, nil, val) if err != nil { return 0, errors.Trace(err) } diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index 33206cabe556f..de3ae9a7d5f94 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -1354,7 +1354,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { sql: "select * from t use index(idx_ag) where a = 1 and g < 21", hist: "column:7 ndv:20 totColSize:196\n" + "num: 13 lower_bound: -838:59:59 upper_bound: 00:00:06 repeats: 0\n" + - "num: 11 lower_bound: 00:00:07 upper_bound: 00:00:13 repeats: 0\n" + + "num: 12 lower_bound: 00:00:07 upper_bound: 00:00:13 repeats: 0\n" + "num: 10 lower_bound: 00:00:14 upper_bound: 00:00:21 repeats: 0", rangeID: tblInfo.Columns[6].ID, idxID: tblInfo.Indices[6].ID, @@ -1365,7 +1365,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { sql: `select * from t use index(idx_ah) where a = 1 and h < "1000-01-21"`, hist: "column:8 ndv:20 totColSize:360\n" + "num: 13 lower_bound: 1000-01-01 upper_bound: 1000-01-07 repeats: 0\n" + - "num: 11 lower_bound: 1000-01-08 upper_bound: 1000-01-14 repeats: 0\n" + + "num: 12 lower_bound: 1000-01-08 upper_bound: 1000-01-14 repeats: 0\n" + "num: 10 lower_bound: 1000-01-15 upper_bound: 1000-01-21 repeats: 0", rangeID: tblInfo.Columns[7].ID, idxID: tblInfo.Indices[7].ID, From cf11ef3e31c836d1687d8c0ce7f275a7ca0c339f Mon Sep 17 00:00:00 2001 From: cfzjywxk Date: Tue, 30 Jul 2019 09:32:27 +0800 Subject: [PATCH 116/196] =?UTF-8?q?executor:=20refactor=20info=20from=20in?= =?UTF-8?q?terface=20to=20string=20to=20avoid=20runT=E2=80=A6=20(#11510)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- session/session.go | 13 ++----------- util/expensivequery/expensivequery.go | 10 +++++----- util/processinfo.go | 16 ++++++++++------ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/session/session.go b/session/session.go index bca00fdf4e7fa..84db86e0e6b72 100644 --- a/session/session.go +++ b/session/session.go @@ -962,23 +962,14 @@ func (s *session) ParseSQL(ctx context.Context, sql, charset, collation string) } func (s *session) SetProcessInfo(sql string, t time.Time, command byte, maxExecutionTime uint64) { - var db interface{} - if len(s.sessionVars.CurrentDB) > 0 { - db = s.sessionVars.CurrentDB - } - - var info interface{} - if len(sql) > 0 { - info = sql - } pi := util.ProcessInfo{ ID: s.sessionVars.ConnectionID, - DB: db, + DB: s.sessionVars.CurrentDB, Command: command, Plan: s.currentPlan, Time: t, State: s.Status(), - Info: info, + Info: sql, CurTxnStartTS: s.sessionVars.TxnCtx.StartTS, StmtCtx: s.sessionVars.StmtCtx, StatsInfo: plannercore.GetStatsInfo, diff --git a/util/expensivequery/expensivequery.go b/util/expensivequery/expensivequery.go index a8d3bb54464d5..c407a27c8ba57 100644 --- a/util/expensivequery/expensivequery.go +++ b/util/expensivequery/expensivequery.go @@ -59,7 +59,7 @@ func (eqh *Handle) Run() { case <-ticker.C: processInfo := eqh.sm.ShowProcessList() for _, info := range processInfo { - if info.Info == nil || info.ExceedExpensiveTimeThresh { + if len(info.Info) == 0 || info.ExceedExpensiveTimeThresh { continue } costTime := time.Since(info.Time) @@ -124,8 +124,8 @@ func logExpensiveQuery(costTime time.Duration, info *util.ProcessInfo) { if len(info.User) > 0 { logFields = append(logFields, zap.String("user", info.User)) } - if info.DB != nil && len(info.DB.(string)) > 0 { - logFields = append(logFields, zap.String("database", info.DB.(string))) + if len(info.DB) > 0 { + logFields = append(logFields, zap.String("database", info.DB)) } var tableIDs, indexIDs string if len(info.StmtCtx.TableIDs) > 0 { @@ -143,8 +143,8 @@ func logExpensiveQuery(costTime time.Duration, info *util.ProcessInfo) { const logSQLLen = 1024 * 8 var sql string - if info.Info != nil { - sql = info.Info.(string) + if len(info.Info) > 0 { + sql = info.Info } if len(sql) > logSQLLen { sql = fmt.Sprintf("%s len(%d)", sql[:logSQLLen], len(sql)) diff --git a/util/processinfo.go b/util/processinfo.go index 9d95ff72b56b9..cff4b20f631dd 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -27,12 +27,12 @@ type ProcessInfo struct { ID uint64 User string Host string - DB interface{} + DB string Command byte Plan interface{} Time time.Time State uint16 - Info interface{} + Info string CurTxnStartTS uint64 StmtCtx *stmtctx.StatementContext StatsInfo func(interface{}) map[string]uint64 @@ -45,19 +45,23 @@ type ProcessInfo struct { // ToRowForShow returns []interface{} for the row data of "SHOW [FULL] PROCESSLIST". func (pi *ProcessInfo) ToRowForShow(full bool) []interface{} { var info interface{} - if pi.Info != nil { + if len(pi.Info) > 0 { if full { - info = pi.Info.(string) + info = pi.Info } else { - info = fmt.Sprintf("%.100v", pi.Info.(string)) + info = fmt.Sprintf("%.100v", pi.Info) } } t := uint64(time.Since(pi.Time) / time.Second) + var db interface{} + if len(pi.DB) > 0 { + db = pi.DB + } return []interface{}{ pi.ID, pi.User, pi.Host, - pi.DB, + db, mysql.Command2Str[pi.Command], t, fmt.Sprintf("%d", pi.State), From a92902aeee3a49c835d5f0beb8f1e6dbb645fc3a Mon Sep 17 00:00:00 2001 From: goroutine Date: Tue, 30 Jul 2019 11:01:04 +0800 Subject: [PATCH 117/196] =?UTF-8?q?load=20data:=20avoid=20an=20unnecessary?= =?UTF-8?q?=20converting=20from=20byte=20slice=20to=E2=80=A6=20(#11516)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executor/load_data.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/executor/load_data.go b/executor/load_data.go index eb612451d36ed..05cd1fe8d11d1 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -14,6 +14,7 @@ package executor import ( + "bytes" "context" "fmt" "strings" @@ -31,6 +32,10 @@ import ( "go.uber.org/zap" ) +var ( + null = []byte("NULL") +) + // LoadDataExec represents a load data executor. type LoadDataExec struct { baseExecutor @@ -501,7 +506,7 @@ func (e *LoadDataInfo) getFieldsFromLine(line []byte) ([]field, error) { for { eol, f := reader.GetField() f = f.escape() - if string(f.str) == "NULL" && !f.enclosed { + if bytes.Compare(f.str, null) == 0 && !f.enclosed { f.str = []byte{'N'} f.maybeNull = true } From 444fb8ce7ec5a06b0aad9dc0f7ef4702d78fe9ca Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Tue, 30 Jul 2019 13:31:50 +0800 Subject: [PATCH 118/196] store/tikv, config: increase pessimistic lock ttl (#11499) --- config/config.go | 4 ++-- config/config.toml.example | 4 ++-- config/config_test.go | 4 ++-- store/tikv/2pc.go | 11 ++++++----- store/tikv/2pc_test.go | 40 ++++++++++++++++++++++++++++++++++++++ store/tikv/txn.go | 5 +++++ 6 files changed, 57 insertions(+), 11 deletions(-) diff --git a/config/config.go b/config/config.go index bbdf24deaed8b..471e83a1bdcc1 100644 --- a/config/config.go +++ b/config/config.go @@ -36,7 +36,7 @@ import ( const ( MaxLogFileSize = 4096 // MB MinPessimisticTTL = time.Second * 15 - MaxPessimisticTTL = time.Second * 60 + MaxPessimisticTTL = time.Second * 120 // DefTxnEntryCountLimit is the default value of TxnEntryCountLimit. DefTxnEntryCountLimit = 300 * 1000 // DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit. @@ -408,7 +408,7 @@ var defaultConf = Config{ Enable: false, Default: false, MaxRetryCount: 256, - TTL: "30s", + TTL: "40s", }, } diff --git a/config/config.toml.example b/config/config.toml.example index 15f079e1de12d..a641fb3486916 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -316,5 +316,5 @@ default = false max-retry-count = 256 # default TTL in milliseconds for pessimistic lock. -# The value must between "15s" and "60s". -ttl = "30s" +# The value must between "15s" and "120s". +ttl = "40s" diff --git a/config/config_test.go b/config/config_test.go index 4e43e22d6916b..00e5031f8db40 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -215,8 +215,8 @@ func (s *testConfigSuite) TestValid(c *C) { }{ {"14s", false}, {"15s", true}, - {"60s", true}, - {"61s", false}, + {"120s", true}, + {"121s", false}, } for _, tt := range tests { c1.PessimisticTxn.TTL = tt.ttl diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 1e11c3f891185..9c750a2678abf 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -103,10 +103,11 @@ type twoPhaseCommitter struct { maxTxnTimeUse uint64 detail *execdetails.CommitDetails // For pessimistic transaction - isPessimistic bool - primaryKey []byte - forUpdateTS uint64 - isFirstLock bool + isPessimistic bool + primaryKey []byte + forUpdateTS uint64 + isFirstLock bool + pessimisticTTL uint64 } type mutationEx struct { @@ -571,7 +572,7 @@ func (c *twoPhaseCommitter) pessimisticLockSingleBatch(bo *Backoffer, batch batc PrimaryLock: c.primary(), StartVersion: c.startTS, ForUpdateTs: c.forUpdateTS, - LockTtl: PessimisticLockTTL, + LockTtl: c.pessimisticTTL, IsFirstLock: c.isFirstLock, }, pb.Context{Priority: c.priority, SyncLog: c.syncLog}) for { diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index 63426bd9fb0e8..8b173375138f2 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -487,3 +487,43 @@ func (s *testCommitterSuite) TestPessimisticLockedKeysDedup(c *C) { c.Assert(err, IsNil) c.Assert(txn.lockKeys, HasLen, 2) } + +func (s *testCommitterSuite) TestPessimisticTTL(c *C) { + key := kv.Key("key") + txn := s.begin(c) + txn.SetOption(kv.Pessimistic, true) + time.Sleep(time.Millisecond * 100) + err := txn.LockKeys(context.Background(), txn.startTS, key) + c.Assert(err, IsNil) + time.Sleep(time.Millisecond * 100) + key2 := kv.Key("key2") + err = txn.LockKeys(context.Background(), txn.startTS, key2) + c.Assert(err, IsNil) + lockInfo := s.getLockInfo(c, key) + elapsedTTL := lockInfo.LockTtl - PessimisticLockTTL + c.Assert(elapsedTTL, GreaterEqual, uint64(100)) + c.Assert(elapsedTTL, Less, uint64(200)) + lockInfo2 := s.getLockInfo(c, key2) + c.Assert(lockInfo2.LockTtl, Equals, lockInfo.LockTtl) +} + +func (s *testCommitterSuite) getLockInfo(c *C, key []byte) *kvrpcpb.LockInfo { + txn := s.begin(c) + err := txn.Set(key, key) + c.Assert(err, IsNil) + commiter, err := newTwoPhaseCommitterWithInit(txn, 1) + c.Assert(err, IsNil) + bo := NewBackoffer(context.Background(), getMaxBackoff) + loc, err := s.store.regionCache.LocateKey(bo, key) + c.Assert(err, IsNil) + batch := batchKeys{region: loc.Region, keys: [][]byte{key}} + req := commiter.buildPrewriteRequest(batch) + resp, err := s.store.SendReq(bo, req, loc.Region, readTimeoutShort) + c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + keyErrs := (resp.Resp.(*kvrpcpb.PrewriteResponse)).Errors + c.Assert(keyErrs, HasLen, 1) + locked := keyErrs[0].Locked + c.Assert(locked, NotNil) + return locked +} diff --git a/store/tikv/txn.go b/store/tikv/txn.go index c842f0aeae6fa..8e14c0980993f 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -390,6 +390,11 @@ func (txn *tikvTxn) LockKeys(ctx context.Context, forUpdateTS uint64, keysInput return err } } + if txn.committer.pessimisticTTL == 0 { + // add elapsed time to pessimistic TTL on the first LockKeys request. + elapsed := uint64(time.Since(txn.startTime) / time.Millisecond) + txn.committer.pessimisticTTL = PessimisticLockTTL + elapsed + } var assignedPrimaryKey bool if txn.committer.primaryKey == nil { txn.committer.primaryKey = keys[0] From fe998656e24a1c60c2b17dccca3b49ab099436ec Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 30 Jul 2019 13:36:56 +0800 Subject: [PATCH 119/196] =?UTF-8?q?executor:=20add=20`PreAlloc`=20and=20re?= =?UTF-8?q?place=20`nullCnt`=20field=20with=20`Nul=E2=80=A6=20(#11485)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/chunk/chunk.go | 5 -- util/chunk/chunk_test.go | 14 ++--- util/chunk/codec.go | 8 +-- util/chunk/codec_test.go | 3 - util/chunk/column.go | 93 ++++++++++++++++++++++++++---- util/chunk/column_test.go | 118 +++++++++++++++++++++++++++++++++++--- 6 files changed, 201 insertions(+), 40 deletions(-) diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index fc8ff8a5a6152..453b57c71b85a 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -468,11 +468,6 @@ func (c *Chunk) TruncateTo(numRows int) { col.data = col.data[:col.offsets[numRows]] col.offsets = col.offsets[:numRows+1] } - for i := numRows; i < col.length; i++ { - if col.IsNull(i) { - col.nullCount-- - } - } col.length = numRows bitmapLen := (col.length + 7) / 8 col.nullBitmap = col.nullBitmap[:bitmapLen] diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 08a34f439c204..e826aa8ef684e 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -160,21 +160,21 @@ func (s *testChunkSuite) TestAppend(c *check.C) { c.Assert(len(dst.columns), check.Equals, 3) c.Assert(dst.columns[0].length, check.Equals, 12) - c.Assert(dst.columns[0].nullCount, check.Equals, 6) + c.Assert(dst.columns[0].nullCount(), check.Equals, 6) c.Assert(string(dst.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(len(dst.columns[0].offsets), check.Equals, 0) c.Assert(len(dst.columns[0].data), check.Equals, 4*12) c.Assert(len(dst.columns[0].elemBuf), check.Equals, 4) c.Assert(dst.columns[1].length, check.Equals, 12) - c.Assert(dst.columns[1].nullCount, check.Equals, 6) + c.Assert(dst.columns[1].nullCount(), check.Equals, 6) c.Assert(string(dst.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(fmt.Sprintf("%v", dst.columns[1].offsets), check.Equals, fmt.Sprintf("%v", []int64{0, 3, 3, 6, 6, 9, 9, 12, 12, 15, 15, 18, 18})) c.Assert(string(dst.columns[1].data), check.Equals, "abcabcabcabcabcabc") c.Assert(len(dst.columns[1].elemBuf), check.Equals, 0) c.Assert(dst.columns[2].length, check.Equals, 12) - c.Assert(dst.columns[2].nullCount, check.Equals, 6) + c.Assert(dst.columns[2].nullCount(), check.Equals, 6) c.Assert(string(dst.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(len(dst.columns[2].offsets), check.Equals, 13) c.Assert(len(dst.columns[2].data), check.Equals, 150) @@ -213,21 +213,21 @@ func (s *testChunkSuite) TestTruncateTo(c *check.C) { c.Assert(len(src.columns), check.Equals, 3) c.Assert(src.columns[0].length, check.Equals, 12) - c.Assert(src.columns[0].nullCount, check.Equals, 6) + c.Assert(src.columns[0].nullCount(), check.Equals, 6) c.Assert(string(src.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(len(src.columns[0].offsets), check.Equals, 0) c.Assert(len(src.columns[0].data), check.Equals, 4*12) c.Assert(len(src.columns[0].elemBuf), check.Equals, 4) c.Assert(src.columns[1].length, check.Equals, 12) - c.Assert(src.columns[1].nullCount, check.Equals, 6) + c.Assert(src.columns[1].nullCount(), check.Equals, 6) c.Assert(string(src.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(fmt.Sprintf("%v", src.columns[1].offsets), check.Equals, fmt.Sprintf("%v", []int64{0, 3, 3, 6, 6, 9, 9, 12, 12, 15, 15, 18, 18})) c.Assert(string(src.columns[1].data), check.Equals, "abcabcabcabcabcabc") c.Assert(len(src.columns[1].elemBuf), check.Equals, 0) c.Assert(src.columns[2].length, check.Equals, 12) - c.Assert(src.columns[2].nullCount, check.Equals, 6) + c.Assert(src.columns[2].nullCount(), check.Equals, 6) c.Assert(string(src.columns[0].nullBitmap), check.Equals, string([]byte{0x55, 0x05})) c.Assert(len(src.columns[2].offsets), check.Equals, 13) c.Assert(len(src.columns[2].data), check.Equals, 150) @@ -655,7 +655,7 @@ func (s *testChunkSuite) TestPreAlloc4RowAndInsert(c *check.C) { c.Assert(len(srcCol.offsets), check.Equals, len(destCol.offsets)) c.Assert(len(srcCol.nullBitmap), check.Equals, len(destCol.nullBitmap)) c.Assert(srcCol.length, check.Equals, destCol.length) - c.Assert(srcCol.nullCount, check.Equals, destCol.nullCount) + c.Assert(srcCol.nullCount(), check.Equals, destCol.nullCount()) for _, val := range destCol.data { c.Assert(val == 0, check.IsTrue) diff --git a/util/chunk/codec.go b/util/chunk/codec.go index 601052cdcd84f..578d1c8b808e9 100644 --- a/util/chunk/codec.go +++ b/util/chunk/codec.go @@ -54,11 +54,11 @@ func (c *Codec) encodeColumn(buffer []byte, col *Column) []byte { buffer = append(buffer, lenBuffer[:4]...) // encode nullCount. - binary.LittleEndian.PutUint32(lenBuffer[:], uint32(col.nullCount)) + binary.LittleEndian.PutUint32(lenBuffer[:], uint32(col.nullCount())) buffer = append(buffer, lenBuffer[:4]...) // encode nullBitmap. - if col.nullCount > 0 { + if col.nullCount() > 0 { numNullBitmapBytes := (col.length + 7) / 8 buffer = append(buffer, col.nullBitmap[:numNullBitmapBytes]...) } @@ -111,11 +111,11 @@ func (c *Codec) decodeColumn(buffer []byte, col *Column, ordinal int) (remained buffer = buffer[4:] // decode nullCount. - col.nullCount = int(binary.LittleEndian.Uint32(buffer)) + nullCount := int(binary.LittleEndian.Uint32(buffer)) buffer = buffer[4:] // decode nullBitmap. - if col.nullCount > 0 { + if nullCount > 0 { numNullBitmapBytes := (col.length + 7) / 8 col.nullBitmap = append(col.nullBitmap[:0], buffer[:numNullBitmapBytes]...) buffer = buffer[numNullBitmapBytes:] diff --git a/util/chunk/codec_test.go b/util/chunk/codec_test.go index bf23a9849b834..b904ed9756f85 100644 --- a/util/chunk/codec_test.go +++ b/util/chunk/codec_test.go @@ -85,7 +85,6 @@ func BenchmarkEncodeChunk(b *testing.B) { for i := 0; i < numCols; i++ { chk.columns[i] = &Column{ length: numRows, - nullCount: 14, nullBitmap: make([]byte, numRows/8+1), data: make([]byte, numRows*8), } @@ -108,7 +107,6 @@ func BenchmarkDecode(b *testing.B) { for i := 0; i < numCols; i++ { chk.columns[i] = &Column{ length: numRows, - nullCount: 14, nullBitmap: make([]byte, numRows/8+1), data: make([]byte, numRows*8), } @@ -136,7 +134,6 @@ func BenchmarkDecodeToChunk(b *testing.B) { for i := 0; i < numCols; i++ { chk.columns[i] = &Column{ length: numRows, - nullCount: 14, nullBitmap: make([]byte, numRows/8+1), data: make([]byte, numRows*8), elemBuf: make([]byte, 8), diff --git a/util/chunk/column.go b/util/chunk/column.go index d6695ba0142dd..7a5379ec5efa2 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -14,6 +14,7 @@ package chunk import ( + "math/bits" "reflect" "time" "unsafe" @@ -58,8 +59,7 @@ func (c *Column) AppendSet(set types.Set) { // See https://arrow.apache.org/docs/memory_layout.html type Column struct { length int - nullCount int - nullBitmap []byte + nullBitmap []byte // bit 0 is null, 1 is not null offsets []int64 data []byte elemBuf []byte @@ -94,7 +94,6 @@ func (c *Column) isFixed() bool { // Reset resets this Column. func (c *Column) Reset() { c.length = 0 - c.nullCount = 0 c.nullBitmap = c.nullBitmap[:0] if len(c.offsets) > 0 { // The first offset is always 0, it makes slicing the data easier, we need to keep it. @@ -114,14 +113,13 @@ func (c *Column) IsNull(rowIdx int) bool { func (c *Column) CopyConstruct(dst *Column) *Column { if dst != nil { dst.length = c.length - dst.nullCount = c.nullCount dst.nullBitmap = append(dst.nullBitmap[:0], c.nullBitmap...) dst.offsets = append(dst.offsets[:0], c.offsets...) dst.data = append(dst.data[:0], c.data...) dst.elemBuf = append(dst.elemBuf[:0], c.elemBuf...) return dst } - newCol := &Column{length: c.length, nullCount: c.nullCount} + newCol := &Column{length: c.length} newCol.nullBitmap = append(newCol.nullBitmap, c.nullBitmap...) newCol.offsets = append(newCol.offsets, c.offsets...) newCol.data = append(newCol.data, c.data...) @@ -137,8 +135,6 @@ func (c *Column) appendNullBitmap(notNull bool) { if notNull { pos := uint(c.length) & 7 c.nullBitmap[idx] |= byte(1 << pos) - } else { - c.nullCount++ } } @@ -155,7 +151,6 @@ func (c *Column) appendMultiSameNullBitmap(notNull bool, num int) { c.nullBitmap = append(c.nullBitmap, b) } if !notNull { - c.nullCount += num return } // 1. Set all the remaining bits in the last slot of old c.numBitMap to 1. @@ -246,6 +241,84 @@ const ( sizeMyDecimal = int(unsafe.Sizeof(types.MyDecimal{})) ) +// preAlloc allocates space for a fixed-length-type slice and resets all slots to null. +func (c *Column) preAlloc(length, typeSize int) { + nData := length * typeSize + if len(c.data) >= nData { + c.data = c.data[:nData] + } else { + c.data = make([]byte, nData) + } + + nBitmap := (length + 7) >> 3 + if len(c.nullBitmap) >= nBitmap { + c.nullBitmap = c.nullBitmap[:nBitmap] + for i := range c.nullBitmap { + // resets all slots to null. + c.nullBitmap[i] = 0 + } + } else { + c.nullBitmap = make([]byte, nBitmap) + } + + if c.elemBuf != nil && len(c.elemBuf) >= typeSize { + c.elemBuf = c.elemBuf[:typeSize] + } else { + c.elemBuf = make([]byte, typeSize) + } + + c.length = length +} + +// SetNull sets the rowIdx to null. +func (c *Column) SetNull(rowIdx int, isNull bool) { + if isNull { + c.nullBitmap[rowIdx>>3] &= ^(1 << uint(rowIdx&7)) + } else { + c.nullBitmap[rowIdx>>3] |= 1 << uint(rowIdx&7) + } +} + +// nullCount returns the number of nulls in this Column. +func (c *Column) nullCount() int { + var cnt, i int + for ; i+8 <= c.length; i += 8 { + // 0 is null and 1 is not null + cnt += 8 - bits.OnesCount8(uint8(c.nullBitmap[i>>3])) + } + for ; i < c.length; i++ { + if c.IsNull(i) { + cnt++ + } + } + return cnt +} + +// PreAllocInt64 allocates space for an int64 slice and resets all slots to null. +func (c *Column) PreAllocInt64(length int) { + c.preAlloc(length, sizeInt64) +} + +// PreAllocUint64 allocates space for a uint64 slice and resets all slots to null. +func (c *Column) PreAllocUint64(length int) { + c.preAlloc(length, sizeUint64) +} + +// PreAllocFloat32 allocates space for a float32 slice and resets all slots to null. +func (c *Column) PreAllocFloat32(length int) { + c.preAlloc(length, sizeFloat32) +} + +// PreAllocFloat64 allocates space for a float64 slice and resets all slots to null. +func (c *Column) PreAllocFloat64(length int) { + c.preAlloc(length, sizeFloat64) +} + +// PreAllocDecimal allocates space for a decimal slice and resets all slots to null. +func (c *Column) PreAllocDecimal(length int) { + c.preAlloc(length, sizeMyDecimal) +} + func (c *Column) castSliceHeader(header *reflect.SliceHeader, typeSize int) { header.Data = uintptr(unsafe.Pointer(&c.data[0])) header.Len = c.length @@ -364,14 +437,12 @@ func (c *Column) reconstruct(sel []int) { if sel == nil { return } - nullCnt := 0 if c.isFixed() { elemLen := len(c.elemBuf) for dst, src := range sel { idx := dst >> 3 pos := uint16(dst & 7) if c.IsNull(src) { - nullCnt++ c.nullBitmap[idx] &= ^byte(1 << pos) } else { copy(c.data[dst*elemLen:dst*elemLen+elemLen], c.data[src*elemLen:src*elemLen+elemLen]) @@ -385,7 +456,6 @@ func (c *Column) reconstruct(sel []int) { idx := dst >> 3 pos := uint(dst & 7) if c.IsNull(src) { - nullCnt++ c.nullBitmap[idx] &= ^byte(1 << pos) c.offsets[dst+1] = int64(tail) } else { @@ -400,7 +470,6 @@ func (c *Column) reconstruct(sel []int) { c.offsets = c.offsets[:len(sel)+1] } c.length = len(sel) - c.nullCount = nullCnt // clean nullBitmap c.nullBitmap = c.nullBitmap[:(len(sel)+7)>>3] diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index b40a91bfac7e6..0b04a8fbd24c9 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -26,7 +26,7 @@ import ( func equalColumn(c1, c2 *Column) bool { if c1.length != c2.length || - c1.nullCount != c2.nullCount { + c1.nullCount() != c2.nullCount() { return false } if len(c1.nullBitmap) != len(c2.nullBitmap) || @@ -105,7 +105,7 @@ func (s *testChunkSuite) TestColumnCopyReconstructFixedLen(c *check.C) { c.Assert(col.GetInt64(n), check.Equals, results[i]) } } - c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(nullCnt, check.Equals, col.nullCount()) c.Assert(col.length, check.Equals, len(sel)) for i := 0; i < 128; i++ { @@ -117,7 +117,7 @@ func (s *testChunkSuite) TestColumnCopyReconstructFixedLen(c *check.C) { } c.Assert(col.length, check.Equals, len(sel)+128) - c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + c.Assert(col.nullCount(), check.Equals, nullCnt+128/2) for i := 0; i < 128; i++ { if i%2 == 0 { c.Assert(col.IsNull(len(sel)+i), check.Equals, true) @@ -161,7 +161,7 @@ func (s *testChunkSuite) TestColumnCopyReconstructVarLen(c *check.C) { c.Assert(col.GetString(n), check.Equals, results[i]) } } - c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(nullCnt, check.Equals, col.nullCount()) c.Assert(col.length, check.Equals, len(sel)) for i := 0; i < 128; i++ { @@ -173,7 +173,7 @@ func (s *testChunkSuite) TestColumnCopyReconstructVarLen(c *check.C) { } c.Assert(col.length, check.Equals, len(sel)+128) - c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + c.Assert(col.nullCount(), check.Equals, nullCnt+128/2) for i := 0; i < 128; i++ { if i%2 == 0 { c.Assert(col.IsNull(len(sel)+i), check.Equals, true) @@ -475,7 +475,7 @@ func (s *testChunkSuite) TestReconstructFixedLen(c *check.C) { c.Assert(col.GetInt64(n), check.Equals, results[i]) } } - c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(nullCnt, check.Equals, col.nullCount()) c.Assert(col.length, check.Equals, len(sel)) for i := 0; i < 128; i++ { @@ -487,7 +487,7 @@ func (s *testChunkSuite) TestReconstructFixedLen(c *check.C) { } c.Assert(col.length, check.Equals, len(sel)+128) - c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + c.Assert(col.nullCount(), check.Equals, nullCnt+128/2) for i := 0; i < 128; i++ { if i%2 == 0 { c.Assert(col.IsNull(len(sel)+i), check.Equals, true) @@ -531,7 +531,7 @@ func (s *testChunkSuite) TestReconstructVarLen(c *check.C) { c.Assert(col.GetString(n), check.Equals, results[i]) } } - c.Assert(nullCnt, check.Equals, col.nullCount) + c.Assert(nullCnt, check.Equals, col.nullCount()) c.Assert(col.length, check.Equals, len(sel)) for i := 0; i < 128; i++ { @@ -543,7 +543,7 @@ func (s *testChunkSuite) TestReconstructVarLen(c *check.C) { } c.Assert(col.length, check.Equals, len(sel)+128) - c.Assert(col.nullCount, check.Equals, nullCnt+128/2) + c.Assert(col.nullCount(), check.Equals, nullCnt+128/2) for i := 0; i < 128; i++ { if i%2 == 0 { c.Assert(col.IsNull(len(sel)+i), check.Equals, true) @@ -553,3 +553,103 @@ func (s *testChunkSuite) TestReconstructVarLen(c *check.C) { } } } + +func (s *testChunkSuite) TestPreAllocInt64(c *check.C) { + col := NewColumn(types.NewFieldType(mysql.TypeLonglong), 128) + col.PreAllocInt64(256) + i64s := col.Int64s() + c.Assert(len(i64s), check.Equals, 256) + for i := 0; i < 256; i++ { + c.Assert(col.IsNull(i), check.Equals, true) + } + col.AppendInt64(2333) + c.Assert(col.IsNull(256), check.Equals, false) + c.Assert(len(col.Int64s()), check.Equals, 257) + c.Assert(col.Int64s()[256], check.Equals, int64(2333)) +} + +func (s *testChunkSuite) TestPreAllocUint64(c *check.C) { + tll := types.NewFieldType(mysql.TypeLonglong) + tll.Flag |= mysql.UnsignedFlag + col := NewColumn(tll, 128) + col.PreAllocUint64(256) + u64s := col.Uint64s() + c.Assert(len(u64s), check.Equals, 256) + for i := 0; i < 256; i++ { + c.Assert(col.IsNull(i), check.Equals, true) + } + col.AppendUint64(2333) + c.Assert(col.IsNull(256), check.Equals, false) + c.Assert(len(col.Uint64s()), check.Equals, 257) + c.Assert(col.Uint64s()[256], check.Equals, uint64(2333)) +} + +func (s *testChunkSuite) TestPreAllocFloat32(c *check.C) { + col := newFixedLenColumn(sizeFloat32, 128) + col.PreAllocFloat32(256) + f32s := col.Float32s() + c.Assert(len(f32s), check.Equals, 256) + for i := 0; i < 256; i++ { + c.Assert(col.IsNull(i), check.Equals, true) + } + col.AppendFloat32(2333) + c.Assert(col.IsNull(256), check.Equals, false) + c.Assert(len(col.Float32s()), check.Equals, 257) + c.Assert(col.Float32s()[256], check.Equals, float32(2333)) +} + +func (s *testChunkSuite) TestPreAllocFloat64(c *check.C) { + col := newFixedLenColumn(sizeFloat64, 128) + col.PreAllocFloat64(256) + f64s := col.Float64s() + c.Assert(len(f64s), check.Equals, 256) + for i := 0; i < 256; i++ { + c.Assert(col.IsNull(i), check.Equals, true) + } + col.AppendFloat64(2333) + c.Assert(col.IsNull(256), check.Equals, false) + c.Assert(len(col.Float64s()), check.Equals, 257) + c.Assert(col.Float64s()[256], check.Equals, float64(2333)) +} + +func (s *testChunkSuite) TestPreAllocDecimal(c *check.C) { + col := newFixedLenColumn(sizeMyDecimal, 128) + col.PreAllocDecimal(256) + ds := col.Decimals() + c.Assert(len(ds), check.Equals, 256) + for i := 0; i < 256; i++ { + c.Assert(col.IsNull(i), check.Equals, true) + } + col.AppendMyDecimal(new(types.MyDecimal)) + c.Assert(col.IsNull(256), check.Equals, false) + c.Assert(len(col.Float64s()), check.Equals, 257) +} + +func (s *testChunkSuite) TestNull(c *check.C) { + col := newFixedLenColumn(sizeFloat64, 32) + col.PreAllocFloat64(1024) + c.Assert(col.nullCount(), check.Equals, 1024) + + notNulls := make(map[int]struct{}) + for i := 0; i < 512; i++ { + idx := rand.Intn(1024) + notNulls[idx] = struct{}{} + col.SetNull(idx, false) + } + + c.Assert(col.nullCount(), check.Equals, 1024-len(notNulls)) + for idx := range notNulls { + c.Assert(col.IsNull(idx), check.Equals, false) + } + + col.PreAllocFloat64(8) + col.SetNull(7, false) + c.Assert(col.nullCount(), check.Equals, 7) + + col.PreAllocFloat64(8) + c.Assert(col.nullCount(), check.Equals, 8) + + col.PreAllocFloat64(9) + col.SetNull(8, false) + c.Assert(col.nullCount(), check.Equals, 8) +} From 1f57a538a4462d326d55bb68e9e3485c925b4fa8 Mon Sep 17 00:00:00 2001 From: SunRunAway Date: Tue, 30 Jul 2019 17:10:52 +0800 Subject: [PATCH 120/196] distsql: make CopStream available (#11311) All tests passed, auto merged by Bot --- distsql/distsql.go | 12 +-- distsql/select_result.go | 2 + distsql/stream.go | 8 ++ executor/analyze_test.go | 13 +++ store/mockstore/mocktikv/cop_handler_dag.go | 14 +-- store/mockstore/mocktikv/rpc.go | 11 ++- store/tikv/client.go | 25 ++++-- store/tikv/coprocessor.go | 21 +++-- store/tikv/sql_fail_test.go | 99 +++++++++++++++++---- store/tikv/tikvrpc/tikvrpc.go | 24 +++-- util/sqlexec/restricted_sql_executor.go | 2 +- 11 files changed, 174 insertions(+), 57 deletions(-) diff --git a/distsql/distsql.go b/distsql/distsql.go index e69ed5f85e466..e34449f9e5b4f 100644 --- a/distsql/distsql.go +++ b/distsql/distsql.go @@ -47,12 +47,19 @@ func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fie return nil, err } + label := metrics.LblGeneral + if sctx.GetSessionVars().InRestrictedSQL { + label = metrics.LblInternal + } + // kvReq.MemTracker is used to trace and control memory usage in DistSQL layer; // for streamResult, since it is a pipeline which has no buffer, it's not necessary to trace it; // for selectResult, we just use the kvReq.MemTracker prepared for co-processor // instead of creating a new one for simplification. if kvReq.Streaming { return &streamResult{ + label: "dag-stream", + sqlType: label, resp: resp, rowLen: len(fieldTypes), fieldTypes: fieldTypes, @@ -60,11 +67,6 @@ func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fie feedback: fb, }, nil } - - label := metrics.LblGeneral - if sctx.GetSessionVars().InRestrictedSQL { - label = metrics.LblInternal - } return &selectResult{ label: "dag", resp: resp, diff --git a/distsql/select_result.go b/distsql/select_result.go index 51035ef8d6df7..273ce7b1d1110 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -96,6 +96,8 @@ func (r *selectResult) fetch(ctx context.Context) { close(r.results) duration := time.Since(startTime) + // TODO: Add a label to distinguish between success or failure. + // https://github.com/pingcap/tidb/issues/11397 metrics.DistSQLQueryHistgram.WithLabelValues(r.label, r.sqlType).Observe(duration.Seconds()) }() for { diff --git a/distsql/stream.go b/distsql/stream.go index ac83afc914bba..78a88bfaa655a 100644 --- a/distsql/stream.go +++ b/distsql/stream.go @@ -15,6 +15,7 @@ package distsql import ( "context" + "time" "github.com/pingcap/errors" "github.com/pingcap/parser/terror" @@ -30,6 +31,9 @@ import ( // streamResult implements the SelectResult interface. type streamResult struct { + label string + sqlType string + resp kv.Response rowLen int fieldTypes []*types.FieldType @@ -64,7 +68,11 @@ func (r *streamResult) Next(ctx context.Context, chk *chunk.Chunk) error { // readDataFromResponse read the data to result. Returns true means the resp is finished. func (r *streamResult) readDataFromResponse(ctx context.Context, resp kv.Response, result *tipb.Chunk) (bool, error) { + startTime := time.Now() resultSubset, err := resp.Next(ctx) + // TODO: Add a label to distinguish between success or failure. + // https://github.com/pingcap/tidb/issues/11397 + metrics.DistSQLQueryHistgram.WithLabelValues(r.label, r.sqlType).Observe(time.Since(startTime).Seconds()) if err != nil { return false, err } diff --git a/executor/analyze_test.go b/executor/analyze_test.go index 26affd809d1b2..d85fd20491f3c 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -318,6 +318,19 @@ func (s *testSuite1) TestFastAnalyze(c *C) { func (s *testSuite1) TestAnalyzeIncremental(c *C) { tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.Se.GetSessionVars().EnableStreaming = false + s.testAnalyzeIncremental(tk, c) +} + +func (s *testSuite1) TestAnalyzeIncrementalStreaming(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.Se.GetSessionVars().EnableStreaming = true + s.testAnalyzeIncremental(tk, c) +} + +func (s *testSuite1) testAnalyzeIncremental(tk *testkit.TestKit, c *C) { tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int, b int, primary key(a), index idx(b))") diff --git a/store/mockstore/mocktikv/cop_handler_dag.go b/store/mockstore/mocktikv/cop_handler_dag.go index d02e87800d9a8..c8b7c6aaf112e 100644 --- a/store/mockstore/mocktikv/cop_handler_dag.go +++ b/store/mockstore/mocktikv/cop_handler_dag.go @@ -507,16 +507,10 @@ func (mock *mockCopStreamClient) Recv() (*coprocessor.Response, error) { } } streamResponse := tipb.StreamResponse{ - Error: toPBError(err), - Data: data, - Warnings: Warnings, - } - // The counts was the output count of each executor, but now it is the scan count of each range, - // so we need a flag to tell them apart. - if counts != nil { - streamResponse.OutputCounts = make([]int64, 1+len(counts)) - copy(streamResponse.OutputCounts, counts) - streamResponse.OutputCounts[len(counts)] = -1 + Error: toPBError(err), + Data: data, + Warnings: Warnings, + OutputCounts: counts, } resp.Data, err = proto.Marshal(&streamResponse) if err != nil { diff --git a/store/mockstore/mocktikv/rpc.go b/store/mockstore/mocktikv/rpc.go index 2081b7f0ece39..8cf5d481bd788 100644 --- a/store/mockstore/mocktikv/rpc.go +++ b/store/mockstore/mocktikv/rpc.go @@ -619,17 +619,20 @@ type RPCClient struct { Cluster *Cluster MvccStore MVCCStore streamTimeout chan *tikvrpc.Lease + done chan struct{} } // NewRPCClient creates an RPCClient. // Note that close the RPCClient may close the underlying MvccStore. func NewRPCClient(cluster *Cluster, mvccStore MVCCStore) *RPCClient { - ch := make(chan *tikvrpc.Lease) - go tikvrpc.CheckStreamTimeoutLoop(ch) + ch := make(chan *tikvrpc.Lease, 1024) + done := make(chan struct{}) + go tikvrpc.CheckStreamTimeoutLoop(ch, done) return &RPCClient{ Cluster: cluster, MvccStore: mvccStore, streamTimeout: ch, + done: done, } } @@ -935,14 +938,14 @@ func (c *RPCClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R Value: strconv.Itoa(len(scanResp.Pairs)), }}} default: - return nil, errors.Errorf("unsupport this request type %v", req.Type) + return nil, errors.Errorf("unsupported this request type %v", req.Type) } return resp, nil } // Close closes the client. func (c *RPCClient) Close() error { - close(c.streamTimeout) + close(c.done) if raw, ok := c.MvccStore.(io.Closer); ok { return raw.Close() } diff --git a/store/tikv/client.go b/store/tikv/client.go index 4f3049236ee4f..102d8cf4db38d 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -43,14 +43,16 @@ import ( var MaxRecvMsgSize = math.MaxInt64 // Timeout durations. -const ( +var ( dialTimeout = 5 * time.Second readTimeoutShort = 20 * time.Second // For requests that read/write several key-values. ReadTimeoutMedium = 60 * time.Second // For requests that may need scan region. ReadTimeoutLong = 150 * time.Second // For requests that may need scan region multiple times. GCTimeout = 5 * time.Minute UnsafeDestroyRangeTimeout = 5 * time.Minute +) +const ( grpcInitialWindowSize = 1 << 30 grpcInitialConnWindowSize = 1 << 30 ) @@ -76,19 +78,19 @@ type connArray struct { *batchConn } -func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32) (*connArray, error) { +func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32, done <-chan struct{}) (*connArray, error) { a := &connArray{ index: 0, v: make([]*grpc.ClientConn, maxSize), streamTimeout: make(chan *tikvrpc.Lease, 1024), } - if err := a.Init(addr, security, idleNotify); err != nil { + if err := a.Init(addr, security, idleNotify, done); err != nil { return nil, err } return a, nil } -func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32) error { +func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32, done <-chan struct{}) error { a.target = addr opt := grpc.WithInsecure() @@ -163,7 +165,7 @@ func (a *connArray) Init(addr string, security config.Security, idleNotify *uint go batchClient.batchRecvLoop(cfg.TiKVClient, &a.tikvTransportLayerLoad) } } - go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout) + go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout, done) if allowBatch { go a.batchSendLoop(cfg.TiKVClient) } @@ -188,7 +190,6 @@ func (a *connArray) Close() { a.v[i] = nil } } - close(a.streamTimeout) } // rpcClient is RPC client struct. @@ -198,6 +199,8 @@ func (a *connArray) Close() { type rpcClient struct { sync.RWMutex isClosed bool + done chan struct{} + conns map[string]*connArray security config.Security @@ -208,6 +211,7 @@ type rpcClient struct { func newRPCClient(security config.Security) *rpcClient { return &rpcClient{ + done: make(chan struct{}, 1), conns: make(map[string]*connArray), security: security, } @@ -243,7 +247,7 @@ func (c *rpcClient) createConnArray(addr string) (*connArray, error) { if !ok { var err error connCount := config.GetGlobalConfig().TiKVClient.GrpcConnectionCount - array, err = newConnArray(connCount, addr, c.security, &c.idleNotify) + array, err = newConnArray(connCount, addr, c.security, &c.idleNotify, c.done) if err != nil { return nil, err } @@ -306,9 +310,12 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R // Coprocessor streaming request. // Use context to support timeout for grpc streaming client. ctx1, cancel := context.WithCancel(ctx) - defer cancel() + // Should NOT call defer cancel() here because it will cancel further stream.Recv() + // We put it in copStream.Lease.Cancel call this cancel at copStream.Close + // TODO: add unit test for SendRequest. resp, err := tikvrpc.CallRPC(ctx1, client, req) if err != nil { + cancel() return nil, errors.Trace(err) } @@ -334,6 +341,8 @@ func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.R } func (c *rpcClient) Close() error { + // TODO: add a unit test for SendRequest After Closed + close(c.done) c.closeConns() return nil } diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 83f9395c4f96b..7b7b04ea2112a 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -658,10 +658,17 @@ func (worker *copIteratorWorker) logTimeCopTask(costTime time.Duration, task *co } var detail *kvrpcpb.ExecDetails if resp.Resp != nil { - detail = resp.Resp.(*coprocessor.Response).ExecDetails - } else if resp.Resp != nil && resp.Resp.(*tikvrpc.CopStreamResponse).Response != nil { - // streaming request returns io.EOF, so the first resp.CopStream.Response maybe nil. - detail = (resp.Resp.(*tikvrpc.CopStreamResponse)).ExecDetails + switch r := resp.Resp.(type) { + case *coprocessor.Response: + detail = r.ExecDetails + case *tikvrpc.CopStreamResponse: + // streaming request returns io.EOF, so the first CopStreamResponse.Response maybe nil. + if r.Response != nil { + detail = r.Response.ExecDetails + } + default: + panic("unreachable") + } } if detail != nil && detail.HandleTime != nil { @@ -718,7 +725,11 @@ func (worker *copIteratorWorker) handleCopStreamResult(bo *Backoffer, rpcCtx *RP } // No coprocessor.Response for network error, rebuild task based on the last success one. - logutil.BgLogger().Info("stream recv timeout", zap.Error(err)) + if errors.Cause(err) == context.Canceled { + logutil.BgLogger().Info("stream recv timeout", zap.Error(err)) + } else { + logutil.BgLogger().Info("stream unknown error", zap.Error(err)) + } return worker.buildCopTasksFromRemain(bo, lastRange, task) } lastRange = resp.Range diff --git a/store/tikv/sql_fail_test.go b/store/tikv/sql_fail_test.go index 7e58706453cd7..ef91aa8b63bb3 100644 --- a/store/tikv/sql_fail_test.go +++ b/store/tikv/sql_fail_test.go @@ -16,7 +16,9 @@ package tikv_test import ( "context" "fmt" + "os" "sync" + "testing" "time" . "github.com/pingcap/check" @@ -99,6 +101,11 @@ func (s *testSQLSuite) TestFailBusyServerCop(c *C) { wg.Wait() } +func TestMain(m *testing.M) { + ReadTimeoutMedium = 2 * time.Second + os.Exit(m.Run()) +} + func (s *testSQLSuite) TestCoprocessorStreamRecvTimeout(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -106,31 +113,85 @@ func (s *testSQLSuite) TestCoprocessorStreamRecvTimeout(c *C) { for i := 0; i < 200; i++ { tk.MustExec(fmt.Sprintf("insert into t values (%d)", i)) } + tk.Se.GetSessionVars().EnableStreaming = true + + { + enable := true + visited := make(chan int, 1) + timeouted := false + timeout := ReadTimeoutMedium + 100*time.Second + ctx := context.WithValue(context.Background(), mock.HookKeyForTest("mockTiKVStreamRecvHook"), func(ctx context.Context) { + if !enable { + return + } + visited <- 1 + + select { + case <-ctx.Done(): + case <-time.After(timeout): + timeouted = true + } + enable = false + }) + + res, err := tk.Se.Execute(ctx, "select * from t") + c.Assert(err, IsNil) - // rowsPerChunk in MockTiKV is 64, so the result will be 4 chunks. - enable := true - ctx := context.WithValue(context.Background(), mock.HookKeyForTest("mockTiKVStreamRecvHook"), func(ctx context.Context) { - if !enable { - return + req := res[0].NewChunk() + for i := 0; ; i++ { + err := res[0].Next(ctx, req) + c.Assert(err, IsNil) + if req.NumRows() == 0 { + break + } + req.Reset() } - select { - case <-ctx.Done(): - case <-time.After(time.Minute): + case <-visited: + // run with mocktikv + c.Assert(timeouted, IsFalse) + default: + // run with real tikv } - enable = false - }) - - res, err := tk.Se.Execute(ctx, "select * from t") - c.Assert(err, IsNil) + } - req := res[0].NewChunk() - for { - err := res[0].Next(ctx, req) + { + enable := true + visited := make(chan int, 1) + timeouted := false + timeout := 1 * time.Millisecond + ctx := context.WithValue(context.Background(), mock.HookKeyForTest("mockTiKVStreamRecvHook"), func(ctx context.Context) { + if !enable { + return + } + visited <- 1 + + select { + case <-ctx.Done(): + case <-time.After(timeout): + timeouted = true + } + enable = false + }) + + res, err := tk.Se.Execute(ctx, "select * from t") c.Assert(err, IsNil) - if req.NumRows() == 0 { - break + + req := res[0].NewChunk() + for i := 0; ; i++ { + err := res[0].Next(ctx, req) + c.Assert(err, IsNil) + if req.NumRows() == 0 { + break + } + req.Reset() + } + select { + case <-visited: + // run with mocktikv + c.Assert(timeouted, IsTrue) + default: + // run with real tikv } - req.Reset() } } diff --git a/store/tikv/tikvrpc/tikvrpc.go b/store/tikv/tikvrpc/tikvrpc.go index 8d2e813e6d9d6..c016fbd694e53 100644 --- a/store/tikv/tikvrpc/tikvrpc.go +++ b/store/tikv/tikvrpc/tikvrpc.go @@ -746,22 +746,36 @@ func (resp *CopStreamResponse) Recv() (*coprocessor.Response, error) { // Close closes the CopStreamResponse object. func (resp *CopStreamResponse) Close() { atomic.StoreInt64(&resp.Lease.deadline, 1) + // We also call cancel here because CheckStreamTimeoutLoop + // is not guaranteed to cancel all items when it exits. + if resp.Lease.Cancel != nil { + resp.Lease.Cancel() + } } // CheckStreamTimeoutLoop runs periodically to check is there any stream request timeouted. // Lease is an object to track stream requests, call this function with "go CheckStreamTimeoutLoop()" -func CheckStreamTimeoutLoop(ch <-chan *Lease) { +// It is not guaranteed to call every Lease.Cancel() putting into channel when exits. +// If grpc-go supports SetDeadline(https://github.com/grpc/grpc-go/issues/2917), we can stop using this method. +func CheckStreamTimeoutLoop(ch <-chan *Lease, done <-chan struct{}) { ticker := time.NewTicker(200 * time.Millisecond) defer ticker.Stop() array := make([]*Lease, 0, 1024) for { select { - case item, ok := <-ch: - if !ok { - // This channel close means goroutine should return. - return + case <-done: + drainLoop: + // Try my best cleaning the channel to make SendRequest which is blocking by it continues. + for { + select { + case <-ch: + default: + break drainLoop + } } + return + case item := <-ch: array = append(array, item) case now := <-ticker.C: array = keepOnlyActive(array, now.UnixNano()) diff --git a/util/sqlexec/restricted_sql_executor.go b/util/sqlexec/restricted_sql_executor.go index cfbe37e4603cb..c5ccc52cacec6 100644 --- a/util/sqlexec/restricted_sql_executor.go +++ b/util/sqlexec/restricted_sql_executor.go @@ -89,7 +89,7 @@ type RecordSet interface { // Next reads records into chunk. Next(ctx context.Context, req *chunk.Chunk) error - //NewChunk create a chunk. + // NewChunk create a chunk. NewChunk() *chunk.Chunk // Close closes the underlying iterator, call Next after Close will From 768f61f52f408f0301f92e3bc2afaef7d29d0400 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 30 Jul 2019 17:16:52 +0800 Subject: [PATCH 121/196] expression: handle builtin time getInterval from Decimal\Real (#11479) All tests passed, auto merged by Bot --- expression/builtin_time.go | 5 +++-- expression/builtin_time_test.go | 26 ++++++++++++++++++++++++++ expression/integration_test.go | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index a0ac0de54d51a..39c45fdd7b131 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -2652,10 +2652,11 @@ func (du *baseDateArithmitical) getIntervalFromString(ctx sessionctx.Context, ar } func (du *baseDateArithmitical) getIntervalFromDecimal(ctx sessionctx.Context, args []Expression, row chunk.Row, unit string) (string, bool, error) { - interval, isNull, err := args[1].EvalString(ctx, row) + decimal, isNull, err := args[1].EvalDecimal(ctx, row) if isNull || err != nil { return "", true, err } + interval := decimal.String() switch strings.ToUpper(unit) { case "HOUR_MINUTE", "MINUTE_SECOND", "YEAR_MONTH", "DAY_HOUR", "DAY_MINUTE", @@ -2721,7 +2722,7 @@ func (du *baseDateArithmitical) getIntervalFromReal(ctx sessionctx.Context, args if isNull || err != nil { return "", true, err } - return strconv.FormatFloat(interval, 'f', -1, 64), false, nil + return strconv.FormatFloat(interval, 'f', args[1].GetType().Decimal, 64), false, nil } func (du *baseDateArithmitical) add(ctx sessionctx.Context, date types.Time, interval string, unit string) (types.Time, bool, error) { diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index 14e7e70094fc4..ed7e784cc231a 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -2767,3 +2767,29 @@ func (s *testEvaluatorSuite) TestTidbParseTso(c *C) { c.Assert(d.IsNull(), IsTrue) } } + +func (s *testEvaluatorSuite) TestGetIntervalFromDecimal(c *C) { + defer testleak.AfterTest(c)() + du := baseDateArithmitical{} + + tests := []struct { + param string + expect string + unit string + }{ + {"1.100", "1:100", "MINUTE_SECOND"}, + {"1.10000", "1-10000", "YEAR_MONTH"}, + {"1.10000", "1 10000", "DAY_HOUR"}, + {"11000", "0 00:00:11000", "DAY_MICROSECOND"}, + {"11000", "00:00:11000", "HOUR_MICROSECOND"}, + {"11.1000", "00:11:1000", "HOUR_SECOND"}, + {"1000", "00:1000", "MINUTE_MICROSECOND"}, + } + + for _, test := range tests { + interval, isNull, err := du.getIntervalFromDecimal(s.ctx, s.datumsToConstants([]types.Datum{types.NewDatum("CURRENT DATE"), types.NewDecimalDatum(newMyDecimal(c, test.param))}), chunk.Row{}, test.unit) + c.Assert(isNull, IsFalse) + c.Assert(err, IsNil) + c.Assert(interval, Equals, test.expect) + } +} diff --git a/expression/integration_test.go b/expression/integration_test.go index e53c37b7729ff..ce2f33a84bc0e 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4711,3 +4711,21 @@ func (s *testIntegrationSuite) TestFuncCaseWithLeftJoin(c *C) { tk.MustQuery("select t1.id from kankan1 t1 left join kankan2 t2 on t1.id = t2.id where (case when t1.name='b' then 'case2' when t1.name='a' then 'case1' else NULL end) = 'case1' order by t1.id").Check(testkit.Rows("1", "2")) } + +func (s *testIntegrationSuite) TestIssue11309(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`CREATE TABLE t (a decimal(6,3),b double(6,3),c float(6,3));`) + tk.MustExec(`INSERT INTO t VALUES (1.100,1.100,1.100);`) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL a MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2003-11-18 07:27:53`)) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL b MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2003-11-18 07:27:53`)) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL c MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2003-11-18 07:27:53`)) + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`CREATE TABLE t (a decimal(11,7),b double(11,7),c float(11,7));`) + tk.MustExec(`INSERT INTO t VALUES (123.9999999,123.9999999,123.9999999),(-123.9999999,-123.9999999,-123.9999999);`) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL a MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2004-03-13 03:14:52`, `2003-07-25 11:35:34`)) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL b MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2004-03-13 03:14:52`, `2003-07-25 11:35:34`)) + tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL c MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2003-11-18 09:29:13`, `2003-11-18 05:21:13`)) + tk.MustExec(`drop table if exists t;`) +} From da8e7d267cf5d6bd01823332c39afa3600afef85 Mon Sep 17 00:00:00 2001 From: crazycs Date: Tue, 30 Jul 2019 17:40:33 +0800 Subject: [PATCH 122/196] executor: refactor union_scan executor by using txn_mem_buffer_reader (#10673) --- executor/builder.go | 36 +-- executor/distsql.go | 13 +- executor/executor.go | 4 +- executor/executor_test.go | 1 + executor/mem_reader.go | 307 ++++++++++++++++++++ executor/table_reader.go | 11 +- executor/union_scan.go | 96 +++++- executor/union_scan_test.go | 107 +++++++ store/mockstore/mocktikv/cop_handler_dag.go | 8 +- store/mockstore/mocktikv/executor.go | 50 +--- tablecodec/tablecodec.go | 52 ++++ 11 files changed, 587 insertions(+), 98 deletions(-) create mode 100644 executor/mem_reader.go diff --git a/executor/builder.go b/executor/builder.go index ebd8295088404..2611aea853d48 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -777,25 +777,20 @@ func (b *executorBuilder) buildUnionScanExec(v *plannercore.PhysicalUnionScan) E if b.err != nil { return nil } - us, err := b.buildUnionScanFromReader(reader, v) - if err != nil { - b.err = err - return nil - } - return us + return b.buildUnionScanFromReader(reader, v) } // buildUnionScanFromReader builds union scan executor from child executor. // Note that this function may be called by inner workers of index lookup join concurrently. // Be careful to avoid data race. -func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannercore.PhysicalUnionScan) (Executor, error) { - var err error +func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannercore.PhysicalUnionScan) Executor { us := &UnionScanExec{baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID(), reader)} // Get the handle column index of the below plannercore. // We can guarantee that there must be only one col in the map. for _, cols := range v.Children()[0].Schema().TblID2Handle { us.belowHandleIndex = cols[0].Index } + us.mutableRow = chunk.MutRowFromTypes(retTypes(us)) switch x := reader.(type) { case *TableReaderExecutor: us.desc = x.desc @@ -809,7 +804,7 @@ func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannerco us.dirty = GetDirtyDB(b.ctx).GetDirtyTable(physicalTableID) us.conditions = v.Conditions us.columns = x.columns - err = us.buildAndSortAddedRows(x.table) + us.table = x.table case *IndexReaderExecutor: us.desc = x.desc for _, ic := range x.index.Columns { @@ -824,7 +819,7 @@ func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannerco us.dirty = GetDirtyDB(b.ctx).GetDirtyTable(physicalTableID) us.conditions = v.Conditions us.columns = x.columns - err = us.buildAndSortAddedRows(x.table) + us.table = x.table case *IndexLookUpExecutor: us.desc = x.desc for _, ic := range x.index.Columns { @@ -839,15 +834,12 @@ func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannerco us.dirty = GetDirtyDB(b.ctx).GetDirtyTable(physicalTableID) us.conditions = v.Conditions us.columns = x.columns - err = us.buildAndSortAddedRows(x.table) + us.table = x.table default: // The mem table will not be written by sql directly, so we can omit the union scan to avoid err reporting. - return reader, nil + return reader } - if err != nil { - return nil, err - } - return us, nil + return us } // buildMergeJoin builds MergeJoinExec executor. @@ -1819,6 +1811,7 @@ func buildNoRangeIndexReader(b *executorBuilder, v *plannercore.PhysicalIndexRea idxCols: is.IdxCols, colLens: is.IdxColLens, plans: v.IndexPlans, + outputColumns: v.OutputColumns, } if containsLimit(dagReq.Executors) { e.feedback = statistics.NewQueryFeedback(0, nil, 0, is.Desc) @@ -1966,13 +1959,9 @@ func (builder *dataReaderBuilder) buildUnionScanForIndexJoin(ctx context.Context if err != nil { return nil, err } - e, err := builder.buildUnionScanFromReader(reader, v) - if err != nil { - return nil, err - } - us := e.(*UnionScanExec) - us.snapshotChunkBuffer = newFirstChunk(us) - return us, nil + us := builder.buildUnionScanFromReader(reader, v).(*UnionScanExec) + err = us.open(ctx) + return us, err } func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Context, v *plannercore.PhysicalTableReader, lookUpContents []*indexJoinLookUpContent) (Executor, error) { @@ -2005,6 +1994,7 @@ func (builder *dataReaderBuilder) buildTableReaderFromHandles(ctx context.Contex if err != nil { return nil, err } + e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) e.resultHandler = &tableResultHandler{} result, err := builder.SelectResult(ctx, builder.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans)) if err != nil { diff --git a/executor/distsql.go b/executor/distsql.go index c4cf91124d696..98f2890cbd799 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -206,14 +206,18 @@ type IndexReaderExecutor struct { keepOrder bool desc bool ranges []*ranger.Range - dagPB *tipb.DAGRequest + // kvRanges are only used for union scan. + kvRanges []kv.KeyRange + dagPB *tipb.DAGRequest // result returns one or more distsql.PartialResult and each PartialResult is returned by one region. result distsql.SelectResult // columns are only required by union scan. - columns []*model.ColumnInfo - streaming bool - feedback *statistics.QueryFeedback + columns []*model.ColumnInfo + // outputColumns are only required by union scan. + outputColumns []*expression.Column + streaming bool + feedback *statistics.QueryFeedback corColInFilter bool corColInAccess bool @@ -277,6 +281,7 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange) collExec := true e.dagPB.CollectExecutionSummaries = &collExec } + e.kvRanges = kvRanges e.memTracker = memory.NewTracker(e.id, e.ctx.GetSessionVars().MemQuotaDistSQL) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) diff --git a/executor/executor.go b/executor/executor.go index 35a6254e49553..7dbc442e49e60 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -1253,7 +1253,9 @@ func (e *UnionExec) Next(ctx context.Context, req *chunk.Chunk) error { // Close implements the Executor Close interface. func (e *UnionExec) Close() error { - close(e.finished) + if e.finished != nil { + close(e.finished) + } e.childrenResults = nil if e.resultPool != nil { for range e.resultPool { diff --git a/executor/executor_test.go b/executor/executor_test.go index 139bb7f8b2629..bda4df0a0c1ba 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2907,6 +2907,7 @@ func (s *testSuite) TestHandleTransfer(c *C) { tk.MustExec("insert into t values(4)") // test single read whose result need handle tk.MustQuery("select * from t use index(idx)").Check(testkit.Rows("1", "2", "3", "4")) + tk.MustQuery("select * from t use index(idx) order by a desc").Check(testkit.Rows("4", "3", "2", "1")) tk.MustExec("update t set a = 5 where a = 3") tk.MustQuery("select * from t use index(idx)").Check(testkit.Rows("1", "2", "4", "5")) tk.MustExec("commit") diff --git a/executor/mem_reader.go b/executor/mem_reader.go new file mode 100644 index 0000000000000..ad408f0e36ec8 --- /dev/null +++ b/executor/mem_reader.go @@ -0,0 +1,307 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/set" +) + +type memIndexReader struct { + ctx sessionctx.Context + index *model.IndexInfo + table *model.TableInfo + kvRanges []kv.KeyRange + desc bool + conditions []expression.Expression + addedRows [][]types.Datum + retFieldTypes []*types.FieldType + outputOffset []int + // cache for decode handle. + handleBytes []byte + // memIdxHandles is uses to store the handle ids that has been read by memIndexReader. + memIdxHandles set.Int64Set + // belowHandleIndex is the handle's position of the below scan plan. + belowHandleIndex int +} + +func buildMemIndexReader(us *UnionScanExec, idxReader *IndexReaderExecutor) *memIndexReader { + kvRanges := idxReader.kvRanges + outputOffset := make([]int, 0, len(us.columns)) + for _, col := range idxReader.outputColumns { + outputOffset = append(outputOffset, col.Index) + } + return &memIndexReader{ + ctx: us.ctx, + index: idxReader.index, + table: idxReader.table.Meta(), + kvRanges: kvRanges, + desc: us.desc, + conditions: us.conditions, + addedRows: make([][]types.Datum, 0, len(us.dirty.addedRows)), + retFieldTypes: retTypes(us), + outputOffset: outputOffset, + handleBytes: make([]byte, 0, 16), + memIdxHandles: set.NewInt64Set(), + belowHandleIndex: us.belowHandleIndex, + } +} + +func (m *memIndexReader) getMemRows() ([][]types.Datum, error) { + tps := make([]*types.FieldType, 0, len(m.index.Columns)+1) + cols := m.table.Columns + for _, col := range m.index.Columns { + tps = append(tps, &cols[col.Offset].FieldType) + } + if m.table.PKIsHandle { + for _, col := range m.table.Columns { + if mysql.HasPriKeyFlag(col.Flag) { + tps = append(tps, &col.FieldType) + break + } + } + } else { + // ExtraHandle Column tp. + tps = append(tps, types.NewFieldType(mysql.TypeLonglong)) + } + + mutableRow := chunk.MutRowFromTypes(m.retFieldTypes) + err := iterTxnMemBuffer(m.ctx, m.kvRanges, func(key, value []byte) error { + data, err := m.decodeIndexKeyValue(key, value, tps) + if err != nil { + return err + } + handle := data[m.belowHandleIndex].GetInt64() + m.memIdxHandles.Insert(handle) + + mutableRow.SetDatums(data...) + matched, _, err := expression.EvalBool(m.ctx, m.conditions, mutableRow.ToRow()) + if err != nil || !matched { + return err + } + m.addedRows = append(m.addedRows, data) + return nil + }) + + if err != nil { + return nil, err + } + // TODO: After refine `IterReverse`, remove below logic and use `IterReverse` when do reverse scan. + if m.desc { + reverseDatumSlice(m.addedRows) + } + return m.addedRows, nil +} + +func (m *memIndexReader) decodeIndexKeyValue(key, value []byte, tps []*types.FieldType) ([]types.Datum, error) { + pkStatus := tablecodec.PrimaryKeyIsSigned + if mysql.HasUnsignedFlag(tps[len(tps)-1].Flag) { + pkStatus = tablecodec.PrimaryKeyIsUnsigned + } + values, err := tablecodec.DecodeIndexKV(key, value, len(m.index.Columns), pkStatus) + if err != nil { + return nil, errors.Trace(err) + } + + ds := make([]types.Datum, 0, len(m.outputOffset)) + for _, offset := range m.outputOffset { + d, err := tablecodec.DecodeColumnValue(values[offset], tps[offset], m.ctx.GetSessionVars().TimeZone) + if err != nil { + return nil, err + } + ds = append(ds, d) + } + return ds, nil +} + +type memTableReader struct { + ctx sessionctx.Context + table *model.TableInfo + columns []*model.ColumnInfo + kvRanges []kv.KeyRange + desc bool + conditions []expression.Expression + addedRows [][]types.Datum + retFieldTypes []*types.FieldType + colIDs map[int64]int + // cache for decode handle. + handleBytes []byte +} + +func buildMemTableReader(us *UnionScanExec, tblReader *TableReaderExecutor) *memTableReader { + kvRanges := tblReader.kvRanges + colIDs := make(map[int64]int) + for i, col := range tblReader.columns { + colIDs[col.ID] = i + } + + return &memTableReader{ + ctx: us.ctx, + table: tblReader.table.Meta(), + columns: us.columns, + kvRanges: kvRanges, + desc: us.desc, + conditions: us.conditions, + addedRows: make([][]types.Datum, 0, len(us.dirty.addedRows)), + retFieldTypes: retTypes(us), + colIDs: colIDs, + handleBytes: make([]byte, 0, 16), + } +} + +// TODO: Try to make memXXXReader lazy, There is no need to decode many rows when parent operator only need 1 row. +func (m *memTableReader) getMemRows() ([][]types.Datum, error) { + mutableRow := chunk.MutRowFromTypes(m.retFieldTypes) + err := iterTxnMemBuffer(m.ctx, m.kvRanges, func(key, value []byte) error { + row, err := m.decodeRecordKeyValue(key, value) + if err != nil { + return err + } + + mutableRow.SetDatums(row...) + matched, _, err := expression.EvalBool(m.ctx, m.conditions, mutableRow.ToRow()) + if err != nil || !matched { + return err + } + m.addedRows = append(m.addedRows, row) + return nil + }) + if err != nil { + return nil, err + } + + // TODO: After refine `IterReverse`, remove below logic and use `IterReverse` when do reverse scan. + if m.desc { + reverseDatumSlice(m.addedRows) + } + return m.addedRows, nil +} + +func (m *memTableReader) decodeRecordKeyValue(key, value []byte) ([]types.Datum, error) { + handle, err := tablecodec.DecodeRowKey(key) + if err != nil { + return nil, errors.Trace(err) + } + return decodeRowData(m.ctx, m.table, m.columns, m.colIDs, handle, m.handleBytes, value) +} + +// decodeRowData uses to decode row data value. +func decodeRowData(ctx sessionctx.Context, tb *model.TableInfo, columns []*model.ColumnInfo, colIDs map[int64]int, handle int64, cacheBytes, value []byte) ([]types.Datum, error) { + values, err := getRowData(ctx.GetSessionVars().StmtCtx, tb, columns, colIDs, handle, cacheBytes, value) + if err != nil { + return nil, err + } + ds := make([]types.Datum, 0, len(columns)) + for _, col := range columns { + offset := colIDs[col.ID] + d, err := tablecodec.DecodeColumnValue(values[offset], &col.FieldType, ctx.GetSessionVars().TimeZone) + if err != nil { + return nil, err + } + ds = append(ds, d) + } + return ds, nil +} + +// getRowData decodes raw byte slice to row data. +func getRowData(ctx *stmtctx.StatementContext, tb *model.TableInfo, columns []*model.ColumnInfo, colIDs map[int64]int, handle int64, cacheBytes, value []byte) ([][]byte, error) { + pkIsHandle := tb.PKIsHandle + values, err := tablecodec.CutRowNew(value, colIDs) + if err != nil { + return nil, errors.Trace(err) + } + if values == nil { + values = make([][]byte, len(colIDs)) + } + // Fill the handle and null columns. + for _, col := range columns { + id := col.ID + offset := colIDs[id] + if (pkIsHandle && mysql.HasPriKeyFlag(col.Flag)) || id == model.ExtraHandleID { + var handleDatum types.Datum + if mysql.HasUnsignedFlag(col.Flag) { + // PK column is Unsigned. + handleDatum = types.NewUintDatum(uint64(handle)) + } else { + handleDatum = types.NewIntDatum(handle) + } + handleData, err1 := codec.EncodeValue(ctx, cacheBytes, handleDatum) + if err1 != nil { + return nil, errors.Trace(err1) + } + values[offset] = handleData + continue + } + if hasColVal(values, colIDs, id) { + continue + } + // no need to fill default value. + values[offset] = []byte{codec.NilFlag} + } + + return values, nil +} + +func hasColVal(data [][]byte, colIDs map[int64]int, id int64) bool { + offset, ok := colIDs[id] + if ok && data[offset] != nil { + return true + } + return false +} + +type processKVFunc func(key, value []byte) error + +func iterTxnMemBuffer(ctx sessionctx.Context, kvRanges []kv.KeyRange, fn processKVFunc) error { + txn, err := ctx.Txn(true) + if err != nil { + return err + } + for _, rg := range kvRanges { + iter, err := txn.GetMemBuffer().Iter(rg.StartKey, rg.EndKey) + if err != nil { + return err + } + for ; iter.Valid(); err = iter.Next() { + if err != nil { + return err + } + // check whether the key was been deleted. + if len(iter.Value()) == 0 { + continue + } + err = fn(iter.Key(), iter.Value()) + if err != nil { + return err + } + } + } + return nil +} + +func reverseDatumSlice(rows [][]types.Datum) { + for i, j := 0, len(rows)-1; i < j; i, j = i+1, j-1 { + rows[i], rows[j] = rows[j], rows[i] + } +} diff --git a/executor/table_reader.go b/executor/table_reader.go index 019957e678247..d6272e8b8bec6 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -16,7 +16,6 @@ package executor import ( "context" "fmt" - "github.com/pingcap/parser/model" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/kv" @@ -56,7 +55,9 @@ type TableReaderExecutor struct { keepOrder bool desc bool ranges []*ranger.Range - dagPB *tipb.DAGRequest + // kvRanges are only use for union scan. + kvRanges []kv.KeyRange + dagPB *tipb.DAGRequest // columns are only required by union scan. columns []*model.ColumnInfo @@ -144,7 +145,10 @@ func (e *TableReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error // Close implements the Executor Close interface. func (e *TableReaderExecutor) Close() error { - err := e.resultHandler.Close() + var err error + if e.resultHandler != nil { + err = e.resultHandler.Close() + } if e.runtimeStats != nil { copStats := e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetRootStats(e.plans[0].ExplainID().String()) copStats.SetRowNum(e.feedback.Actual()) @@ -168,6 +172,7 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra if err != nil { return nil, err } + e.kvRanges = append(e.kvRanges, kvReq.KeyRanges...) result, err := e.SelectResult(ctx, e.ctx, kvReq, retTypes(e), e.feedback, getPhysicalPlanIDs(e.plans)) if err != nil { return nil, err diff --git a/executor/union_scan.go b/executor/union_scan.go index ac3eba067c3e0..c9377cbd5e49c 100644 --- a/executor/union_scan.go +++ b/executor/union_scan.go @@ -22,9 +22,9 @@ import ( "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" - "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/set" ) // DirtyDB stores uncommitted write operations for a transaction. @@ -98,16 +98,19 @@ type UnionScanExec struct { desc bool conditions []expression.Expression columns []*model.ColumnInfo - + table table.Table // belowHandleIndex is the handle's position of the below scan plan. belowHandleIndex int - addedRows [][]types.Datum + addedRows [][]types.Datum + // memIdxHandles is uses to store the handle ids that has been read by memIndexReader. + memIdxHandles set.Int64Set cursor4AddRows int sortErr error snapshotRows [][]types.Datum cursor4SnapshotRows int snapshotChunkBuffer *chunk.Chunk + mutableRow chunk.MutRow } // Open implements the Executor Open interface. @@ -115,6 +118,26 @@ func (us *UnionScanExec) Open(ctx context.Context) error { if err := us.baseExecutor.Open(ctx); err != nil { return err } + return us.open(ctx) +} + +func (us *UnionScanExec) open(ctx context.Context) error { + var err error + reader := us.children[0] + switch x := reader.(type) { + case *TableReaderExecutor: + us.addedRows, err = buildMemTableReader(us, x).getMemRows() + case *IndexReaderExecutor: + mIdxReader := buildMemIndexReader(us, x) + us.addedRows, err = mIdxReader.getMemRows() + us.memIdxHandles = mIdxReader.memIdxHandles + case *IndexLookUpExecutor: + us.memIdxHandles = set.NewInt64Set() + err = us.buildAndSortAddedRows(x.table) + } + if err != nil { + return err + } us.snapshotChunkBuffer = newFirstChunk(us) return nil } @@ -196,11 +219,19 @@ func (us *UnionScanExec) getSnapshotRow(ctx context.Context) ([]types.Datum, err for row := iter.Begin(); row != iter.End(); row = iter.Next() { snapshotHandle := row.GetInt64(us.belowHandleIndex) if _, ok := us.dirty.deletedRows[snapshotHandle]; ok { + err = us.getMissIndexRowsByHandle(snapshotHandle) + if err != nil { + return nil, err + } continue } if _, ok := us.dirty.addedRows[snapshotHandle]; ok { // If src handle appears in added rows, it means there is conflict and the transaction will fail to // commit, but for simplicity, we don't handle it here. + err = us.getMissIndexRowsByHandle(snapshotHandle) + if err != nil { + return nil, err + } continue } us.snapshotRows = append(us.snapshotRows, row.GetDatumRow(retTypes(us.children[0]))) @@ -209,6 +240,30 @@ func (us *UnionScanExec) getSnapshotRow(ctx context.Context) ([]types.Datum, err return us.snapshotRows[0], nil } +// For index reader and index look up reader, update doesn't write index to txn memBuffer when the idx column +// is unchanged. So the `memIndexReader` and `memIndexLookUpReader` can't read the index from txn memBuffer. +// This function is used to get the missing row by the handle if the handle is in dirtyTable.addedRows. +func (us *UnionScanExec) getMissIndexRowsByHandle(handle int64) error { + reader := us.children[0] + switch reader.(type) { + case *TableReaderExecutor: + return nil + } + if _, ok := us.dirty.addedRows[handle]; !ok { + return nil + } + // Don't miss in memBuffer reader. + if us.memIdxHandles.Exist(handle) { + return nil + } + memRow, err := us.getMemRow(handle) + if memRow == nil || err != nil { + return err + } + us.snapshotRows = append(us.snapshotRows, memRow) + return nil +} + func (us *UnionScanExec) getAddedRow() []types.Datum { var addedRow []types.Datum if us.cursor4AddRows < len(us.addedRows) { @@ -265,7 +320,7 @@ func (us *UnionScanExec) compare(a, b []types.Datum) (int, error) { } // rowWithColsInTxn gets the row from the transaction buffer. -func (us *UnionScanExec) rowWithColsInTxn(t table.Table, h int64, cols []*table.Column) ([]types.Datum, error) { +func (us *UnionScanExec) rowWithColsInTxn(t table.Table, h int64) ([]types.Datum, error) { key := t.RecordKey(h) txn, err := us.ctx.Txn(true) if err != nil { @@ -275,30 +330,39 @@ func (us *UnionScanExec) rowWithColsInTxn(t table.Table, h int64, cols []*table. if err != nil { return nil, err } - v, _, err := tables.DecodeRawRowData(us.ctx, t.Meta(), h, cols, value) + colIDs := make(map[int64]int) + for i, col := range us.columns { + colIDs[col.ID] = i + } + return decodeRowData(us.ctx, us.table.Meta(), us.columns, colIDs, h, []byte{}, value) +} + +func (us *UnionScanExec) getMemRow(h int64) ([]types.Datum, error) { + data, err := us.rowWithColsInTxn(us.table, h) if err != nil { return nil, err } - return v, nil + us.mutableRow.SetDatums(data...) + matched, _, err := expression.EvalBool(us.ctx, us.conditions, us.mutableRow.ToRow()) + if err != nil { + return nil, err + } + if !matched { + return nil, nil + } + return data, nil } +// TODO: remove `buildAndSortAddedRows` functions and `DirtyTable`. func (us *UnionScanExec) buildAndSortAddedRows(t table.Table) error { us.addedRows = make([][]types.Datum, 0, len(us.dirty.addedRows)) mutableRow := chunk.MutRowFromTypes(retTypes(us)) - cols := t.WritableCols() for h := range us.dirty.addedRows { - newData := make([]types.Datum, 0, us.schema.Len()) - data, err := us.rowWithColsInTxn(t, h, cols) + us.memIdxHandles.Insert(h) + newData, err := us.rowWithColsInTxn(t, h) if err != nil { return err } - for _, col := range us.columns { - if col.ID == model.ExtraHandleID { - newData = append(newData, types.NewIntDatum(h)) - } else { - newData = append(newData, data[col.Offset]) - } - } mutableRow.SetDatums(newData...) matched, _, err := expression.EvalBool(us.ctx, us.conditions, mutableRow.ToRow()) if err != nil { diff --git a/executor/union_scan_test.go b/executor/union_scan_test.go index 7fc90083cf144..73b6ee04e3c24 100644 --- a/executor/union_scan_test.go +++ b/executor/union_scan_test.go @@ -106,3 +106,110 @@ func (s *testSuite4) TestUnionScanWithCastCondition(c *C) { tk.MustQuery("select * from ta where a = 1").Check(testkit.Rows("1")) tk.MustExec("rollback") } + +func (s *testSuite4) TestUnionScanForMemBufferReader(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int,b int, index idx(b))") + tk.MustExec("insert t values (1,1),(2,2)") + + // Test for delete in union scan + tk.MustExec("begin") + tk.MustExec("delete from t") + tk.MustQuery("select * from t").Check(testkit.Rows()) + tk.MustExec("insert t values (1,1)") + tk.MustQuery("select a,b from t").Check(testkit.Rows("1 1")) + tk.MustQuery("select a,b from t use index(idx)").Check(testkit.Rows("1 1")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // Test update with untouched index columns. + tk.MustExec("delete from t") + tk.MustExec("insert t values (1,1),(2,2)") + tk.MustExec("begin") + tk.MustExec("update t set a=a+1") + tk.MustQuery("select * from t").Check(testkit.Rows("2 1", "3 2")) + tk.MustQuery("select * from t use index (idx)").Check(testkit.Rows("2 1", "3 2")) + tk.MustQuery("select * from t use index (idx) order by b desc").Check(testkit.Rows("3 2", "2 1")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // Test update with index column. + tk.MustQuery("select * from t").Check(testkit.Rows("2 1", "3 2")) + tk.MustExec("begin") + tk.MustExec("update t set b=b+1 where a=2") + tk.MustQuery("select * from t").Check(testkit.Rows("2 2", "3 2")) + tk.MustQuery("select * from t use index(idx)").Check(testkit.Rows("2 2", "3 2")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // Test index reader order. + tk.MustQuery("select * from t").Check(testkit.Rows("2 2", "3 2")) + tk.MustExec("begin") + tk.MustExec("insert t values (3,3),(1,1),(4,4),(-1,-1);") + tk.MustQuery("select * from t use index (idx)").Check(testkit.Rows("-1 -1", "1 1", "2 2", "3 2", "3 3", "4 4")) + tk.MustQuery("select b from t use index (idx) order by b desc").Check(testkit.Rows("4", "3", "2", "2", "1", "-1")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // test for update unique index. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int,b int, unique index idx(b))") + tk.MustExec("insert t values (1,1),(2,2)") + tk.MustExec("begin") + _, err := tk.Exec("update t set b=b+1") + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[kv:1062]Duplicate entry '2' for key 'idx'") + // update with unchange index column. + tk.MustExec("update t set a=a+1") + tk.MustQuery("select * from t use index (idx)").Check(testkit.Rows("2 1", "3 2")) + tk.MustExec("update t set b=b+2 where a=2") + tk.MustQuery("select * from t").Check(testkit.Rows("2 3", "3 2")) + tk.MustQuery("select * from t use index (idx) order by b desc").Check(testkit.Rows("2 3", "3 2")) + tk.MustQuery("select * from t use index (idx)").Check(testkit.Rows("3 2", "2 3")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // Test for getMissIndexRowsByHandle return nil. + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int,b int, index idx(a))") + tk.MustExec("insert into t values (1,1),(2,2),(3,3)") + tk.MustExec("begin") + tk.MustExec("update t set b=0 where a=2") + tk.MustQuery("select * from t ignore index (idx) where a>0 and b>0;").Check(testkit.Rows("1 1", "3 3")) + tk.MustQuery("select * from t use index (idx) where a>0 and b>0;").Check(testkit.Rows("1 1", "3 3")) + tk.MustExec("commit") + tk.MustExec("admin check table t") + + // Test index lookup reader corner case. + tk.MustExec("drop table if exists tt") + tk.MustExec("create table tt (a bigint, b int,c int,primary key (a,b));") + tk.MustExec("insert into tt set a=1,b=1;") + tk.MustExec("begin;") + tk.MustExec("update tt set c=1;") + tk.MustQuery("select * from tt use index (PRIMARY) where c is not null;").Check(testkit.Rows("1 1 1")) + tk.MustExec("commit") + tk.MustExec("admin check table tt") + + // Test index reader corner case. + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (a int,b int,primary key(a,b));") + tk.MustExec("begin;") + tk.MustExec("insert into t1 values(1, 1);") + tk.MustQuery("select * from t1 use index(primary) where a=1;").Check(testkit.Rows("1 1")) + tk.MustExec("commit") + tk.MustExec("admin check table t1;") + + // Test index reader with pk handle. + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (a int unsigned key,b int,c varchar(10), index idx(b,a,c));") + tk.MustExec("begin;") + tk.MustExec("insert into t1 (a,b) values (0, 0), (1, 1);") + tk.MustQuery("select a,b from t1 use index(idx) where b>0;").Check(testkit.Rows("1 1")) + tk.MustQuery("select a,b,c from t1 ignore index(idx) where a>=1 order by a desc").Check(testkit.Rows("1 1 ")) + tk.MustExec("insert into t1 values (2, 2, null), (3, 3, 'a');") + tk.MustQuery("select a,b from t1 use index(idx) where b>1 and c is not null;").Check(testkit.Rows("3 3")) + tk.MustExec("commit") + tk.MustExec("admin check table t1;") +} diff --git a/store/mockstore/mocktikv/cop_handler_dag.go b/store/mockstore/mocktikv/cop_handler_dag.go index c8b7c6aaf112e..aa2d49de53a20 100644 --- a/store/mockstore/mocktikv/cop_handler_dag.go +++ b/store/mockstore/mocktikv/cop_handler_dag.go @@ -216,17 +216,17 @@ func (h *rpcHandler) buildIndexScan(ctx *dagContext, executor *tipb.Executor) (* columns := executor.IdxScan.Columns ctx.evalCtx.setColumnInfo(columns) length := len(columns) - pkStatus := pkColNotExists + pkStatus := tablecodec.PrimaryKeyNotExists // The PKHandle column info has been collected in ctx. if columns[length-1].GetPkHandle() { if mysql.HasUnsignedFlag(uint(columns[length-1].GetFlag())) { - pkStatus = pkColIsUnsigned + pkStatus = tablecodec.PrimaryKeyIsUnsigned } else { - pkStatus = pkColIsSigned + pkStatus = tablecodec.PrimaryKeyIsSigned } columns = columns[:length-1] } else if columns[length-1].ColumnId == model.ExtraHandleID { - pkStatus = pkColIsSigned + pkStatus = tablecodec.PrimaryKeyIsSigned columns = columns[:length-1] } ranges, err := h.extractKVRanges(ctx.keyRanges, executor.IdxScan.Desc) diff --git a/store/mockstore/mocktikv/executor.go b/store/mockstore/mocktikv/executor.go index f4f0e7a1d8eba..7808e18cff554 100644 --- a/store/mockstore/mocktikv/executor.go +++ b/store/mockstore/mocktikv/executor.go @@ -16,7 +16,6 @@ package mocktikv import ( "bytes" "context" - "encoding/binary" "sort" "time" @@ -246,12 +245,6 @@ func (e *tableScanExec) getRowFromRange(ran kv.KeyRange) ([][]byte, error) { return row, nil } -const ( - pkColNotExists = iota - pkColIsSigned - pkColIsUnsigned -) - type indexScanExec struct { *tipb.IndexScan colsLen int @@ -261,7 +254,7 @@ type indexScanExec struct { mvccStore MVCCStore cursor int seekKey []byte - pkStatus int + pkStatus tablecodec.PrimaryKeyStatus start int counts []int64 execDetail *execDetail @@ -373,37 +366,7 @@ func (e *indexScanExec) getRowFromPoint(ran kv.KeyRange) ([][]byte, error) { if len(val) == 0 { return nil, nil } - return e.decodeIndexKV(Pair{Key: ran.StartKey, Value: val}) -} - -func (e *indexScanExec) decodeIndexKV(pair Pair) ([][]byte, error) { - values, b, err := tablecodec.CutIndexKeyNew(pair.Key, e.colsLen) - if err != nil { - return nil, errors.Trace(err) - } - if len(b) > 0 { - if e.pkStatus != pkColNotExists { - values = append(values, b) - } - } else if e.pkStatus != pkColNotExists { - handle, err := decodeHandle(pair.Value) - if err != nil { - return nil, errors.Trace(err) - } - var handleDatum types.Datum - if e.pkStatus == pkColIsUnsigned { - handleDatum = types.NewUintDatum(uint64(handle)) - } else { - handleDatum = types.NewIntDatum(handle) - } - handleBytes, err := codec.EncodeValue(nil, nil, handleDatum) - if err != nil { - return nil, errors.Trace(err) - } - values = append(values, handleBytes) - } - - return values, nil + return tablecodec.DecodeIndexKV(ran.StartKey, val, e.colsLen, e.pkStatus) } func (e *indexScanExec) getRowFromRange(ran kv.KeyRange) ([][]byte, error) { @@ -443,7 +406,7 @@ func (e *indexScanExec) getRowFromRange(ran kv.KeyRange) ([][]byte, error) { e.seekKey = []byte(kv.Key(pair.Key).PrefixNext()) } - return e.decodeIndexKV(pair) + return tablecodec.DecodeIndexKV(pair.Key, pair.Value, e.colsLen, e.pkStatus) } type selectionExec struct { @@ -757,10 +720,3 @@ func convertToExprs(sc *stmtctx.StatementContext, fieldTps []*types.FieldType, p } return exprs, nil } - -func decodeHandle(data []byte) (int64, error) { - var h int64 - buf := bytes.NewBuffer(data) - err := binary.Read(buf, binary.BigEndian, &h) - return h, errors.Trace(err) -} diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index fbbf6b232e736..568cc51f3f529 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -14,6 +14,7 @@ package tablecodec import ( + "bytes" "encoding/binary" "math" "time" @@ -516,6 +517,57 @@ func CutIndexKeyNew(key kv.Key, length int) (values [][]byte, b []byte, err erro return } +// PrimaryKeyStatus is the primary key column status. +type PrimaryKeyStatus int + +const ( + // PrimaryKeyNotExists means no need to decode primary key column value when DecodeIndexKV. + PrimaryKeyNotExists PrimaryKeyStatus = iota + // PrimaryKeyIsSigned means decode primary key column value as int64 when DecodeIndexKV. + PrimaryKeyIsSigned + // PrimaryKeyIsUnsigned means decode primary key column value as uint64 when DecodeIndexKV. + PrimaryKeyIsUnsigned +) + +// DecodeIndexKV uses to decode index key values. +func DecodeIndexKV(key, value []byte, colsLen int, pkStatus PrimaryKeyStatus) ([][]byte, error) { + values, b, err := CutIndexKeyNew(key, colsLen) + if err != nil { + return nil, errors.Trace(err) + } + if len(b) > 0 { + if pkStatus != PrimaryKeyNotExists { + values = append(values, b) + } + } else if pkStatus != PrimaryKeyNotExists { + handle, err := DecodeIndexValueAsHandle(value) + if err != nil { + return nil, errors.Trace(err) + } + var handleDatum types.Datum + if pkStatus == PrimaryKeyIsUnsigned { + handleDatum = types.NewUintDatum(uint64(handle)) + } else { + handleDatum = types.NewIntDatum(handle) + } + handleBytes := make([]byte, 0, 8) + handleBytes, err = codec.EncodeValue(nil, handleBytes, handleDatum) + if err != nil { + return nil, errors.Trace(err) + } + values = append(values, handleBytes) + } + return values, nil +} + +// DecodeIndexValueAsHandle uses to decode index value as handle id. +func DecodeIndexValueAsHandle(data []byte) (int64, error) { + var h int64 + buf := bytes.NewBuffer(data) + err := binary.Read(buf, binary.BigEndian, &h) + return h, errors.Trace(err) +} + // EncodeTableIndexPrefix encodes index prefix with tableID and idxID. func EncodeTableIndexPrefix(tableID, idxID int64) kv.Key { key := make([]byte, 0, prefixLen) From 8bc63973dfa2b6e7c10631360537c64c0f457362 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Tue, 30 Jul 2019 18:32:05 +0800 Subject: [PATCH 123/196] executor: move failpoint test case to seq test suite (#11525) --- executor/seqtest/seq_executor_test.go | 19 +++++++++++++++++++ executor/write_test.go | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index 0aeee16fceaf1..3936675404586 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -1031,3 +1031,22 @@ func (s *seqTestSuite1) TestCoprocessorPriority(c *C) { cli.mu.checkPrio = false cli.mu.Unlock() } + +func (s *seqTestSuite) TestAutoIDInRetry(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (id int not null auto_increment primary key)") + + tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") + tk.MustExec("begin") + tk.MustExec("insert into t values ()") + tk.MustExec("insert into t values (),()") + tk.MustExec("insert into t values ()") + + c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/mockCommitRetryForAutoID", `return(true)`), IsNil) + tk.MustExec("commit") + c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/mockCommitRetryForAutoID"), IsNil) + + tk.MustExec("insert into t values ()") + tk.MustQuery(`select * from t`).Check(testkit.Rows("1", "2", "3", "4", "5")) +} diff --git a/executor/write_test.go b/executor/write_test.go index 43408d57ab03a..14e86b09a3c05 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -19,7 +19,6 @@ import ( "fmt" . "github.com/pingcap/check" - "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/executor" @@ -2469,24 +2468,6 @@ func (s *testSuite4) TestDefEnumInsert(c *C) { tk.MustQuery("select prescription_type from test").Check(testkit.Rows("a")) } -func (s *testSuite4) TestAutoIDInRetry(c *C) { - tk := testkit.NewTestKitWithInit(c, s.store) - tk.MustExec("create table t (id int not null auto_increment primary key)") - - tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") - tk.MustExec("begin") - tk.MustExec("insert into t values ()") - tk.MustExec("insert into t values (),()") - tk.MustExec("insert into t values ()") - - c.Assert(failpoint.Enable("github.com/pingcap/tidb/session/mockCommitRetryForAutoID", `return(true)`), IsNil) - tk.MustExec("commit") - c.Assert(failpoint.Disable("github.com/pingcap/tidb/session/mockCommitRetryForAutoID"), IsNil) - - tk.MustExec("insert into t values ()") - tk.MustQuery(`select * from t`).Check(testkit.Rows("1", "2", "3", "4", "5")) -} - func (s *testSuite4) TestIssue11059(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("create table t (pk int primary key, uk int unique, v int)") From 60bb04b8f0635f326db54fed4eea6ca3a49a20b2 Mon Sep 17 00:00:00 2001 From: Luffbee Date: Tue, 30 Jul 2019 19:39:02 +0800 Subject: [PATCH 124/196] store/helper, infoschema: add table info for region info mem tables (#11444) * add table regions distribution mem table * fix a small bug * add table info for TIKV_REGION_* mem tables * fix IsBefore() * add test for GetRegionsTableInfo() * remove Table*RegionsDistribution which can be calculated from TiKVRegionPeers * unexported some stuff; remove a unused function --- infoschema/tables.go | 158 ++++++++++++++++++++-------- store/helper/helper.go | 203 ++++++++++++++++++++++++++++++++++-- store/helper/helper_test.go | 198 +++++++++++++++++++++++++++++------ 3 files changed, 474 insertions(+), 85 deletions(-) diff --git a/infoschema/tables.go b/infoschema/tables.go index e31db88b9af68..6743c7af880f6 100644 --- a/infoschema/tables.go +++ b/infoschema/tables.go @@ -616,6 +616,12 @@ var tableTiKVRegionStatusCols = []columnInfo{ {"REGION_ID", mysql.TypeLonglong, 21, 0, nil, nil}, {"START_KEY", mysql.TypeBlob, types.UnspecifiedLength, 0, nil, nil}, {"END_KEY", mysql.TypeBlob, types.UnspecifiedLength, 0, nil, nil}, + {"TABLE_ID", mysql.TypeLonglong, 21, 0, nil, nil}, + {"DB_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"TABLE_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"IS_INDEX", mysql.TypeTiny, 1, mysql.NotNullFlag, 0, nil}, + {"INDEX_ID", mysql.TypeLonglong, 21, 0, nil, nil}, + {"INDEX_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, {"EPOCH_CONF_VER", mysql.TypeLonglong, 21, 0, nil, nil}, {"EPOCH_VERSION", mysql.TypeLonglong, 21, 0, nil, nil}, {"WRITTEN_BYTES", mysql.TypeLonglong, 21, 0, nil, nil}, @@ -626,6 +632,12 @@ var tableTiKVRegionStatusCols = []columnInfo{ var tableTiKVRegionPeersCols = []columnInfo{ {"REGION_ID", mysql.TypeLonglong, 21, 0, nil, nil}, + {"TABLE_ID", mysql.TypeLonglong, 21, 0, nil, nil}, + {"DB_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"TABLE_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, + {"IS_INDEX", mysql.TypeTiny, 1, mysql.NotNullFlag, 0, nil}, + {"INDEX_ID", mysql.TypeLonglong, 21, 0, nil, nil}, + {"INDEX_NAME", mysql.TypeVarchar, 64, 0, nil, nil}, {"PEER_ID", mysql.TypeLonglong, 21, 0, nil, nil}, {"STORE_ID", mysql.TypeLonglong, 21, 0, nil, nil}, {"IS_LEARNER", mysql.TypeTiny, 1, mysql.NotNullFlag, 0, nil}, @@ -643,26 +655,51 @@ func dataForTiKVRegionStatus(ctx sessionctx.Context) (records [][]types.Datum, e Store: tikvStore, RegionCache: tikvStore.GetRegionCache(), } - regionsStat, err := tikvHelper.GetRegionsInfo() + regionsInfo, err := tikvHelper.GetRegionsInfo() if err != nil { return nil, err } - for _, regionStat := range regionsStat.Regions { - row := make([]types.Datum, len(tableTiKVRegionStatusCols)) - row[0].SetInt64(regionStat.ID) - row[1].SetString(regionStat.StartKey) - row[2].SetString(regionStat.EndKey) - row[3].SetInt64(regionStat.Epoch.ConfVer) - row[4].SetInt64(regionStat.Epoch.Version) - row[5].SetInt64(regionStat.WrittenBytes) - row[6].SetInt64(regionStat.ReadBytes) - row[7].SetInt64(regionStat.ApproximateSize) - row[8].SetInt64(regionStat.ApproximateKeys) - records = append(records, row) + allSchemas := ctx.GetSessionVars().TxnCtx.InfoSchema.(InfoSchema).AllSchemas() + tableInfos := tikvHelper.GetRegionsTableInfo(regionsInfo, allSchemas) + for _, region := range regionsInfo.Regions { + tableList := tableInfos[region.ID] + if len(tableList) == 0 { + records = append(records, newTiKVRegionStatusCol(®ion, nil)) + } + for _, table := range tableList { + row := newTiKVRegionStatusCol(®ion, &table) + records = append(records, row) + } } return records, nil } +func newTiKVRegionStatusCol(region *helper.RegionInfo, table *helper.TableInfo) []types.Datum { + row := make([]types.Datum, len(tableTiKVRegionStatusCols)) + row[0].SetInt64(region.ID) + row[1].SetString(region.StartKey) + row[2].SetString(region.EndKey) + if table != nil { + row[3].SetInt64(table.Table.ID) + row[4].SetString(table.DB.Name.O) + row[5].SetString(table.Table.Name.O) + if table.IsIndex { + row[6].SetInt64(1) + row[7].SetInt64(table.Index.ID) + row[8].SetString(table.Index.Name.O) + } else { + row[6].SetInt64(0) + } + } + row[9].SetInt64(region.Epoch.ConfVer) + row[10].SetInt64(region.Epoch.Version) + row[11].SetInt64(region.WrittenBytes) + row[12].SetInt64(region.ReadBytes) + row[13].SetInt64(region.ApproximateSize) + row[14].SetInt64(region.ApproximateKeys) + return row +} + const ( normalPeer = "NORMAL" pendingPeer = "PENDING" @@ -678,48 +715,77 @@ func dataForTikVRegionPeers(ctx sessionctx.Context) (records [][]types.Datum, er Store: tikvStore, RegionCache: tikvStore.GetRegionCache(), } - regionsStat, err := tikvHelper.GetRegionsInfo() + regionsInfo, err := tikvHelper.GetRegionsInfo() if err != nil { return nil, err } - for _, regionStat := range regionsStat.Regions { - pendingPeerIDSet := set.NewInt64Set() - for _, peer := range regionStat.PendingPeers { - pendingPeerIDSet.Insert(peer.ID) + allSchemas := ctx.GetSessionVars().TxnCtx.InfoSchema.(InfoSchema).AllSchemas() + tableInfos := tikvHelper.GetRegionsTableInfo(regionsInfo, allSchemas) + for _, region := range regionsInfo.Regions { + tableList := tableInfos[region.ID] + if len(tableList) == 0 { + records = append(records, newTiKVRegionPeersCols(®ion, nil)...) } - downPeerMap := make(map[int64]int64) - for _, peerStat := range regionStat.DownPeers { - downPeerMap[peerStat.ID] = peerStat.DownSec - } - for _, peer := range regionStat.Peers { - row := make([]types.Datum, len(tableTiKVRegionPeersCols)) - row[0].SetInt64(regionStat.ID) - row[1].SetInt64(peer.ID) - row[2].SetInt64(peer.StoreID) - if peer.IsLearner { - row[3].SetInt64(1) - } else { - row[3].SetInt64(0) - } - if peer.ID == regionStat.Leader.ID { - row[4].SetInt64(1) - } else { - row[3].SetInt64(0) - } - if pendingPeerIDSet.Exist(peer.ID) { - row[4].SetString(pendingPeer) - } else if downSec, ok := downPeerMap[peer.ID]; ok { - row[5].SetString(downPeer) - row[6].SetInt64(downSec) - } else { - row[5].SetString(normalPeer) - } - records = append(records, row) + for _, table := range tableList { + rows := newTiKVRegionPeersCols(®ion, &table) + records = append(records, rows...) } } return records, nil } +func newTiKVRegionPeersCols(region *helper.RegionInfo, table *helper.TableInfo) [][]types.Datum { + records := make([][]types.Datum, 0, len(region.Peers)) + pendingPeerIDSet := set.NewInt64Set() + for _, peer := range region.PendingPeers { + pendingPeerIDSet.Insert(peer.ID) + } + downPeerMap := make(map[int64]int64) + for _, peerStat := range region.DownPeers { + downPeerMap[peerStat.ID] = peerStat.DownSec + } + template := make([]types.Datum, 7) + template[0].SetInt64(region.ID) + if table != nil { + template[1].SetInt64(table.Table.ID) + template[2].SetString(table.DB.Name.O) + template[3].SetString(table.Table.Name.O) + if table.IsIndex { + template[4].SetInt64(1) + template[5].SetInt64(table.Index.ID) + template[6].SetString(table.Index.Name.O) + } else { + template[4].SetInt64(0) + } + } + for _, peer := range region.Peers { + row := make([]types.Datum, len(tableTiKVRegionPeersCols)) + copy(row, template) + row[7].SetInt64(peer.ID) + row[8].SetInt64(peer.StoreID) + if peer.IsLearner { + row[9].SetInt64(1) + } else { + row[9].SetInt64(0) + } + if peer.ID == region.Leader.ID { + row[10].SetInt64(1) + } else { + row[10].SetInt64(0) + } + if pendingPeerIDSet.Exist(peer.ID) { + row[11].SetString(pendingPeer) + } else if downSec, ok := downPeerMap[peer.ID]; ok { + row[11].SetString(downPeer) + row[12].SetInt64(downSec) + } else { + row[11].SetString(normalPeer) + } + records = append(records, row) + } + return records +} + func dataForTiKVStoreStatus(ctx sessionctx.Context) (records [][]types.Datum, err error) { tikvStore, ok := ctx.GetStore().(tikv.Storage) if !ok { diff --git a/store/helper/helper.go b/store/helper/helper.go index 4be8a343cfec8..b0e2c0e9c7327 100644 --- a/store/helper/helper.go +++ b/store/helper/helper.go @@ -16,9 +16,13 @@ package helper import ( "bytes" "context" + "encoding/hex" "encoding/json" + "io" "math" "net/http" + "sort" + "strings" "time" "github.com/pingcap/errors" @@ -28,6 +32,7 @@ import ( "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/pdapi" "go.uber.org/zap" @@ -43,6 +48,14 @@ type Helper struct { RegionCache *tikv.RegionCache } +// NewHelper get a Helper from Storage +func NewHelper(store tikv.Storage) *Helper { + return &Helper{ + Store: store, + RegionCache: store.GetRegionCache(), + } +} + // GetMvccByEncodedKey get the MVCC value by the specific encoded key. func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyResponse, error) { keyLocation, err := h.RegionCache.LocateKey(tikv.NewBackoffer(context.Background(), 500), encodedKey) @@ -399,36 +412,208 @@ type RegionsInfo struct { Regions []RegionInfo `json:"regions"` } +// TableInfo stores the information of a table or an index +type TableInfo struct { + DB *model.DBInfo + Table *model.TableInfo + IsIndex bool + Index *model.IndexInfo +} + +type withKeyRange interface { + getStartKey() string + getEndKey() string +} + +// isIntersecting returns true if x and y intersect. +func isIntersecting(x, y withKeyRange) bool { + return isIntersectingKeyRange(x, y.getStartKey(), y.getEndKey()) +} + +// isIntersectingKeyRange returns true if [startKey, endKey) intersect with x. +func isIntersectingKeyRange(x withKeyRange, startKey, endKey string) bool { + return !isBeforeKeyRange(x, startKey, endKey) && !isBehindKeyRange(x, startKey, endKey) +} + +// IsBefore returns true is x is before y +func inBefore(x, y withKeyRange) bool { + return isBeforeKeyRange(x, y.getStartKey(), y.getEndKey()) +} + +// isBehind returns true is x is behind y +func isBehind(x, y withKeyRange) bool { + return isBehindKeyRange(x, y.getStartKey(), y.getEndKey()) +} + +// IsBefore returns true is x is before [startKey, endKey) +func isBeforeKeyRange(x withKeyRange, startKey, endKey string) bool { + return x.getEndKey() != "" && x.getEndKey() <= startKey +} + +// IsBehind returns true is x is behind [startKey, endKey) +func isBehindKeyRange(x withKeyRange, startKey, endKey string) bool { + return endKey != "" && x.getStartKey() >= endKey +} + +func (r *RegionInfo) getStartKey() string { return r.StartKey } +func (r *RegionInfo) getEndKey() string { return r.EndKey } + +// for sorting +type byRegionStartKey []*RegionInfo + +func (xs byRegionStartKey) Len() int { return len(xs) } +func (xs byRegionStartKey) Swap(i, j int) { xs[i], xs[j] = xs[j], xs[i] } +func (xs byRegionStartKey) Less(i, j int) bool { + return xs[i].getStartKey() < xs[j].getStartKey() +} + +// tableInfoWithKeyRange stores table or index informations with its key range. +type tableInfoWithKeyRange struct { + *TableInfo + StartKey string + EndKey string +} + +func (t tableInfoWithKeyRange) getStartKey() string { return t.StartKey } +func (t tableInfoWithKeyRange) getEndKey() string { return t.EndKey } + +// for sorting +type byTableStartKey []tableInfoWithKeyRange + +func (xs byTableStartKey) Len() int { return len(xs) } +func (xs byTableStartKey) Swap(i, j int) { xs[i], xs[j] = xs[j], xs[i] } +func (xs byTableStartKey) Less(i, j int) bool { + return xs[i].getStartKey() < xs[j].getStartKey() +} + +func newTableWithKeyRange(db *model.DBInfo, table *model.TableInfo) tableInfoWithKeyRange { + sk, ek := tablecodec.GetTableHandleKeyRange(table.ID) + startKey := bytesKeyToHex(codec.EncodeBytes(nil, sk)) + endKey := bytesKeyToHex(codec.EncodeBytes(nil, ek)) + return tableInfoWithKeyRange{ + &TableInfo{ + DB: db, + Table: table, + IsIndex: false, + Index: nil, + }, + startKey, + endKey, + } +} + +func newIndexWithKeyRange(db *model.DBInfo, table *model.TableInfo, index *model.IndexInfo) tableInfoWithKeyRange { + sk, ek := tablecodec.GetTableIndexKeyRange(table.ID, index.ID) + startKey := bytesKeyToHex(codec.EncodeBytes(nil, sk)) + endKey := bytesKeyToHex(codec.EncodeBytes(nil, ek)) + return tableInfoWithKeyRange{ + &TableInfo{ + DB: db, + Table: table, + IsIndex: true, + Index: index, + }, + startKey, + endKey, + } +} + +// GetRegionsTableInfo returns a map maps region id to its tables or indices. +// Assuming tables or indices key ranges never intersect. +// Regions key ranges can intersect. +func (h *Helper) GetRegionsTableInfo(regionsInfo *RegionsInfo, schemas []*model.DBInfo) map[int64][]TableInfo { + tableInfos := make(map[int64][]TableInfo) + + regions := []*RegionInfo{} + for i := 0; i < len(regionsInfo.Regions); i++ { + tableInfos[regionsInfo.Regions[i].ID] = []TableInfo{} + regions = append(regions, ®ionsInfo.Regions[i]) + } + + tables := []tableInfoWithKeyRange{} + for _, db := range schemas { + for _, table := range db.Tables { + tables = append(tables, newTableWithKeyRange(db, table)) + for _, index := range table.Indices { + tables = append(tables, newIndexWithKeyRange(db, table, index)) + } + } + } + + if len(tables) == 0 || len(regions) == 0 { + return tableInfos + } + + sort.Sort(byRegionStartKey(regions)) + sort.Sort(byTableStartKey(tables)) + + idx := 0 +OutLoop: + for _, region := range regions { + id := region.ID + for isBehind(region, &tables[idx]) { + idx++ + if idx >= len(tables) { + break OutLoop + } + } + for i := idx; i < len(tables) && isIntersecting(region, &tables[i]); i++ { + tableInfos[id] = append(tableInfos[id], *tables[i].TableInfo) + } + } + + return tableInfos +} + +func bytesKeyToHex(key []byte) string { + return strings.ToUpper(hex.EncodeToString(key)) +} + +func hexKeyToBytes(key string) ([]byte, error) { + return hex.DecodeString(key) +} + // GetRegionsInfo gets the region information of current store by using PD's api. func (h *Helper) GetRegionsInfo() (*RegionsInfo, error) { + var regionsInfo RegionsInfo + err := h.requestPD("GET", pdapi.Regions, nil, ®ionsInfo) + return ®ionsInfo, err +} + +// request PD API, decode the response body into res +func (h *Helper) requestPD(method, uri string, body io.Reader, res interface{}) error { etcd, ok := h.Store.(tikv.EtcdBackend) if !ok { - return nil, errors.WithStack(errors.New("not implemented")) + return errors.WithStack(errors.New("not implemented")) } pdHosts := etcd.EtcdAddrs() if len(pdHosts) == 0 { - return nil, errors.New("pd unavailable") + return errors.New("pd unavailable") } - req, err := http.NewRequest("GET", protocol+pdHosts[0]+pdapi.Regions, nil) + + logutil.BgLogger().Debug("RequestPD URL", zap.String("url", protocol+pdHosts[0]+uri)) + req, err := http.NewRequest(method, protocol+pdHosts[0]+uri, body) if err != nil { - return nil, errors.Trace(err) + return errors.Trace(err) } resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, errors.Trace(err) + return errors.Trace(err) } + defer func() { err = resp.Body.Close() if err != nil { logutil.BgLogger().Error("close body failed", zap.Error(err)) } }() - var regionsInfo RegionsInfo - err = json.NewDecoder(resp.Body).Decode(®ionsInfo) + + err = json.NewDecoder(resp.Body).Decode(res) if err != nil { - return nil, errors.Trace(err) + return errors.Trace(err) } - return ®ionsInfo, nil + + return nil } // StoresStat stores all information get from PD's api. diff --git a/store/helper/helper_test.go b/store/helper/helper_test.go index baa9415188786..eb2977cb8e6ab 100644 --- a/store/helper/helper_test.go +++ b/store/helper/helper_test.go @@ -89,6 +89,15 @@ func (s *HelperTestSuite) TestHotRegion(c *C) { c.Assert(err, IsNil, Commentf("err: %+v", err)) } +func (s *HelperTestSuite) TestGetRegionsTableInfo(c *C) { + h := helper.NewHelper(s.store) + regionsInfo := getMockTiKVRegionsInfo() + schemas := getMockRegionsTableInfoSchema() + tableInfos := h.GetRegionsTableInfo(regionsInfo, schemas) + ans := getRegionsTableInfoAns(schemas) + c.Assert(fmt.Sprintf("%v", tableInfos), Equals, fmt.Sprintf("%v", ans)) +} + func (s *HelperTestSuite) TestTiKVRegionsInfo(c *C) { h := helper.Helper{ Store: s.store, @@ -96,7 +105,7 @@ func (s *HelperTestSuite) TestTiKVRegionsInfo(c *C) { } regionsInfo, err := h.GetRegionsInfo() c.Assert(err, IsNil, Commentf("err: %+v", err)) - c.Assert(fmt.Sprintf("%v", regionsInfo), Equals, "&{1 [{1 test testtest {1 1} [{2 1 false}] {2 1 false} [] [] 100 1000 500 200}]}") + c.Assert(fmt.Sprintf("%v", regionsInfo), Equals, fmt.Sprintf("%v", getMockTiKVRegionsInfo())) } func (s *HelperTestSuite) TestTiKVStoresStat(c *C) { @@ -150,41 +159,170 @@ func (s *HelperTestSuite) mockHotRegionResponse(w http.ResponseWriter, req *http } -func (s *HelperTestSuite) mockTiKVRegionsInfoResponse(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - resp := helper.RegionsInfo{ - Count: 1, - Regions: []helper.RegionInfo{ - { - ID: 1, - StartKey: "test", - EndKey: "testtest", - Epoch: helper.RegionEpoch{ - ConfVer: 1, - Version: 1, +func getMockRegionsTableInfoSchema() []*model.DBInfo { + return []*model.DBInfo{ + { + Name: model.NewCIStr("test"), + Tables: []*model.TableInfo{ + { + ID: 41, + Indices: []*model.IndexInfo{{ID: 1}}, }, - Peers: []helper.RegionPeer{ - { - ID: 2, - StoreID: 1, - IsLearner: false, - }, + { + ID: 63, + Indices: []*model.IndexInfo{{ID: 1}, {ID: 2}}, }, - Leader: helper.RegionPeer{ - ID: 2, - StoreID: 1, - IsLearner: false, + { + ID: 66, + Indices: []*model.IndexInfo{{ID: 1}, {ID: 2}, {ID: 3}}, + }, + }, + }, + } +} + +func getRegionsTableInfoAns(dbs []*model.DBInfo) map[uint64][]helper.TableInfo { + ans := make(map[uint64][]helper.TableInfo) + db := dbs[0] + ans[1] = []helper.TableInfo{} + ans[2] = []helper.TableInfo{ + {db, db.Tables[0], true, db.Tables[0].Indices[0]}, + {db, db.Tables[0], false, nil}, + } + ans[3] = []helper.TableInfo{ + {db, db.Tables[1], true, db.Tables[1].Indices[0]}, + {db, db.Tables[1], true, db.Tables[1].Indices[1]}, + {db, db.Tables[1], false, nil}, + } + ans[4] = []helper.TableInfo{ + {db, db.Tables[2], false, nil}, + } + ans[5] = []helper.TableInfo{ + {db, db.Tables[2], true, db.Tables[2].Indices[2]}, + {db, db.Tables[2], false, nil}, + } + ans[6] = []helper.TableInfo{ + {db, db.Tables[2], true, db.Tables[2].Indices[0]}, + } + ans[7] = []helper.TableInfo{ + {db, db.Tables[2], true, db.Tables[2].Indices[1]}, + } + ans[8] = []helper.TableInfo{ + {db, db.Tables[2], true, db.Tables[2].Indices[1]}, + {db, db.Tables[2], true, db.Tables[2].Indices[2]}, + {db, db.Tables[2], false, nil}, + } + return ans +} + +func getMockTiKVRegionsInfo() *helper.RegionsInfo { + regions := []helper.RegionInfo{ + { + ID: 1, + StartKey: "", + EndKey: "12341234", + Epoch: helper.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: []helper.RegionPeer{ + {ID: 2, StoreID: 1}, + {ID: 15, StoreID: 51}, + {ID: 66, StoreID: 99, IsLearner: true}, + {ID: 123, StoreID: 111, IsLearner: true}, + }, + Leader: helper.RegionPeer{ + ID: 2, + StoreID: 1, + }, + DownPeers: []helper.RegionPeerStat{ + { + helper.RegionPeer{ID: 66, StoreID: 99, IsLearner: true}, + 120, }, - DownPeers: nil, - PendingPeers: nil, - WrittenBytes: 100, - ReadBytes: 1000, - ApproximateKeys: 200, - ApproximateSize: 500, }, + PendingPeers: []helper.RegionPeer{ + {ID: 15, StoreID: 51}, + }, + WrittenBytes: 100, + ReadBytes: 1000, + ApproximateKeys: 200, + ApproximateSize: 500, + }, + // table: 41, record + index: 1 + { + ID: 2, + StartKey: "7480000000000000FF295F698000000000FF0000010000000000FA", + EndKey: "7480000000000000FF2B5F698000000000FF0000010000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 3, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 3, StoreID: 1}, + }, + // table: 63, record + index: 1, 2 + { + ID: 3, + StartKey: "7480000000000000FF3F5F698000000000FF0000010000000000FA", + EndKey: "7480000000000000FF425F698000000000FF0000010000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 4, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 4, StoreID: 1}, + }, + // table: 66, record + { + ID: 4, + StartKey: "7480000000000000FF425F72C000000000FF0000000000000000FA", + EndKey: "", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 5, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 5, StoreID: 1}, }, + // table: 66, record + index: 3 + { + ID: 5, + StartKey: "7480000000000000FF425F698000000000FF0000030000000000FA", + EndKey: "7480000000000000FF425F72C000000000FF0000000000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 6, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 6, StoreID: 1}, + }, + // table: 66, index: 1 + { + ID: 6, + StartKey: "7480000000000000FF425F698000000000FF0000010000000000FA", + EndKey: "7480000000000000FF425F698000000000FF0000020000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 7, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 7, StoreID: 1}, + }, + // table: 66, index: 2 + { + ID: 7, + StartKey: "7480000000000000FF425F698000000000FF0000020000000000FA", + EndKey: "7480000000000000FF425F698000000000FF0000030000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 8, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 8, StoreID: 1}, + }, + // merge region 7, 5 + { + ID: 8, + StartKey: "7480000000000000FF425F698000000000FF0000020000000000FA", + EndKey: "7480000000000000FF425F72C000000000FF0000000000000000FA", + Epoch: helper.RegionEpoch{ConfVer: 1, Version: 1}, + Peers: []helper.RegionPeer{{ID: 9, StoreID: 1}}, + Leader: helper.RegionPeer{ID: 9, StoreID: 1}, + }, + } + return &helper.RegionsInfo{ + Count: int64(len(regions)), + Regions: regions, } +} + +func (s *HelperTestSuite) mockTiKVRegionsInfoResponse(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + resp := getMockTiKVRegionsInfo() data, err := json.MarshalIndent(resp, "", " ") if err != nil { log.Panic("json marshal failed", zap.Error(err)) From d91ed2b7a7b7eb28a8ce62b04df7476998deb78e Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 31 Jul 2019 10:47:40 +0800 Subject: [PATCH 125/196] config: add split-region-max-num in config to control the maximum split region number (#11427) All tests passed, auto merged by Bot --- config/config.go | 2 ++ config/config.toml.example | 3 +++ config/config_test.go | 2 ++ planner/core/planbuilder.go | 5 +++-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 471e83a1bdcc1..8db018422171c 100644 --- a/config/config.go +++ b/config/config.go @@ -96,6 +96,7 @@ type Config struct { // TODO: remove this after table lock features stable. EnableTableLock bool `toml:"enable-table-lock" json:"enable-table-lock"` DelayCleanTableLock uint64 `toml:"delay-clean-table-lock" json:"delay-clean-table-lock"` + SplitRegionMaxNum uint64 `toml:"split-region-max-num" json:"split-region-max-num"` } // Log is the log section of config. @@ -334,6 +335,7 @@ var defaultConf = Config{ TreatOldVersionUTF8AsUTF8MB4: true, EnableTableLock: false, DelayCleanTableLock: 0, + SplitRegionMaxNum: 1000, TxnLocalLatches: TxnLocalLatches{ Enabled: true, Capacity: 2048000, diff --git a/config/config.toml.example b/config/config.toml.example index a641fb3486916..045d5a64a90a5 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -60,6 +60,9 @@ enable-table-lock = false # delay-clean-table-lock is used to control the time (Milliseconds) of delay before unlock the table in the abnormal situation. delay-clean-table-lock = 0 +# Maximum number of the splitting region, which is used by the split region statement. +split-region-max-num = 1000 + [log] # Log level: debug, info, warn, error, fatal. level = "info" diff --git a/config/config_test.go b/config/config_test.go index 00e5031f8db40..5a0d1519e7d9f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -64,6 +64,7 @@ unrecognized-option-test = true token-limit = 0 enable-table-lock = true delay-clean-table-lock = 5 +split-region-max-num=10000 [performance] txn-entry-count-limit=2000 txn-total-size-limit=2000 @@ -90,6 +91,7 @@ max-batch-size=128 c.Assert(conf.TokenLimit, Equals, uint(1000)) c.Assert(conf.EnableTableLock, IsTrue) c.Assert(conf.DelayCleanTableLock, Equals, uint64(5)) + c.Assert(conf.SplitRegionMaxNum, Equals, uint64(10000)) c.Assert(f.Close(), IsNil) c.Assert(os.Remove(configFile), IsNil) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 48b9a76669577..269490558dbe7 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/opcode" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/infoschema" @@ -1698,8 +1699,6 @@ func (b *PlanBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { return p } -const maxSplitRegionNum = 1000 - func (b *PlanBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) { if len(node.IndexName.L) != 0 { return b.buildSplitIndexRegion(node) @@ -1760,6 +1759,7 @@ func (b *PlanBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, er p.Lower = lowerValues p.Upper = upperValues + maxSplitRegionNum := int64(config.GetGlobalConfig().SplitRegionMaxNum) if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split index region num exceeded the limit %v", maxSplitRegionNum) } else if node.SplitOpt.Num < 1 { @@ -1870,6 +1870,7 @@ func (b *PlanBuilder) buildSplitTableRegion(node *ast.SplitRegionStmt) (Plan, er p.Lower = []types.Datum{lowerValues} p.Upper = []types.Datum{upperValue} + maxSplitRegionNum := int64(config.GetGlobalConfig().SplitRegionMaxNum) if node.SplitOpt.Num > maxSplitRegionNum { return nil, errors.Errorf("Split table region num exceeded the limit %v", maxSplitRegionNum) } else if node.SplitOpt.Num < 1 { From ab45834fd259eec2dc6dcc8522c1ee1585a4a2c8 Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Wed, 31 Jul 2019 11:09:11 +0800 Subject: [PATCH 126/196] fix error in ConvertJSONToInt error msg (#11493) All tests passed, auto merged by Bot --- expression/integration_test.go | 9 ++++++++- types/convert.go | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index ce2f33a84bc0e..1e0da39536561 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2186,6 +2186,13 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { tk.MustExec(`insert into tb5 (a, b) select * from (select cast(a as json) as a1, b from tb5) as t where t.a1 = t.b;`) tk.MustExec(`drop table tb5;`) + tk.MustExec(`create table tb5(a float(64));`) + tk.MustExec(`insert into tb5(a) values (13835058055282163712);`) + err := tk.QueryToErr(`select convert(t.a1, signed int) from (select convert(a, json) as a1 from tb5) as t`) + msg := strings.Split(err.Error(), " ") + last := msg[len(msg)-1] + c.Assert(last, Equals, "bigint") + // Test corner cases of cast string as datetime result = tk.MustQuery(`select cast("170102034" as datetime);`) result.Check(testkit.Rows("2017-01-02 03:04:00")) @@ -2372,7 +2379,7 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result.Check(testkit.Rows("99999.99")) result = tk.MustQuery("select cast(s1 as decimal(8, 2)) from t1;") result.Check(testkit.Rows("111111.00")) - _, err := tk.Exec("insert into t1 values(cast('111111.00' as decimal(7, 2)));") + _, err = tk.Exec("insert into t1 values(cast('111111.00' as decimal(7, 2)));") c.Assert(err, NotNil) result = tk.MustQuery(`select CAST(0x8fffffffffffffff as signed) a, diff --git a/types/convert.go b/types/convert.go index 4c427c5522a4f..079a2b0e152ba 100644 --- a/types/convert.go +++ b/types/convert.go @@ -560,10 +560,10 @@ func ConvertJSONToInt(sc *stmtctx.StatementContext, j json.BinaryJSON, unsigned if !unsigned { lBound := IntergerSignedLowerBound(mysql.TypeLonglong) uBound := IntergerSignedUpperBound(mysql.TypeLonglong) - return ConvertFloatToInt(f, lBound, uBound, mysql.TypeDouble) + return ConvertFloatToInt(f, lBound, uBound, mysql.TypeLonglong) } bound := IntergerUnsignedUpperBound(mysql.TypeLonglong) - u, err := ConvertFloatToUint(sc, f, bound, mysql.TypeDouble) + u, err := ConvertFloatToUint(sc, f, bound, mysql.TypeLonglong) return int64(u), errors.Trace(err) case json.TypeCodeString: str := string(hack.String(j.GetString())) From 1913237f2620781997d4b2f1e23c67d365ab24d3 Mon Sep 17 00:00:00 2001 From: zhaoyanxing Date: Wed, 31 Jul 2019 11:29:08 +0800 Subject: [PATCH 127/196] planner: function name in `ErrWindowInvalidWindowFuncUse` should be lowercase --- planner/core/expression_rewriter.go | 2 +- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plan_test.go | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index 4a79a68e3f7d9..b19901f9031a9 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -323,7 +323,7 @@ func (er *expressionRewriter) Enter(inNode ast.Node) (ast.Node, bool) { index, ok = er.windowMap[v] } if !ok { - er.err = ErrWindowInvalidWindowFuncUse.GenWithStackByArgs(v.F) + er.err = ErrWindowInvalidWindowFuncUse.GenWithStackByArgs(strings.ToLower(v.F)) return inNode, true } er.ctxStack = append(er.ctxStack, er.schema.Columns[index]) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index bc2087d1190b3..d434c8fecec19 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -1241,7 +1241,7 @@ func (a *havingWindowAndOrderbyExprResolver) Leave(n ast.Node) (node ast.Node, o case *ast.WindowFuncExpr: a.inWindowFunc = false if a.curClause == havingClause { - a.err = ErrWindowInvalidWindowFuncUse.GenWithStackByArgs(v.F) + a.err = ErrWindowInvalidWindowFuncUse.GenWithStackByArgs(strings.ToLower(v.F)) return node, false } if a.curClause == orderByClause { diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index fac77b8e6c31a..910f4c8a26044 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2411,6 +2411,14 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { sql: "delete from t order by (sum(a) over())", result: "[planner:3593]You cannot use the window function 'sum' in this context.'", }, + { + sql: "delete from t order by (SUM(a) over())", + result: "[planner:3593]You cannot use the window function 'sum' in this context.'", + }, + { + sql: "SELECT * from t having ROW_NUMBER() over()", + result: "[planner:3593]You cannot use the window function 'row_number' in this context.'", + }, { // The best execution order should be (a,c), (a, b, c), (a, b), (), it requires only 2 sort operations. sql: "select sum(a) over (partition by a order by b), sum(b) over (order by a, b, c), sum(c) over(partition by a order by c), sum(d) over() from t", From 13778fe51b713f005e1de848e7994f0a8031678f Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Wed, 31 Jul 2019 11:50:10 +0800 Subject: [PATCH 128/196] planner: fix compatibility bug for window function (#11488) All tests passed, auto merged by Bot --- planner/core/logical_plan_builder.go | 114 ++++++++++++++++++--------- planner/core/logical_plan_test.go | 28 +++++++ 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index d434c8fecec19..2c1cdad0fdd4c 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3099,14 +3099,7 @@ func (b *PlanBuilder) buildWindowFunctionFrameBound(ctx context.Context, spec *a if bound.Type == ast.CurrentRow { return bound, nil } - // Rows type does not support interval range. - if boundClause.Unit != nil { - return nil, ErrWindowRowsIntervalUse.GenWithStackByArgs(getWindowName(spec.Name.O)) - } - numRows, isNull, isExpectedType := getUintFromNode(b.ctx, boundClause.Expr) - if isNull || !isExpectedType { - return nil, ErrWindowFrameIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) - } + numRows, _, _ := getUintFromNode(b.ctx, boundClause.Expr) bound.Num = numRows return bound, nil } @@ -3122,23 +3115,7 @@ func (b *PlanBuilder) buildWindowFunctionFrameBound(ctx context.Context, spec *a return bound, nil } - if len(orderByItems) != 1 { - return nil, ErrWindowRangeFrameOrderType.GenWithStackByArgs(getWindowName(spec.Name.O)) - } col := orderByItems[0].Col - isNumeric, isTemporal := types.IsTypeNumeric(col.RetType.Tp), types.IsTypeTemporal(col.RetType.Tp) - if !isNumeric && !isTemporal { - return nil, ErrWindowRangeFrameOrderType.GenWithStackByArgs(getWindowName(spec.Name.O)) - } - // Interval bounds only support order by temporal types. - if boundClause.Unit != nil && isNumeric { - return nil, ErrWindowRangeFrameNumericType.GenWithStackByArgs(getWindowName(spec.Name.O)) - } - // Non-interval bound only support order by numeric types. - if boundClause.Unit == nil && !isNumeric { - return nil, ErrWindowRangeFrameTemporalType.GenWithStackByArgs(getWindowName(spec.Name.O)) - } - // TODO: We also need to raise error for non-deterministic expressions, like rand(). val, err := evalAstExpr(b.ctx, boundClause.Expr) if err != nil { @@ -3223,25 +3200,13 @@ func (b *PlanBuilder) buildWindowFunctionFrame(ctx context.Context, spec *ast.Wi if frameClause == nil { return nil, nil } - if frameClause.Type == ast.Groups { - return nil, ErrNotSupportedYet.GenWithStackByArgs("GROUPS") - } frame := &WindowFrame{Type: frameClause.Type} - start := frameClause.Extent.Start - if start.Type == ast.Following && start.UnBounded { - return nil, ErrWindowFrameStartIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) - } var err error - frame.Start, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &start) + frame.Start, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &frameClause.Extent.Start) if err != nil { return nil, err } - - end := frameClause.Extent.End - if end.Type == ast.Preceding && end.UnBounded { - return nil, ErrWindowFrameEndIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) - } - frame.End, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &end) + frame.End, err = b.buildWindowFunctionFrameBound(ctx, spec, orderByItems, &frameClause.Extent.End) return frame, err } @@ -3316,6 +3281,10 @@ func (b *PlanBuilder) buildWindowFunctions(ctx context.Context, p LogicalPlan, g if err != nil { return nil, nil, err } + err = b.checkOriginWindowSpecs(funcs, orderBy) + if err != nil { + return nil, nil, err + } frame, err := b.buildWindowFunctionFrame(ctx, spec, orderBy) if err != nil { return nil, nil, err @@ -3356,6 +3325,74 @@ func (b *PlanBuilder) buildWindowFunctions(ctx context.Context, p LogicalPlan, g return p, windowMap, nil } +// checkOriginWindowSpecs checks the validation for origin window specifications for a group of functions. +// Because of the grouped specification is different from it, we should especially check them before build window frame. +func (b *PlanBuilder) checkOriginWindowSpecs(funcs []*ast.WindowFuncExpr, orderByItems []property.Item) error { + for _, f := range funcs { + spec := f.Spec + if spec.Frame == nil { + continue + } + if spec.Frame.Type == ast.Groups { + return ErrNotSupportedYet.GenWithStackByArgs("GROUPS") + } + start, end := spec.Frame.Extent.Start, spec.Frame.Extent.End + if start.Type == ast.Following && start.UnBounded { + return ErrWindowFrameStartIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + if end.Type == ast.Preceding && end.UnBounded { + return ErrWindowFrameEndIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + if start.Type == ast.Following && end.Type == ast.Preceding { + return ErrWindowFrameIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + + err := b.checkOriginWindowFrameBound(&start, &spec, orderByItems) + if err != nil { + return err + } + err = b.checkOriginWindowFrameBound(&end, &spec, orderByItems) + if err != nil { + return err + } + } + return nil +} + +func (b *PlanBuilder) checkOriginWindowFrameBound(bound *ast.FrameBound, spec *ast.WindowSpec, orderByItems []property.Item) error { + if bound.Type == ast.CurrentRow || bound.UnBounded { + return nil + } + + frameType := spec.Frame.Type + if frameType == ast.Rows { + if bound.Unit != nil { + return ErrWindowRowsIntervalUse.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + _, isNull, isExpectedType := getUintFromNode(b.ctx, bound.Expr) + if isNull || !isExpectedType { + return ErrWindowFrameIllegal.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + return nil + } + + if len(orderByItems) != 1 { + return ErrWindowRangeFrameOrderType.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + orderItemType := orderByItems[0].Col.RetType.Tp + isNumeric, isTemporal := types.IsTypeNumeric(orderItemType), types.IsTypeTemporal(orderItemType) + if !isNumeric && !isTemporal { + return ErrWindowRangeFrameOrderType.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + if bound.Unit != nil && !isTemporal { + return ErrWindowRangeFrameNumericType.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + if bound.Unit == nil && !isNumeric { + return ErrWindowRangeFrameTemporalType.GenWithStackByArgs(getWindowName(spec.Name.O)) + } + return nil +} + func extractWindowFuncs(fields []*ast.SelectField) []*ast.WindowFuncExpr { extractor := &WindowFuncExtractor{} for _, f := range fields { @@ -3422,6 +3459,7 @@ func (b *PlanBuilder) groupWindowFuncs(windowFuncs []*ast.WindowFuncExpr) (map[* if !ok { return nil, ErrWindowNoSuchWindow.GenWithStackByArgs(windowFunc.Spec.Name.O) } + windowFunc.Spec = *spec newSpec, updated := b.handleDefaultFrame(spec, windowFunc.F) if !updated { groupedWindow[spec] = append(groupedWindow[spec], windowFunc) diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index 910f4c8a26044..c47d4c5a74bd0 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2424,6 +2424,34 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { sql: "select sum(a) over (partition by a order by b), sum(b) over (order by a, b, c), sum(c) over(partition by a order by c), sum(d) over() from t", result: "TableReader(Table(t))->Sort->Window(sum(cast(test.t.c)) over(partition by test.t.a order by test.t.c asc range between unbounded preceding and current row))->Sort->Window(sum(cast(test.t.b)) over(order by test.t.a asc, test.t.b asc, test.t.c asc range between unbounded preceding and current row))->Window(sum(cast(test.t.a)) over(partition by test.t.a order by test.t.b asc range between unbounded preceding and current row))->Window(sum(cast(test.t.d)) over())->Projection", }, + // Test issue 11010. + { + sql: "select dense_rank() over w1, a, b from t window w1 as (partition by t.b order by t.a desc, t.b desc range between current row and 1 following)", + result: "[planner:3587]Window 'w1' with RANGE N PRECEDING/FOLLOWING frame requires exactly one ORDER BY expression, of numeric or temporal type", + }, + { + sql: "select dense_rank() over w1, a, b from t window w1 as (partition by t.b order by t.a desc, t.b desc range between current row and unbounded following)", + result: "TableReader(Table(t))->Sort->Window(dense_rank() over(partition by test.t.b order by test.t.a desc, test.t.b desc))->Projection", + }, + { + sql: "select dense_rank() over w1, a, b from t window w1 as (partition by t.b order by t.a desc, t.b desc range between 1 preceding and 1 following)", + result: "[planner:3587]Window 'w1' with RANGE N PRECEDING/FOLLOWING frame requires exactly one ORDER BY expression, of numeric or temporal type", + }, + // Test issue 11001. + { + sql: "SELECT PERCENT_RANK() OVER w1 AS 'percent_rank', fieldA, fieldB FROM ( SELECT a AS fieldA, b AS fieldB FROM t ) t1 WINDOW w1 AS ( ROWS BETWEEN 0 FOLLOWING AND UNBOUNDED PRECEDING)", + result: "[planner:3585]Window 'w1': frame end cannot be UNBOUNDED PRECEDING.", + }, + // Test issue 11002. + { + sql: "SELECT PERCENT_RANK() OVER w1 AS 'percent_rank', fieldA, fieldB FROM ( SELECT a AS fieldA, b AS fieldB FROM t ) as t1 WINDOW w1 AS ( ROWS BETWEEN UNBOUNDED FOLLOWING AND UNBOUNDED FOLLOWING)", + result: "[planner:3584]Window 'w1': frame start cannot be UNBOUNDED FOLLOWING.", + }, + // Test issue 11011. + { + sql: "select dense_rank() over w1, a, b from t window w1 as (partition by t.b order by t.a asc range between 1250951168 following AND 1250951168 preceding)", + result: "[planner:3586]Window 'w1': frame start or end is negative, NULL or of non-integral type", + }, } s.Parser.EnableWindowFunc(true) From a13671a02e8ca85209bc806629767e006c5f1115 Mon Sep 17 00:00:00 2001 From: wjHuang Date: Wed, 31 Jul 2019 13:52:39 +0800 Subject: [PATCH 129/196] server: fix format for select constant (#9689) --- server/column.go | 3 +++ server/conn_test.go | 4 ++-- server/tidb_test.go | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/column.go b/server/column.go index 850b02eb3097c..195061c3be875 100644 --- a/server/column.go +++ b/server/column.go @@ -66,6 +66,9 @@ func dumpFlag(tp byte, flag uint16) uint16 { case mysql.TypeEnum: return flag | uint16(mysql.EnumFlag) default: + if mysql.HasBinaryFlag(uint(flag)) { + return flag | uint16(mysql.NotNullFlag) + } return flag } } diff --git a/server/conn_test.go b/server/conn_test.go index 3faed0280a994..b7dc6019c3a5f 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -242,7 +242,7 @@ func (ts *ConnTestSuite) TestDispatch(c *C) { out: []byte{ 0xc, 0x0, 0x0, 0x3, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x18, 0x0, 0x0, 0x4, 0x3, 0x64, 0x65, 0x66, 0x0, 0x0, 0x0, 0x1, 0x31, 0x1, 0x31, 0xc, 0x3f, - 0x0, 0x1, 0x0, 0x0, 0x0, 0x8, 0x80, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x5, 0xfe, + 0x0, 0x1, 0x0, 0x0, 0x0, 0x8, 0x81, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x5, 0xfe, }, }, { @@ -251,7 +251,7 @@ func (ts *ConnTestSuite) TestDispatch(c *C) { err: nil, out: []byte{ 0x1, 0x0, 0x0, 0x6, 0x1, 0x18, 0x0, 0x0, 0x7, 0x3, 0x64, 0x65, 0x66, 0x0, 0x0, 0x0, - 0x1, 0x31, 0x1, 0x31, 0xc, 0x3f, 0x0, 0x1, 0x0, 0x0, 0x0, 0x8, 0x80, 0x0, 0x0, 0x0, + 0x1, 0x31, 0x1, 0x31, 0xc, 0x3f, 0x0, 0x1, 0x0, 0x0, 0x0, 0x8, 0x81, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x8, 0xfe, }, }, diff --git a/server/tidb_test.go b/server/tidb_test.go index fb22dcb77b9b9..84bb7c5921055 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -550,3 +550,17 @@ func (ts *TidbTestSuite) TestSumAvg(c *C) { c.Parallel() runTestSumAvg(c) } + +func (ts *TidbTestSuite) TestNullFlag(c *C) { + // issue #9689 + qctx, err := ts.tidbdrv.OpenCtx(uint64(0), 0, uint8(tmysql.DefaultCollationID), "test", nil) + c.Assert(err, IsNil) + + ctx := context.Background() + rs, err := qctx.Execute(ctx, "select 1") + c.Assert(err, IsNil) + cols := rs[0].Columns() + c.Assert(len(cols), Equals, 1) + expectFlag := uint16(tmysql.NotNullFlag | tmysql.BinaryFlag) + c.Assert(dumpFlag(cols[0].Type, cols[0].Flag), Equals, expectFlag) +} From 9b0eb1a0066e1541a28d6a02b98970aed14cecd7 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 31 Jul 2019 14:51:10 +0800 Subject: [PATCH 130/196] privilege: make 'grant all privileges' work right (#11449) --- go.mod | 2 +- go.sum | 4 ++-- privilege/privileges/cache.go | 7 ++++++- privilege/privileges/privileges_test.go | 26 +++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a6b13ca525c92..d35cff7ab4845 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2 + github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index da1a87c8df089..a9cc96d8b451c 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2 h1:uPLL91Kf8Sh86ht/ijNj6iUWw7JBArTcJtyn8aR+uDs= -github.com/pingcap/parser v0.0.0-20190726045944-46d1b3d054b2/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8 h1:Htdqrie9rVA/x2yqnXjqYu+VNvff3Cnj7clCNvIyXVU= +github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/privilege/privileges/cache.go b/privilege/privileges/cache.go index 699bfe616dacc..284db28e05762 100644 --- a/privilege/privileges/cache.go +++ b/privilege/privileges/cache.go @@ -254,7 +254,12 @@ func (p *MySQLPrivilege) LoadRoleGraph(ctx sessionctx.Context) error { // LoadUserTable loads the mysql.user table from database. func (p *MySQLPrivilege) LoadUserTable(ctx sessionctx.Context) error { - err := p.loadTable(ctx, "select HIGH_PRIORITY Host,User,Password,Select_priv,Insert_priv,Update_priv,Delete_priv,Create_priv,Drop_priv,Process_priv,Grant_priv,References_priv,Alter_priv,Show_db_priv,Super_priv,Execute_priv,Create_view_priv,Show_view_priv,Index_priv,Create_user_priv,Trigger_priv,Create_role_priv,Drop_role_priv,account_locked from mysql.user;", p.decodeUserTableRow) + userPrivCols := make([]string, 0, len(mysql.Priv2UserCol)) + for _, v := range mysql.Priv2UserCol { + userPrivCols = append(userPrivCols, v) + } + query := fmt.Sprintf("select HIGH_PRIORITY Host,User,Password,%s,account_locked from mysql.user;", strings.Join(userPrivCols, ", ")) + err := p.loadTable(ctx, query, p.decodeUserTableRow) if err != nil { return errors.Trace(err) } diff --git a/privilege/privileges/privileges_test.go b/privilege/privileges/privileges_test.go index 383b52d515a02..ac96ed9417eec 100644 --- a/privilege/privileges/privileges_test.go +++ b/privilege/privileges/privileges_test.go @@ -14,6 +14,7 @@ package privileges_test import ( + "bytes" "context" "fmt" "strings" @@ -30,6 +31,7 @@ import ( "github.com/pingcap/tidb/session" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" "github.com/pingcap/tidb/util/testutil" ) @@ -650,6 +652,30 @@ func (s *testPrivilegeSuite) TestDefaultRoles(c *C) { c.Assert(len(ret), Equals, 0) } +func (s *testPrivilegeSuite) TestUserTableConsistency(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create user superadmin") + tk.MustExec("grant all privileges on *.* to 'superadmin'") + + c.Assert(len(mysql.Priv2UserCol), Equals, len(mysql.AllGlobalPrivs)) + + var buf bytes.Buffer + var res bytes.Buffer + buf.WriteString("select ") + i := 0 + for _, priv := range mysql.Priv2UserCol { + if i != 0 { + buf.WriteString(", ") + res.WriteString(" ") + } + buf.WriteString(priv) + res.WriteString("Y") + i++ + } + buf.WriteString(" from mysql.user where user = 'superadmin'") + tk.MustQuery(buf.String()).Check(testkit.Rows(res.String())) +} + func mustExec(c *C, se session.Session, sql string) { _, err := se.Execute(context.Background(), sql) c.Assert(err, IsNil) From 730e1408cfbf75ed3c45fd98e3a8bdcf5b15d89a Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Wed, 31 Jul 2019 16:54:15 +0800 Subject: [PATCH 131/196] server: remove the error stack in logs when connection reset by client (#11542) All tests passed, auto merged by Bot --- server/conn.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/conn.go b/server/conn.go index 11516087fc266..6b3767d3156a9 100644 --- a/server/conn.go +++ b/server/conn.go @@ -636,7 +636,8 @@ func (cc *clientConn) Run(ctx context.Context) { } else { errStack := errors.ErrorStack(err) if !strings.Contains(errStack, "use of closed network connection") { - logutil.Logger(ctx).Error("read packet failed, close this connection", zap.Error(err)) + logutil.Logger(ctx).Warn("read packet failed, close this connection", + zap.Error(errors.SuspendStack(err))) } } } From 59e3eb75cd423e123e72d79b067ac17ea9c74f3e Mon Sep 17 00:00:00 2001 From: Lynn Date: Wed, 31 Jul 2019 17:11:30 +0800 Subject: [PATCH 132/196] *: speed up the operation of "admin check table" (#8572) --- executor/admin_test.go | 116 +++++++++++++++- executor/builder.go | 55 +++++++- executor/distsql.go | 189 +++++++++++++++++++------ executor/executor.go | 140 +++++++++++++------ executor/executor_test.go | 2 +- planner/core/common_plans.go | 7 +- planner/core/physical_plans.go | 5 + planner/core/planbuilder.go | 244 ++++++++++++++++++++++++--------- util/admin/admin.go | 40 ++++-- util/admin/admin_test.go | 8 +- 10 files changed, 632 insertions(+), 174 deletions(-) diff --git a/executor/admin_test.go b/executor/admin_test.go index f2317570e23b7..b5109ebae6bc2 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -399,6 +399,112 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) { tk.MustExec("admin check table admin_test") } +func (s *testSuite2) TestAdminCheckTableFailed(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists admin_test") + tk.MustExec("create table admin_test (c1 int, c2 int, c3 varchar(255) default '1', primary key(c1), key(c3), unique key(c2), key(c2, c3))") + tk.MustExec("insert admin_test (c1, c2, c3) values (-10, -20, 'y'), (-1, -10, 'z'), (1, 11, 'a'), (2, 12, 'b'), (5, 15, 'c'), (10, 20, 'd'), (20, 30, 'e')") + + // Make some corrupted index. Build the index information. + s.ctx = mock.NewContext() + s.ctx.Store = s.store + is := s.domain.InfoSchema() + dbName := model.NewCIStr("test") + tblName := model.NewCIStr("admin_test") + tbl, err := is.TableByName(dbName, tblName) + c.Assert(err, IsNil) + tblInfo := tbl.Meta() + idxInfo := tblInfo.Indices[1] + indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo) + sc := s.ctx.GetSessionVars().StmtCtx + tk.Se.GetSessionVars().IndexLookupSize = 3 + tk.Se.GetSessionVars().MaxChunkSize = 3 + + // Reduce one row of index. + // Table count > index count. + // Index c2 is missing 11. + txn, err := s.store.Begin() + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(-10), -1, nil) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + _, err = tk.Exec("admin check table admin_test") + c.Assert(err.Error(), Equals, + "[executor:8003]admin_test err:[admin:1]index: != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:-10, b:[]uint8(nil), x:interface {}(nil)}}}") + c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue) + r := tk.MustQuery("admin recover index admin_test c2") + r.Check(testkit.Rows("1 7")) + tk.MustExec("admin check table admin_test") + + // Add one row of index. + // Table count < index count. + // Index c2 has one more values ​​than table data: 0, and the handle 0 hasn't correlative record. + txn, err = s.store.Begin() + c.Assert(err, IsNil) + _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(0), 0) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + _, err = tk.Exec("admin check table admin_test") + c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:0, b:[]uint8(nil), x:interface {}(nil)} != record:") + + // Add one row of index. + // Table count < index count. + // Index c2 has two more values ​​than table data: 10, 13, and these handles have correlative record. + txn, err = s.store.Begin() + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(0), 0, nil) + c.Assert(err, IsNil) + // Make sure the index value "19" is smaller "21". Then we scan to "19" before "21". + _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(19), 10) + c.Assert(err, IsNil) + _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(13), 2) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + _, err = tk.Exec("admin check table admin_test") + c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:13, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:12, b:[]uint8(nil), x:interface {}(nil)}") + + // Table count = index count. + // Two indices have the same handle. + txn, err = s.store.Begin() + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(13), 2, nil) + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(12), 2, nil) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + _, err = tk.Exec("admin check table admin_test") + c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}") + + // Table count = index count. + // Index c2 has one line of data is 19, the corresponding table data is 20. + txn, err = s.store.Begin() + c.Assert(err, IsNil) + _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(12), 2) + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(20), 10, nil) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + _, err = tk.Exec("admin check table admin_test") + c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}") + + // Recover records. + txn, err = s.store.Begin() + c.Assert(err, IsNil) + err = indexOpr.Delete(sc, txn, types.MakeDatums(19), 10, nil) + c.Assert(err, IsNil) + _, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(20), 10) + c.Assert(err, IsNil) + err = txn.Commit(context.Background()) + c.Assert(err, IsNil) + tk.MustExec("admin check table admin_test") +} + func (s *testSuite1) TestAdminCheckTable(c *C) { // test NULL value. tk := testkit.NewTestKit(c, s.store) @@ -455,16 +561,16 @@ func (s *testSuite1) TestAdminCheckTable(c *C) { // Test index in virtual generated column. tk.MustExec(`drop table if exists test`) - tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')) , index idxc(c));`) + tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')), index idxc(c));`) tk.MustExec(`INSERT INTO test set b='{"d": 100}';`) tk.MustExec(`admin check table test;`) // Test prefix index. tk.MustExec(`drop table if exists t`) tk.MustExec(`CREATE TABLE t ( - ID CHAR(32) NOT NULL, - name CHAR(32) NOT NULL, - value CHAR(255), - INDEX indexIDname (ID(8),name(8)));`) + ID CHAR(32) NOT NULL, + name CHAR(32) NOT NULL, + value CHAR(255), + INDEX indexIDname (ID(8),name(8)));`) tk.MustExec(`INSERT INTO t VALUES ('keyword','urlprefix','text/ /text');`) tk.MustExec(`admin check table t;`) diff --git a/executor/builder.go b/executor/builder.go index 2611aea853d48..d2192c66512a7 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -308,8 +308,8 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor { b.err = err return nil } - readerExec.ranges = ranger.FullRange() - readerExec.isCheckOp = true + + buildIndexLookUpChecker(b, v.IndexLookUpReader, readerExec) e := &CheckIndexExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), @@ -322,12 +322,59 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor { return e } +// buildIndexLookUpChecker builds check information to IndexLookUpReader. +func buildIndexLookUpChecker(b *executorBuilder, readerPlan *plannercore.PhysicalIndexLookUpReader, + readerExec *IndexLookUpExecutor) { + is := readerPlan.IndexPlans[0].(*plannercore.PhysicalIndexScan) + readerExec.dagPB.OutputOffsets = make([]uint32, 0, len(is.Index.Columns)) + for i := 0; i <= len(is.Index.Columns); i++ { + readerExec.dagPB.OutputOffsets = append(readerExec.dagPB.OutputOffsets, uint32(i)) + } + readerExec.ranges = ranger.FullRange() + ts := readerPlan.TablePlans[0].(*plannercore.PhysicalTableScan) + readerExec.handleIdx = ts.HandleIdx + + tps := make([]*types.FieldType, 0, len(is.Columns)+1) + for _, col := range is.Columns { + tps = append(tps, &col.FieldType) + } + tps = append(tps, types.NewFieldType(mysql.TypeLonglong)) + readerExec.checkIndexValue = &checkIndexValue{genExprs: is.GenExprs, idxColTps: tps} + + colNames := make([]string, 0, len(is.Columns)) + for _, col := range is.Columns { + colNames = append(colNames, col.Name.O) + } + var err error + readerExec.idxTblCols, err = table.FindCols(readerExec.table.Cols(), colNames, true) + if err != nil { + b.err = errors.Trace(err) + return + } +} + func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor { + readerExecs := make([]*IndexLookUpExecutor, 0, len(v.IndexLookUpReaders)) + for _, readerPlan := range v.IndexLookUpReaders { + readerExec, err := buildNoRangeIndexLookUpReader(b, readerPlan) + if err != nil { + b.err = errors.Trace(err) + return nil + } + buildIndexLookUpChecker(b, readerPlan, readerExec) + + readerExecs = append(readerExecs, readerExec) + } + e := &CheckTableExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), - tables: v.Tables, + dbName: v.DBName, + tblInfo: v.TblInfo, + indices: v.Indices, is: b.is, - genExprs: v.GenExprs, + srcs: readerExecs, + exitCh: make(chan struct{}), + retCh: make(chan error, len(v.Indices)), } return e } diff --git a/executor/distsql.go b/executor/distsql.go index 98f2890cbd799..ffb28e46fb784 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/table" + "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" @@ -61,6 +62,7 @@ type lookupTableTask struct { handles []int64 rowIdx []int // rowIdx represents the handle index for every row. Only used when keep order. rows []chunk.Row + idxRows *chunk.Chunk cursor int doneCh chan error @@ -70,6 +72,9 @@ type lookupTableTask struct { // The handles fetched from index is originally ordered by index, but we need handles to be ordered by itself // to do table request. indexOrder map[int64]int + // duplicatedIndexOrder map likes indexOrder. But it's used when checkIndexValue isn't nil and + // the same handle of index has multiple values. + duplicatedIndexOrder map[int64]int // memUsage records the memory usage of this task calculated by table worker. // memTracker is used to release memUsage after task is done and unused. @@ -341,8 +346,8 @@ type IndexLookUpExecutor struct { // memTracker is used to track the memory usage of this executor. memTracker *memory.Tracker - // isCheckOp is used to determine whether we need to check the consistency of the index data. - isCheckOp bool + // checkIndexValue is used to check the consistency of the index data. + *checkIndexValue corColInIdxSide bool idxPlans []plannercore.PhysicalPlan @@ -353,6 +358,12 @@ type IndexLookUpExecutor struct { colLens []int } +type checkIndexValue struct { + idxColTps []*types.FieldType + idxTblCols []*table.Column + genExprs map[model.TableColumnID]expression.Expression +} + // Open implements the Executor Open interface. func (e *IndexLookUpExecutor) Open(ctx context.Context) error { var err error @@ -437,21 +448,26 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, kvRanges []k if err != nil { return err } + tps := []*types.FieldType{types.NewFieldType(mysql.TypeLonglong)} + if e.checkIndexValue != nil { + tps = e.idxColTps + } // Since the first read only need handle information. So its returned col is only 1. - result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, []*types.FieldType{types.NewFieldType(mysql.TypeLonglong)}, e.feedback, getPhysicalPlanIDs(e.idxPlans)) + result, err := distsql.SelectWithRuntimeStats(ctx, e.ctx, kvReq, tps, e.feedback, getPhysicalPlanIDs(e.idxPlans)) if err != nil { return err } result.Fetch(ctx) worker := &indexWorker{ - idxLookup: e, - workCh: workCh, - finished: e.finished, - resultCh: e.resultCh, - keepOrder: e.keepOrder, - batchSize: initBatchSize, - maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize, - maxChunkSize: e.maxChunkSize, + idxLookup: e, + workCh: workCh, + finished: e.finished, + resultCh: e.resultCh, + keepOrder: e.keepOrder, + batchSize: initBatchSize, + checkIndexValue: e.checkIndexValue, + maxBatchSize: e.ctx.GetSessionVars().IndexLookupSize, + maxChunkSize: e.maxChunkSize, } if worker.batchSize > worker.maxBatchSize { worker.batchSize = worker.maxBatchSize @@ -487,13 +503,13 @@ func (e *IndexLookUpExecutor) startTableWorker(ctx context.Context, workCh <-cha e.tblWorkerWg.Add(lookupConcurrencyLimit) for i := 0; i < lookupConcurrencyLimit; i++ { worker := &tableWorker{ - idxLookup: e, - workCh: workCh, - finished: e.finished, - buildTblReader: e.buildTableReader, - keepOrder: e.keepOrder, - handleIdx: e.handleIdx, - isCheckOp: e.isCheckOp, + idxLookup: e, + workCh: workCh, + finished: e.finished, + buildTblReader: e.buildTableReader, + keepOrder: e.keepOrder, + handleIdx: e.handleIdx, + checkIndexValue: e.checkIndexValue, memTracker: memory.NewTracker(stringutil.MemoizeStr(func() string { return "TableWorker_" + strconv.Itoa(i) }), e.ctx.GetSessionVars().MemQuotaIndexLookupReader), } @@ -610,6 +626,9 @@ type indexWorker struct { batchSize int maxBatchSize int maxChunkSize int + + // checkIndexValue is used to check the consistency of the index data. + *checkIndexValue } // fetchHandles fetches a batch of handles from index data and builds the index lookup tasks. @@ -633,9 +652,14 @@ func (w *indexWorker) fetchHandles(ctx context.Context, result distsql.SelectRes } } }() - chk := chunk.NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong)}, w.idxLookup.maxChunkSize) + var chk *chunk.Chunk + if w.checkIndexValue != nil { + chk = chunk.NewChunkWithCapacity(w.idxColTps, w.maxChunkSize) + } else { + chk = chunk.NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeLonglong)}, w.idxLookup.maxChunkSize) + } for { - handles, err := w.extractTaskHandles(ctx, chk, result) + handles, retChunk, err := w.extractTaskHandles(ctx, chk, result) if err != nil { doneCh := make(chan error, 1) doneCh <- err @@ -648,7 +672,7 @@ func (w *indexWorker) fetchHandles(ctx context.Context, result distsql.SelectRes return count, nil } count += int64(len(handles)) - task := w.buildTableTask(handles) + task := w.buildTableTask(handles, retChunk) select { case <-ctx.Done(): return count, nil @@ -660,30 +684,40 @@ func (w *indexWorker) fetchHandles(ctx context.Context, result distsql.SelectRes } } -func (w *indexWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, idxResult distsql.SelectResult) (handles []int64, err error) { +func (w *indexWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, idxResult distsql.SelectResult) ( + handles []int64, retChk *chunk.Chunk, err error) { + handleOffset := chk.NumCols() - 1 handles = make([]int64, 0, w.batchSize) for len(handles) < w.batchSize { chk.SetRequiredRows(w.batchSize-len(handles), w.maxChunkSize) - err = idxResult.Next(ctx, chk) + err = errors.Trace(idxResult.Next(ctx, chk)) if err != nil { - return handles, err + return handles, nil, err } if chk.NumRows() == 0 { - return handles, nil + return handles, retChk, nil } for i := 0; i < chk.NumRows(); i++ { - handles = append(handles, chk.GetRow(i).GetInt64(0)) + h := chk.GetRow(i).GetInt64(handleOffset) + handles = append(handles, h) + } + if w.checkIndexValue != nil { + if retChk == nil { + retChk = chunk.NewChunkWithCapacity(w.idxColTps, w.batchSize) + } + retChk.Append(chk, 0, chk.NumRows()) } } w.batchSize *= 2 if w.batchSize > w.maxBatchSize { w.batchSize = w.maxBatchSize } - return handles, nil + return handles, retChk, nil } -func (w *indexWorker) buildTableTask(handles []int64) *lookupTableTask { +func (w *indexWorker) buildTableTask(handles []int64, retChk *chunk.Chunk) *lookupTableTask { var indexOrder map[int64]int + var duplicatedIndexOrder map[int64]int if w.keepOrder { // Save the index order. indexOrder = make(map[int64]int, len(handles)) @@ -691,10 +725,27 @@ func (w *indexWorker) buildTableTask(handles []int64) *lookupTableTask { indexOrder[h] = i } } + + if w.checkIndexValue != nil { + // Save the index order. + indexOrder = make(map[int64]int, len(handles)) + duplicatedIndexOrder = make(map[int64]int) + for i, h := range handles { + if _, ok := indexOrder[h]; ok { + duplicatedIndexOrder[h] = i + } else { + indexOrder[h] = i + } + } + } + task := &lookupTableTask{ - handles: handles, - indexOrder: indexOrder, + handles: handles, + indexOrder: indexOrder, + duplicatedIndexOrder: duplicatedIndexOrder, + idxRows: retChk, } + task.doneCh = make(chan error, 1) return task } @@ -711,8 +762,8 @@ type tableWorker struct { // memTracker is used to track the memory usage of this executor. memTracker *memory.Tracker - // isCheckOp is used to determine whether we need to check the consistency of the index data. - isCheckOp bool + // checkIndexValue is used to check the consistency of the index data. + *checkIndexValue } // pickAndExecTask picks tasks from workCh, and execute them. @@ -745,6 +796,66 @@ func (w *tableWorker) pickAndExecTask(ctx context.Context) { } } +func (w *tableWorker) compareData(ctx context.Context, task *lookupTableTask, tableReader Executor) error { + chk := newFirstChunk(tableReader) + tblInfo := w.idxLookup.table.Meta() + vals := make([]types.Datum, 0, len(w.idxTblCols)) + for { + err := tableReader.Next(ctx, chk) + if err != nil { + return errors.Trace(err) + } + if chk.NumRows() == 0 { + for h := range task.indexOrder { + idxRow := task.idxRows.GetRow(task.indexOrder[h]) + return errors.Errorf("handle %#v, index:%#v != record:%#v", h, idxRow.GetDatum(0, w.idxColTps[0]), nil) + } + break + } + + tblReaderExec := tableReader.(*TableReaderExecutor) + iter := chunk.NewIterator4Chunk(chk) + for row := iter.Begin(); row != iter.End(); row = iter.Next() { + handle := row.GetInt64(w.handleIdx) + offset, ok := task.indexOrder[handle] + if !ok { + offset = task.duplicatedIndexOrder[handle] + } + delete(task.indexOrder, handle) + idxRow := task.idxRows.GetRow(offset) + vals = vals[:0] + for i, col := range w.idxTblCols { + if col.IsGenerated() && !col.GeneratedStored { + expr := w.genExprs[model.TableColumnID{TableID: tblInfo.ID, ColumnID: col.ID}] + // Eval the column value + val, err := expr.Eval(row) + if err != nil { + return errors.Trace(err) + } + val, err = table.CastValue(tblReaderExec.ctx, val, col.ColumnInfo) + if err != nil { + return errors.Trace(err) + } + vals = append(vals, val) + } else { + vals = append(vals, row.GetDatum(i, &col.FieldType)) + } + } + vals = tables.TruncateIndexValuesIfNeeded(tblInfo, w.idxLookup.index, vals) + for i, val := range vals { + col := w.idxTblCols[i] + tp := &col.FieldType + ret := chunk.Compare(idxRow, i, &val) + if ret != 0 { + return errors.Errorf("col %s, handle %#v, index:%#v != record:%#v", col.Name, handle, idxRow.GetDatum(i, tp), val) + } + } + } + } + + return nil +} + // executeTask executes the table look up tasks. We will construct a table reader and send request by handles. // Then we hold the returning rows and finish this task. func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) error { @@ -755,6 +866,10 @@ func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) er } defer terror.Call(tableReader.Close) + if w.checkIndexValue != nil { + return w.compareData(ctx, task, tableReader) + } + task.memTracker = w.memTracker memUsage := int64(cap(task.handles) * 8) task.memUsage = memUsage @@ -796,16 +911,6 @@ func (w *tableWorker) executeTask(ctx context.Context, task *lookupTableTask) er } if handleCnt != len(task.rows) { - if w.isCheckOp { - obtainedHandlesMap := make(map[int64]struct{}, len(task.rows)) - for _, row := range task.rows { - handle := row.GetInt64(w.handleIdx) - obtainedHandlesMap[handle] = struct{}{} - } - return errors.Errorf("inconsistent index %s handle count %d isn't equal to value count %d, missing handles %v in a batch", - w.idxLookup.index.Name.O, handleCnt, len(task.rows), GetLackHandles(task.handles, obtainedHandlesMap)) - } - if len(w.idxLookup.tblPlans) == 1 { obtainedHandlesMap := make(map[int64]struct{}, len(task.rows)) for _, row := range task.rows { diff --git a/executor/executor.go b/executor/executor.go index 7dbc442e49e60..5c668dec7f18c 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -41,6 +41,7 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util" "github.com/pingcap/tidb/util/admin" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/execdetails" @@ -455,11 +456,14 @@ func getTableName(is infoschema.InfoSchema, id int64) string { type CheckTableExec struct { baseExecutor - tables []*ast.TableName - done bool - is infoschema.InfoSchema - - genExprs map[model.TableColumnID]expression.Expression + dbName string + tblInfo *model.TableInfo + indices []table.Index + srcs []*IndexLookUpExecutor + done bool + is infoschema.InfoSchema + exitCh chan struct{} + retCh chan error } // Open implements the Executor Open interface. @@ -467,63 +471,121 @@ func (e *CheckTableExec) Open(ctx context.Context) error { if err := e.baseExecutor.Open(ctx); err != nil { return err } + for _, src := range e.srcs { + if err := src.Open(ctx); err != nil { + return errors.Trace(err) + } + } e.done = false return nil } +func (e *CheckTableExec) checkIndexHandle(ctx context.Context, num int, src *IndexLookUpExecutor) error { + cols := src.schema.Columns + retFieldTypes := make([]*types.FieldType, len(cols)) + for i := range cols { + retFieldTypes[i] = cols[i].RetType + } + chk := chunk.New(retFieldTypes, e.initCap, e.maxChunkSize) + + var err error + for { + err = src.Next(ctx, chk) + if err != nil { + break + } + if chk.NumRows() == 0 { + break + } + + select { + case <-e.exitCh: + return nil + default: + } + } + e.retCh <- errors.Trace(err) + return errors.Trace(err) +} + +func (e *CheckTableExec) handlePanic(r interface{}) { + if r != nil { + e.retCh <- errors.Errorf("%v", r) + } +} + // Next implements the Executor Next interface. func (e *CheckTableExec) Next(ctx context.Context, req *chunk.Chunk) error { - if e.done { + if e.done || len(e.srcs) == 0 { return nil } defer func() { e.done = true }() - for _, t := range e.tables { - dbName := t.DBInfo.Name - tb, err := e.is.TableByName(dbName, t.Name) - if err != nil { - return err + + idxNames := make([]string, 0, len(e.indices)) + for _, idx := range e.indices { + idxNames = append(idxNames, idx.Meta().Name.O) + } + greater, idxOffset, err := admin.CheckIndicesCount(e.ctx, e.dbName, e.tblInfo.Name.O, idxNames) + if err != nil { + tbl := e.srcs[idxOffset].table + if greater == admin.IdxCntGreater { + err = e.checkIndexHandle(ctx, idxOffset, e.srcs[idxOffset]) + } else if greater == admin.TblCntGreater { + err = e.checkTableRecord(tbl, idxOffset) } - if tb.Meta().GetPartitionInfo() != nil { - err = e.doCheckPartitionedTable(tb.(table.PartitionedTable)) - } else { - err = e.doCheckTable(tb) + if err != nil && admin.ErrDataInConsistent.Equal(err) { + return ErrAdminCheckTable.GenWithStack("%v err:%v", tbl.Meta().Name, err) } - if err != nil { - logutil.Logger(ctx).Warn("check table failed", zap.String("tableName", t.Name.O), zap.Error(err)) - if admin.ErrDataInConsistent.Equal(err) { - return ErrAdminCheckTable.GenWithStack("%v err:%v", t.Name, err) - } + return errors.Trace(err) + } + + // The number of table rows is equal to the number of index rows. + // TODO: Make the value of concurrency adjustable. And we can consider the number of records. + concurrency := 3 + wg := sync.WaitGroup{} + for i := range e.srcs { + wg.Add(1) + go func(num int) { + defer wg.Done() + util.WithRecovery(func() { + err1 := e.checkIndexHandle(ctx, num, e.srcs[num]) + if err1 != nil { + logutil.Logger(ctx).Info("check index handle failed", zap.Error(err)) + } + }, e.handlePanic) + }(i) - return errors.Errorf("%v err:%v", t.Name, err) + if (i+1)%concurrency == 0 { + wg.Wait() } } - return nil -} -func (e *CheckTableExec) doCheckPartitionedTable(tbl table.PartitionedTable) error { - info := tbl.Meta().GetPartitionInfo() - for _, def := range info.Definitions { - pid := def.ID - partition := tbl.GetPartition(pid) - if err := e.doCheckTable(partition); err != nil { - return err + for i := 0; i < len(e.srcs); i++ { + err = <-e.retCh + if err != nil { + return errors.Trace(err) } } return nil } -func (e *CheckTableExec) doCheckTable(tbl table.Table) error { +func (e *CheckTableExec) checkTableRecord(tbl table.Table, idxOffset int) error { + idx := e.indices[idxOffset] + genExprs := e.srcs[idxOffset].genExprs txn, err := e.ctx.Txn(true) if err != nil { return err } - for _, idx := range tbl.Indices() { - if idx.Meta().State != model.StatePublic { - continue - } - err := admin.CompareIndexData(e.ctx, txn, tbl, idx, e.genExprs) - if err != nil { - return err + if tbl.Meta().GetPartitionInfo() == nil { + return admin.CheckRecordAndIndex(e.ctx, txn, tbl, idx, genExprs) + } + + info := tbl.Meta().GetPartitionInfo() + for _, def := range info.Definitions { + pid := def.ID + partition := tbl.(table.PartitionedTable).GetPartition(pid) + if err := admin.CheckRecordAndIndex(e.ctx, txn, partition, idx, genExprs); err != nil { + return errors.Trace(err) } } return nil @@ -567,7 +629,7 @@ func (e *CheckIndexExec) Next(ctx context.Context, req *chunk.Chunk) error { } defer func() { e.done = true }() - err := admin.CheckIndicesCount(e.ctx, e.dbName, e.tableName, []string{e.idxName}) + _, _, err := admin.CheckIndicesCount(e.ctx, e.dbName, e.tableName, []string{e.idxName}) if err != nil { return err } diff --git a/executor/executor_test.go b/executor/executor_test.go index bda4df0a0c1ba..96544c0451cfb 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3170,7 +3170,7 @@ func (s *testSuite) TestCheckIndex(c *C) { c.Assert(err, IsNil) _, err = se.Execute(context.Background(), "admin check index t c") c.Assert(err, NotNil) - c.Assert(strings.Contains(err.Error(), "isn't equal to value count"), IsTrue) + c.Assert(err.Error(), Equals, "handle 3, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:30, b:[]uint8(nil), x:interface {}(nil)} != record:") // set data to: // index data (handle, data): (1, 10), (2, 20), (3, 30), (4, 40) diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 71e720f3c3b18..725519de0322c 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -75,9 +75,10 @@ type ShowNextRowID struct { type CheckTable struct { baseSchemaProducer - Tables []*ast.TableName - - GenExprs map[model.TableColumnID]expression.Expression + DBName string + TblInfo *model.TableInfo + Indices []table.Index + IndexLookUpReaders []*PhysicalIndexLookUpReader } // RecoverIndex is used for backfilling corrupted index data. diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 67048b70f7e7c..a43f3fa03f340 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -119,6 +119,8 @@ type PhysicalIndexScan struct { // The index scan may be on a partition. isPartition bool physicalTableID int64 + + GenExprs map[model.TableColumnID]expression.Expression } // PhysicalMemTable reads memory table. @@ -160,6 +162,9 @@ type PhysicalTableScan struct { physicalTableID int64 rangeDecidedBy []*expression.Column + + // HandleIdx is the index of handle, which is only used for admin check table. + HandleIdx int } // IsPartition returns true and partition ID if it's actually a partition. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 269490558dbe7..20d8c4f1f6fbb 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -40,7 +40,9 @@ import ( "github.com/pingcap/tidb/types" driver "github.com/pingcap/tidb/types/parser_driver" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/ranger" + "go.uber.org/zap" ) type visitInfo struct { @@ -547,7 +549,7 @@ func (b *PlanBuilder) buildPrepare(x *ast.PrepareStmt) Plan { return p } -func (b *PlanBuilder) buildCheckIndex(dbName model.CIStr, as *ast.AdminStmt) (Plan, error) { +func (b *PlanBuilder) buildCheckIndex(ctx context.Context, dbName model.CIStr, as *ast.AdminStmt) (Plan, error) { tblName := as.Tables[0] tbl, err := b.is.TableByName(dbName, tblName.Name) if err != nil { @@ -570,42 +572,7 @@ func (b *PlanBuilder) buildCheckIndex(dbName model.CIStr, as *ast.AdminStmt) (Pl return nil, errors.Errorf("index %s state %s isn't public", as.Index, idx.State) } - id := 1 - columns := make([]*model.ColumnInfo, 0, len(idx.Columns)) - schema := expression.NewSchema(make([]*expression.Column, 0, len(idx.Columns))...) - for _, idxCol := range idx.Columns { - for _, col := range tblInfo.Columns { - if idxCol.Name.L == col.Name.L { - columns = append(columns, col) - schema.Append(&expression.Column{ - ColName: col.Name, - UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), - RetType: &col.FieldType, - }) - } - } - } - is := PhysicalIndexScan{ - Table: tblInfo, - TableAsName: &tblName.Name, - DBName: dbName, - Columns: columns, - Index: idx, - dataSourceSchema: schema, - Ranges: ranger.FullRange(), - KeepOrder: false, - }.Init(b.ctx) - is.stats = &property.StatsInfo{} - cop := &copTask{indexPlan: is} - // It's double read case. - ts := PhysicalTableScan{Columns: columns, Table: is.Table}.Init(b.ctx) - ts.SetSchema(is.dataSourceSchema) - cop.tablePlan = ts - is.initSchema(id, idx, true) - t := finishCopTask(b.ctx, cop) - - rootT := t.(*rootTask) - return rootT.p, nil + return b.buildPhysicalIndexLookUpReader(ctx, dbName, tbl, idx, 1) } func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (Plan, error) { @@ -619,7 +586,7 @@ func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (Plan, } case ast.AdminCheckIndex: dbName := as.Tables[0].Schema - readerPlan, err := b.buildCheckIndex(dbName, as) + readerPlan, err := b.buildCheckIndex(ctx, dbName, as) if err != nil { return ret, err } @@ -691,43 +658,188 @@ func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (Plan, return ret, nil } -func (b *PlanBuilder) buildAdminCheckTable(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { - p := &CheckTable{Tables: as.Tables} - p.GenExprs = make(map[model.TableColumnID]expression.Expression, len(p.Tables)) - +// getGenExprs gets generated expressions map. +func (b *PlanBuilder) getGenExprs(ctx context.Context, dbName model.CIStr, tbl table.Table, idx *model.IndexInfo) ( + map[model.TableColumnID]expression.Expression, error) { + tblInfo := tbl.Meta() + genExprsMap := make(map[model.TableColumnID]expression.Expression) + exprs := make([]expression.Expression, 0, len(tbl.Cols())) + genExprIdxs := make([]model.TableColumnID, len(tbl.Cols())) mockTablePlan := LogicalTableDual{}.Init(b.ctx) - for _, tbl := range p.Tables { - tableInfo := tbl.TableInfo - schema := expression.TableInfo2SchemaWithDBName(b.ctx, tbl.Schema, tableInfo) - table, ok := b.is.TableByID(tableInfo.ID) - if !ok { - return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tbl.DBInfo.Name.O, tableInfo.Name.O) + mockTablePlan.SetSchema(expression.TableInfo2SchemaWithDBName(b.ctx, dbName, tblInfo)) + for i, colExpr := range mockTablePlan.Schema().Columns { + col := tbl.Cols()[i] + var expr expression.Expression + expr = colExpr + if col.IsGenerated() && !col.GeneratedStored { + var err error + expr, _, err = b.rewrite(ctx, col.GeneratedExpr, mockTablePlan, nil, true) + if err != nil { + return nil, errors.Trace(err) + } + expr = expression.BuildCastFunction(b.ctx, expr, colExpr.GetType()) + found := false + for _, column := range idx.Columns { + if strings.EqualFold(col.Name.L, column.Name.L) { + found = true + break + } + } + if found { + genColumnID := model.TableColumnID{TableID: tblInfo.ID, ColumnID: col.ColumnInfo.ID} + genExprsMap[genColumnID] = expr + genExprIdxs[i] = genColumnID + } + } + exprs = append(exprs, expr) + } + // Re-iterate expressions to handle those virtual generated columns that refers to the other generated columns. + for i, expr := range exprs { + exprs[i] = expression.ColumnSubstitute(expr, mockTablePlan.Schema(), exprs) + if _, ok := genExprsMap[genExprIdxs[i]]; ok { + genExprsMap[genExprIdxs[i]] = exprs[i] } + } + return genExprsMap, nil +} - mockTablePlan.SetSchema(schema) +func (b *PlanBuilder) buildPhysicalIndexLookUpReader(ctx context.Context, dbName model.CIStr, tbl table.Table, idx *model.IndexInfo, id int) (Plan, error) { + genExprsMap, err := b.getGenExprs(ctx, dbName, tbl, idx) + if err != nil { + return nil, errors.Trace(err) + } - // Calculate generated columns. - columns := table.Cols() - for _, column := range columns { - if !column.IsGenerated() { - continue + // Get generated columns. + var genCols []*expression.Column + pkOffset := -1 + tblInfo := tbl.Meta() + colsMap := make(map[int64]struct{}) + schema := expression.NewSchema(make([]*expression.Column, 0, len(idx.Columns))...) + idxReaderCols := make([]*model.ColumnInfo, 0, len(idx.Columns)) + tblReaderCols := make([]*model.ColumnInfo, 0, len(tbl.Cols())) + for _, idxCol := range idx.Columns { + for _, col := range tblInfo.Columns { + if idxCol.Name.L == col.Name.L { + idxReaderCols = append(idxReaderCols, col) + tblReaderCols = append(tblReaderCols, col) + schema.Append(&expression.Column{ + ColName: col.Name, + UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), + RetType: &col.FieldType}) + colsMap[col.ID] = struct{}{} + if mysql.HasPriKeyFlag(col.Flag) { + pkOffset = len(tblReaderCols) - 1 + } } - columnName := &ast.ColumnName{Name: column.Name} - columnName.SetText(column.Name.O) - - colExpr, _, err := mockTablePlan.findColumn(columnName) - if err != nil { - return nil, err + genColumnID := model.TableColumnID{TableID: tblInfo.ID, ColumnID: col.ID} + if expr, ok := genExprsMap[genColumnID]; ok { + cols := expression.ExtractColumns(expr) + genCols = append(genCols, cols...) } - - expr, _, err := b.rewrite(ctx, column.GeneratedExpr, mockTablePlan, nil, true) - if err != nil { - return nil, err + } + } + // Add generated columns to tblSchema and tblReaderCols. + tblSchema := schema.Clone() + for _, col := range genCols { + if _, ok := colsMap[col.ID]; !ok { + c := table.FindCol(tbl.Cols(), col.ColName.O) + if c != nil { + col.Index = len(tblReaderCols) + tblReaderCols = append(tblReaderCols, c.ColumnInfo) + tblSchema.Append(&expression.Column{ + ColName: c.Name, + UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), + RetType: &c.FieldType}) + colsMap[c.ID] = struct{}{} + if mysql.HasPriKeyFlag(c.Flag) { + pkOffset = len(tblReaderCols) - 1 + } } - expr = expression.BuildCastFunction(b.ctx, expr, colExpr.GetType()) - p.GenExprs[model.TableColumnID{TableID: tableInfo.ID, ColumnID: column.ColumnInfo.ID}] = expr } } + if !tbl.Meta().PKIsHandle || pkOffset == -1 { + tblReaderCols = append(tblReaderCols, model.NewExtraHandleColInfo()) + handleCol := &expression.Column{ + DBName: dbName, + TblName: tblInfo.Name, + ColName: model.ExtraHandleName, + RetType: types.NewFieldType(mysql.TypeLonglong), + UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), + ID: model.ExtraHandleID, + } + tblSchema.Append(handleCol) + pkOffset = len(tblReaderCols) - 1 + } + + is := PhysicalIndexScan{ + Table: tblInfo, + TableAsName: &tblInfo.Name, + DBName: dbName, + Columns: idxReaderCols, + Index: idx, + dataSourceSchema: schema, + Ranges: ranger.FullRange(), + GenExprs: genExprsMap, + }.Init(b.ctx) + is.stats = property.NewSimpleStats(0) + // It's double read case. + ts := PhysicalTableScan{Columns: tblReaderCols, Table: is.Table}.Init(b.ctx) + ts.SetSchema(tblSchema) + cop := &copTask{indexPlan: is, tablePlan: ts} + ts.HandleIdx = pkOffset + is.initSchema(id, idx, true) + rootT := finishCopTask(b.ctx, cop).(*rootTask) + return rootT.p, nil +} + +func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbName model.CIStr, tbl table.Table) ([]Plan, []table.Index, error) { + tblInfo := tbl.Meta() + // get index information + indices := make([]table.Index, 0, len(tblInfo.Indices)) + indexLookUpReaders := make([]Plan, 0, len(tblInfo.Indices)) + for i, idx := range tbl.Indices() { + idxInfo := idx.Meta() + if idxInfo.State != model.StatePublic { + logutil.Logger(context.Background()).Info("build physical index lookup reader, the index isn't public", + zap.String("index", idxInfo.Name.O), zap.Stringer("state", idxInfo.State), zap.String("table", tblInfo.Name.O)) + continue + } + indices = append(indices, idx) + reader, err := b.buildPhysicalIndexLookUpReader(ctx, dbName, tbl, idxInfo, i) + if err != nil { + return nil, nil, err + } + indexLookUpReaders = append(indexLookUpReaders, reader) + } + if len(indexLookUpReaders) == 0 { + return nil, nil, nil + } + return indexLookUpReaders, indices, nil +} + +func (b *PlanBuilder) buildAdminCheckTable(ctx context.Context, as *ast.AdminStmt) (*CheckTable, error) { + tbl := as.Tables[0] + p := &CheckTable{ + DBName: tbl.Schema.O, + TblInfo: tbl.TableInfo, + } + + tableInfo := as.Tables[0].TableInfo + table, ok := b.is.TableByID(tableInfo.ID) + if !ok { + return nil, infoschema.ErrTableNotExists.GenWithStackByArgs(tbl.DBInfo.Name.O, tableInfo.Name.O) + } + + readerPlans, indices, err := b.buildPhysicalIndexLookUpReaders(ctx, tbl.Schema, table) + if err != nil { + return nil, errors.Trace(err) + } + readers := make([]*PhysicalIndexLookUpReader, 0, len(readerPlans)) + for _, plan := range readerPlans { + readers = append(readers, plan.(*PhysicalIndexLookUpReader)) + } + p.Indices = indices + p.IndexLookUpReaders = readers return p, nil } diff --git a/util/admin/admin.go b/util/admin/admin.go index ad6f0774465ce..fce94e497bc93 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -14,8 +14,10 @@ package admin import ( + "context" "fmt" "io" + "math" "sort" "time" @@ -266,28 +268,46 @@ func getCount(ctx sessionctx.Context, sql string) (int64, error) { return rows[0].GetInt64(0), nil } +// Count greater Types +const ( + // TblCntGreater means that the number of table rows is more than the number of index rows. + TblCntGreater byte = 1 + // IdxCntGreater means that the number of index rows is more than the number of table rows. + IdxCntGreater byte = 2 +) + // CheckIndicesCount compares indices count with table count. +// It returns the count greater type, the index offset and an error. // It returns nil if the count from the index is equal to the count from the table columns, -// otherwise it returns an error with a different information. -func CheckIndicesCount(ctx sessionctx.Context, dbName, tableName string, indices []string) error { +// otherwise it returns an error and the corresponding index's offset. +func CheckIndicesCount(ctx sessionctx.Context, dbName, tableName string, indices []string) (byte, int, error) { // Add `` for some names like `table name`. sql := fmt.Sprintf("SELECT COUNT(*) FROM `%s`.`%s`", dbName, tableName) tblCnt, err := getCount(ctx, sql) if err != nil { - return errors.Trace(err) + return 0, 0, errors.Trace(err) } - for _, idx := range indices { + for i, idx := range indices { sql = fmt.Sprintf("SELECT COUNT(*) FROM `%s`.`%s` USE INDEX(`%s`)", dbName, tableName, idx) idxCnt, err := getCount(ctx, sql) if err != nil { - return errors.Trace(err) + return 0, i, errors.Trace(err) } - if tblCnt != idxCnt { - return errors.Errorf("table count %d != index(%s) count %d", tblCnt, idx, idxCnt) + logutil.Logger(context.Background()).Info("check indices count, table %s cnt %d, index %s cnt %d", + zap.String("table", tableName), zap.Int64("cnt", tblCnt), zap.Reflect("index", idx), zap.Int64("cnt", idxCnt)) + if tblCnt == idxCnt { + continue } - } - return nil + var ret byte + if tblCnt > idxCnt { + ret = TblCntGreater + } else if idxCnt > tblCnt { + ret = IdxCntGreater + } + return ret, i, errors.Errorf("table count %d != index(%s) count %d", tblCnt, idx, idxCnt) + } + return 0, 0, nil } // ScanIndexData scans the index handles and values in a limited number, according to the index information. @@ -443,7 +463,7 @@ func CheckRecordAndIndex(sessCtx sessionctx.Context, txn kv.Transaction, t table cols[i] = t.Cols()[col.Offset] } - startKey := t.RecordKey(0) + startKey := t.RecordKey(math.MinInt64) filterFunc := func(h1 int64, vals1 []types.Datum, cols []*table.Column) (bool, error) { for i, val := range vals1 { col := cols[i] diff --git a/util/admin/admin_test.go b/util/admin/admin_test.go index ae756b1f68be0..aea080ebbdda5 100644 --- a/util/admin/admin_test.go +++ b/util/admin/admin_test.go @@ -458,7 +458,7 @@ func (s *testSuite) testIndex(c *C, ctx sessionctx.Context, dbName string, tb ta c.Assert(err, IsNil) idxNames := []string{idx.Meta().Name.L} - err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) + _, _, err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) c.Assert(err, IsNil) mockCtx := mock.NewContext() @@ -480,7 +480,7 @@ func (s *testSuite) testIndex(c *C, ctx sessionctx.Context, dbName string, tb ta diffMsg := newDiffRetError("index", record1, nil) c.Assert(err.Error(), DeepEquals, diffMsg) - err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) + _, _, err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) c.Assert(err, IsNil) // set data to: @@ -539,7 +539,7 @@ func (s *testSuite) testIndex(c *C, ctx sessionctx.Context, dbName string, tb ta diffMsg = newDiffRetError("index", record1, nil) c.Assert(err.Error(), DeepEquals, diffMsg) - err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) + _, _, err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) c.Assert(err.Error(), Equals, "table count 3 != index(c) count 4") // set data to: @@ -559,7 +559,7 @@ func (s *testSuite) testIndex(c *C, ctx sessionctx.Context, dbName string, tb ta diffMsg = newDiffRetError("index", nil, record1) c.Assert(err.Error(), DeepEquals, diffMsg) - err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) + _, _, err = CheckIndicesCount(ctx, dbName, tb.Meta().Name.L, idxNames) c.Assert(err.Error(), Equals, "table count 4 != index(c) count 3") } From 9f68166c939e1ff6c685b697d7f9ba7660a0edb0 Mon Sep 17 00:00:00 2001 From: lysu Date: Wed, 31 Jul 2019 17:18:40 +0800 Subject: [PATCH 133/196] tikv: make region_request_test stable (#11541) --- store/tikv/region_request_test.go | 39 +++++++++++++------------------ 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/store/tikv/region_request_test.go b/store/tikv/region_request_test.go index 444e5410a1c8f..2bb824126fbbb 100644 --- a/store/tikv/region_request_test.go +++ b/store/tikv/region_request_test.go @@ -94,40 +94,33 @@ func (s *testRegionRequestSuite) TestOnSendFailedWithCloseKnownStoreThenUseNewOn Key: []byte("key"), Value: []byte("value"), }) - region, err := s.cache.LocateRegionByID(s.bo, s.region) - c.Assert(err, IsNil) - c.Assert(region, NotNil) - resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) - c.Assert(err, IsNil) - c.Assert(resp.Resp, NotNil) - // add new unknown region + // add new store2 and make store2 as leader. store2 := s.cluster.AllocID() peer2 := s.cluster.AllocID() s.cluster.AddStore(store2, fmt.Sprintf("store%d", store2)) - s.cluster.AddPeer(region.Region.id, store2, peer2) - - // stop known region - s.cluster.StopStore(s.store) + s.cluster.AddPeer(s.region, store2, peer2) + s.cluster.ChangeLeader(s.region, peer2) - // send to failed store - resp, err = s.regionRequestSender.SendReq(NewBackoffer(context.Background(), 100), req, region.Region, time.Second) - c.Assert(err, IsNil) - regionErr, err := resp.GetRegionError() + region, err := s.cache.LocateRegionByID(s.bo, s.region) c.Assert(err, IsNil) - c.Assert(regionErr, NotNil) - - // retry to send store by old region info - region, err = s.cache.LocateRegionByID(s.bo, s.region) c.Assert(region, NotNil) + resp, err := s.regionRequestSender.SendReq(s.bo, req, region.Region, time.Second) c.Assert(err, IsNil) + c.Assert(resp.Resp, NotNil) + + // stop store2 and make store1 as new leader. + s.cluster.StopStore(store2) + s.cluster.ChangeLeader(s.region, s.peer) - // retry again, reload region info and send to new store. - resp, err = s.regionRequestSender.SendReq(NewBackoffer(context.Background(), 100), req, region.Region, time.Second) + // send to store2 fail and send to new leader store1. + bo2 := NewBackoffer(context.Background(), 100) + resp, err = s.regionRequestSender.SendReq(bo2, req, region.Region, time.Second) c.Assert(err, IsNil) - regionErr, err = resp.GetRegionError() + regionErr, err := resp.GetRegionError() c.Assert(err, IsNil) - c.Assert(regionErr, NotNil) + c.Assert(regionErr, IsNil) + c.Assert(resp.Resp, NotNil) } func (s *testRegionRequestSuite) TestSendReqCtx(c *C) { From 8f512145337eb41a2f4e68eb23f12690a1029065 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 31 Jul 2019 17:24:44 +0800 Subject: [PATCH 134/196] server: move log operation out of lock scope (#11536) It's a bad style to print log under a lock, tiny refactor to make the code better. Lock() is also changed to RLock() to reduce lock contend --- server/server.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/server.go b/server/server.go index 6d1945fb6b22a..aca5125bef656 100644 --- a/server/server.go +++ b/server/server.go @@ -513,11 +513,11 @@ func (s *Server) GetProcessInfo(id uint64) (*util.ProcessInfo, bool) { // Kill implements the SessionManager interface. func (s *Server) Kill(connectionID uint64, query bool) { - s.rwlock.Lock() - defer s.rwlock.Unlock() logutil.BgLogger().Info("kill", zap.Uint64("connID", connectionID), zap.Bool("query", query)) metrics.ServerEventCounter.WithLabelValues(metrics.EventKill).Inc() + s.rwlock.RLock() + defer s.rwlock.RUnlock() conn, ok := s.clients[uint32(connectionID)] if !ok { return @@ -538,13 +538,15 @@ func killConn(conn *clientConn) { // KillAllConnections kills all connections when server is not gracefully shutdown. func (s *Server) KillAllConnections() { - s.rwlock.Lock() - defer s.rwlock.Unlock() logutil.BgLogger().Info("[server] kill all connections.") + s.rwlock.RLock() + defer s.rwlock.RUnlock() for _, conn := range s.clients { atomic.StoreInt32(&conn.status, connStatusShutdown) - terror.Log(errors.Trace(conn.closeWithoutLock())) + if err := conn.closeWithoutLock(); err != nil { + terror.Log(err) + } killConn(conn) } } From 1c0b366295fa67f424aa190c6f6824d4348cd8ec Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Thu, 1 Aug 2019 10:40:04 +0800 Subject: [PATCH 135/196] fix ConvertJSONToInt unsigned bug (#11483) --- expression/integration_test.go | 1 + types/convert.go | 6 +++++- types/convert_test.go | 36 ++++++++++++++++++++-------------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 1e0da39536561..04b4526e3cb53 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2180,6 +2180,7 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result.Check(testkit.Rows("2017-01-01 00:00:00.00")) result = tk.MustQuery(`select cast(20170118.999 as datetime);`) result.Check(testkit.Rows("2017-01-18 00:00:00")) + tk.MustQuery(`select convert(a2.a, unsigned int) from (select cast('"9223372036854775808"' as json) as a) as a2;`) tk.MustExec(`create table tb5(a bigint(64) unsigned, b double);`) tk.MustExec(`insert into tb5 (a, b) values (9223372036854776000, 9223372036854776000);`) diff --git a/types/convert.go b/types/convert.go index 079a2b0e152ba..4a8368332b5f7 100644 --- a/types/convert.go +++ b/types/convert.go @@ -567,7 +567,11 @@ func ConvertJSONToInt(sc *stmtctx.StatementContext, j json.BinaryJSON, unsigned return int64(u), errors.Trace(err) case json.TypeCodeString: str := string(hack.String(j.GetString())) - return StrToInt(sc, str) + if !unsigned { + return StrToInt(sc, str) + } + u, err := StrToUint(sc, str) + return int64(u), errors.Trace(err) } return 0, errors.New("Unknown type code in JSON") } diff --git a/types/convert_test.go b/types/convert_test.go index 2d5c97cc2a215..de86c398f7195 100644 --- a/types/convert_test.go +++ b/types/convert_test.go @@ -721,22 +721,24 @@ func (s *testTypeConvertSuite) TestGetValidInt(c *C) { tests := []struct { origin string valid string + signed bool warning bool }{ - {"100", "100", false}, - {"-100", "-100", false}, - {"1abc", "1", true}, - {"-1-1", "-1", true}, - {"+1+1", "+1", true}, - {"123..34", "123", true}, - {"123.23E-10", "123", true}, - {"1.1e1.3", "1", true}, - {"11e1.3", "11", true}, - {"1.", "1", true}, - {".1", "0", true}, - {"", "0", true}, - {"123e+", "123", true}, - {"123de", "123", true}, + {"100", "100", true, false}, + {"-100", "-100", true, false}, + {"9223372036854775808", "9223372036854775808", false, false}, + {"1abc", "1", true, true}, + {"-1-1", "-1", true, true}, + {"+1+1", "+1", true, true}, + {"123..34", "123", true, true}, + {"123.23E-10", "123", true, true}, + {"1.1e1.3", "1", true, true}, + {"11e1.3", "11", true, true}, + {"1.", "1", true, true}, + {".1", "0", true, true}, + {"", "0", true, true}, + {"123e+", "123", true, true}, + {"123de", "123", true, true}, } sc := new(stmtctx.StatementContext) sc.TruncateAsWarning = true @@ -746,7 +748,11 @@ func (s *testTypeConvertSuite) TestGetValidInt(c *C) { prefix, err := getValidIntPrefix(sc, tt.origin) c.Assert(err, IsNil) c.Assert(prefix, Equals, tt.valid) - _, err = strconv.ParseInt(prefix, 10, 64) + if tt.signed { + _, err = strconv.ParseInt(prefix, 10, 64) + } else { + _, err = strconv.ParseUint(prefix, 10, 64) + } c.Assert(err, IsNil) warnings := sc.GetWarnings() if tt.warning { From 1846eba0741846874b6f25f72f1ff98fd07bc7ff Mon Sep 17 00:00:00 2001 From: leoppro Date: Thu, 1 Aug 2019 11:50:50 +0800 Subject: [PATCH 136/196] improve asserts oftest cases in `helper_test.go` and fix ci (#11553) --- store/helper/helper_test.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/store/helper/helper_test.go b/store/helper/helper_test.go index eb2977cb8e6ab..0163434ca14d2 100644 --- a/store/helper/helper_test.go +++ b/store/helper/helper_test.go @@ -16,7 +16,6 @@ package helper_test import ( "crypto/tls" "encoding/json" - "fmt" "net/http" "testing" "time" @@ -74,18 +73,24 @@ func (s *HelperTestSuite) SetUpSuite(c *C) { } func (s *HelperTestSuite) TestHotRegion(c *C) { - helper := helper.Helper{ + h := helper.Helper{ Store: s.store, RegionCache: s.store.GetRegionCache(), } - regionMetric, err := helper.FetchHotRegion(pdapi.HotRead) + regionMetric, err := h.FetchHotRegion(pdapi.HotRead) c.Assert(err, IsNil, Commentf("err: %+v", err)) - c.Assert(fmt.Sprintf("%v", regionMetric), Equals, "map[1:{100 1 0}]") + expected := make(map[uint64]helper.RegionMetric) + expected[1] = helper.RegionMetric{ + FlowBytes: 100, + MaxHotDegree: 1, + Count: 0, + } + c.Assert(regionMetric, DeepEquals, expected) dbInfo := &model.DBInfo{ Name: model.NewCIStr("test"), } c.Assert(err, IsNil) - _, err = helper.FetchRegionTableIndex(regionMetric, []*model.DBInfo{dbInfo}) + _, err = h.FetchRegionTableIndex(regionMetric, []*model.DBInfo{dbInfo}) c.Assert(err, IsNil, Commentf("err: %+v", err)) } @@ -94,8 +99,7 @@ func (s *HelperTestSuite) TestGetRegionsTableInfo(c *C) { regionsInfo := getMockTiKVRegionsInfo() schemas := getMockRegionsTableInfoSchema() tableInfos := h.GetRegionsTableInfo(regionsInfo, schemas) - ans := getRegionsTableInfoAns(schemas) - c.Assert(fmt.Sprintf("%v", tableInfos), Equals, fmt.Sprintf("%v", ans)) + c.Assert(tableInfos, DeepEquals, getRegionsTableInfoAns(schemas)) } func (s *HelperTestSuite) TestTiKVRegionsInfo(c *C) { @@ -105,7 +109,7 @@ func (s *HelperTestSuite) TestTiKVRegionsInfo(c *C) { } regionsInfo, err := h.GetRegionsInfo() c.Assert(err, IsNil, Commentf("err: %+v", err)) - c.Assert(fmt.Sprintf("%v", regionsInfo), Equals, fmt.Sprintf("%v", getMockTiKVRegionsInfo())) + c.Assert(regionsInfo, DeepEquals, getMockTiKVRegionsInfo()) } func (s *HelperTestSuite) TestTiKVStoresStat(c *C) { @@ -117,7 +121,7 @@ func (s *HelperTestSuite) TestTiKVStoresStat(c *C) { c.Assert(err, IsNil, Commentf("err: %+v", err)) data, err := json.Marshal(stat) c.Assert(err, IsNil) - c.Assert(fmt.Sprintf("%s", data), Equals, "{\"count\":1,\"stores\":[{\"store\":{\"id\":1,\"address\":\"127.0.0.1:20160\",\"state\":0,\"state_name\":\"Up\",\"version\":\"3.0.0-beta\",\"labels\":[{\"key\":\"test\",\"value\":\"test\"}]},\"status\":{\"capacity\":\"60 GiB\",\"available\":\"100 GiB\",\"leader_count\":10,\"leader_weight\":1,\"leader_score\":1000,\"leader_size\":1000,\"region_count\":200,\"region_weight\":1,\"region_score\":1000,\"region_size\":1000,\"start_ts\":\"2019-04-23T19:30:30+08:00\",\"last_heartbeat_ts\":\"2019-04-23T19:31:30+08:00\",\"uptime\":\"1h30m\"}}]}") + c.Assert(string(data), Equals, `{"count":1,"stores":[{"store":{"id":1,"address":"127.0.0.1:20160","state":0,"state_name":"Up","version":"3.0.0-beta","labels":[{"key":"test","value":"test"}]},"status":{"capacity":"60 GiB","available":"100 GiB","leader_count":10,"leader_weight":1,"leader_score":1000,"leader_size":1000,"region_count":200,"region_weight":1,"region_score":1000,"region_size":1000,"start_ts":"2019-04-23T19:30:30+08:00","last_heartbeat_ts":"2019-04-23T19:31:30+08:00","uptime":"1h30m"}}]}`) } func (s *HelperTestSuite) mockPDHTTPServer(c *C) { @@ -181,8 +185,8 @@ func getMockRegionsTableInfoSchema() []*model.DBInfo { } } -func getRegionsTableInfoAns(dbs []*model.DBInfo) map[uint64][]helper.TableInfo { - ans := make(map[uint64][]helper.TableInfo) +func getRegionsTableInfoAns(dbs []*model.DBInfo) map[int64][]helper.TableInfo { + ans := make(map[int64][]helper.TableInfo) db := dbs[0] ans[1] = []helper.TableInfo{} ans[2] = []helper.TableInfo{ From 7b632931803d69fcd57010203c01ea0d4e270f5a Mon Sep 17 00:00:00 2001 From: Lingyu Song Date: Thu, 1 Aug 2019 13:10:39 +0800 Subject: [PATCH 137/196] http: add region-id when getting mvcc by handle (#11436) --- server/http_handler.go | 48 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/server/http_handler.go b/server/http_handler.go index ece6aba284eea..446cd027b3a84 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -132,17 +132,33 @@ func (s *Server) newTikvHandlerTool() *tikvHandlerTool { } type mvccKV struct { - Key string `json:"key"` - Value *kvrpcpb.MvccGetByKeyResponse `json:"value"` + Key string `json:"key"` + RegionID uint64 `json:"region_id"` + Value *kvrpcpb.MvccGetByKeyResponse `json:"value"` +} + +func (t *tikvHandlerTool) getRegionIDByKey(encodedKey []byte) (uint64, error) { + keyLocation, err := t.RegionCache.LocateKey(tikv.NewBackoffer(context.Background(), 500), encodedKey) + if err != nil { + return 0, err + } + return keyLocation.Region.GetID(), nil } func (t *tikvHandlerTool) getMvccByHandle(tableID, handle int64) (*mvccKV, error) { encodedKey := tablecodec.EncodeRowKeyWithHandle(tableID, handle) data, err := t.GetMvccByEncodedKey(encodedKey) - return &mvccKV{Key: strings.ToUpper(hex.EncodeToString(encodedKey)), Value: data}, err + if err != nil { + return nil, err + } + regionID, err := t.getRegionIDByKey(encodedKey) + if err != nil { + return nil, err + } + return &mvccKV{Key: strings.ToUpper(hex.EncodeToString(encodedKey)), Value: data, RegionID: regionID}, err } -func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey []byte) (*kvrpcpb.MvccGetByStartTsResponse, error) { +func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey []byte) (*mvccKV, error) { bo := tikv.NewBackoffer(context.Background(), 5000) for { curRegion, err := t.RegionCache.LocateKey(bo, startKey) @@ -194,7 +210,8 @@ func (t *tikvHandlerTool) getMvccByStartTs(startTS uint64, startKey, endKey []by key := data.GetKey() if len(key) > 0 { - return data, nil + resp := &kvrpcpb.MvccGetByKeyResponse{Info: data.Info, RegionError: data.RegionError, Error: data.Error} + return &mvccKV{Key: strings.ToUpper(hex.EncodeToString(key)), Value: resp, RegionID: curRegion.Region.GetID()}, nil } if len(endKey) > 0 && curRegion.Contains(endKey) { @@ -225,7 +242,14 @@ func (t *tikvHandlerTool) getMvccByIdxValue(idx table.Index, values url.Values, return nil, errors.Trace(err) } data, err := t.GetMvccByEncodedKey(encodedKey) - return &mvccKV{strings.ToUpper(hex.EncodeToString(encodedKey)), data}, err + if err != nil { + return nil, err + } + regionID, err := t.getRegionIDByKey(encodedKey) + if err != nil { + return nil, err + } + return &mvccKV{strings.ToUpper(hex.EncodeToString(encodedKey)), regionID, data}, err } // formValue2DatumRow converts URL query string to a Datum Row. @@ -284,12 +308,20 @@ func (t *tikvHandlerTool) schema() (infoschema.InfoSchema, error) { return domain.GetDomain(session.(sessionctx.Context)).InfoSchema(), nil } -func (t *tikvHandlerTool) handleMvccGetByHex(params map[string]string) (interface{}, error) { +func (t *tikvHandlerTool) handleMvccGetByHex(params map[string]string) (*mvccKV, error) { encodedKey, err := hex.DecodeString(params[pHexKey]) if err != nil { return nil, errors.Trace(err) } - return t.GetMvccByEncodedKey(encodedKey) + data, err := t.GetMvccByEncodedKey(encodedKey) + if err != nil { + return nil, errors.Trace(err) + } + regionID, err := t.getRegionIDByKey(encodedKey) + if err != nil { + return nil, err + } + return &mvccKV{Key: strings.ToUpper(params[pHexKey]), Value: data, RegionID: regionID}, nil } // settingsHandler is the handler for list tidb server settings. From d54f0a2bd4f2dfd14daf39ac8da993ab26743ae1 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 1 Aug 2019 13:28:23 +0800 Subject: [PATCH 138/196] expression: add vectorized evaluation methods to `Expression` (#11530) --- expression/builtin.go | 27 +++-- expression/column.go | 11 ++ expression/constant.go | 8 ++ expression/constant_test.go | 55 +++++++++ expression/expression.go | 7 ++ expression/scalar_function.go | 5 + expression/util_test.go | 47 +++----- expression/vectorized.go | 203 ++++++++++++++++++++++++++++++++++ util/chunk/chunk.go | 15 +++ util/chunk/column.go | 18 +++ util/chunk/column_test.go | 28 +++++ 11 files changed, 388 insertions(+), 36 deletions(-) create mode 100644 expression/vectorized.go diff --git a/expression/builtin.go b/expression/builtin.go index 98049ba148ba5..dc1c1d40bd683 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -18,6 +18,7 @@ package expression import ( + "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/mysql" @@ -171,32 +172,36 @@ func (b *baseBuiltinFunc) getArgs() []Expression { return b.args } +func (b *baseBuiltinFunc) vecEval(input *chunk.Chunk, result *chunk.Column) error { + return errors.Errorf("baseBuiltinFunc.vecEval() should never be called, please contact the TiDB team for help") +} + func (b *baseBuiltinFunc) evalInt(row chunk.Row) (int64, bool, error) { - panic("baseBuiltinFunc.evalInt() should never be called.") + return 0, false, errors.Errorf("baseBuiltinFunc.evalInt() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalReal(row chunk.Row) (float64, bool, error) { - panic("baseBuiltinFunc.evalReal() should never be called.") + return 0, false, errors.Errorf("baseBuiltinFunc.evalReal() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalString(row chunk.Row) (string, bool, error) { - panic("baseBuiltinFunc.evalString() should never be called.") + return "", false, errors.Errorf("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalDecimal(row chunk.Row) (*types.MyDecimal, bool, error) { - panic("baseBuiltinFunc.evalDecimal() should never be called.") + return nil, false, errors.Errorf("baseBuiltinFunc.evalDecimal() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalTime(row chunk.Row) (types.Time, bool, error) { - panic("baseBuiltinFunc.evalTime() should never be called.") + return types.Time{}, false, errors.Errorf("baseBuiltinFunc.evalTime() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalDuration(row chunk.Row) (types.Duration, bool, error) { - panic("baseBuiltinFunc.evalDuration() should never be called.") + return types.Duration{}, false, errors.Errorf("baseBuiltinFunc.evalDuration() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) evalJSON(row chunk.Row) (json.BinaryJSON, bool, error) { - panic("baseBuiltinFunc.evalJSON() should never be called.") + return json.BinaryJSON{}, false, errors.Errorf("baseBuiltinFunc.evalJSON() should never be called, please contact the TiDB team for help") } func (b *baseBuiltinFunc) getRetTp() *types.FieldType { @@ -276,8 +281,16 @@ func newBaseBuiltinCastFunc(builtinFunc baseBuiltinFunc, inUnion bool) baseBuilt } } +// vecBuiltinFunc contains all vectorized methods for a builtin function. +type vecBuiltinFunc interface { + // vecEval evaluates this builtin function in a vectorized manner. + vecEval(input *chunk.Chunk, result *chunk.Column) error +} + // builtinFunc stands for a particular function signature. type builtinFunc interface { + vecBuiltinFunc + // evalInt evaluates int result of builtinFunc by given row. evalInt(row chunk.Row) (val int64, isNull bool, err error) // evalReal evaluates real representation of builtinFunc by given row. diff --git a/expression/column.go b/expression/column.go index 99b94e0ea4414..e89eb93ebe62f 100644 --- a/expression/column.go +++ b/expression/column.go @@ -40,6 +40,11 @@ func (col *CorrelatedColumn) Clone() Expression { return col } +// VecEval evaluates this expression in a vectorized manner. +func (col *CorrelatedColumn) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) (err error) { + return genVecFromConstExpr(ctx, col, input, result) +} + // Eval implements Expression interface. func (col *CorrelatedColumn) Eval(row chunk.Row) (types.Datum, error) { return *col.Data, nil @@ -181,6 +186,12 @@ func (col *Column) Equal(_ sessionctx.Context, expr Expression) bool { return false } +// VecEval evaluates this expression in a vectorized manner. +func (col *Column) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error { + input.Column(col.Index).CopyConstruct(result) + return nil +} + // String implements Stringer interface. func (col *Column) String() string { result := col.ColName.L diff --git a/expression/constant.go b/expression/constant.go index 7e7948cf387d0..360da451325bd 100644 --- a/expression/constant.go +++ b/expression/constant.go @@ -88,6 +88,14 @@ func (c *Constant) GetType() *types.FieldType { return c.RetType } +// VecEval evaluates this expression in a vectorized manner. +func (c *Constant) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error { + if c.DeferredExpr == nil { + return genVecFromConstExpr(ctx, c, input, result) + } + return c.DeferredExpr.VecEval(ctx, input, result) +} + // Eval implements Expression interface. func (c *Constant) Eval(_ chunk.Row) (types.Datum, error) { if c.DeferredExpr != nil { diff --git a/expression/constant_test.go b/expression/constant_test.go index e5fe7b5681e6c..84e241b3f6491 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -423,3 +423,58 @@ func (*testExpressionSuite) TestDeferredExprNotNull(c *C) { cln := cst.Clone().(*Constant) c.Assert(cln.DeferredExpr, Equals, cst.DeferredExpr) } + +func (*testExpressionSuite) TestVectorizedConstant(c *C) { + // fixed-length type with/without Sel + for _, cst := range []*Constant{ + {RetType: newIntFieldType(), Value: types.NewIntDatum(2333)}, + {RetType: newIntFieldType(), DeferredExpr: &Constant{RetType: newIntFieldType(), Value: types.NewIntDatum(2333)}}} { + chk := chunk.New([]*types.FieldType{newIntFieldType()}, 1024, 1024) + for i := 0; i < 1024; i++ { + chk.AppendInt64(0, int64(i)) + } + col := chunk.NewColumn(newIntFieldType(), 1024) + ctx := mock.NewContext() + c.Assert(cst.VecEval(ctx, chk, col), IsNil) + i64s := col.Int64s() + c.Assert(len(i64s), Equals, 1024) + for _, v := range i64s { + c.Assert(v, Equals, int64(2333)) + } + + // fixed-length type with Sel + sel := []int{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + chk.SetSel(sel) + c.Assert(cst.VecEval(ctx, chk, col), IsNil) + i64s = col.Int64s() + for _, i := range sel { + c.Assert(i64s[i], Equals, int64(2333)) + } + } + + // var-length type with/without Sel + for _, cst := range []*Constant{ + {RetType: newStringFieldType(), Value: types.NewStringDatum("hello")}, + {RetType: newStringFieldType(), DeferredExpr: &Constant{RetType: newStringFieldType(), Value: types.NewStringDatum("hello")}}} { + chk := chunk.New([]*types.FieldType{newIntFieldType()}, 1024, 1024) + for i := 0; i < 1024; i++ { + chk.AppendInt64(0, int64(i)) + } + cst = &Constant{DeferredExpr: nil, RetType: newStringFieldType(), Value: types.NewStringDatum("hello")} + chk.SetSel(nil) + col := chunk.NewColumn(newStringFieldType(), 1024) + ctx := mock.NewContext() + c.Assert(cst.VecEval(ctx, chk, col), IsNil) + for i := 0; i < 1024; i++ { + c.Assert(col.GetString(i), Equals, "hello") + } + + // var-length type with Sel + sel := []int{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + chk.SetSel(sel) + c.Assert(cst.VecEval(ctx, chk, col), IsNil) + for _, i := range sel { + c.Assert(col.GetString(i), Equals, "hello") + } + } +} diff --git a/expression/expression.go b/expression/expression.go index d1ea0e24d7730..14c3edcb640fa 100644 --- a/expression/expression.go +++ b/expression/expression.go @@ -38,10 +38,17 @@ const ( // EvalAstExpr evaluates ast expression directly. var EvalAstExpr func(sctx sessionctx.Context, expr ast.ExprNode) (types.Datum, error) +// VecExpr contains all vectorized evaluation methods. +type VecExpr interface { + // VecEval evaluates this expression in a vectorized manner. + VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error +} + // Expression represents all scalar expression in SQL. type Expression interface { fmt.Stringer goJSON.Marshaler + VecExpr // Eval evaluates an expression through a row. Eval(row chunk.Row) (types.Datum, error) diff --git a/expression/scalar_function.go b/expression/scalar_function.go index 200360323df44..873ed8a3c3660 100644 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -41,6 +41,11 @@ type ScalarFunction struct { hashcode []byte } +// VecEval evaluates this expression in a vectorized manner. +func (sf *ScalarFunction) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error { + return sf.Function.vecEval(input, result) +} + // GetArgs gets arguments of function. func (sf *ScalarFunction) GetArgs() []Expression { return sf.Function.getArgs() diff --git a/expression/util_test.go b/expression/util_test.go index dcd5f62c1144c..1311ca13a0dc7 100644 --- a/expression/util_test.go +++ b/expression/util_test.go @@ -46,37 +46,22 @@ func (s *testUtilSuite) checkPanic(f func()) (ret bool) { } func (s *testUtilSuite) TestBaseBuiltin(c *check.C) { - c.Assert(s.checkPanic(func() { - newBaseBuiltinFuncWithTp(nil, nil, types.ETTimestamp) - }), check.IsTrue) - ctx := mock.NewContext() - c.Assert(s.checkPanic(func() { - newBaseBuiltinFuncWithTp(ctx, nil, types.ETTimestamp, types.ETTimestamp) - }), check.IsTrue) - bf := newBaseBuiltinFuncWithTp(ctx, nil, types.ETTimestamp) - c.Assert(s.checkPanic(func() { - bf.evalInt(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalReal(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalString(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalDecimal(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalTime(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalDuration(chunk.Row{}) - }), check.IsTrue) - c.Assert(s.checkPanic(func() { - bf.evalJSON(chunk.Row{}) - }), check.IsTrue) + _, _, err := bf.evalInt(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalReal(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalString(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalDecimal(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalTime(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalDuration(chunk.Row{}) + c.Assert(err, check.NotNil) + _, _, err = bf.evalJSON(chunk.Row{}) + c.Assert(err, check.NotNil) } func (s *testUtilSuite) TestClone(c *check.C) { @@ -400,6 +385,10 @@ type MockExpr struct { i interface{} } +func (m *MockExpr) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error { + return nil +} + func (m *MockExpr) String() string { return "" } func (m *MockExpr) MarshalJSON() ([]byte, error) { return nil, nil } func (m *MockExpr) Eval(row chunk.Row) (types.Datum, error) { return types.NewDatum(m.i), m.err } diff --git a/expression/vectorized.go b/expression/vectorized.go new file mode 100644 index 0000000000000..366394ee79919 --- /dev/null +++ b/expression/vectorized.go @@ -0,0 +1,203 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package expression + +import ( + "github.com/pingcap/errors" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" +) + +func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.Chunk, result *chunk.Column) error { + n := input.NumEffectiveRows() + sel := input.Sel() + tp := expr.GetType() + switch tp.EvalType() { + case types.ETInt: + result.PreAllocInt64(n) + v, isNull, err := expr.EvalInt(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { // all slots are set to null by PreAlloc() + return nil + } + i64s := result.Int64s() + if sel == nil { + for i := range i64s { + i64s[i] = v + } + result.SetNulls(0, n, false) + } else { + for _, i := range sel { + i64s[i] = v + result.SetNull(i, false) + } + } + case types.ETReal: + result.PreAllocFloat64(n) + v, isNull, err := expr.EvalReal(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { // all slots are set to null by PreAlloc() + return nil + } + f64s := result.Float64s() + if sel == nil { + for i := range f64s { + f64s[i] = v + } + result.SetNulls(0, n, false) + } else { + for _, i := range sel { + f64s[i] = v + result.SetNull(i, false) + } + } + case types.ETDecimal: + result.PreAllocDecimal(n) + v, isNull, err := expr.EvalDecimal(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { // all slots are set to null by PreAlloc() + return nil + } + ds := result.Decimals() + if sel == nil { + for i := range ds { + ds[i] = *v + } + result.SetNulls(0, n, false) + } else { + for _, i := range sel { + ds[i] = *v + result.SetNull(i, false) + } + } + case types.ETDatetime, types.ETTimestamp: + result.Reset() + v, isNull, err := expr.EvalTime(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { + for i := 0; i < n; i++ { + result.AppendNull() + } + } else { + if sel == nil { + for i := 0; i < n; i++ { + result.AppendTime(v) + } + } else { + pos := 0 + for _, i := range sel { + for pos < i { + result.AppendNull() + pos++ + } + result.AppendTime(v) + pos++ + } + } + } + case types.ETDuration: + result.Reset() + v, isNull, err := expr.EvalDuration(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { + for i := 0; i < n; i++ { + result.AppendNull() + } + } else { + if sel == nil { + for i := 0; i < n; i++ { + result.AppendDuration(v) + } + } else { + pos := 0 + for _, i := range sel { + for pos < i { + result.AppendNull() + pos++ + } + result.AppendDuration(v) + pos++ + } + } + } + case types.ETJson: + result.Reset() + v, isNull, err := expr.EvalJSON(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { + for i := 0; i < n; i++ { + result.AppendNull() + } + } else { + if sel == nil { + for i := 0; i < n; i++ { + result.AppendJSON(v) + } + } else { + pos := 0 + for _, i := range sel { + for pos < i { + result.AppendNull() + pos++ + } + result.AppendJSON(v) + pos++ + } + } + } + case types.ETString: + result.Reset() + v, isNull, err := expr.EvalString(ctx, chunk.Row{}) + if err != nil { + return err + } + if isNull { + for i := 0; i < n; i++ { + result.AppendNull() + } + } else { + if sel == nil { + for i := 0; i < n; i++ { + result.AppendString(v) + } + } else { + pos := 0 + for _, i := range sel { + for pos < i { + result.AppendNull() + pos++ + } + result.AppendString(v) + pos++ + } + } + } + default: + return errors.Errorf("unsupported Constant type for vectorized evaluation") + } + return nil +} diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 453b57c71b85a..914a88210d17a 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -311,6 +311,21 @@ func (c *Chunk) NumRows() int { return c.columns[0].length } +// NumEffectiveRows returns the effective number of rows physically stored in this Chunk. +// It is different with NumRows when sel is not nil. +// For example: if sel is [2, 3, 5, 7, 9], then +// NumRow() returns 5 to indicate that 5 rows are selected logically in this Chunk, while +// NumEffectiveRows() returns 10(9+1) to indicate that at least 10 rows are stored in this Chunk physically. +func (c *Chunk) NumEffectiveRows() int { + if c.sel == nil { + return c.NumRows() + } + if len(c.sel) == 0 { + return 0 + } + return c.sel[len(c.sel)-1] + 1 +} + // GetRow gets the Row in the chunk with the row index. func (c *Chunk) GetRow(idx int) Row { if c.sel != nil { diff --git a/util/chunk/column.go b/util/chunk/column.go index 7a5379ec5efa2..66d1778ced7ea 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -279,6 +279,24 @@ func (c *Column) SetNull(rowIdx int, isNull bool) { } } +// SetNulls sets rows in [begin, end) to null. +func (c *Column) SetNulls(begin, end int, isNull bool) { + i := ((begin + 7) >> 3) << 3 + for ; begin < i && begin < end; begin++ { + c.SetNull(begin, isNull) + } + var v uint8 + if !isNull { + v = (1 << 8) - 1 + } + for ; begin+8 <= end; begin += 8 { + c.nullBitmap[begin>>3] = v + } + for ; begin < end; begin++ { + c.SetNull(begin, isNull) + } +} + // nullCount returns the number of nulls in this Column. func (c *Column) nullCount() int { var cnt, i int diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 0b04a8fbd24c9..277cde93664b2 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -653,3 +653,31 @@ func (s *testChunkSuite) TestNull(c *check.C) { col.SetNull(8, false) c.Assert(col.nullCount(), check.Equals, 8) } + +func (s *testChunkSuite) TestSetNulls(c *check.C) { + col := newFixedLenColumn(sizeFloat64, 32) + col.PreAllocFloat64(1024) + c.Assert(col.nullCount(), check.Equals, 1024) + + col.SetNulls(0, 1024, false) + c.Assert(col.nullCount(), check.Equals, 0) + + nullMap := make(map[int]struct{}) + for i := 0; i < 100; i++ { + begin := rand.Intn(1024) + l := rand.Intn(37) + end := begin + l + if end > 1024 { + end = 1024 + } + for i := begin; i < end; i++ { + nullMap[i] = struct{}{} + } + col.SetNulls(begin, end, true) + + c.Assert(col.nullCount(), check.Equals, len(nullMap)) + for k := range nullMap { + c.Assert(col.IsNull(k), check.Equals, true) + } + } +} From 1c43f55f909594cc2e7a109179ca34f018ba3789 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Thu, 1 Aug 2019 14:30:07 +0800 Subject: [PATCH 139/196] parser: support cast as float (#11519) --- expression/integration_test.go | 28 ++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 04b4526e3cb53..785457b8a0097 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2372,6 +2372,34 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result = tk.MustQuery("select cast(0x12345678 as double)") result.Check(testkit.Rows("305419896")) + // test cast as float + result = tk.MustQuery("select cast(1 as float)") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(12345 as unsigned) as float)") + result.Check(testkit.Rows("12345")) + result = tk.MustQuery("select cast(1.1 as float) = 1.1") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(-1.1 as float) = -1.1") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast('123.321' as float) =123.321") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast('12345678901234567890' as float) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(-1 as float)") + result.Check(testkit.Rows("-1")) + result = tk.MustQuery("select cast(null as float)") + result.Check(testkit.Rows("")) + result = tk.MustQuery("select cast(12345678901234567890 as float) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(-1 as unsigned) as float) = 1.8446744073709552e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(1e100 as float(40)) = 1e100") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(123456789012345678901234567890 as float(40)) = 1.2345678901234568e29") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(0x12345678 as float(40)) = 305419896") + result.Check(testkit.Rows("1")) + // test cast time as decimal overflow tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(s1 time);") diff --git a/go.mod b/go.mod index d35cff7ab4845..ce0b1ec56a1a9 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8 + github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index a9cc96d8b451c..37382ecbbf34e 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8 h1:Htdqrie9rVA/x2yqnXjqYu+VNvff3Cnj7clCNvIyXVU= -github.com/pingcap/parser v0.0.0-20190730091357-5238015a66f8/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66 h1:RQlXn268eNxAh4ZcDGHSdDkv20x1eJDQu8vYvQh3x5c= +github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From e3c9abdf6c6776c4524130b6adf771bd46dbd299 Mon Sep 17 00:00:00 2001 From: Maxwell Date: Thu, 1 Aug 2019 15:42:26 +0800 Subject: [PATCH 140/196] util: add missing logutil dependence in `signal_windows` (#11559) Co-authored-by: Lynn --- util/signal/signal_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util/signal/signal_windows.go b/util/signal/signal_windows.go index 52c02023d63f4..79134b17d219b 100644 --- a/util/signal/signal_windows.go +++ b/util/signal/signal_windows.go @@ -19,6 +19,7 @@ import ( "os/signal" "syscall" + "github.com/pingcap/tidb/util/logutil" "go.uber.org/zap" ) From f13f7bbdbdb2ae7df995d310f772294326da060f Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 1 Aug 2019 16:24:36 +0800 Subject: [PATCH 141/196] ranger: BuildColumnRange should merge ranges when column has prefix len (#11563) --- executor/join_test.go | 10 +++++++++ util/ranger/ranger.go | 4 ++++ util/ranger/ranger_test.go | 43 +++++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/executor/join_test.go b/executor/join_test.go index 2b768d3d53e6f..0f0b66214bf9d 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -1381,3 +1381,13 @@ func (s *testSuite2) TestInjectProjOnTopN(c *C) { "2", )) } + +func (s *testSuite2) TestIssue11544(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table 11544t(a int)") + tk.MustExec("create table 11544tt(a int, b varchar(10), index idx(a, b(3)))") + tk.MustExec("insert into 11544t values(1)") + tk.MustExec("insert into 11544tt values(1, 'aaaaaaa'), (1, 'aaaabbb'), (1, 'aaaacccc')") + tk.MustQuery("select /*+ TIDB_INLJ(tt) */ * from 11544t t, 11544tt tt where t.a=tt.a and (tt.b = 'aaaaaaa' or tt.b = 'aaaabbb')").Check(testkit.Rows("1 1 aaaaaaa", "1 1 aaaabbb")) +} diff --git a/util/ranger/ranger.go b/util/ranger/ranger.go index 4b9b12e0b6e13..6f858af480a39 100644 --- a/util/ranger/ranger.go +++ b/util/ranger/ranger.go @@ -270,6 +270,10 @@ func buildColumnRange(accessConditions []expression.Expression, sc *stmtctx.Stat ran.HighExclude = false } } + ranges, err = unionRanges(sc, ranges) + if err != nil { + return nil, err + } } return ranges, nil } diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 6fc8d102aff25..4fe376d2b5697 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -747,6 +747,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds string filterConds string resultStr string + length int }{ { colPos: 0, @@ -754,6 +755,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[eq(test.t.a, 1)]", filterConds: "[gt(test.t.b, 1)]", resultStr: "[[1,1]]", + length: types.UnspecifiedLength, }, { colPos: 1, @@ -761,6 +763,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.b, 1)]", filterConds: "[]", resultStr: "[(1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -768,6 +771,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[eq(1, test.t.a)]", filterConds: "[]", resultStr: "[[1,1]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -775,6 +779,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ne(test.t.a, 1)]", filterConds: "[]", resultStr: "[[-inf,1) (1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -782,6 +787,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ne(1, test.t.a)]", filterConds: "[]", resultStr: "[[-inf,1) (1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -789,6 +795,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.a, 1)]", filterConds: "[]", resultStr: "[(1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -796,6 +803,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[lt(1, test.t.a)]", filterConds: "[]", resultStr: "[(1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -803,6 +811,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ge(test.t.a, 1)]", filterConds: "[]", resultStr: "[[1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -810,6 +819,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[le(1, test.t.a)]", filterConds: "[]", resultStr: "[[1,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -817,6 +827,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[lt(test.t.a, 1)]", filterConds: "[]", resultStr: "[[-inf,1)]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -824,6 +835,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(1, test.t.a)]", filterConds: "[]", resultStr: "[[-inf,1)]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -831,6 +843,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[le(test.t.a, 1)]", filterConds: "[]", resultStr: "[[-inf,1]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -838,6 +851,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ge(1, test.t.a)]", filterConds: "[]", resultStr: "[[-inf,1]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -845,6 +859,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[test.t.a]", filterConds: "[]", resultStr: "[[-inf,0) (0,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -852,6 +867,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[in(test.t.a, 1, 3, , 2)]", filterConds: "[]", resultStr: "[[1,1] [2,2] [3,3]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -859,6 +875,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[in(test.t.a, 8, 8, 81, 45)]", filterConds: "[]", resultStr: `[[8,8] [45,45] [81,81]]`, + length: types.UnspecifiedLength, }, { colPos: 0, @@ -866,6 +883,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ge(test.t.a, 1) le(test.t.a, 2)]", filterConds: "[]", resultStr: "[[1,2]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -873,6 +891,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[or(lt(test.t.a, 1), gt(test.t.a, 2))]", filterConds: "[]", resultStr: "[[-inf,1) (2,+inf]]", + length: types.UnspecifiedLength, }, //{ // `a > null` will be converted to `castAsString(a) > null` which can not be extracted as access condition. @@ -885,6 +904,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[ge(test.t.a, 2) le(test.t.a, 1)]", filterConds: "[]", resultStr: "[]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -892,6 +912,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[or(lt(test.t.a, 2), gt(test.t.a, 1))]", filterConds: "[]", resultStr: "[[-inf,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -899,6 +920,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[isnull(test.t.a)]", filterConds: "[]", resultStr: "[[NULL,NULL]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -906,6 +928,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[not(isnull(test.t.a))]", filterConds: "[]", resultStr: "[[-inf,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -913,6 +936,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[istrue(test.t.a)]", filterConds: "[]", resultStr: "[[-inf,0) (0,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -920,6 +944,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[not(istrue(test.t.a))]", filterConds: "[]", resultStr: "[[NULL,NULL] [0,0]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -927,6 +952,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[isfalse(test.t.a)]", filterConds: "[]", resultStr: "[[0,0]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -934,6 +960,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[not(isfalse(test.t.a))]", filterConds: "[]", resultStr: "[[NULL,0) (0,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 1, @@ -941,6 +968,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[in(test.t.b, 1, 2.1)]", filterConds: "[]", resultStr: "[[1,1] [2.1,2.1]]", + length: types.UnspecifiedLength, }, { colPos: 0, @@ -948,6 +976,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.a, 9223372036854775807)]", filterConds: "[]", resultStr: "[(9223372036854775807,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 2, @@ -955,6 +984,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.c, 111.11111111)]", filterConds: "[]", resultStr: "[[111.111115,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 3, @@ -962,6 +992,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.d, aaaaaaaaaaaaaa)]", filterConds: "[]", resultStr: "[(\"aaaaaaaaaaaaaa\",+inf]]", + length: types.UnspecifiedLength, }, { colPos: 4, @@ -969,6 +1000,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.e, 18446744073709500000)]", filterConds: "[]", resultStr: "[(18446744073709500000,+inf]]", + length: types.UnspecifiedLength, }, { colPos: 4, @@ -976,6 +1008,15 @@ func (s *testRangerSuite) TestColumnRange(c *C) { accessConds: "[gt(test.t.e, -2147483648)]", filterConds: "[]", resultStr: "[[0,+inf]]", + length: types.UnspecifiedLength, + }, + { + colPos: 3, + exprStr: "d = 'aab' or d = 'aac'", + accessConds: "[or(eq(test.t.d, aab), eq(test.t.d, aac))]", + filterConds: "[]", + resultStr: "[[\"a\",\"a\"]]", + length: 1, }, } @@ -1002,7 +1043,7 @@ func (s *testRangerSuite) TestColumnRange(c *C) { c.Assert(col, NotNil) conds = ranger.ExtractAccessConditionsForColumn(conds, col.UniqueID) c.Assert(fmt.Sprintf("%s", conds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr)) - result, err := ranger.BuildColumnRange(conds, new(stmtctx.StatementContext), col.RetType, types.UnspecifiedLength) + result, err := ranger.BuildColumnRange(conds, new(stmtctx.StatementContext), col.RetType, tt.length) c.Assert(err, IsNil) got := fmt.Sprintf("%v", result) c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s, col: %v", tt.exprStr, col)) From 10e8498dd607baacc8a65c45c608da1ab7deaccc Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 1 Aug 2019 17:29:35 +0800 Subject: [PATCH 142/196] ddl: fix a bug that adding range partition meets error when value is negative (#11407) --- ddl/db_partition_test.go | 16 ++++++++++++++++ ddl/ddl_api.go | 26 ++++++++------------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/ddl/db_partition_test.go b/ddl/db_partition_test.go index 8b741ad8ff605..8f00bedbfbbfc 100644 --- a/ddl/db_partition_test.go +++ b/ddl/db_partition_test.go @@ -512,6 +512,19 @@ func (s *testIntegrationSuite5) TestAlterTableAddPartition(c *C) { sql10 := "alter table table3 add partition partitions 4;" assertErrorCode(c, tk, sql10, tmysql.ErrPartitionsMustBeDefined) + + tk.MustExec("alter table table3 add partition (partition p3 values less than (2001 + 10))") + + // less than value can be negative or expression. + tk.MustExec(`CREATE TABLE tt5 ( + c3 bigint(20) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin + PARTITION BY RANGE ( c3 ) ( + PARTITION p0 VALUES LESS THAN (-3), + PARTITION p1 VALUES LESS THAN (-2) + );`) + tk.MustExec(`ALTER TABLE tt5 add partition ( partition p2 values less than (-1) );`) + tk.MustExec(`ALTER TABLE tt5 add partition ( partition p3 values less than (5-1) );`) } func (s *testIntegrationSuite5) TestAlterTableDropPartition(c *C) { @@ -1453,6 +1466,9 @@ func (s *testIntegrationSuite3) TestPartitionErrorCode(c *C) { _, err := tk.Exec("alter table employees add partition partitions 8;") c.Assert(ddl.ErrUnsupportedAddPartition.Equal(err), IsTrue) + _, err = tk.Exec("alter table employees add partition (partition p5 values less than (42));") + c.Assert(ddl.ErrUnsupportedAddPartition.Equal(err), IsTrue) + // coalesce partition tk.MustExec(`create table clients ( id int, diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index f96c1723dac62..62f7d56444b8a 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2156,7 +2156,8 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec * } meta := t.Meta() - if meta.GetPartitionInfo() == nil { + pi := meta.GetPartitionInfo() + if pi == nil { return errors.Trace(ErrPartitionMgmtOnNonpartitioned) } @@ -2179,7 +2180,11 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec * return errors.Trace(err) } - err = checkAddPartitionValue(meta, partInfo) + // partInfo contains only the new added partition, we have to combine it with the + // old partitions to check all partitions is strictly increasing. + tmp := *partInfo + tmp.Definitions = append(pi.Definitions, tmp.Definitions...) + err = checkCreatePartitionValue(ctx, meta, &tmp, t.Cols()) if err != nil { return errors.Trace(err) } @@ -3422,32 +3427,17 @@ func buildPartitionInfo(meta *model.TableInfo, d *ddl, spec *ast.AlterTableSpec) Columns: meta.Partition.Columns, Enable: meta.Partition.Enable, } + genIDs, err := d.genGlobalIDs(len(spec.PartDefinitions)) if err != nil { return nil, err } - buf := new(bytes.Buffer) for ith, def := range spec.PartDefinitions { if err := def.Clause.Validate(part.Type, len(part.Columns)); err != nil { return nil, errors.Trace(err) } // For RANGE partition only VALUES LESS THAN should be possible. clause := def.Clause.(*ast.PartitionDefinitionClauseLessThan) - for _, expr := range clause.Exprs { - tp := expr.GetType().Tp - if len(part.Columns) == 0 { - // Partition by range. - if !(tp == mysql.TypeLong || tp == mysql.TypeLonglong) { - expr.Format(buf) - if strings.EqualFold(buf.String(), "MAXVALUE") { - continue - } - buf.Reset() - return nil, infoschema.ErrColumnNotExists.GenWithStackByArgs(buf.String(), "partition function") - } - } - // Partition by range columns if len(part.Columns) != 0. - } comment, _ := def.Comment() piDef := model.PartitionDefinition{ Name: def.Name, From 7715341cb7ffed0fb2343a1e7be8d361e9fab9d8 Mon Sep 17 00:00:00 2001 From: goroutine Date: Thu, 1 Aug 2019 17:36:05 +0800 Subject: [PATCH 143/196] server: tiny refactor, more go style (#11478) --- executor/insert_common.go | 8 ++------ server/server.go | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/executor/insert_common.go b/executor/insert_common.go index 71ad672830378..77c703532acaa 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -447,7 +447,6 @@ func (e *InsertValues) fillColValue(ctx context.Context, datum types.Datum, idx // fillRow fills generated columns, auto_increment column and empty column. // For NOT NULL column, it will return error or use zero value based on sql_mode. func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue []bool) ([]types.Datum, error) { - gIdx := 0 gCols := make([]*table.Column, 0) for i, c := range e.Table.Cols() { var err error @@ -466,12 +465,9 @@ func (e *InsertValues) fillRow(ctx context.Context, row []types.Datum, hasValue } } } - for _, gCol := range gCols { - var err error - var val types.Datum + for i, gCol := range gCols { colIdx := gCol.ColumnInfo.Offset - val, err = e.GenExprs[gIdx].Eval(chunk.MutRowFromDatums(row).ToRow()) - gIdx++ + val, err := e.GenExprs[i].Eval(chunk.MutRowFromDatums(row).ToRow()) if e.filterErr(err) != nil { return nil, err } diff --git a/server/server.go b/server/server.go index aca5125bef656..906ffdc48dec6 100644 --- a/server/server.go +++ b/server/server.go @@ -121,9 +121,8 @@ type Server struct { // ConnectionCount gets current connection count. func (s *Server) ConnectionCount() int { - var cnt int s.rwlock.RLock() - cnt = len(s.clients) + cnt := len(s.clients) s.rwlock.RUnlock() return cnt } @@ -294,7 +293,6 @@ func (s *Server) loadTLSCertificates() { Certificates: []tls.Certificate{tlsCert}, ClientCAs: certPool, ClientAuth: clientAuthPolicy, - MinVersion: 0, } } @@ -471,11 +469,9 @@ func (cc *clientConn) connectInfo() *variable.ConnectionInfo { ClientPort: cc.peerPort, ServerID: 1, ServerPort: int(cc.server.cfg.Port), - Duration: 0, User: cc.user, ServerOSLoginUser: osUser, OSVersion: osVersion, - ClientVersion: "", ServerVersion: mysql.TiDBReleaseVersion, SSLVersion: "v1.2.0", // for current go version PID: serverPID, From 576cdcdd0db99f76a29e9dd2eb8ea4a97b0ef3c7 Mon Sep 17 00:00:00 2001 From: Lynn Date: Thu, 1 Aug 2019 18:52:59 +0800 Subject: [PATCH 144/196] executor: fix data race of "admin check table" (#11568) --- executor/admin_test.go | 50 +++++++++++++++++++++--------------------- executor/executor.go | 11 ++++++++++ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/executor/admin_test.go b/executor/admin_test.go index b5109ebae6bc2..843d39a67c239 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -91,10 +91,10 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue) - _, err = tk.Exec("admin check index admin_test c2") + err = tk.ExecToErr("admin check index admin_test c2") c.Assert(err, NotNil) r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)") @@ -115,7 +115,7 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check index admin_test c2") + err = tk.ExecToErr("admin check index admin_test c2") c.Assert(err, NotNil) r = tk.MustQuery("admin recover index admin_test c2") r.Check(testkit.Rows("1 5")) @@ -137,9 +137,9 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test c2") + err = tk.ExecToErr("admin check index admin_test c2") c.Assert(err, NotNil) r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)") @@ -261,9 +261,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test c2") + err = tk.ExecToErr("admin check index admin_test c2") c.Assert(err, NotNil) r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)") r.Check(testkit.Rows("11")) @@ -273,9 +273,9 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) { r.Check(testkit.Rows("6")) tk.MustExec("admin check index admin_test c2") - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test c3") + err = tk.ExecToErr("admin check index admin_test c3") c.Assert(err, NotNil) r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c3)") r.Check(testkit.Rows("9")) @@ -322,9 +322,9 @@ func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test `primary`") + err = tk.ExecToErr("admin check index admin_test `primary`") c.Assert(err, NotNil) r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(`primary`)") r.Check(testkit.Rows("6")) @@ -374,11 +374,11 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) { err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test c1") + err = tk.ExecToErr("admin check index admin_test c1") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_test c2") + err = tk.ExecToErr("admin check index admin_test c2") c.Assert(err, NotNil) r := tk.MustQuery("SELECT COUNT(*) FROM admin_test") r.Check(testkit.Rows("3")) @@ -430,7 +430,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err.Error(), Equals, "[executor:8003]admin_test err:[admin:1]index: != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:-10, b:[]uint8(nil), x:interface {}(nil)}}}") c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue) @@ -447,7 +447,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:0, b:[]uint8(nil), x:interface {}(nil)} != record:") // Add one row of index. @@ -464,7 +464,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:13, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:12, b:[]uint8(nil), x:interface {}(nil)}") // Table count = index count. @@ -477,7 +477,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}") // Table count = index count. @@ -490,7 +490,7 @@ func (s *testSuite2) TestAdminCheckTableFailed(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_test") + err = tk.ExecToErr("admin check table admin_test") c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}") // Recover records. @@ -576,7 +576,7 @@ func (s *testSuite1) TestAdminCheckTable(c *C) { tk.MustExec("use mysql") tk.MustExec(`admin check table test.t;`) - _, err := tk.Exec("admin check table t") + err := tk.ExecToErr("admin check table t") c.Assert(err, NotNil) // test add index on time type column which have default value @@ -616,7 +616,7 @@ func (s *testSuite1) TestAdminCheckTable(c *C) { tk.MustExec(`drop table if exists t1`) tk.MustExec(`create table t1 (a decimal(2,1), index(a))`) tk.MustExec(`insert into t1 set a='1.9'`) - _, err = tk.Exec(`alter table t1 modify column a decimal(3,2);`) + err = tk.ExecToErr(`alter table t1 modify column a decimal(3,2);`) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify decimal column precision") tk.MustExec(`delete from t1;`) @@ -659,9 +659,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) { c.Assert(err, IsNil) err = txn.Commit(context.Background()) c.Assert(err, IsNil) - _, err = tk.Exec("admin check table admin_t_s") + err = tk.ExecToErr("admin check table admin_t_s") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_t_s a") + err = tk.ExecToErr("admin check index admin_t_s a") c.Assert(err, NotNil) // For mocktikv, safe point is not initialized, we manually insert it for snapshot to use. @@ -678,9 +678,9 @@ func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) { tk.MustExec("admin check index admin_t_s a;") tk.MustExec("set @@tidb_snapshot = ''") - _, err = tk.Exec("admin check table admin_t_s") + err = tk.ExecToErr("admin check table admin_t_s") c.Assert(err, NotNil) - _, err = tk.Exec("admin check index admin_t_s a") + err = tk.ExecToErr("admin check index admin_t_s a") c.Assert(err, NotNil) r := tk.MustQuery("admin cleanup index admin_t_s a") diff --git a/executor/executor.go b/executor/executor.go index 5c668dec7f18c..f6d3d336d6f31 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -480,6 +480,17 @@ func (e *CheckTableExec) Open(ctx context.Context) error { return nil } +// Close implements the Executor Close interface. +func (e *CheckTableExec) Close() error { + var firstErr error + for _, src := range e.srcs { + if err := src.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + func (e *CheckTableExec) checkIndexHandle(ctx context.Context, num int, src *IndexLookUpExecutor) error { cols := src.schema.Columns retFieldTypes := make([]*types.FieldType, len(cols)) From ea16273d44ca25f6a2cfa5e6477608a67719f6c6 Mon Sep 17 00:00:00 2001 From: Lynn Date: Fri, 2 Aug 2019 10:50:07 +0800 Subject: [PATCH 145/196] executor: fix baseExecutor'children may not close (#11570) --- executor/executor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index f6d3d336d6f31..7a8b020117811 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -105,13 +105,13 @@ func (e *baseExecutor) Open(ctx context.Context) error { // Close closes all executors and release all resources. func (e *baseExecutor) Close() error { - for _, child := range e.children { - err := child.Close() - if err != nil { - return err + var firstErr error + for _, src := range e.children { + if err := src.Close(); err != nil && firstErr == nil { + firstErr = err } } - return nil + return firstErr } // Schema returns the current baseExecutor's schema. If it is nil, then create and return a new one. From 479db57ba0a60a2580491ea0707df6d2cff63097 Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 2 Aug 2019 11:34:13 +0800 Subject: [PATCH 146/196] types/time: optimize time convertDateFormat function by remove fprintf (#11523) --- types/time.go | 62 ++++++++++++++++++++++++++++------------------ types/time_test.go | 31 +++++++++++++++++++++++ 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/types/time.go b/types/time.go index b44c261597521..72373ee61ff57 100644 --- a/types/time.go +++ b/types/time.go @@ -1987,37 +1987,38 @@ func (t Time) convertDateFormat(b rune, buf *bytes.Buffer) error { } buf.WriteString(MonthNames[m-1]) case 'm': - fmt.Fprintf(buf, "%02d", t.Time.Month()) + buf.WriteString(FormatIntWidthN(t.Time.Month(), 2)) case 'c': - fmt.Fprintf(buf, "%d", t.Time.Month()) + buf.WriteString(strconv.FormatInt(int64(t.Time.Month()), 10)) case 'D': - fmt.Fprintf(buf, "%d%s", t.Time.Day(), abbrDayOfMonth(t.Time.Day())) + buf.WriteString(strconv.FormatInt(int64(t.Time.Day()), 10)) + buf.WriteString(abbrDayOfMonth(t.Time.Day())) case 'd': - fmt.Fprintf(buf, "%02d", t.Time.Day()) + buf.WriteString(FormatIntWidthN(t.Time.Day(), 2)) case 'e': - fmt.Fprintf(buf, "%d", t.Time.Day()) + buf.WriteString(strconv.FormatInt(int64(t.Time.Day()), 10)) case 'j': fmt.Fprintf(buf, "%03d", t.Time.YearDay()) case 'H': - fmt.Fprintf(buf, "%02d", t.Time.Hour()) + buf.WriteString(FormatIntWidthN(t.Time.Hour(), 2)) case 'k': - fmt.Fprintf(buf, "%d", t.Time.Hour()) + buf.WriteString(strconv.FormatInt(int64(t.Time.Hour()), 10)) case 'h', 'I': t := t.Time.Hour() if t%12 == 0 { - fmt.Fprintf(buf, "%02d", 12) + buf.WriteString("12") } else { - fmt.Fprintf(buf, "%02d", t%12) + buf.WriteString(FormatIntWidthN(t%12, 2)) } case 'l': t := t.Time.Hour() if t%12 == 0 { - fmt.Fprintf(buf, "%d", 12) + buf.WriteString("12") } else { - fmt.Fprintf(buf, "%d", t%12) + buf.WriteString(strconv.FormatInt(int64(t%12), 10)) } case 'i': - fmt.Fprintf(buf, "%02d", t.Time.Minute()) + buf.WriteString(FormatIntWidthN(t.Time.Minute(), 2)) case 'p': hour := t.Time.Hour() if hour/12%2 == 0 { @@ -2041,46 +2042,46 @@ func (t Time) convertDateFormat(b rune, buf *bytes.Buffer) error { case 'T': fmt.Fprintf(buf, "%02d:%02d:%02d", t.Time.Hour(), t.Time.Minute(), t.Time.Second()) case 'S', 's': - fmt.Fprintf(buf, "%02d", t.Time.Second()) + buf.WriteString(FormatIntWidthN(t.Time.Second(), 2)) case 'f': fmt.Fprintf(buf, "%06d", t.Time.Microsecond()) case 'U': w := t.Time.Week(0) - fmt.Fprintf(buf, "%02d", w) + buf.WriteString(FormatIntWidthN(w, 2)) case 'u': w := t.Time.Week(1) - fmt.Fprintf(buf, "%02d", w) + buf.WriteString(FormatIntWidthN(w, 2)) case 'V': w := t.Time.Week(2) - fmt.Fprintf(buf, "%02d", w) + buf.WriteString(FormatIntWidthN(w, 2)) case 'v': _, w := t.Time.YearWeek(3) - fmt.Fprintf(buf, "%02d", w) + buf.WriteString(FormatIntWidthN(w, 2)) case 'a': weekday := t.Time.Weekday() buf.WriteString(abbrevWeekdayName[weekday]) case 'W': buf.WriteString(t.Time.Weekday().String()) case 'w': - fmt.Fprintf(buf, "%d", t.Time.Weekday()) + buf.WriteString(strconv.FormatInt(int64(t.Time.Weekday()), 10)) case 'X': year, _ := t.Time.YearWeek(2) if year < 0 { - fmt.Fprintf(buf, "%v", uint64(math.MaxUint32)) + buf.WriteString(strconv.FormatUint(uint64(math.MaxUint32), 10)) } else { - fmt.Fprintf(buf, "%04d", year) + buf.WriteString(FormatIntWidthN(year, 4)) } case 'x': year, _ := t.Time.YearWeek(3) if year < 0 { - fmt.Fprintf(buf, "%v", uint64(math.MaxUint32)) + buf.WriteString(strconv.FormatUint(uint64(math.MaxUint32), 10)) } else { - fmt.Fprintf(buf, "%04d", year) + buf.WriteString(FormatIntWidthN(year, 4)) } case 'Y': - fmt.Fprintf(buf, "%04d", t.Time.Year()) + buf.WriteString(FormatIntWidthN(t.Time.Year(), 4)) case 'y': - str := fmt.Sprintf("%04d", t.Time.Year()) + str := FormatIntWidthN(t.Time.Year(), 4) buf.WriteString(str[2:]) default: buf.WriteRune(b) @@ -2089,6 +2090,19 @@ func (t Time) convertDateFormat(b rune, buf *bytes.Buffer) error { return nil } +// FormatIntWidthN uses to format int with width. Insufficient digits are filled by 0. +func FormatIntWidthN(num, n int) string { + numString := strconv.FormatInt(int64(num), 10) + if len(numString) >= n { + return numString + } + padBytes := make([]byte, n-len(numString)) + for i := range padBytes { + padBytes[i] = '0' + } + return string(padBytes) + numString +} + func abbrDayOfMonth(day int) string { var str string switch day { diff --git a/types/time_test.go b/types/time_test.go index 57c6cca351d6f..eb3283054dd8e 100644 --- a/types/time_test.go +++ b/types/time_test.go @@ -15,6 +15,7 @@ package types_test import ( "math" + "testing" "time" . "github.com/pingcap/check" @@ -1640,3 +1641,33 @@ func (s *testTimeSuite) TestCheckMonthDay(c *C) { } } } +func (s *testTimeSuite) TestFormatIntWidthN(c *C) { + cases := []struct { + num int + width int + result string + }{ + {0, 0, "0"}, + {1, 0, "1"}, + {1, 1, "1"}, + {1, 2, "01"}, + {10, 2, "10"}, + {99, 3, "099"}, + {100, 3, "100"}, + {999, 3, "999"}, + {1000, 3, "1000"}, + } + for _, ca := range cases { + re := types.FormatIntWidthN(ca.num, ca.width) + c.Assert(re, Equals, ca.result) + } +} + +func BenchmarkFormat(b *testing.B) { + var t1 types.Time + t1.Type = mysql.TypeTimestamp + t1.Time = types.FromGoTime(time.Now()) + for i := 0; i < b.N; i++ { + t1.DateFormat("%Y-%m-%d %H:%i:%s") + } +} From b493fb63d4e65f885468c3347e28907390eec0a6 Mon Sep 17 00:00:00 2001 From: lysu Date: Fri, 2 Aug 2019 14:13:12 +0800 Subject: [PATCH 147/196] tikv: invalidate store's regions when store be removed in kv (#11567) --- store/tikv/region_cache.go | 7 +++++++ store/tikv/region_cache_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index 04950d0ccafc6..7f398af45a97e 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -49,6 +49,7 @@ var ( tikvRegionCacheCounterWithScanRegionsError = metrics.TiKVRegionCacheCounter.WithLabelValues("scan_regions", "err") tikvRegionCacheCounterWithGetStoreOK = metrics.TiKVRegionCacheCounter.WithLabelValues("get_store", "ok") tikvRegionCacheCounterWithGetStoreError = metrics.TiKVRegionCacheCounter.WithLabelValues("get_store", "err") + tikvRegionCacheCounterWithInvalidateStoreRegionsOK = metrics.TiKVRegionCacheCounter.WithLabelValues("invalidate_store_regions", "ok") ) const ( @@ -967,6 +968,7 @@ func (c *RegionCache) switchNextPeer(r *Region, currentPeerIdx int, err error) { epoch := rs.storeFails[rs.workStoreIdx] if atomic.CompareAndSwapUint32(&s.fail, epoch, epoch+1) { logutil.BgLogger().Info("mark store's regions need be refill", zap.String("store", s.addr)) + tikvRegionCacheCounterWithInvalidateStoreRegionsOK.Inc() } } @@ -1099,6 +1101,11 @@ func (s *Store) reResolve(c *RegionCache) { return } if store == nil { + // store has be removed in PD, we should invalidate all regions using those store. + logutil.BgLogger().Info("invalidate regions in removed store", + zap.Uint64("store", s.storeID), zap.String("add", s.addr)) + atomic.AddUint32(&s.fail, 1) + tikvRegionCacheCounterWithInvalidateStoreRegionsOK.Inc() return } diff --git a/store/tikv/region_cache_test.go b/store/tikv/region_cache_test.go index 68788fd6c897a..4d9ebecd768f2 100644 --- a/store/tikv/region_cache_test.go +++ b/store/tikv/region_cache_test.go @@ -483,6 +483,35 @@ func (s *testRegionCacheSuite) TestUpdateStoreAddr(c *C) { c.Assert(getVal, BytesEquals, testValue) } +func (s *testRegionCacheSuite) TestReplaceAddrWithNewStore(c *C) { + mvccStore := mocktikv.MustNewMVCCStore() + defer mvccStore.Close() + + client := &RawKVClient{ + clusterID: 0, + regionCache: NewRegionCache(mocktikv.NewPDClient(s.cluster)), + rpcClient: mocktikv.NewRPCClient(s.cluster, mvccStore), + } + defer client.Close() + testKey := []byte("test_key") + testValue := []byte("test_value") + err := client.Put(testKey, testValue) + c.Assert(err, IsNil) + + // make store2 using store1's addr and store1 offline + store1Addr := s.storeAddr(s.store1) + s.cluster.UpdateStoreAddr(s.store1, s.storeAddr(s.store2)) + s.cluster.UpdateStoreAddr(s.store2, store1Addr) + s.cluster.RemoveStore(s.store1) + s.cluster.ChangeLeader(s.region1, s.peer2) + s.cluster.RemovePeer(s.region1, s.store1) + + getVal, err := client.Get(testKey) + + c.Assert(err, IsNil) + c.Assert(getVal, BytesEquals, testValue) +} + func (s *testRegionCacheSuite) TestListRegionIDsInCache(c *C) { // ['' - 'm' - 'z'] region2 := s.cluster.AllocID() From fcae7fa62ca68501a239ac09101776f96893d337 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Fri, 2 Aug 2019 14:19:42 +0800 Subject: [PATCH 148/196] store/tikv: make test stable by running `TestPanicInRecvLoop` unparallel with others (#11572) --- store/tikv/client_fail_test.go | 15 ++++++++++++++- store/tikv/client_test.go | 1 + store/tikv/ticlient_test.go | 2 +- store/tikv/tikv_test.go | 11 +++++++---- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/store/tikv/client_fail_test.go b/store/tikv/client_fail_test.go index 198bd7e889906..7eab9ecfc3395 100644 --- a/store/tikv/client_fail_test.go +++ b/store/tikv/client_fail_test.go @@ -25,13 +25,26 @@ import ( "github.com/pingcap/tidb/store/tikv/tikvrpc" ) +type testClientFailSuite struct { + OneByOneSuite +} + +func (s *testClientFailSuite) SetUpSuite(c *C) { + // This lock make testClientFailSuite runs exclusively. + withTiKVGlobalLock.Lock() +} + +func (s testClientFailSuite) TearDownSuite(c *C) { + withTiKVGlobalLock.Unlock() +} + func setGrpcConnectionCount(count uint) { newConf := config.NewConfig() newConf.TiKVClient.GrpcConnectionCount = count config.StoreGlobalConfig(newConf) } -func (s *testClientSuite) TestPanicInRecvLoop(c *C) { +func (s *testClientFailSuite) TestPanicInRecvLoop(c *C) { c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/panicInFailPendingRequests", `panic`), IsNil) c.Assert(failpoint.Enable("github.com/pingcap/tidb/store/tikv/gotErrorInRecvLoop", `return("0")`), IsNil) diff --git a/store/tikv/client_test.go b/store/tikv/client_test.go index 00bc0bbee0f95..1f3e4abf19582 100644 --- a/store/tikv/client_test.go +++ b/store/tikv/client_test.go @@ -36,6 +36,7 @@ type testClientSuite struct { } var _ = Suite(&testClientSuite{}) +var _ = Suite(&testClientFailSuite{}) func setMaxBatchSize(size uint) { newConf := config.NewConfig() diff --git a/store/tikv/ticlient_test.go b/store/tikv/ticlient_test.go index 6662bb9a8ad20..44ca2d40f7f41 100644 --- a/store/tikv/ticlient_test.go +++ b/store/tikv/ticlient_test.go @@ -28,7 +28,7 @@ import ( ) var ( - withTiKVGlobalLock sync.Mutex + withTiKVGlobalLock sync.RWMutex withTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") ) diff --git a/store/tikv/tikv_test.go b/store/tikv/tikv_test.go index a4db0b7df60ec..d4a5bfe1d6a6c 100644 --- a/store/tikv/tikv_test.go +++ b/store/tikv/tikv_test.go @@ -18,18 +18,21 @@ import ( ) // OneByOneSuite is a suite, When with-tikv flag is true, there is only one storage, so the test suite have to run one by one. -type OneByOneSuite struct { -} +type OneByOneSuite struct{} -func (s OneByOneSuite) SetUpSuite(c *C) { +func (s *OneByOneSuite) SetUpSuite(c *C) { if *withTiKV { withTiKVGlobalLock.Lock() + } else { + withTiKVGlobalLock.RLock() } } -func (s OneByOneSuite) TearDownSuite(c *C) { +func (s *OneByOneSuite) TearDownSuite(c *C) { if *withTiKV { withTiKVGlobalLock.Unlock() + } else { + withTiKVGlobalLock.RUnlock() } } From d2b5387183a591847f952aa4e3a3c4597c34dcba Mon Sep 17 00:00:00 2001 From: crazycs Date: Fri, 2 Aug 2019 15:03:24 +0800 Subject: [PATCH 149/196] infoschema: fix load drop database schema bug and refine db-table api error. (#11573) --- infoschema/builder.go | 12 ++++++------ infoschema/tables_test.go | 20 ++++++++++++++++++++ server/http_handler.go | 4 ++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/infoschema/builder.go b/infoschema/builder.go index 7b90e1f23cd12..72d0d9ce29e72 100644 --- a/infoschema/builder.go +++ b/infoschema/builder.go @@ -154,22 +154,22 @@ func (b *Builder) applyDropSchema(schemaID int64) []int64 { delete(b.is.schemaMap, di.Name.L) // Copy the sortedTables that contain the table we are going to drop. + tableIDs := make([]int64, 0, len(di.Tables)) bucketIdxMap := make(map[int]struct{}) for _, tbl := range di.Tables { bucketIdxMap[tableBucketIdx(tbl.ID)] = struct{}{} + // TODO: If the table ID doesn't exist. + tableIDs = append(tableIDs, tbl.ID) } for bucketIdx := range bucketIdxMap { b.copySortedTablesBucket(bucketIdx) } - ids := make([]int64, 0, len(di.Tables)) di = di.Clone() - for _, tbl := range di.Tables { - b.applyDropTable(di, tbl.ID) - // TODO: If the table ID doesn't exist. - ids = append(ids, tbl.ID) + for _, id := range tableIDs { + b.applyDropTable(di, id) } - return ids + return tableIDs } func (b *Builder) copySortedTablesBucket(bucketIdx int) { diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index c2c7c84cb1ea5..13fa50cf8d15b 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -22,7 +22,9 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/parser/auth" + "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/infoschema" "github.com/pingcap/tidb/kv" @@ -525,3 +527,21 @@ func (s *testTableSuite) TestColumnStatistics(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustQuery("select * from information_schema.column_statistics").Check(testkit.Rows()) } + +func (s *testTableSuite) TestReloadDropDatabase(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database test_dbs") + tk.MustExec("use test_dbs") + tk.MustExec("create table t1 (a int)") + tk.MustExec("create table t2 (a int)") + tk.MustExec("create table t3 (a int)") + is := domain.GetDomain(tk.Se).InfoSchema() + t2, err := is.TableByName(model.NewCIStr("test_dbs"), model.NewCIStr("t2")) + c.Assert(err, IsNil) + tk.MustExec("drop database test_dbs") + is = domain.GetDomain(tk.Se).InfoSchema() + _, err = is.TableByName(model.NewCIStr("test_dbs"), model.NewCIStr("t2")) + c.Assert(terror.ErrorEqual(infoschema.ErrTableNotExists, err), IsTrue) + _, ok := is.TableByID(t2.Meta().ID) + c.Assert(ok, IsFalse) +} diff --git a/server/http_handler.go b/server/http_handler.go index 446cd027b3a84..3c59ea613901e 100644 --- a/server/http_handler.go +++ b/server/http_handler.go @@ -1531,8 +1531,8 @@ func (h dbTableHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { dbTblInfo.TableInfo = tbl.Meta() dbInfo, ok := schema.SchemaByTable(dbTblInfo.TableInfo) if !ok { - log.Warnf("can not find the database of table id: %v, table name: %v", dbTblInfo.TableInfo.ID, dbTblInfo.TableInfo.Name) - writeData(w, dbTblInfo) + logutil.BgLogger().Error("can not find the database of the table", zap.Int64("table id", dbTblInfo.TableInfo.ID), zap.String("table name", dbTblInfo.TableInfo.Name.L)) + writeError(w, infoschema.ErrTableNotExists.GenWithStack("Table which ID = %s does not exist.", tableID)) return } dbTblInfo.DBInfo = dbInfo From 129c0afdc6f96fd99b5f5b14485ba2e14c929d1b Mon Sep 17 00:00:00 2001 From: Tanner Date: Fri, 2 Aug 2019 16:50:10 +0800 Subject: [PATCH 150/196] executor, infoschema: fix display of on update CURRENT_TIMESTAMP with decimal (#11480) --- ddl/db_integration_test.go | 8 ++++++++ ddl/ddl_api.go | 4 ++-- executor/show.go | 1 + executor/show_test.go | 10 ++++++++-- expression/helper.go | 22 +++++++++++++++++----- expression/helper_test.go | 24 +++++++++++++++++++++--- infoschema/tables_test.go | 24 ++++++++++++++---------- table/column.go | 12 +++++++++++- 8 files changed, 82 insertions(+), 23 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index 477961ffb7320..ce8fde13972f8 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -416,6 +416,14 @@ func (s *testIntegrationSuite5) TestMySQLErrorCode(c *C) { assertErrorCode(c, tk, sql, tmysql.ErrPrimaryCantHaveNull) sql = "create table t2 (id int auto_increment);" assertErrorCode(c, tk, sql, tmysql.ErrWrongAutoKey) + sql = "create table t2 (a datetime(2) default current_timestamp(3))" + assertErrorCode(c, tk, sql, tmysql.ErrInvalidDefault) + sql = "create table t2 (a datetime(2) default current_timestamp(2) on update current_timestamp)" + assertErrorCode(c, tk, sql, tmysql.ErrInvalidOnUpdate) + sql = "create table t2 (a datetime default current_timestamp on update current_timestamp(2))" + assertErrorCode(c, tk, sql, tmysql.ErrInvalidOnUpdate) + sql = "create table t2 (a datetime(2) default current_timestamp(2) on update current_timestamp(3))" + assertErrorCode(c, tk, sql, tmysql.ErrInvalidOnUpdate) sql = "create table t2 (id int primary key , age int);" tk.MustExec(sql) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index 62f7d56444b8a..bfbc6c8da13bb 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -540,7 +540,7 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o case ast.ColumnOptionOnUpdate: // TODO: Support other time functions. if col.Tp == mysql.TypeTimestamp || col.Tp == mysql.TypeDatetime { - if !expression.IsCurrentTimestampExpr(v.Expr) { + if !expression.IsValidCurrentTimestampExpr(v.Expr, colDef.Tp) { return nil, nil, ErrInvalidOnUpdate.GenWithStackByArgs(col.Name) } } else { @@ -2532,7 +2532,7 @@ func processColumnOptions(ctx sessionctx.Context, col *table.Column, options []* case ast.ColumnOptionOnUpdate: // TODO: Support other time functions. if col.Tp == mysql.TypeTimestamp || col.Tp == mysql.TypeDatetime { - if !expression.IsCurrentTimestampExpr(opt.Expr) { + if !expression.IsValidCurrentTimestampExpr(opt.Expr, &col.FieldType) { return ErrInvalidOnUpdate.GenWithStackByArgs(col.Name) } } else { diff --git a/executor/show.go b/executor/show.go index c97e15e2b7a6d..d28fbbf1e70b5 100644 --- a/executor/show.go +++ b/executor/show.go @@ -721,6 +721,7 @@ func (e *ShowExec) fetchShowCreateTable() error { } if mysql.HasOnUpdateNowFlag(col.Flag) { buf.WriteString(" ON UPDATE CURRENT_TIMESTAMP") + buf.WriteString(table.OptionalFsp(&col.FieldType)) } } if len(col.Comment) > 0 { diff --git a/executor/show_test.go b/executor/show_test.go index c7306f63cd44d..6d0eefe75e991 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -198,6 +198,7 @@ func (s *testSuite2) TestShow2(c *C) { c_timestamp timestamp, c_timestamp_default timestamp default current_timestamp, c_timestamp_default_3 timestamp(3) default current_timestamp(3), + c_timestamp_default_4 timestamp(3) default current_timestamp(3) on update current_timestamp(3), c_blob blob, c_tinyblob tinyblob, c_mediumblob mediumblob, @@ -233,6 +234,7 @@ func (s *testSuite2) TestShow2(c *C) { "[c_timestamp timestamp YES select,insert,update,references ]\n" + "[c_timestamp_default timestamp YES CURRENT_TIMESTAMP select,insert,update,references ]\n" + "[c_timestamp_default_3 timestamp(3) YES CURRENT_TIMESTAMP(3) select,insert,update,references ]\n" + + "[c_timestamp_default_4 timestamp(3) YES CURRENT_TIMESTAMP(3) DEFAULT_GENERATED on update CURRENT_TIMESTAMP(3) select,insert,update,references ]\n" + "[c_blob blob YES select,insert,update,references ]\n" + "[c_tinyblob tinyblob YES select,insert,update,references ]\n" + "[c_mediumblob mediumblob YES select,insert,update,references ]\n" + @@ -488,7 +490,9 @@ func (s *testSuite2) TestShowCreateTable(c *C) { "`b` timestamp(3) default current_timestamp(3),\n" + "`c` datetime default current_timestamp,\n" + "`d` datetime(4) default current_timestamp(4),\n" + - "`e` varchar(20) default 'cUrrent_tImestamp')") + "`e` varchar(20) default 'cUrrent_tImestamp',\n" + + "`f` datetime(2) default current_timestamp(2) on update current_timestamp(2),\n" + + "`g` timestamp(2) default current_timestamp(2) on update current_timestamp(2))") tk.MustQuery("show create table `t`").Check(testutil.RowsWithSep("|", ""+ "t CREATE TABLE `t` (\n"+ @@ -496,7 +500,9 @@ func (s *testSuite2) TestShowCreateTable(c *C) { " `b` timestamp(3) DEFAULT CURRENT_TIMESTAMP(3),\n"+ " `c` datetime DEFAULT CURRENT_TIMESTAMP,\n"+ " `d` datetime(4) DEFAULT CURRENT_TIMESTAMP(4),\n"+ - " `e` varchar(20) DEFAULT 'cUrrent_tImestamp'\n"+ + " `e` varchar(20) DEFAULT 'cUrrent_tImestamp',\n"+ + " `f` datetime(2) DEFAULT CURRENT_TIMESTAMP(2) ON UPDATE CURRENT_TIMESTAMP(2),\n"+ + " `g` timestamp(2) DEFAULT CURRENT_TIMESTAMP(2) ON UPDATE CURRENT_TIMESTAMP(2)\n"+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin", )) tk.MustExec("drop table t") diff --git a/expression/helper.go b/expression/helper.go index 7b0f49e71a71d..f5268a6619107 100644 --- a/expression/helper.go +++ b/expression/helper.go @@ -34,12 +34,24 @@ func boolToInt64(v bool) int64 { return 0 } -// IsCurrentTimestampExpr returns whether e is CurrentTimestamp expression. -func IsCurrentTimestampExpr(e ast.ExprNode) bool { - if fn, ok := e.(*ast.FuncCallExpr); ok && fn.FnName.L == ast.CurrentTimestamp { - return true +// IsValidCurrentTimestampExpr returns true if exprNode is a valid CurrentTimestamp expression. +// Here `valid` means it is consistent with the given fieldType's Decimal. +func IsValidCurrentTimestampExpr(exprNode ast.ExprNode, fieldType *types.FieldType) bool { + fn, isFuncCall := exprNode.(*ast.FuncCallExpr) + if !isFuncCall || fn.FnName.L != ast.CurrentTimestamp { + return false } - return false + + containsArg := len(fn.Args) > 0 + // Fsp represents fractional seconds precision. + containsFsp := fieldType != nil && fieldType.Decimal > 0 + var isConsistent bool + if containsArg { + v, ok := fn.Args[0].(*driver.ValueExpr) + isConsistent = ok && fieldType != nil && v.Datum.GetInt64() == int64(fieldType.Decimal) + } + + return (containsArg && isConsistent) || (!containsArg && !containsFsp) } // GetTimeValue gets the time value with type tp. diff --git a/expression/helper_test.go b/expression/helper_test.go index e0acecfdf817b..21db9667f1063 100644 --- a/expression/helper_test.go +++ b/expression/helper_test.go @@ -14,6 +14,7 @@ package expression import ( + driver "github.com/pingcap/tidb/types/parser_driver" "strings" "time" @@ -108,11 +109,28 @@ func (s *testExpressionSuite) TestGetTimeValue(c *C) { func (s *testExpressionSuite) TestIsCurrentTimestampExpr(c *C) { defer testleak.AfterTest(c)() - v := IsCurrentTimestampExpr(ast.NewValueExpr("abc")) - c.Assert(v, IsFalse) + buildTimestampFuncCallExpr := func(i int64) *ast.FuncCallExpr { + var args []ast.ExprNode + if i != 0 { + args = []ast.ExprNode{&driver.ValueExpr{Datum: types.NewIntDatum(i)}} + } + return &ast.FuncCallExpr{FnName: model.NewCIStr("CURRENT_TIMESTAMP"), Args: args} + } - v = IsCurrentTimestampExpr(&ast.FuncCallExpr{FnName: model.NewCIStr("CURRENT_TIMESTAMP")}) + v := IsValidCurrentTimestampExpr(ast.NewValueExpr("abc"), nil) + c.Assert(v, IsFalse) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(0), nil) + c.Assert(v, IsTrue) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(3), &types.FieldType{Decimal: 3}) c.Assert(v, IsTrue) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(1), &types.FieldType{Decimal: 3}) + c.Assert(v, IsFalse) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(0), &types.FieldType{Decimal: 3}) + c.Assert(v, IsFalse) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(2), &types.FieldType{Decimal: 0}) + c.Assert(v, IsFalse) + v = IsValidCurrentTimestampExpr(buildTimestampFuncCallExpr(2), nil) + c.Assert(v, IsFalse) } func (s *testExpressionSuite) TestCurrentTimestampTimeZone(c *C) { diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index 13fa50cf8d15b..020cfd5c80551 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -274,23 +274,27 @@ func (s *testTableSuite) TestCurrentTimestampAsDefault(c *C) { c_timestamp_default_3 timestamp(3) default current_timestamp(3), c_varchar_default varchar(20) default "current_timestamp", c_varchar_default_3 varchar(20) default "current_timestamp(3)", + c_varchar_default_on_update datetime default current_timestamp on update current_timestamp, + c_varchar_default_on_update_fsp datetime(3) default current_timestamp(3) on update current_timestamp(3), c_varchar_default_with_case varchar(20) default "cUrrent_tImestamp" );`) - tk.MustQuery(`SELECT column_name, column_default + tk.MustQuery(`SELECT column_name, column_default, extra FROM information_schema.COLUMNS WHERE table_schema = "default_time_test" AND table_name = "default_time_table" ORDER BY column_name`, ).Check(testkit.Rows( - "c_datetime ", - "c_datetime_default CURRENT_TIMESTAMP", - "c_datetime_default_2 CURRENT_TIMESTAMP(2)", - "c_timestamp ", - "c_timestamp_default CURRENT_TIMESTAMP", - "c_timestamp_default_3 CURRENT_TIMESTAMP(3)", - "c_varchar_default current_timestamp", - "c_varchar_default_3 current_timestamp(3)", - "c_varchar_default_with_case cUrrent_tImestamp", + "c_datetime ", + "c_datetime_default CURRENT_TIMESTAMP ", + "c_datetime_default_2 CURRENT_TIMESTAMP(2) ", + "c_timestamp ", + "c_timestamp_default CURRENT_TIMESTAMP ", + "c_timestamp_default_3 CURRENT_TIMESTAMP(3) ", + "c_varchar_default current_timestamp ", + "c_varchar_default_3 current_timestamp(3) ", + "c_varchar_default_on_update CURRENT_TIMESTAMP DEFAULT_GENERATED on update CURRENT_TIMESTAMP", + "c_varchar_default_on_update_fsp CURRENT_TIMESTAMP(3) DEFAULT_GENERATED on update CURRENT_TIMESTAMP(3)", + "c_varchar_default_with_case cUrrent_tImestamp ", )) tk.MustExec("DROP DATABASE default_time_test") } diff --git a/table/column.go b/table/column.go index 1ebacaf51876a..21c961dbbfc0a 100644 --- a/table/column.go +++ b/table/column.go @@ -19,6 +19,7 @@ package table import ( "fmt" + "strconv" "strings" "time" "unicode/utf8" @@ -271,7 +272,7 @@ func NewColDesc(col *Column) *ColDesc { } else if mysql.HasOnUpdateNowFlag(col.Flag) { //in order to match the rules of mysql 8.0.16 version //see https://github.com/pingcap/tidb/issues/10337 - extra = "DEFAULT_GENERATED on update CURRENT_TIMESTAMP" + extra = "DEFAULT_GENERATED on update CURRENT_TIMESTAMP" + OptionalFsp(&col.FieldType) } else if col.IsGenerated() { if col.GeneratedStored { extra = "STORED GENERATED" @@ -490,3 +491,12 @@ func GetZeroValue(col *model.ColumnInfo) types.Datum { } return d } + +// OptionalFsp convert a FieldType.Decimal to string. +func OptionalFsp(fieldType *types.FieldType) string { + fsp := fieldType.Decimal + if fsp == 0 { + return "" + } + return "(" + strconv.Itoa(fsp) + ")" +} From cffd04518cebf0a130166dd4046b4d915864398f Mon Sep 17 00:00:00 2001 From: SunRunAway Date: Fri, 2 Aug 2019 17:07:43 +0800 Subject: [PATCH 151/196] executor: no need to wait for fetchInnerRows if buildHashTableForList fails, and clean code (#11571) --- executor/builder.go | 1 - executor/executor.go | 2 +- executor/join.go | 45 ++++++++++++++++---------------------------- executor/joiner.go | 18 +++++++++--------- executor/pkg_test.go | 1 - 5 files changed, 26 insertions(+), 41 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index d2192c66512a7..ffa4a8318e4de 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -990,7 +990,6 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo concurrency: v.Concurrency, joinType: v.JoinType, isOuterJoin: v.JoinType.IsOuterJoin(), - innerIdx: v.InnerChildIdx, } defaultValues := v.DefaultValues diff --git a/executor/executor.go b/executor/executor.go index 7a8b020117811..911b7005f88f9 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -134,7 +134,7 @@ func retTypes(e Executor) []*types.FieldType { return base.retFieldTypes } -// Next fills mutiple rows into a chunk. +// Next fills multiple rows into a chunk. func (e *baseExecutor) Next(ctx context.Context, req *chunk.Chunk) error { return nil } diff --git a/executor/join.go b/executor/join.go index d6f8e9042ca4a..50f444ad82e29 100644 --- a/executor/join.go +++ b/executor/join.go @@ -60,7 +60,6 @@ type HashJoinExec struct { // closeCh add a lock for closing executor. closeCh chan struct{} joinType plannercore.JoinType - innerIdx int isOuterJoin bool requiredRows int64 @@ -262,37 +261,31 @@ var innerResultLabel fmt.Stringer = stringutil.StringerStr("innerResult") // fetchInnerRows fetches all rows from inner executor, // and append them to e.innerResult. -func (e *HashJoinExec) fetchInnerRows(ctx context.Context, chkCh chan<- *chunk.Chunk, doneCh chan struct{}) { +func (e *HashJoinExec) fetchInnerRows(ctx context.Context, chkCh chan<- *chunk.Chunk, doneCh <-chan struct{}) { defer close(chkCh) e.innerResult = chunk.NewList(e.innerExec.base().retFieldTypes, e.initCap, e.maxChunkSize) e.innerResult.GetMemTracker().AttachTo(e.memTracker) e.innerResult.GetMemTracker().SetLabel(innerResultLabel) var err error for { + if e.finished.Load().(bool) { + return + } + chk := chunk.NewChunkWithCapacity(e.innerExec.base().retFieldTypes, e.ctx.GetSessionVars().MaxChunkSize) + err = e.innerExec.Next(ctx, chk) + if err != nil { + e.innerFinished <- errors.Trace(err) + return + } + if chk.NumRows() == 0 { + return + } select { case <-doneCh: return case <-e.closeCh: return - default: - if e.finished.Load().(bool) { - return - } - chk := chunk.NewChunkWithCapacity(e.children[e.innerIdx].base().retFieldTypes, e.ctx.GetSessionVars().MaxChunkSize) - err = e.innerExec.Next(ctx, chk) - if err != nil { - e.innerFinished <- errors.Trace(err) - return - } - if chk.NumRows() == 0 { - return - } - select { - case chkCh <- chk: - break - case <-e.closeCh: - return - } + case chkCh <- chk: e.innerResult.Add(chk) } } @@ -518,9 +511,6 @@ func (e *HashJoinExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { atomic.StoreInt64(&e.requiredRows, int64(req.RequiredRows())) } req.Reset() - if e.joinResultCh == nil { - return nil - } result, ok := <-e.joinResultCh if !ok { @@ -552,17 +542,14 @@ func (e *HashJoinExec) fetchInnerAndBuildHashTable(ctx context.Context) { err := e.buildHashTableForList(innerResultCh) if err != nil { e.innerFinished <- errors.Trace(err) - close(doneCh) - } - // wait fetchInnerRows be finished. - for range innerResultCh { } + close(doneCh) } // buildHashTableForList builds hash table from `list`. // key of hash table: hash value of key columns // value of hash table: RowPtr of the corresponded row -func (e *HashJoinExec) buildHashTableForList(innerResultCh chan *chunk.Chunk) error { +func (e *HashJoinExec) buildHashTableForList(innerResultCh <-chan *chunk.Chunk) error { e.hashTable = mvmap.NewMVMap() e.innerKeyColIdx = make([]int, len(e.innerKeys)) for i := range e.innerKeys { diff --git a/executor/joiner.go b/executor/joiner.go index 75f3b39a1f670..2f04fd91c05ce 100644 --- a/executor/joiner.go +++ b/executor/joiner.go @@ -53,13 +53,13 @@ type joiner interface { // rows are appended to `chk`. The size of `chk` is limited to MaxChunkSize. // Note that when the outer row is considered unmatched, we need to differentiate // whether the join conditions return null or false, because that matters for - // AntiSemiJoin/LeftOuterSemiJoin/AntiLeftOuterSemijoin, and the result is reflected + // AntiSemiJoin/LeftOuterSemiJoin/AntiLeftOuterSemiJoin, and the result is reflected // by the second return value; for other join types, we always return false. // // NOTE: Callers need to call this function multiple times to consume all - // the inner rows for an outer row, and dicide whether the outer row can be + // the inner rows for an outer row, and decide whether the outer row can be // matched with at lease one inner row. - tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (bool, bool, error) + tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (matched bool, isNull bool, err error) // onMissMatch operates on the unmatched outer row according to the join // type. An outer row can be considered miss matched if: @@ -354,7 +354,7 @@ type leftOuterJoiner struct { } // tryToMatch implements joiner interface. -func (j *leftOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (bool, bool, error) { +func (j *leftOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (matched bool, hasNull bool, err error) { if inners.Len() == 0 { return false, false, nil } @@ -374,7 +374,7 @@ func (j *leftOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk } // reach here, chkForJoin is j.chk - matched, err := j.filter(chkForJoin, chk, outer.Len()) + matched, err = j.filter(chkForJoin, chk, outer.Len()) if err != nil { return false, false, err } @@ -391,7 +391,7 @@ type rightOuterJoiner struct { } // tryToMatch implements joiner interface. -func (j *rightOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (bool, bool, error) { +func (j *rightOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (matched bool, hasNull bool, err error) { if inners.Len() == 0 { return false, false, nil } @@ -411,7 +411,7 @@ func (j *rightOuterJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, ch return true, false, nil } - matched, err := j.filter(chkForJoin, chk, outer.Len()) + matched, err = j.filter(chkForJoin, chk, outer.Len()) if err != nil { return false, false, err } @@ -428,7 +428,7 @@ type innerJoiner struct { } // tryToMatch implements joiner interface. -func (j *innerJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (bool, bool, error) { +func (j *innerJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *chunk.Chunk) (matched bool, hasNull bool, err error) { if inners.Len() == 0 { return false, false, nil } @@ -450,7 +450,7 @@ func (j *innerJoiner) tryToMatch(outer chunk.Row, inners chunk.Iterator, chk *ch } // reach here, chkForJoin is j.chk - matched, err := j.filter(chkForJoin, chk, outer.Len()) + matched, err = j.filter(chkForJoin, chk, outer.Len()) if err != nil { return false, false, err } diff --git a/executor/pkg_test.go b/executor/pkg_test.go index f06896ed7d9f5..637661ed3e40b 100644 --- a/executor/pkg_test.go +++ b/executor/pkg_test.go @@ -135,7 +135,6 @@ func prepare4RadixPartition(sctx sessionctx.Context, rowCount int) *HashJoinExec baseExecutor: newBaseExecutor(sctx, joinSchema, stringutil.StringerStr("HashJoin"), childExec0, childExec1), concurrency: 4, joinType: 0, // InnerJoin - innerIdx: 0, innerKeys: []*expression.Column{{Index: 0, RetType: types.NewFieldType(mysql.TypeLong)}}, innerKeyColIdx: []int{0}, outerKeys: []*expression.Column{{Index: 0, RetType: types.NewFieldType(mysql.TypeLong)}}, From cfee8c09843b4b061880fa4d83529b2cfdbe3be2 Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Fri, 2 Aug 2019 17:14:42 +0800 Subject: [PATCH 152/196] planner: fix bug that window function do not check ignore nulls. (#11554) --- planner/core/logical_plan_builder.go | 9 +++++++++ planner/core/logical_plan_test.go | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 2c1cdad0fdd4c..a53150bcf5332 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3329,6 +3329,15 @@ func (b *PlanBuilder) buildWindowFunctions(ctx context.Context, p LogicalPlan, g // Because of the grouped specification is different from it, we should especially check them before build window frame. func (b *PlanBuilder) checkOriginWindowSpecs(funcs []*ast.WindowFuncExpr, orderByItems []property.Item) error { for _, f := range funcs { + if f.IgnoreNull { + return ErrNotSupportedYet.GenWithStackByArgs("IGNORE NULLS") + } + if f.Distinct { + return ErrNotSupportedYet.GenWithStackByArgs("(DISTINCT ..)") + } + if f.FromLast { + return ErrNotSupportedYet.GenWithStackByArgs("FROM LAST") + } spec := f.Spec if spec.Frame == nil { continue diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index c47d4c5a74bd0..e9334b16ee206 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2452,6 +2452,23 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { sql: "select dense_rank() over w1, a, b from t window w1 as (partition by t.b order by t.a asc range between 1250951168 following AND 1250951168 preceding)", result: "[planner:3586]Window 'w1': frame start or end is negative, NULL or of non-integral type", }, + // Test issue 10556. + { + sql: "SELECT FIRST_VALUE(a) IGNORE NULLS OVER () FROM t", + result: "[planner:1235]This version of TiDB doesn't yet support 'IGNORE NULLS'", + }, + { + sql: "SELECT SUM(DISTINCT a) OVER () FROM t", + result: "[planner:1235]This version of TiDB doesn't yet support '(DISTINCT ..)'", + }, + { + sql: "SELECT NTH_VALUE(a, 1) FROM LAST over (partition by b order by b), a FROM t", + result: "[planner:1235]This version of TiDB doesn't yet support 'FROM LAST'", + }, + { + sql: "SELECT NTH_VALUE(a, 1) FROM LAST IGNORE NULLS over (partition by b order by b), a FROM t", + result: "[planner:1235]This version of TiDB doesn't yet support 'IGNORE NULLS'", + }, } s.Parser.EnableWindowFunc(true) From 3bd64ba92f14d155a95a7851c096868fb91bc375 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 4 Aug 2019 14:14:01 +0800 Subject: [PATCH 153/196] =?UTF-8?q?planner,expression:=20recognize=20the?= =?UTF-8?q?=20three=20new=20expressions=20from=20pi=E2=80=A6=20(#8891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- expression/simple_rewriter.go | 16 ++++++++++++++++ expression/simple_rewriter_test.go | 15 +++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- planner/core/expression_rewriter.go | 16 ++++++++++++++++ planner/core/logical_plan_builder.go | 17 ++++++++++------- 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/expression/simple_rewriter.go b/expression/simple_rewriter.go index 28e2c38e46f7f..fee1980a7575a 100644 --- a/expression/simple_rewriter.go +++ b/expression/simple_rewriter.go @@ -167,6 +167,22 @@ func (sr *simpleRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok boo sr.rowToScalarFunc(v) case *ast.ParenthesesExpr: case *ast.ColumnName: + // TODO: Perhaps we don't need to transcode these back to generic integers/strings + case *ast.TrimDirectionExpr: + sr.push(&Constant{ + Value: types.NewIntDatum(int64(v.Direction)), + RetType: types.NewFieldType(mysql.TypeTiny), + }) + case *ast.TimeUnitExpr: + sr.push(&Constant{ + Value: types.NewStringDatum(v.Unit.String()), + RetType: types.NewFieldType(mysql.TypeVarchar), + }) + case *ast.GetFormatSelectorExpr: + sr.push(&Constant{ + Value: types.NewStringDatum(v.Selector.String()), + RetType: types.NewFieldType(mysql.TypeVarchar), + }) default: sr.err = errors.Errorf("UnknownType: %T", v) return retNode, false diff --git a/expression/simple_rewriter_test.go b/expression/simple_rewriter_test.go index bfcadc3142f64..37486b1de1aec 100644 --- a/expression/simple_rewriter_test.go +++ b/expression/simple_rewriter_test.go @@ -150,4 +150,19 @@ func (s *testEvaluatorSuite) TestSimpleRewriter(c *C) { c.Assert(err, IsNil) num, _, _ = exprs[0].EvalInt(ctx, chunk.Row{}) c.Assert(num, Equals, int64(1)) + + exprs, err = ParseSimpleExprsWithSchema(ctx, "trim(leading 'z' from 'zxyz')", sch) + c.Assert(err, IsNil) + str, _, _ := exprs[0].EvalString(ctx, chunk.Row{}) + c.Assert(str, Equals, "xyz") + + exprs, err = ParseSimpleExprsWithSchema(ctx, "get_format(datetime, 'ISO')", sch) + c.Assert(err, IsNil) + str, _, _ = exprs[0].EvalString(ctx, chunk.Row{}) + c.Assert(str, Equals, "%Y-%m-%d %H:%i:%s") + + exprs, err = ParseSimpleExprsWithSchema(ctx, "extract(day_minute from '2184-07-03 18:42:18.895059')", sch) + c.Assert(err, IsNil) + num, _, _ = exprs[0].EvalInt(ctx, chunk.Row{}) + c.Assert(num, Equals, int64(31842)) } diff --git a/go.mod b/go.mod index ce0b1ec56a1a9..26f5acaea5bbc 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66 + github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 37382ecbbf34e..0ffbb6f66eeef 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66 h1:RQlXn268eNxAh4ZcDGHSdDkv20x1eJDQu8vYvQh3x5c= -github.com/pingcap/parser v0.0.0-20190801033650-4849eb88ee66/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9 h1:oEbyHJedKpf7wcD7lL45HgD24E/W0Qo/HBW6Zv9kPTI= +github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index b19901f9031a9..ef197daa5a63b 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -906,6 +906,22 @@ func (er *expressionRewriter) Leave(originInNode ast.Node) (retNode ast.Node, ok er.isTrueToScalarFunc(v) case *ast.DefaultExpr: er.evalDefaultExpr(v) + // TODO: Perhaps we don't need to transcode these back to generic integers/strings + case *ast.TrimDirectionExpr: + er.ctxStack = append(er.ctxStack, &expression.Constant{ + Value: types.NewIntDatum(int64(v.Direction)), + RetType: types.NewFieldType(mysql.TypeTiny), + }) + case *ast.TimeUnitExpr: + er.ctxStack = append(er.ctxStack, &expression.Constant{ + Value: types.NewStringDatum(v.Unit.String()), + RetType: types.NewFieldType(mysql.TypeVarchar), + }) + case *ast.GetFormatSelectorExpr: + er.ctxStack = append(er.ctxStack, &expression.Constant{ + Value: types.NewStringDatum(v.Selector.String()), + RetType: types.NewFieldType(mysql.TypeVarchar), + }) default: er.err = errors.Errorf("UnknownType: %T", v) return retNode, false diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index a53150bcf5332..69e831a015009 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3139,10 +3139,13 @@ func (b *PlanBuilder) buildWindowFunctionFrameBound(ctx context.Context, spec *a } desc := orderByItems[0].Desc - if boundClause.Unit != nil { - // It can be guaranteed by the parser. - unitVal := boundClause.Unit.(*driver.ValueExpr) - unit := expression.Constant{Value: unitVal.Datum, RetType: unitVal.GetType()} + if boundClause.Unit != ast.TimeUnitInvalid { + // TODO: Perhaps we don't need to transcode this back to generic string + unitVal := boundClause.Unit.String() + unit := expression.Constant{ + Value: types.NewStringDatum(unitVal), + RetType: types.NewFieldType(mysql.TypeVarchar), + } // When the order is asc: // `+` for following, and `-` for the preceding @@ -3375,7 +3378,7 @@ func (b *PlanBuilder) checkOriginWindowFrameBound(bound *ast.FrameBound, spec *a frameType := spec.Frame.Type if frameType == ast.Rows { - if bound.Unit != nil { + if bound.Unit != ast.TimeUnitInvalid { return ErrWindowRowsIntervalUse.GenWithStackByArgs(getWindowName(spec.Name.O)) } _, isNull, isExpectedType := getUintFromNode(b.ctx, bound.Expr) @@ -3393,10 +3396,10 @@ func (b *PlanBuilder) checkOriginWindowFrameBound(bound *ast.FrameBound, spec *a if !isNumeric && !isTemporal { return ErrWindowRangeFrameOrderType.GenWithStackByArgs(getWindowName(spec.Name.O)) } - if bound.Unit != nil && !isTemporal { + if bound.Unit != ast.TimeUnitInvalid && !isTemporal { return ErrWindowRangeFrameNumericType.GenWithStackByArgs(getWindowName(spec.Name.O)) } - if bound.Unit == nil && !isNumeric { + if bound.Unit == ast.TimeUnitInvalid && !isNumeric { return ErrWindowRangeFrameTemporalType.GenWithStackByArgs(getWindowName(spec.Name.O)) } return nil From 03bb8d7818b85996d6a85664223857a4b395030e Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 5 Aug 2019 10:45:51 +0800 Subject: [PATCH 154/196] stats: convert small range to points in selectivity (#11524) --- cmd/explaintest/r/explain_easy.result | 9 ++ cmd/explaintest/t/explain_easy.test | 6 ++ statistics/handle/update_test.go | 8 +- statistics/histogram.go | 12 +++ statistics/scalar.go | 121 ++++++++++++++++++++++++++ statistics/scalar_test.go | 81 +++++++++++++++++ statistics/table.go | 59 +++++++++---- 7 files changed, 277 insertions(+), 19 deletions(-) diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index aa9b6ace34e26..5d33b516c4cb8 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -703,3 +703,12 @@ TableReader_7 1.00 root data:Selection_6 └─Selection_6 1.00 cop eq(test.t.b, 1000-01-01 00:00:00.000000) └─TableScan_5 3.00 cop table:t, range:[-inf,+inf], keep order:false drop table t; +create table t(a int); +insert into t values (1),(2),(2),(2),(9),(9),(9),(10); +analyze table t with 1 buckets; +explain select * from t where a >= 3 and a <= 8; +id count task operator info +TableReader_7 0.00 root data:Selection_6 +└─Selection_6 0.00 cop ge(test.t.a, 3), le(test.t.a, 8) + └─TableScan_5 8.00 cop table:t, range:[-inf,+inf], keep order:false +drop table t; diff --git a/cmd/explaintest/t/explain_easy.test b/cmd/explaintest/t/explain_easy.test index 3510b4fabb8f6..8c1ff5e5502f7 100644 --- a/cmd/explaintest/t/explain_easy.test +++ b/cmd/explaintest/t/explain_easy.test @@ -159,3 +159,9 @@ analyze table t; explain select * from t where a = 1; explain select * from t where b = "1000-01-01"; drop table t; + +create table t(a int); +insert into t values (1),(2),(2),(2),(9),(9),(9),(10); +analyze table t with 1 buckets; +explain select * from t where a >= 3 and a <= 8; +drop table t; diff --git a/statistics/handle/update_test.go b/statistics/handle/update_test.go index de3ae9a7d5f94..5baeff26fed83 100644 --- a/statistics/handle/update_test.go +++ b/statistics/handle/update_test.go @@ -1300,11 +1300,11 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { hist: "index:1 ndv:20\n" + "num: 16 lower_bound: -inf upper_bound: 7 repeats: 0\n" + "num: 16 lower_bound: 8 upper_bound: 15 repeats: 0\n" + - "num: 8 lower_bound: 16 upper_bound: 21 repeats: 0", + "num: 9 lower_bound: 16 upper_bound: 21 repeats: 0", rangeID: tblInfo.Indices[0].ID, idxID: tblInfo.Indices[1].ID, idxCols: 1, - eqCount: 39, + eqCount: 32, }, { sql: "select * from t use index(idx_ac) where a = 1 and c < 21", @@ -1315,7 +1315,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { rangeID: tblInfo.Columns[2].ID, idxID: tblInfo.Indices[2].ID, idxCols: 0, - eqCount: 35, + eqCount: 32, }, { sql: "select * from t use index(idx_ad) where a = 1 and d < 21", @@ -1359,7 +1359,7 @@ func (s *testStatsSuite) TestIndexQueryFeedback(c *C) { rangeID: tblInfo.Columns[6].ID, idxID: tblInfo.Indices[6].ID, idxCols: 0, - eqCount: 32, + eqCount: 30, }, { sql: `select * from t use index(idx_ah) where a = 1 and h < "1000-01-21"`, diff --git a/statistics/histogram.go b/statistics/histogram.go index 8bdb2b25e00b0..503ce178ce362 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -742,6 +742,18 @@ func (c *Column) GetColumnRowCount(sc *stmtctx.StatementContext, ranges []*range } continue } + rangeVals := enumRangeValues(rg.LowVal[0], rg.HighVal[0], rg.LowExclude, rg.HighExclude) + // The small range case. + if rangeVals != nil { + for _, val := range rangeVals { + cnt, err := c.equalRowCount(sc, val, modifyCount) + if err != nil { + return 0, err + } + rowCount += cnt + } + continue + } // The interval case. cnt := c.BetweenRowCount(rg.LowVal[0], rg.HighVal[0]) if (c.outOfRange(rg.LowVal[0]) && !rg.LowVal[0].IsNull()) || c.outOfRange(rg.HighVal[0]) { diff --git a/statistics/scalar.go b/statistics/scalar.go index 446a78a7383f6..42b98143e9cee 100644 --- a/statistics/scalar.go +++ b/statistics/scalar.go @@ -16,7 +16,9 @@ package statistics import ( "encoding/binary" "math" + "time" + "github.com/cznic/mathutil" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/types" @@ -175,3 +177,122 @@ func calcFraction4Datums(lower, upper, value *types.Datum) float64 { } return 0.5 } + +const maxNumStep = 10 + +func enumRangeValues(low, high types.Datum, lowExclude, highExclude bool) []types.Datum { + if low.Kind() != high.Kind() { + return nil + } + exclude := 0 + if lowExclude { + exclude++ + } + if highExclude { + exclude++ + } + switch low.Kind() { + case types.KindInt64: + // Overflow check. + lowVal, highVal := low.GetInt64(), high.GetInt64() + if lowVal < 0 && highVal > 0 { + if lowVal <= -maxNumStep || highVal >= maxNumStep { + return nil + } + } + remaining := highVal - lowVal + if remaining >= maxNumStep+1 { + return nil + } + remaining = remaining + 1 - int64(exclude) + if remaining >= maxNumStep { + return nil + } + values := make([]types.Datum, 0, remaining) + startValue := lowVal + if lowExclude { + startValue++ + } + for i := int64(0); i < remaining; i++ { + values = append(values, types.NewIntDatum(startValue+i)) + } + return values + case types.KindUint64: + remaining := high.GetUint64() - low.GetUint64() + if remaining >= maxNumStep+1 { + return nil + } + remaining = remaining + 1 - uint64(exclude) + if remaining >= maxNumStep { + return nil + } + values := make([]types.Datum, 0, remaining) + startValue := low.GetUint64() + if lowExclude { + startValue++ + } + for i := uint64(0); i < remaining; i++ { + values = append(values, types.NewUintDatum(startValue+i)) + } + return values + case types.KindMysqlDuration: + lowDur, highDur := low.GetMysqlDuration(), high.GetMysqlDuration() + fsp := mathutil.Max(lowDur.Fsp, highDur.Fsp) + stepSize := int64(math.Pow10(types.MaxFsp-fsp)) * int64(time.Microsecond) + lowDur.Duration = lowDur.Duration.Round(time.Duration(stepSize)) + remaining := int64(highDur.Duration-lowDur.Duration)/stepSize + 1 - int64(exclude) + if remaining >= maxNumStep { + return nil + } + startValue := int64(lowDur.Duration) + if lowExclude { + startValue += stepSize + } + values := make([]types.Datum, 0, remaining) + for i := int64(0); i < remaining; i++ { + values = append(values, types.NewDurationDatum(types.Duration{Duration: time.Duration(startValue + i*stepSize), Fsp: fsp})) + } + return values + case types.KindMysqlTime: + lowTime, highTime := low.GetMysqlTime(), high.GetMysqlTime() + if lowTime.Type != highTime.Type { + return nil + } + fsp := mathutil.Max(lowTime.Fsp, highTime.Fsp) + var stepSize int64 + sc := &stmtctx.StatementContext{TimeZone: time.UTC} + if lowTime.Type == mysql.TypeDate { + stepSize = 24 * int64(time.Hour) + lowTime.Time = types.FromDate(lowTime.Time.Year(), lowTime.Time.Month(), lowTime.Time.Day(), 0, 0, 0, 0) + } else { + var err error + lowTime, err = lowTime.RoundFrac(sc, fsp) + if err != nil { + return nil + } + stepSize = int64(math.Pow10(types.MaxFsp-fsp)) * int64(time.Microsecond) + } + remaining := int64(highTime.Sub(sc, &lowTime).Duration)/stepSize + 1 - int64(exclude) + if remaining >= maxNumStep { + return nil + } + startValue := lowTime + var err error + if lowExclude { + startValue, err = lowTime.Add(sc, types.Duration{Duration: time.Duration(stepSize), Fsp: fsp}) + if err != nil { + return nil + } + } + values := make([]types.Datum, 0, remaining) + for i := int64(0); i < remaining; i++ { + value, err := startValue.Add(sc, types.Duration{Duration: time.Duration(i * stepSize), Fsp: fsp}) + if err != nil { + return nil + } + values = append(values, types.NewTimeDatum(value)) + } + return values + } + return nil +} diff --git a/statistics/scalar_test.go b/statistics/scalar_test.go index 027294a075b62..ee91e935a53bd 100644 --- a/statistics/scalar_test.go +++ b/statistics/scalar_test.go @@ -42,6 +42,14 @@ func getTime(year, month, day int, timeType byte) types.Time { return ret } +func getTimeStamp(hour, min, sec int, timeType byte) types.Time { + ret := types.Time{ + Time: types.FromDate(2017, int(1), 1, hour, min, sec, 0), + Type: timeType, + Fsp: 0} + return ret +} + func getBinaryLiteral(value string) types.BinaryLiteral { b, _ := types.ParseBitStr(value) return b @@ -168,3 +176,76 @@ func (s *testStatisticsSuite) TestCalcFraction(c *C) { c.Check(math.Abs(fraction-test.fraction) < eps, IsTrue) } } + +func (s *testStatisticsSuite) TestEnumRangeValues(c *C) { + tests := []struct { + low types.Datum + high types.Datum + lowExclude bool + highExclude bool + res string + }{ + { + low: types.NewIntDatum(0), + high: types.NewIntDatum(5), + lowExclude: false, + highExclude: true, + res: "(0, 1, 2, 3, 4)", + }, + { + low: types.NewIntDatum(math.MinInt64), + high: types.NewIntDatum(math.MaxInt64), + lowExclude: false, + highExclude: false, + res: "", + }, + { + low: types.NewUintDatum(0), + high: types.NewUintDatum(5), + lowExclude: false, + highExclude: true, + res: "(0, 1, 2, 3, 4)", + }, + { + low: types.NewDurationDatum(getDuration("0:00:00")), + high: types.NewDurationDatum(getDuration("0:00:05")), + lowExclude: false, + highExclude: true, + res: "(00:00:00, 00:00:01, 00:00:02, 00:00:03, 00:00:04)", + }, + { + low: types.NewDurationDatum(getDuration("0:00:00")), + high: types.NewDurationDatum(getDuration("0:00:05")), + lowExclude: false, + highExclude: true, + res: "(00:00:00, 00:00:01, 00:00:02, 00:00:03, 00:00:04)", + }, + { + low: types.NewTimeDatum(getTime(2017, 1, 1, mysql.TypeDate)), + high: types.NewTimeDatum(getTime(2017, 1, 5, mysql.TypeDate)), + lowExclude: false, + highExclude: true, + res: "(2017-01-01, 2017-01-02, 2017-01-03, 2017-01-04)", + }, + { + low: types.NewTimeDatum(getTimeStamp(0, 0, 0, mysql.TypeTimestamp)), + high: types.NewTimeDatum(getTimeStamp(0, 0, 5, mysql.TypeTimestamp)), + lowExclude: false, + highExclude: true, + res: "(2017-01-01 00:00:00, 2017-01-01 00:00:01, 2017-01-01 00:00:02, 2017-01-01 00:00:03, 2017-01-01 00:00:04)", + }, + { + low: types.NewTimeDatum(getTimeStamp(0, 0, 0, mysql.TypeDatetime)), + high: types.NewTimeDatum(getTimeStamp(0, 0, 5, mysql.TypeDatetime)), + lowExclude: false, + highExclude: true, + res: "(2017-01-01 00:00:00, 2017-01-01 00:00:01, 2017-01-01 00:00:02, 2017-01-01 00:00:03, 2017-01-01 00:00:04)", + }, + } + for _, t := range tests { + vals := enumRangeValues(t.low, t.high, t.lowExclude, t.highExclude) + str, err := types.DatumsToString(vals, true) + c.Assert(err, IsNil) + c.Assert(t.res, Equals, str) + } +} diff --git a/statistics/table.go b/statistics/table.go index 5805b177e347b..06ee867413903 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -361,11 +361,36 @@ func isSingleColIdxNullRange(idx *Index, ran *ranger.Range) bool { return false } +// getEqualCondSelectivity gets the selectivity of the equal conditions. `coverAll` means if the conditions +// have covered all the index columns. +func (coll *HistColl) getEqualCondSelectivity(idx *Index, bytes []byte, coverAll bool) float64 { + val := types.NewBytesDatum(bytes) + if idx.outOfRange(val) { + // When the value is out of range, we could not found this value in the CM Sketch, + // so we use heuristic methods to estimate the selectivity. + if idx.NDV > 0 && coverAll { + // for equality queries + return float64(coll.ModifyCount) / float64(idx.NDV) / idx.TotalRowCount() + } + // for range queries + return float64(coll.ModifyCount) / outOfRangeBetweenRate / idx.TotalRowCount() + } + return float64(idx.CMSketch.QueryBytes(bytes)) / float64(idx.TotalRowCount()) +} + func (coll *HistColl) getIndexRowCount(sc *stmtctx.StatementContext, idxID int64, indexRanges []*ranger.Range) (float64, error) { idx := coll.Indices[idxID] totalCount := float64(0) for _, ran := range indexRanges { rangePosition := GetOrdinalOfRangeCond(sc, ran) + var rangeVals []types.Datum + // Try to enum the last range values. + if rangePosition != len(ran.LowVal) { + rangeVals = enumRangeValues(ran.LowVal[rangePosition], ran.HighVal[rangePosition], ran.LowExclude, ran.HighExclude) + if rangeVals != nil { + rangePosition++ + } + } // If first one is range, just use the previous way to estimate; if it is [NULL, NULL] range // on single-column index, use previous way as well, because CMSketch does not contain null // values in this case. @@ -378,24 +403,28 @@ func (coll *HistColl) getIndexRowCount(sc *stmtctx.StatementContext, idxID int64 continue } var selectivity float64 + coverAll := len(ran.LowVal) == len(idx.Info.Columns) && rangePosition == len(ran.LowVal) // use CM Sketch to estimate the equal conditions - bytes, err := codec.EncodeKey(sc, nil, ran.LowVal[:rangePosition]...) - if err != nil { - return 0, errors.Trace(err) - } - val := types.NewBytesDatum(bytes) - if idx.outOfRange(val) { - // When the value is out of range, we could not found this value in the CM Sketch, - // so we use heuristic methods to estimate the selectivity. - if idx.NDV > 0 && len(ran.LowVal) == len(idx.Info.Columns) && rangePosition == len(ran.LowVal) { - // for equality queries - selectivity = float64(coll.ModifyCount) / float64(idx.NDV) / idx.TotalRowCount() - } else { - // for range queries - selectivity = float64(coll.ModifyCount) / outOfRangeBetweenRate / idx.TotalRowCount() + if rangeVals == nil { + bytes, err := codec.EncodeKey(sc, nil, ran.LowVal[:rangePosition]...) + if err != nil { + return 0, errors.Trace(err) } + selectivity = coll.getEqualCondSelectivity(idx, bytes, coverAll) } else { - selectivity = float64(idx.CMSketch.QueryBytes(bytes)) / float64(idx.TotalRowCount()) + bytes, err := codec.EncodeKey(sc, nil, ran.LowVal[:rangePosition-1]...) + if err != nil { + return 0, errors.Trace(err) + } + prefixLen := len(bytes) + for _, val := range rangeVals { + bytes = bytes[:prefixLen] + bytes, err = codec.EncodeKey(sc, bytes, val) + if err != nil { + return 0, err + } + selectivity += coll.getEqualCondSelectivity(idx, bytes, coverAll) + } } // use histogram to estimate the range condition if rangePosition != len(ran.LowVal) { From ddafdff8f74a7429237925467381a8dfe1226de3 Mon Sep 17 00:00:00 2001 From: maiyang Date: Mon, 5 Aug 2019 15:22:16 +0800 Subject: [PATCH 155/196] expression: fix date_add func in SECOND INTERVAL (#11312) --- expression/integration_test.go | 31 ++++++++++++++++++++++++++++++- types/time.go | 1 + 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index 785457b8a0097..a5a684748ded3 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4748,7 +4748,7 @@ func (s *testIntegrationSuite) TestFuncCaseWithLeftJoin(c *C) { tk.MustQuery("select t1.id from kankan1 t1 left join kankan2 t2 on t1.id = t2.id where (case when t1.name='b' then 'case2' when t1.name='a' then 'case1' else NULL end) = 'case1' order by t1.id").Check(testkit.Rows("1", "2")) } -func (s *testIntegrationSuite) TestIssue11309(c *C) { +func (s *testIntegrationSuite) TestIssue11309And11319(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec(`drop table if exists t;`) @@ -4764,4 +4764,33 @@ func (s *testIntegrationSuite) TestIssue11309(c *C) { tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL b MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2004-03-13 03:14:52`, `2003-07-25 11:35:34`)) tk.MustQuery(`SELECT DATE_ADD('2003-11-18 07:25:13',INTERVAL c MINUTE_SECOND) FROM t`).Check(testkit.Rows(`2003-11-18 09:29:13`, `2003-11-18 05:21:13`)) tk.MustExec(`drop table if exists t;`) + + // for https://github.com/pingcap/tidb/issues/11319 + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MINUTE_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 SECOND_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 SECOND)`).Check(testkit.Rows("2007-03-28 22:08:25.800000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR_SECOND)`).Check(testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_SECOND)`).Check(testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MINUTE_SECOND)`).Check(testkit.Rows("2007-03-28 22:06:26")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 MINUTE)`).Check(testkit.Rows("2007-03-28 22:06:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_MINUTE)`).Check(testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 HOUR_MINUTE)`).Check(testkit.Rows("2007-03-28 20:06:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 DAY_HOUR)`).Check(testkit.Rows("2007-03-26 20:08:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL -2.2 YEAR_MONTH)`).Check(testkit.Rows("2005-01-28 22:08:28")) + + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MINUTE_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 SECOND_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_MICROSECOND)`).Check(testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 SECOND)`).Check(testkit.Rows("2007-03-28 22:08:30.200000")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR_SECOND)`).Check(testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_SECOND)`).Check(testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MINUTE_SECOND)`).Check(testkit.Rows("2007-03-28 22:10:30")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 MINUTE)`).Check(testkit.Rows("2007-03-28 22:10:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_MINUTE)`).Check(testkit.Rows("2007-03-29 00:10:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 HOUR_MINUTE)`).Check(testkit.Rows("2007-03-29 00:10:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 DAY_HOUR)`).Check(testkit.Rows("2007-03-31 00:08:28")) + tk.MustQuery(`SELECT DATE_ADD('2007-03-28 22:08:28',INTERVAL 2.2 YEAR_MONTH)`).Check(testkit.Rows("2009-05-28 22:08:28")) } diff --git a/types/time.go b/types/time.go index 72373ee61ff57..73889fccefc46 100644 --- a/types/time.go +++ b/types/time.go @@ -1649,6 +1649,7 @@ func parseSingleTimeValue(unit string, format string, strictCheck bool) (int64, if unit != "SECOND" { err = ErrTruncatedWrongValue.GenWithStackByArgs(format) } + dv *= sign } switch strings.ToUpper(unit) { case "MICROSECOND": From 6dbc8cf651c2bdcdcc507e852247b38441a3450a Mon Sep 17 00:00:00 2001 From: "Zhuomin(Charming) Liu" Date: Mon, 5 Aug 2019 16:46:31 +0800 Subject: [PATCH 156/196] =?UTF-8?q?statistics:=20the=20overflow=20check=20?= =?UTF-8?q?is=20not=20correct=20in=20`0-Math.MinIn=E2=80=A6=20(#11611)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- statistics/scalar.go | 4 ++-- statistics/scalar_test.go | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/statistics/scalar.go b/statistics/scalar.go index 42b98143e9cee..53073e16967ce 100644 --- a/statistics/scalar.go +++ b/statistics/scalar.go @@ -195,8 +195,8 @@ func enumRangeValues(low, high types.Datum, lowExclude, highExclude bool) []type case types.KindInt64: // Overflow check. lowVal, highVal := low.GetInt64(), high.GetInt64() - if lowVal < 0 && highVal > 0 { - if lowVal <= -maxNumStep || highVal >= maxNumStep { + if lowVal <= 0 && highVal >= 0 { + if lowVal < -maxNumStep || highVal > maxNumStep { return nil } } diff --git a/statistics/scalar_test.go b/statistics/scalar_test.go index ee91e935a53bd..adafa96fb0649 100644 --- a/statistics/scalar_test.go +++ b/statistics/scalar_test.go @@ -241,11 +241,19 @@ func (s *testStatisticsSuite) TestEnumRangeValues(c *C) { highExclude: true, res: "(2017-01-01 00:00:00, 2017-01-01 00:00:01, 2017-01-01 00:00:02, 2017-01-01 00:00:03, 2017-01-01 00:00:04)", }, + // fix issue 11610 + { + low: types.NewIntDatum(math.MinInt64), + high: types.NewIntDatum(0), + lowExclude: false, + highExclude: false, + res: "", + }, } for _, t := range tests { vals := enumRangeValues(t.low, t.high, t.lowExclude, t.highExclude) str, err := types.DatumsToString(vals, true) c.Assert(err, IsNil) - c.Assert(t.res, Equals, str) + c.Assert(str, Equals, t.res) } } From 265b654809d9113ead58b9cd496a0a64f62e09de Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 5 Aug 2019 17:30:23 +0800 Subject: [PATCH 157/196] model: add schema name, table name to ddl job. (#11561) --- ddl/ddl_api.go | 27 +++++++++++++++++++++++ executor/executor.go | 26 +++++++++++++++++++--- executor/executor_test.go | 44 +++++++++++++++++++++++++++++++++++-- planner/core/planbuilder.go | 3 ++- 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index bfbc6c8da13bb..c82e8c3599995 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -86,6 +86,7 @@ func (d *ddl) CreateSchema(ctx sessionctx.Context, schema model.CIStr, charsetIn job := &model.Job{ SchemaID: schemaID, + SchemaName: dbInfo.Name.L, Type: model.ActionCreateSchema, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{dbInfo}, @@ -144,6 +145,7 @@ func (d *ddl) AlterSchema(ctx sessionctx.Context, stmt *ast.AlterDatabaseStmt) ( // Do the DDL job. job := &model.Job{ SchemaID: dbInfo.ID, + SchemaName: dbInfo.Name.L, Type: model.ActionModifySchemaCharsetAndCollate, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{toCharset, toCollate}, @@ -161,6 +163,7 @@ func (d *ddl) DropSchema(ctx sessionctx.Context, schema model.CIStr) (err error) } job := &model.Job{ SchemaID: old.ID, + SchemaName: old.Name.L, Type: model.ActionDropSchema, BinlogInfo: &model.HistoryInfo{}, } @@ -1183,6 +1186,7 @@ func (d *ddl) CreateTableWithLike(ctx sessionctx.Context, ident, referIdent ast. job := &model.Job{ SchemaID: schema.ID, TableID: tblInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tblInfo}, @@ -1355,6 +1359,7 @@ func (d *ddl) CreateTable(ctx sessionctx.Context, s *ast.CreateTableStmt) (err e job := &model.Job{ SchemaID: schema.ID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo}, @@ -1422,6 +1427,7 @@ func (d *ddl) RecoverTable(ctx sessionctx.Context, tbInfo *model.TableInfo, sche job := &model.Job{ SchemaID: schemaID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionRecoverTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo, autoID, dropJobID, snapshotTS, recoverTableCheckFlagNone}, @@ -1477,6 +1483,7 @@ func (d *ddl) CreateView(ctx sessionctx.Context, s *ast.CreateViewStmt) (err err job := &model.Job{ SchemaID: schema.ID, TableID: tbInfo.ID, + SchemaName: schema.Name.L, Type: model.ActionCreateView, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{tbInfo, s.OrReplace, oldViewTblID}, @@ -1985,6 +1992,7 @@ func (d *ddl) RebaseAutoID(ctx sessionctx.Context, ident ast.Ident, newBase int6 job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionRebaseAutoID, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{newBase}, @@ -2015,6 +2023,7 @@ func (d *ddl) ShardRowID(ctx sessionctx.Context, tableIdent ast.Ident, uVal uint Type: model.ActionShardRowID, SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{uVal}, } @@ -2128,6 +2137,7 @@ func (d *ddl) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTab job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{col, spec.Position, 0}, @@ -2192,6 +2202,7 @@ func (d *ddl) AddTablePartitions(ctx sessionctx.Context, ident ast.Ident, spec * job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionAddTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{partInfo}, @@ -2268,6 +2279,7 @@ func (d *ddl) TruncateTablePartition(ctx sessionctx.Context, ident ast.Ident, sp job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionTruncateTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{pid}, @@ -2314,6 +2326,7 @@ func (d *ddl) DropTablePartition(ctx sessionctx.Context, ident ast.Ident, spec * job := &model.Job{ SchemaID: schema.ID, TableID: meta.ID, + SchemaName: schema.Name.L, Type: model.ActionDropTablePartition, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{partName}, @@ -2362,6 +2375,7 @@ func (d *ddl) DropColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTa job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{colName}, @@ -2694,6 +2708,7 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyColumn, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{&newCol, originalColName, spec.Position, modifyColumnTp}, @@ -2846,6 +2861,7 @@ func (d *ddl) AlterColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionSetDefaultValue, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{col}, @@ -2872,6 +2888,7 @@ func (d *ddl) AlterTableComment(ctx sessionctx.Context, ident ast.Ident, spec *a job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyTableComment, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{spec.Comment}, @@ -2923,6 +2940,7 @@ func (d *ddl) AlterTableCharsetAndCollate(ctx sessionctx.Context, ident ast.Iden job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionModifyTableCharsetAndCollate, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{toCharset, toCollate}, @@ -3015,6 +3033,7 @@ func (d *ddl) RenameIndex(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionRenameIndex, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{spec.FromKey, spec.ToKey}, @@ -3035,6 +3054,7 @@ func (d *ddl) DropTable(ctx sessionctx.Context, ti ast.Ident) (err error) { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropTable, BinlogInfo: &model.HistoryInfo{}, } @@ -3067,6 +3087,7 @@ func (d *ddl) DropView(ctx sessionctx.Context, ti ast.Ident) (err error) { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropView, BinlogInfo: &model.HistoryInfo{}, } @@ -3089,6 +3110,7 @@ func (d *ddl) TruncateTable(ctx sessionctx.Context, ti ast.Ident) error { job := &model.Job{ SchemaID: schema.ID, TableID: tb.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionTruncateTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{newTableID}, @@ -3154,6 +3176,7 @@ func (d *ddl) RenameTable(ctx sessionctx.Context, oldIdent, newIdent ast.Ident, job := &model.Job{ SchemaID: newSchema.ID, TableID: oldTbl.Meta().ID, + SchemaName: newSchema.Name.L, Type: model.ActionRenameTable, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{oldSchema.ID, newIdent.Name}, @@ -3233,6 +3256,7 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, unique bool, ind job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddIndex, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{unique, indexName, idxColNames, indexOption}, @@ -3294,6 +3318,7 @@ func (d *ddl) CreateForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName mode job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionAddForeignKey, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{fkInfo}, @@ -3320,6 +3345,7 @@ func (d *ddl) DropForeignKey(ctx sessionctx.Context, ti ast.Ident, fkName model. job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropForeignKey, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{fkName}, @@ -3361,6 +3387,7 @@ func (d *ddl) DropIndex(ctx sessionctx.Context, ti ast.Ident, indexName model.CI job := &model.Job{ SchemaID: schema.ID, TableID: t.Meta().ID, + SchemaName: schema.Name.L, Type: model.ActionDropIndex, BinlogInfo: &model.HistoryInfo{}, Args: []interface{}{indexName}, diff --git a/executor/executor.go b/executor/executor.go index 911b7005f88f9..32f1735f09795 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -414,15 +414,35 @@ func (e *ShowDDLJobsExec) Next(ctx context.Context, req *chunk.Chunk) error { numCurBatch := mathutil.Min(req.Capacity(), len(e.jobs)-e.cursor) for i := e.cursor; i < e.cursor+numCurBatch; i++ { req.AppendInt64(0, e.jobs[i].ID) - req.AppendString(1, getSchemaName(e.is, e.jobs[i].SchemaID)) - req.AppendString(2, getTableName(e.is, e.jobs[i].TableID)) + schemaName := e.jobs[i].SchemaName + tableName := "" + finishTS := uint64(0) + if e.jobs[i].BinlogInfo != nil { + finishTS = e.jobs[i].BinlogInfo.FinishedTS + if e.jobs[i].BinlogInfo.TableInfo != nil { + tableName = e.jobs[i].BinlogInfo.TableInfo.Name.L + } + if len(schemaName) == 0 && e.jobs[i].BinlogInfo.DBInfo != nil { + schemaName = e.jobs[i].BinlogInfo.DBInfo.Name.L + } + } + // For compatibility, the old version of DDL Job wasn't store the schema name and table name. + if len(schemaName) == 0 { + schemaName = getSchemaName(e.is, e.jobs[i].SchemaID) + } + if len(tableName) == 0 { + tableName = getTableName(e.is, e.jobs[i].TableID) + } + req.AppendString(1, schemaName) + req.AppendString(2, tableName) req.AppendString(3, e.jobs[i].Type.String()) req.AppendString(4, e.jobs[i].SchemaState.String()) req.AppendInt64(5, e.jobs[i].SchemaID) req.AppendInt64(6, e.jobs[i].TableID) req.AppendInt64(7, e.jobs[i].RowCount) req.AppendString(8, model.TSConvert2Time(e.jobs[i].StartTS).String()) - req.AppendString(9, e.jobs[i].State.String()) + req.AppendString(9, model.TSConvert2Time(finishTS).String()) + req.AppendString(10, e.jobs[i].State.String()) } e.cursor += numCurBatch return nil diff --git a/executor/executor_test.go b/executor/executor_test.go index 96544c0451cfb..fba261cb887b3 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -315,7 +315,7 @@ func (s *testSuiteP1) TestAdmin(c *C) { err = r.Next(ctx, req) c.Assert(err, IsNil) row = req.GetRow(0) - c.Assert(row.Len(), Equals, 10) + c.Assert(row.Len(), Equals, 11) txn, err = s.store.Begin() c.Assert(err, IsNil) historyJobs, err := admin.GetHistoryDDLJobs(txn, admin.DefNumHistoryJobs) @@ -331,7 +331,7 @@ func (s *testSuiteP1) TestAdmin(c *C) { err = r.Next(ctx, req) c.Assert(err, IsNil) row = req.GetRow(0) - c.Assert(row.Len(), Equals, 10) + c.Assert(row.Len(), Equals, 11) c.Assert(row.GetInt64(0), Equals, historyJobs[0].ID) c.Assert(err, IsNil) @@ -400,6 +400,13 @@ func (s *testSuiteP1) TestAdmin(c *C) { tk.MustExec("ALTER TABLE t1 ADD INDEX idx3 (c4);") tk.MustExec("admin check table t1;") + // Test admin show ddl jobs table name after table has been droped. + tk.MustExec("drop table if exists t1;") + re := tk.MustQuery("admin show ddl jobs 1") + rows := re.Rows() + c.Assert(len(rows), Equals, 1) + c.Assert(rows[0][2], Equals, "t1") + // Test for reverse scan get history ddl jobs when ddl history jobs queue has multiple regions. txn, err = s.store.Begin() c.Assert(err, IsNil) @@ -417,6 +424,39 @@ func (s *testSuiteP1) TestAdmin(c *C) { c.Assert(historyJobs, DeepEquals, historyJobs2) } +func (s *testSuite) TestAdminShowDDLJobs(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database if not exists test_admin_show_ddl_jobs") + tk.MustExec("use test_admin_show_ddl_jobs") + tk.MustExec("create table t (a int);") + + re := tk.MustQuery("admin show ddl jobs 1") + row := re.Rows()[0] + c.Assert(row[1], Equals, "test_admin_show_ddl_jobs") + jobID, err := strconv.Atoi(row[0].(string)) + c.Assert(err, IsNil) + + c.Assert(tk.Se.NewTxn(context.Background()), IsNil) + txn, err := tk.Se.Txn(true) + c.Assert(err, IsNil) + t := meta.NewMeta(txn) + job, err := t.GetHistoryDDLJob(int64(jobID)) + c.Assert(err, IsNil) + c.Assert(job, NotNil) + // Test for compatibility. Old TiDB version doesn't have SchemaName field, and the BinlogInfo maybe nil. + // See PR: 11561. + job.BinlogInfo = nil + job.SchemaName = "" + err = t.AddHistoryDDLJob(job) + c.Assert(err, IsNil) + err = tk.Se.CommitTxn(context.Background()) + c.Assert(err, IsNil) + + re = tk.MustQuery("admin show ddl jobs 1") + row = re.Rows()[0] + c.Assert(row[1], Equals, "test_admin_show_ddl_jobs") +} + func (s *testSuite) TestAdminChecksumOfPartitionedTable(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("USE test;") diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 20d8c4f1f6fbb..446c381a92bee 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1126,7 +1126,7 @@ func buildCleanupIndexFields() *expression.Schema { } func buildShowDDLJobsFields() *expression.Schema { - schema := expression.NewSchema(make([]*expression.Column, 0, 10)...) + schema := expression.NewSchema(make([]*expression.Column, 0, 11)...) schema.Append(buildColumn("", "JOB_ID", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "DB_NAME", mysql.TypeVarchar, 64)) schema.Append(buildColumn("", "TABLE_NAME", mysql.TypeVarchar, 64)) @@ -1136,6 +1136,7 @@ func buildShowDDLJobsFields() *expression.Schema { schema.Append(buildColumn("", "TABLE_ID", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "ROW_COUNT", mysql.TypeLonglong, 4)) schema.Append(buildColumn("", "START_TIME", mysql.TypeVarchar, 64)) + schema.Append(buildColumn("", "END_TIME", mysql.TypeVarchar, 64)) schema.Append(buildColumn("", "STATE", mysql.TypeVarchar, 64)) return schema } From 829ba98907d463edfa29809672710270e5e608c6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 5 Aug 2019 17:41:40 +0800 Subject: [PATCH 158/196] expression: remove the NotNullFlag for aggregation func MAX/MIN when inferring type (#11343) --- cmd/explaintest/r/tpch.result | 65 ++++++++++++------------ expression/aggregation/base_func.go | 4 ++ expression/aggregation/base_func_test.go | 11 ++++ expression/integration_test.go | 5 ++ 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index eefcc37ffaf98..45f0cdc023f06 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -182,38 +182,39 @@ s_name, p_partkey limit 100; id count task operator info -Projection_36 100.00 root tpch.supplier.s_acctbal, tpch.supplier.s_name, tpch.nation.n_name, tpch.part.p_partkey, tpch.part.p_mfgr, tpch.supplier.s_address, tpch.supplier.s_phone, tpch.supplier.s_comment -└─TopN_39 100.00 root tpch.supplier.s_acctbal:desc, tpch.nation.n_name:asc, tpch.supplier.s_name:asc, tpch.part.p_partkey:asc, offset:0, count:100 - └─HashRightJoin_44 155496.00 root inner join, inner:HashLeftJoin_50, equal:[eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey) eq(tpch.partsupp.ps_supplycost, min(ps_supplycost))] - ├─HashLeftJoin_50 155496.00 root inner join, inner:TableReader_73, equal:[eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)] - │ ├─HashRightJoin_53 8155010.44 root inner join, inner:HashRightJoin_55, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] - │ │ ├─HashRightJoin_55 100000.00 root inner join, inner:HashRightJoin_61, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] - │ │ │ ├─HashRightJoin_61 5.00 root inner join, inner:TableReader_66, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] - │ │ │ │ ├─TableReader_66 1.00 root data:Selection_65 - │ │ │ │ │ └─Selection_65 1.00 cop eq(tpch.region.r_name, "ASIA") - │ │ │ │ │ └─TableScan_64 5.00 cop table:region, range:[-inf,+inf], keep order:false - │ │ │ │ └─TableReader_63 25.00 root data:TableScan_62 - │ │ │ │ └─TableScan_62 25.00 cop table:nation, range:[-inf,+inf], keep order:false - │ │ │ └─TableReader_68 500000.00 root data:TableScan_67 - │ │ │ └─TableScan_67 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false - │ │ └─TableReader_70 40000000.00 root data:TableScan_69 - │ │ └─TableScan_69 40000000.00 cop table:partsupp, range:[-inf,+inf], keep order:false - │ └─TableReader_73 155496.00 root data:Selection_72 - │ └─Selection_72 155496.00 cop eq(tpch.part.p_size, 30), like(tpch.part.p_type, "%STEEL", 92) - │ └─TableScan_71 10000000.00 cop table:part, range:[-inf,+inf], keep order:false - └─HashAgg_76 8155010.44 root group by:tpch.partsupp.ps_partkey, funcs:min(tpch.partsupp.ps_supplycost), firstrow(tpch.partsupp.ps_partkey) - └─HashRightJoin_80 8155010.44 root inner join, inner:HashRightJoin_82, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] - ├─HashRightJoin_82 100000.00 root inner join, inner:HashRightJoin_88, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] - │ ├─HashRightJoin_88 5.00 root inner join, inner:TableReader_93, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] - │ │ ├─TableReader_93 1.00 root data:Selection_92 - │ │ │ └─Selection_92 1.00 cop eq(tpch.region.r_name, "ASIA") - │ │ │ └─TableScan_91 5.00 cop table:region, range:[-inf,+inf], keep order:false - │ │ └─TableReader_90 25.00 root data:TableScan_89 - │ │ └─TableScan_89 25.00 cop table:nation, range:[-inf,+inf], keep order:false - │ └─TableReader_95 500000.00 root data:TableScan_94 - │ └─TableScan_94 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false - └─TableReader_97 40000000.00 root data:TableScan_96 - └─TableScan_96 40000000.00 cop table:partsupp, range:[-inf,+inf], keep order:false +Projection_37 100.00 root tpch.supplier.s_acctbal, tpch.supplier.s_name, tpch.nation.n_name, tpch.part.p_partkey, tpch.part.p_mfgr, tpch.supplier.s_address, tpch.supplier.s_phone, tpch.supplier.s_comment +└─TopN_40 100.00 root tpch.supplier.s_acctbal:desc, tpch.nation.n_name:asc, tpch.supplier.s_name:asc, tpch.part.p_partkey:asc, offset:0, count:100 + └─HashRightJoin_45 155496.00 root inner join, inner:HashLeftJoin_51, equal:[eq(tpch.part.p_partkey, tpch.partsupp.ps_partkey) eq(tpch.partsupp.ps_supplycost, min(ps_supplycost))] + ├─HashLeftJoin_51 155496.00 root inner join, inner:TableReader_74, equal:[eq(tpch.partsupp.ps_partkey, tpch.part.p_partkey)] + │ ├─HashRightJoin_54 8155010.44 root inner join, inner:HashRightJoin_56, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] + │ │ ├─HashRightJoin_56 100000.00 root inner join, inner:HashRightJoin_62, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] + │ │ │ ├─HashRightJoin_62 5.00 root inner join, inner:TableReader_67, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] + │ │ │ │ ├─TableReader_67 1.00 root data:Selection_66 + │ │ │ │ │ └─Selection_66 1.00 cop eq(tpch.region.r_name, "ASIA") + │ │ │ │ │ └─TableScan_65 5.00 cop table:region, range:[-inf,+inf], keep order:false + │ │ │ │ └─TableReader_64 25.00 root data:TableScan_63 + │ │ │ │ └─TableScan_63 25.00 cop table:nation, range:[-inf,+inf], keep order:false + │ │ │ └─TableReader_69 500000.00 root data:TableScan_68 + │ │ │ └─TableScan_68 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false + │ │ └─TableReader_71 40000000.00 root data:TableScan_70 + │ │ └─TableScan_70 40000000.00 cop table:partsupp, range:[-inf,+inf], keep order:false + │ └─TableReader_74 155496.00 root data:Selection_73 + │ └─Selection_73 155496.00 cop eq(tpch.part.p_size, 30), like(tpch.part.p_type, "%STEEL", 92) + │ └─TableScan_72 10000000.00 cop table:part, range:[-inf,+inf], keep order:false + └─Selection_75 6524008.35 root not(isnull(min(ps_supplycost))) + └─HashAgg_78 8155010.44 root group by:tpch.partsupp.ps_partkey, funcs:min(tpch.partsupp.ps_supplycost), firstrow(tpch.partsupp.ps_partkey) + └─HashRightJoin_82 8155010.44 root inner join, inner:HashRightJoin_84, equal:[eq(tpch.supplier.s_suppkey, tpch.partsupp.ps_suppkey)] + ├─HashRightJoin_84 100000.00 root inner join, inner:HashRightJoin_90, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] + │ ├─HashRightJoin_90 5.00 root inner join, inner:TableReader_95, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] + │ │ ├─TableReader_95 1.00 root data:Selection_94 + │ │ │ └─Selection_94 1.00 cop eq(tpch.region.r_name, "ASIA") + │ │ │ └─TableScan_93 5.00 cop table:region, range:[-inf,+inf], keep order:false + │ │ └─TableReader_92 25.00 root data:TableScan_91 + │ │ └─TableScan_91 25.00 cop table:nation, range:[-inf,+inf], keep order:false + │ └─TableReader_97 500000.00 root data:TableScan_96 + │ └─TableScan_96 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false + └─TableReader_99 40000000.00 root data:TableScan_98 + └─TableScan_98 40000000.00 cop table:partsupp, range:[-inf,+inf], keep order:false /* Q3 Shipping Priority Query This query retrieves the 10 unshipped orders with the highest value. diff --git a/expression/aggregation/base_func.go b/expression/aggregation/base_func.go index 1a4a971c02294..6706eeea99d9d 100644 --- a/expression/aggregation/base_func.go +++ b/expression/aggregation/base_func.go @@ -184,6 +184,10 @@ func (a *baseFuncDesc) typeInfer4MaxMin(ctx sessionctx.Context) { a.Args[0] = expression.BuildCastFunction(ctx, a.Args[0], tp) } a.RetTp = a.Args[0].GetType() + if (a.Name == ast.AggFuncMax || a.Name == ast.AggFuncMin) && a.RetTp.Tp != mysql.TypeBit { + a.RetTp = a.Args[0].GetType().Clone() + a.RetTp.Flag &^= mysql.NotNullFlag + } if a.RetTp.Tp == mysql.TypeEnum || a.RetTp.Tp == mysql.TypeSet { a.RetTp = &types.FieldType{Tp: mysql.TypeString, Flen: mysql.MaxFieldCharLength} } diff --git a/expression/aggregation/base_func_test.go b/expression/aggregation/base_func_test.go index ba7fd757fdbaa..bf6e96364fe39 100644 --- a/expression/aggregation/base_func_test.go +++ b/expression/aggregation/base_func_test.go @@ -39,3 +39,14 @@ func (s *testBaseFuncSuite) TestClone(c *check.C) { c.Assert(desc.Args[0], check.Equals, col) c.Assert(desc.equal(s.ctx, cloned), check.IsFalse) } + +func (s *testBaseFuncSuite) TestMaxMin(c *check.C) { + col := &expression.Column{ + UniqueID: 0, + RetType: types.NewFieldType(mysql.TypeLonglong), + } + col.RetType.Flag |= mysql.NotNullFlag + desc, err := newBaseFuncDesc(s.ctx, ast.AggFuncMax, []expression.Expression{col}) + c.Assert(err, check.IsNil) + c.Assert(mysql.HasNotNullFlag(desc.RetTp.Flag), check.IsFalse) +} diff --git a/expression/integration_test.go b/expression/integration_test.go index a5a684748ded3..ca73a617beaf8 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2823,6 +2823,11 @@ func (s *testIntegrationSuite) TestControlBuiltin(c *C) { result = tk.MustQuery("select ifnull(null, null)") result.Check(testkit.Rows("")) + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1(a bigint not null)") + result = tk.MustQuery("select ifnull(max(a),0) from t1") + result.Check(testkit.Rows("0")) + tk.MustExec("drop table if exists t1") tk.MustExec("drop table if exists t2") tk.MustExec("create table t1(a decimal(20,4))") From f0ba5087be922509a3274d7f03966c81cfe7a033 Mon Sep 17 00:00:00 2001 From: crazycs Date: Mon, 5 Aug 2019 17:51:40 +0800 Subject: [PATCH 159/196] util: update the minimum version of tikv information (#11608) --- util/printer/printer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/printer/printer.go b/util/printer/printer.go index dc521ce15057e..ec8eee90532d6 100644 --- a/util/printer/printer.go +++ b/util/printer/printer.go @@ -32,7 +32,7 @@ var ( TiDBGitBranch = "None" GoVersion = "None" // TiKVMinVersion is the minimum version of TiKV that can be compatible with the current TiDB. - TiKVMinVersion = "2.1.0-alpha.1-ff3dd160846b7d1aed9079c389fc188f7f5ea13e" + TiKVMinVersion = "v3.0.0-60965b006877ca7234adaced7890d7b029ed1306" ) // PrintTiDBInfo prints the TiDB version information. From 5f9fe27a1475479bff1788c2c45e1f199af5dd9a Mon Sep 17 00:00:00 2001 From: gaoxingliang Date: Mon, 5 Aug 2019 19:05:37 +0800 Subject: [PATCH 160/196] =?UTF-8?q?expression:=20the=20quote=20function=20?= =?UTF-8?q?should=20treat=20null=20expr=20as=20NULL=E2=80=A6=20(#11592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- expression/builtin_string.go | 5 ++++- expression/builtin_string_test.go | 2 +- expression/integration_test.go | 9 ++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/expression/builtin_string.go b/expression/builtin_string.go index 38cd2c09faffc..7288ae1afbf0e 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -2695,8 +2695,11 @@ func (b *builtinQuoteSig) Clone() builtinFunc { // See https://dev.mysql.com/doc/refman/5.7/en/string-functions.html#function_quote func (b *builtinQuoteSig) evalString(row chunk.Row) (string, bool, error) { str, isNull, err := b.args[0].EvalString(b.ctx, row) - if isNull || err != nil { + if err != nil { return "", true, err + } else if isNull { + // If the argument is NULL, the return value is the word "NULL" without enclosing single quotation marks. see ref. + return "NULL", false, err } runes := []rune(str) diff --git a/expression/builtin_string_test.go b/expression/builtin_string_test.go index a9bcaf581ae15..7a47ac1b4cd7a 100644 --- a/expression/builtin_string_test.go +++ b/expression/builtin_string_test.go @@ -2171,7 +2171,7 @@ func (s *testEvaluatorSuite) TestQuote(c *C) { {`萌萌哒(๑•ᴗ•๑)😊`, `'萌萌哒(๑•ᴗ•๑)😊'`}, {`㍿㌍㍑㌫`, `'㍿㌍㍑㌫'`}, {string([]byte{0, 26}), `'\0\Z'`}, - {nil, nil}, + {nil, "NULL"}, } for _, t := range tbl { diff --git a/expression/integration_test.go b/expression/integration_test.go index ca73a617beaf8..52137a5096ac4 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -901,7 +901,14 @@ func (s *testIntegrationSuite) TestStringBuiltin(c *C) { result = tk.MustQuery(`select quote("aaaa"), quote(""), quote("\"\""), quote("\n\n");`) result.Check(testkit.Rows("'aaaa' '' '\"\"' '\n\n'")) result = tk.MustQuery(`select quote(0121), quote(0000), quote("中文"), quote(NULL);`) - result.Check(testkit.Rows("'121' '0' '中文' ")) + result.Check(testkit.Rows("'121' '0' '中文' NULL")) + tk.MustQuery(`select quote(null) is NULL;`).Check(testkit.Rows(`0`)) + tk.MustQuery(`select quote(null) is NOT NULL;`).Check(testkit.Rows(`1`)) + tk.MustQuery(`select length(quote(null));`).Check(testkit.Rows(`4`)) + tk.MustQuery(`select quote(null) REGEXP binary 'null'`).Check(testkit.Rows(`0`)) + tk.MustQuery(`select quote(null) REGEXP binary 'NULL'`).Check(testkit.Rows(`1`)) + tk.MustQuery(`select quote(null) REGEXP 'NULL'`).Check(testkit.Rows(`1`)) + tk.MustQuery(`select quote(null) REGEXP 'null'`).Check(testkit.Rows(`1`)) // for convert result = tk.MustQuery(`select convert("123" using "binary"), convert("中文" using "binary"), convert("中文" using "utf8"), convert("中文" using "utf8mb4"), convert(cast("中文" as binary) using "utf8");`) From 3f3ead2d594507a24e1e86b8318463f3352fac16 Mon Sep 17 00:00:00 2001 From: miraclesu Date: Mon, 5 Aug 2019 06:14:11 -0500 Subject: [PATCH 161/196] executor: fix #10513, alias identifier maximum length compatible with MySQL (#10597) --- executor/adapter.go | 9 +++++++++ server/tidb_test.go | 16 ++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/executor/adapter.go b/executor/adapter.go index a7c63c854d081..cf61065498f1c 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -90,6 +90,15 @@ func schema2ResultFields(schema *expression.Schema, defaultDB string) (rfs []*as Name: origColName, }, } + // This is for compatibility. + if len(rf.ColumnAsName.O) > mysql.MaxAliasIdentifierLen { + rf.ColumnAsName.O = rf.ColumnAsName.O[:mysql.MaxAliasIdentifierLen] + } + // Usually the length of O equals the length of L. + // Add this len judgement to avoid panic. + if len(rf.ColumnAsName.L) > mysql.MaxAliasIdentifierLen { + rf.ColumnAsName.L = rf.ColumnAsName.L[:mysql.MaxAliasIdentifierLen] + } rfs = append(rfs, rf) } return rfs diff --git a/server/tidb_test.go b/server/tidb_test.go index 84bb7c5921055..cf7677c46473c 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -544,6 +544,22 @@ func (ts *TidbTestSuite) TestFieldList(c *C) { // c_decimal decimal(6, 3) c.Assert(colInfos[5].Decimal, Equals, uint8(3)) + + // for issue#10513 + tooLongColumnAsName := "COALESCE(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)" + columnAsName := tooLongColumnAsName[:tmysql.MaxAliasIdentifierLen] + + rs, err := qctx.Execute(ctx, "select "+tooLongColumnAsName) + c.Assert(err, IsNil) + cols := rs[0].Columns() + c.Assert(cols[0].OrgName, Equals, tooLongColumnAsName) + c.Assert(cols[0].Name, Equals, columnAsName) + + rs, err = qctx.Execute(ctx, "select c_bit as '"+tooLongColumnAsName+"' from t") + c.Assert(err, IsNil) + cols = rs[0].Columns() + c.Assert(cols[0].OrgName, Equals, "c_bit") + c.Assert(cols[0].Name, Equals, columnAsName) } func (ts *TidbTestSuite) TestSumAvg(c *C) { From cb4b778268a5e0770b25b3b67ad71858135a3dae Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 5 Aug 2019 19:30:10 +0800 Subject: [PATCH 162/196] expression: deduce result type for multi-argument functions like `IF` wrongly in some cases (#11605) --- expression/builtin_control.go | 3 +-- expression/integration_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/expression/builtin_control.go b/expression/builtin_control.go index 015f9705f2844..80e8d5e233e14 100644 --- a/expression/builtin_control.go +++ b/expression/builtin_control.go @@ -67,9 +67,8 @@ func InferType4ControlFuncs(lhs, rhs *types.FieldType) *types.FieldType { } else if rhs.Tp == mysql.TypeNull { *resultFieldType = *lhs } else { - var unsignedFlag uint - evalType := types.AggregateEvalType([]*types.FieldType{lhs, rhs}, &unsignedFlag) resultFieldType = types.AggFieldType([]*types.FieldType{lhs, rhs}) + evalType := types.AggregateEvalType([]*types.FieldType{lhs, rhs}, &resultFieldType.Flag) if evalType == types.ETInt { resultFieldType.Decimal = 0 } else { diff --git a/expression/integration_test.go b/expression/integration_test.go index 52137a5096ac4..c35e1d8cdcfa5 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4760,6 +4760,18 @@ func (s *testIntegrationSuite) TestFuncCaseWithLeftJoin(c *C) { tk.MustQuery("select t1.id from kankan1 t1 left join kankan2 t2 on t1.id = t2.id where (case when t1.name='b' then 'case2' when t1.name='a' then 'case1' else NULL end) = 'case1' order by t1.id").Check(testkit.Rows("1", "2")) } +func (s *testIntegrationSuite) TestIssue11594(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec(`drop table if exists t1;`) + tk.MustExec("CREATE TABLE t1 (v bigint(20) UNSIGNED NOT NULL);") + tk.MustExec("INSERT INTO t1 VALUES (1), (2);") + tk.MustQuery("SELECT SUM(IF(v > 1, v, -v)) FROM t1;").Check(testkit.Rows("1")) + tk.MustQuery("SELECT sum(IFNULL(cast(null+rand() as unsigned), -v)) FROM t1;").Check(testkit.Rows("-3")) + tk.MustQuery("SELECT sum(COALESCE(cast(null+rand() as unsigned), -v)) FROM t1;").Check(testkit.Rows("-3")) + tk.MustQuery("SELECT sum(COALESCE(cast(null+rand() as unsigned), v)) FROM t1;").Check(testkit.Rows("3")) +} + func (s *testIntegrationSuite) TestIssue11309And11319(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") From 050172c248cbcce064d106821c174c63ae95712a Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Mon, 5 Aug 2019 19:38:10 +0800 Subject: [PATCH 163/196] parser: support cast as real (#11564) --- expression/integration_test.go | 28 ++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/expression/integration_test.go b/expression/integration_test.go index c35e1d8cdcfa5..c3578ad96d9f7 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2407,6 +2407,34 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { result = tk.MustQuery("select cast(0x12345678 as float(40)) = 305419896") result.Check(testkit.Rows("1")) + // test cast as real + result = tk.MustQuery("select cast(1 as real)") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(12345 as unsigned) as real)") + result.Check(testkit.Rows("12345")) + result = tk.MustQuery("select cast(1.1 as real) = 1.1") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(-1.1 as real) = -1.1") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast('123.321' as real) =123.321") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast('12345678901234567890' as real) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(-1 as real)") + result.Check(testkit.Rows("-1")) + result = tk.MustQuery("select cast(null as real)") + result.Check(testkit.Rows("")) + result = tk.MustQuery("select cast(12345678901234567890 as real) = 1.2345678901234567e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(cast(-1 as unsigned) as real) = 1.8446744073709552e19") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(1e100 as real) = 1e100") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(123456789012345678901234567890 as real) = 1.2345678901234568e29") + result.Check(testkit.Rows("1")) + result = tk.MustQuery("select cast(0x12345678 as real) = 305419896") + result.Check(testkit.Rows("1")) + // test cast time as decimal overflow tk.MustExec("drop table if exists t1") tk.MustExec("create table t1(s1 time);") diff --git a/go.mod b/go.mod index 26f5acaea5bbc..e9493dc3a1392 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9 + github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0 github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 0ffbb6f66eeef..8b1e18c303219 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9 h1:oEbyHJedKpf7wcD7lL45HgD24E/W0Qo/HBW6Zv9kPTI= -github.com/pingcap/parser v0.0.0-20190803082711-3db5c165d5d9/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0 h1:INh0Y2k/XxocdBYzWtyeFgGZB4H4OuPBBKHZLb36Bm0= +github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= From c9502f06672a6a9a37cc4bc5826eaae32e911edc Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Tue, 6 Aug 2019 11:38:33 +0800 Subject: [PATCH 164/196] executor: support analyze option for samples (#11609) --- executor/analyze.go | 25 +++++++++++++------------ executor/analyze_test.go | 11 ++++++----- go.mod | 2 +- go.sum | 4 ++-- planner/core/planbuilder.go | 2 ++ 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index ee4a91a49fce8..9c95df3397d29 100755 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -62,9 +62,6 @@ type AnalyzeExec struct { } var ( - // MaxSampleSize is the size of samples for once analyze. - // It's public for test. - MaxSampleSize = 10000 // RandSeed is the seed for randing package. // It's public for test. RandSeed = int64(1) @@ -464,7 +461,7 @@ func (e *AnalyzeColumnsExec) buildStats(ranges []*ranger.Range) (hists []*statis collectors[i] = &statistics.SampleCollector{ IsMerger: true, FMSketch: statistics.NewFMSketch(maxSketchSize), - MaxSampleSize: int64(MaxSampleSize), + MaxSampleSize: int64(e.opts[ast.AnalyzeOptNumSamples]), CMSketch: statistics.NewCMSketch(int32(e.opts[ast.AnalyzeOptCMSketchDepth]), int32(e.opts[ast.AnalyzeOptCMSketchWidth])), } } @@ -898,17 +895,18 @@ func (e *AnalyzeFastExec) handleScanIter(iter kv.Iterator) (scanKeysSize int, er hasPKInfo = 1 } rander := rand.New(rand.NewSource(e.randSeed + int64(e.rowCount))) + sampleSize := int64(e.opts[ast.AnalyzeOptNumSamples]) for ; iter.Valid() && err == nil; err = iter.Next() { // reservoir sampling e.rowCount++ scanKeysSize++ randNum := rander.Int63n(int64(e.rowCount)) - if randNum > int64(MaxSampleSize) && e.sampCursor == int32(MaxSampleSize) { + if randNum > sampleSize && e.sampCursor == int32(sampleSize) { continue } - p := rander.Int31n(int32(MaxSampleSize)) - if e.sampCursor < int32(MaxSampleSize) { + p := rander.Int31n(int32(sampleSize)) + if e.sampCursor < int32(sampleSize) { p = e.sampCursor e.sampCursor++ } @@ -1058,8 +1056,8 @@ func (e *AnalyzeFastExec) runTasks() ([]*statistics.Histogram, []*statistics.CMS e.collectors = make([]*statistics.SampleCollector, length) for i := range e.collectors { e.collectors[i] = &statistics.SampleCollector{ - MaxSampleSize: int64(MaxSampleSize), - Samples: make([]*statistics.SampleItem, MaxSampleSize), + MaxSampleSize: int64(e.opts[ast.AnalyzeOptNumSamples]), + Samples: make([]*statistics.SampleItem, e.opts[ast.AnalyzeOptNumSamples]), } } @@ -1141,7 +1139,8 @@ func (e *AnalyzeFastExec) buildStats() (hists []*statistics.Histogram, cms []*st // If total row count of the table is smaller than 2*MaxSampleSize, we // translate all the sample tasks to scan tasks. - if e.rowCount < uint64(MaxSampleSize)*2 { + sampleSize := e.opts[ast.AnalyzeOptNumSamples] + if e.rowCount < sampleSize*2 { for _, task := range e.sampTasks { e.scanTasks = append(e.scanTasks, task.Location) } @@ -1150,8 +1149,8 @@ func (e *AnalyzeFastExec) buildStats() (hists []*statistics.Histogram, cms []*st return e.runTasks() } - randPos := make([]uint64, 0, MaxSampleSize+1) - for i := 0; i < MaxSampleSize; i++ { + randPos := make([]uint64, 0, sampleSize+1) + for i := 0; i < int(sampleSize); i++ { randPos = append(randPos, uint64(rander.Int63n(int64(e.rowCount)))) } sort.Slice(randPos, func(i, j int) bool { return randPos[i] < randPos[j] }) @@ -1175,6 +1174,7 @@ type AnalyzeTestFastExec struct { Concurrency int Collectors []*statistics.SampleCollector TblInfo *model.TableInfo + Opts map[ast.AnalyzeOptionType]uint64 } // TestFastSample only test the fast sample in unit test. @@ -1188,6 +1188,7 @@ func (e *AnalyzeTestFastExec) TestFastSample() error { e.wg = &sync.WaitGroup{} e.job = &statistics.AnalyzeJob{} e.tblInfo = e.TblInfo + e.opts = e.Opts _, _, err := e.buildStats() e.Collectors = e.collectors return err diff --git a/executor/analyze_test.go b/executor/analyze_test.go index d85fd20491f3c..ca1b6356fca78 100644 --- a/executor/analyze_test.go +++ b/executor/analyze_test.go @@ -22,6 +22,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/failpoint" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/domain" @@ -126,8 +127,7 @@ func (s *testSuite1) TestAnalyzeParameters(c *C) { tk.MustExec("insert into t values (19), (19), (19)") tk.MustExec("set @@tidb_enable_fast_analyze = 1") - executor.MaxSampleSize = 30 - tk.MustExec("analyze table t") + tk.MustExec("analyze table t with 30 samples") is := executor.GetInfoSchema(tk.Se.(sessionctx.Context)) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) @@ -181,7 +181,6 @@ func (s *testSuite1) TestAnalyzeFastSample(c *C) { dom, err = session.BootstrapSession(store) c.Assert(err, IsNil) tk := testkit.NewTestKit(c, store) - executor.MaxSampleSize = 20 executor.RandSeed = 123 tk.MustExec("use test") @@ -214,6 +213,8 @@ func (s *testSuite1) TestAnalyzeFastSample(c *C) { indicesInfo = append(indicesInfo, idx) } } + opts := make(map[ast.AnalyzeOptionType]uint64) + opts[ast.AnalyzeOptNumSamples] = 20 mockExec := &executor.AnalyzeTestFastExec{ Ctx: tk.Se.(sessionctx.Context), PKInfo: pkCol, @@ -222,6 +223,7 @@ func (s *testSuite1) TestAnalyzeFastSample(c *C) { Concurrency: 1, PhysicalTableID: tbl.(table.PhysicalTable).GetPhysicalID(), TblInfo: tblInfo, + Opts: opts, } err = mockExec.TestFastSample() c.Assert(err, IsNil) @@ -252,7 +254,6 @@ func (s *testSuite1) TestFastAnalyze(c *C) { dom, err = session.BootstrapSession(store) c.Assert(err, IsNil) tk := testkit.NewTestKit(c, store) - executor.MaxSampleSize = 6 executor.RandSeed = 123 tk.MustExec("use test") @@ -271,7 +272,7 @@ func (s *testSuite1) TestFastAnalyze(c *C) { for i := 0; i < 20; i++ { tk.MustExec(fmt.Sprintf(`insert into t values (%d, %d, "char")`, i*3, i*3)) } - tk.MustExec("analyze table t with 5 buckets") + tk.MustExec("analyze table t with 5 buckets, 6 samples") is := executor.GetInfoSchema(tk.Se.(sessionctx.Context)) table, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) diff --git a/go.mod b/go.mod index e9493dc3a1392..0a373f0dc22d6 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0 + github.com/pingcap/parser v0.0.0-20190805112544-2b054e14336b github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 diff --git a/go.sum b/go.sum index 8b1e18c303219..b4170e6ca9006 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/pingcap/kvproto v0.0.0-20190703131923-d9830856b531/go.mod h1:QMdbTAXC github.com/pingcap/log v0.0.0-20190214045112-b37da76f67a7/go.mod h1:xsfkWVaFVV5B8e1K9seWfyJWFrIhbtUTAD8NV1Pq3+w= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0 h1:INh0Y2k/XxocdBYzWtyeFgGZB4H4OuPBBKHZLb36Bm0= -github.com/pingcap/parser v0.0.0-20190805082930-d93f654611e0/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190805112544-2b054e14336b h1:CAfB8gFHcyyNUEVBqv/L8uNGfCWnJGhxAXqApAk5ICA= +github.com/pingcap/parser v0.0.0-20190805112544-2b054e14336b/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b h1:oS9PftxQqgcRouKhhdaB52tXhVLEP7Ng3Qqsd6Z18iY= github.com/pingcap/pd v0.0.0-20190712044914-75a1f9f3062b/go.mod h1:3DlDlFT7EF64A1bmb/tulZb6wbPSagm5G4p1AlhaEDs= github.com/pingcap/tidb-tools v2.1.3-0.20190321065848-1e8b48f5c168+incompatible h1:MkWCxgZpJBgY2f4HtwWMMFzSBb3+JPzeJgF3VrXE/bU= diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 446c381a92bee..0b351cd360dfb 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1031,6 +1031,7 @@ var analyzeOptionLimit = map[ast.AnalyzeOptionType]uint64{ ast.AnalyzeOptNumTopN: 1024, ast.AnalyzeOptCMSketchWidth: uint64(cmSketchSizeLimit), ast.AnalyzeOptCMSketchDepth: uint64(cmSketchSizeLimit), + ast.AnalyzeOptNumSamples: 100000, } var analyzeOptionDefault = map[ast.AnalyzeOptionType]uint64{ @@ -1038,6 +1039,7 @@ var analyzeOptionDefault = map[ast.AnalyzeOptionType]uint64{ ast.AnalyzeOptNumTopN: 20, ast.AnalyzeOptCMSketchWidth: 2048, ast.AnalyzeOptCMSketchDepth: 5, + ast.AnalyzeOptNumSamples: 10000, } func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint64, error) { From 02ab1ed7ee3031d8fbe2e3f82f0e2fd54e255470 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 6 Aug 2019 14:18:31 +0800 Subject: [PATCH 165/196] executor,session: display the tree-like format of the trace statement (#11633) --- executor/adapter.go | 1 + executor/compiler.go | 1 + executor/executor.go | 2 ++ executor/trace.go | 24 ------------------------ executor/update.go | 6 ------ session/session.go | 2 ++ session/tidb.go | 1 + session/txn.go | 1 + 8 files changed, 8 insertions(+), 30 deletions(-) diff --git a/executor/adapter.go b/executor/adapter.go index cf61065498f1c..c48a607f696b9 100644 --- a/executor/adapter.go +++ b/executor/adapter.go @@ -398,6 +398,7 @@ func (a *ExecStmt) handleNoDelayExecutor(ctx context.Context, e Executor) (sqlex if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("executor.handleNoDelayExecutor", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } // Check if "tidb_snapshot" is set for the write executors. diff --git a/executor/compiler.go b/executor/compiler.go index 9190763fe68fd..9c2bae6d34afd 100644 --- a/executor/compiler.go +++ b/executor/compiler.go @@ -64,6 +64,7 @@ func (c *Compiler) compile(ctx context.Context, stmtNode ast.StmtNode, skipBind if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("executor.Compile", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } if !skipBind { diff --git a/executor/executor.go b/executor/executor.go index 32f1735f09795..73890001eee50 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -195,6 +195,7 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) error { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan(fmt.Sprintf("%T.Next", e), opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } return e.Next(ctx, req) } @@ -907,6 +908,7 @@ func init() { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("executor.EvalSubQuery", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } e := &executorBuilder{is: is, ctx: sctx} diff --git a/executor/trace.go b/executor/trace.go index 9d30e21ebd94d..92644c28015f5 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -16,7 +16,6 @@ package executor import ( "context" "encoding/json" - "sort" "time" "github.com/opentracing/basictracer-go" @@ -91,7 +90,6 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { return nil } trace := traces[0] - sortTraceByStartTime(trace) dfsTree(trace, "", false, req) e.exhausted = true return nil @@ -131,28 +129,6 @@ func drainRecordSet(ctx context.Context, sctx sessionctx.Context, rs sqlexec.Rec } } -type sortByStartTime []*appdash.Trace - -func (t sortByStartTime) Len() int { return len(t) } -func (t sortByStartTime) Less(i, j int) bool { - return getStartTime(t[j]).After(getStartTime(t[i])) -} -func (t sortByStartTime) Swap(i, j int) { t[i], t[j] = t[j], t[i] } - -func getStartTime(trace *appdash.Trace) (t time.Time) { - if e, err := trace.TimespanEvent(); err == nil { - t = e.Start() - } - return -} - -func sortTraceByStartTime(trace *appdash.Trace) { - sort.Sort(sortByStartTime(trace.Sub)) - for _, t := range trace.Sub { - sortTraceByStartTime(t) - } -} - func dfsTree(t *appdash.Trace, prefix string, isLast bool, chk *chunk.Chunk) { var newPrefix, suffix string if len(prefix) == 0 { diff --git a/executor/update.go b/executor/update.go index 2ac64ad2ed07a..11538c1c8df12 100644 --- a/executor/update.go +++ b/executor/update.go @@ -17,7 +17,6 @@ import ( "context" "fmt" - "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" @@ -129,11 +128,6 @@ func (e *UpdateExec) canNotUpdate(handle types.Datum) bool { // Next implements the Executor Next interface. func (e *UpdateExec) Next(ctx context.Context, req *chunk.Chunk) error { - if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { - span1 := span.Tracer().StartSpan("update.Next", opentracing.ChildOf(span.Context())) - defer span1.Finish() - } - req.Reset() if !e.fetched { err := e.fetchChunkRows(ctx) diff --git a/session/session.go b/session/session.go index 84db86e0e6b72..236c78955e645 100644 --- a/session/session.go +++ b/session/session.go @@ -499,6 +499,7 @@ func (s *session) CommitTxn(ctx context.Context) error { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("session.CommitTxn", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } var commitDetail *execdetails.CommitDetails @@ -1027,6 +1028,7 @@ func (s *session) Execute(ctx context.Context, sql string) (recordSets []sqlexec if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("session.Execute", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } if recordSets, err = s.execute(ctx, sql); err != nil { s.sessionVars.StmtCtx.AppendError(err) diff --git a/session/tidb.go b/session/tidb.go index 70c3f9a9665c8..1834367da4e32 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -207,6 +207,7 @@ func runStmt(ctx context.Context, sctx sessionctx.Context, s sqlexec.Statement) span1 := span.Tracer().StartSpan("session.runStmt", opentracing.ChildOf(span.Context())) span1.LogKV("sql", s.OriginText()) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } se := sctx.(*session) defer func() { diff --git a/session/txn.go b/session/txn.go index ca1473975995e..fca3be468b7f4 100644 --- a/session/txn.go +++ b/session/txn.go @@ -399,6 +399,7 @@ func (s *session) getTxnFuture(ctx context.Context) *txnFuture { if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan("session.getTxnFuture", opentracing.ChildOf(span.Context())) defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) } oracleStore := s.store.GetOracle() From 45790b5799d535dab0df7b7c6959d947be5f5fe1 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Tue, 6 Aug 2019 15:42:56 +0800 Subject: [PATCH 166/196] fixup (#11636) --- expression/column.go | 2 +- expression/constant_test.go | 4 +- expression/vectorized.go | 106 ++++++------------------------------ util/chunk/chunk.go | 15 ----- 4 files changed, 21 insertions(+), 106 deletions(-) diff --git a/expression/column.go b/expression/column.go index e89eb93ebe62f..50d53c0d3452a 100644 --- a/expression/column.go +++ b/expression/column.go @@ -188,7 +188,7 @@ func (col *Column) Equal(_ sessionctx.Context, expr Expression) bool { // VecEval evaluates this expression in a vectorized manner. func (col *Column) VecEval(ctx sessionctx.Context, input *chunk.Chunk, result *chunk.Column) error { - input.Column(col.Index).CopyConstruct(result) + input.Column(col.Index).CopyReconstruct(input.Sel(), result) return nil } diff --git a/expression/constant_test.go b/expression/constant_test.go index 84e241b3f6491..e6b443366070f 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -447,7 +447,7 @@ func (*testExpressionSuite) TestVectorizedConstant(c *C) { chk.SetSel(sel) c.Assert(cst.VecEval(ctx, chk, col), IsNil) i64s = col.Int64s() - for _, i := range sel { + for i := range sel { c.Assert(i64s[i], Equals, int64(2333)) } } @@ -473,7 +473,7 @@ func (*testExpressionSuite) TestVectorizedConstant(c *C) { sel := []int{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} chk.SetSel(sel) c.Assert(cst.VecEval(ctx, chk, col), IsNil) - for _, i := range sel { + for i := range sel { c.Assert(col.GetString(i), Equals, "hello") } } diff --git a/expression/vectorized.go b/expression/vectorized.go index 366394ee79919..ae9c5a23777ab 100644 --- a/expression/vectorized.go +++ b/expression/vectorized.go @@ -21,8 +21,7 @@ import ( ) func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.Chunk, result *chunk.Column) error { - n := input.NumEffectiveRows() - sel := input.Sel() + n := input.NumRows() tp := expr.GetType() switch tp.EvalType() { case types.ETInt: @@ -35,17 +34,10 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C return nil } i64s := result.Int64s() - if sel == nil { - for i := range i64s { - i64s[i] = v - } - result.SetNulls(0, n, false) - } else { - for _, i := range sel { - i64s[i] = v - result.SetNull(i, false) - } + for i := range i64s { + i64s[i] = v } + result.SetNulls(0, n, false) case types.ETReal: result.PreAllocFloat64(n) v, isNull, err := expr.EvalReal(ctx, chunk.Row{}) @@ -56,17 +48,10 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C return nil } f64s := result.Float64s() - if sel == nil { - for i := range f64s { - f64s[i] = v - } - result.SetNulls(0, n, false) - } else { - for _, i := range sel { - f64s[i] = v - result.SetNull(i, false) - } + for i := range f64s { + f64s[i] = v } + result.SetNulls(0, n, false) case types.ETDecimal: result.PreAllocDecimal(n) v, isNull, err := expr.EvalDecimal(ctx, chunk.Row{}) @@ -77,17 +62,10 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C return nil } ds := result.Decimals() - if sel == nil { - for i := range ds { - ds[i] = *v - } - result.SetNulls(0, n, false) - } else { - for _, i := range sel { - ds[i] = *v - result.SetNull(i, false) - } + for i := range ds { + ds[i] = *v } + result.SetNulls(0, n, false) case types.ETDatetime, types.ETTimestamp: result.Reset() v, isNull, err := expr.EvalTime(ctx, chunk.Row{}) @@ -99,20 +77,8 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C result.AppendNull() } } else { - if sel == nil { - for i := 0; i < n; i++ { - result.AppendTime(v) - } - } else { - pos := 0 - for _, i := range sel { - for pos < i { - result.AppendNull() - pos++ - } - result.AppendTime(v) - pos++ - } + for i := 0; i < n; i++ { + result.AppendTime(v) } } case types.ETDuration: @@ -126,20 +92,8 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C result.AppendNull() } } else { - if sel == nil { - for i := 0; i < n; i++ { - result.AppendDuration(v) - } - } else { - pos := 0 - for _, i := range sel { - for pos < i { - result.AppendNull() - pos++ - } - result.AppendDuration(v) - pos++ - } + for i := 0; i < n; i++ { + result.AppendDuration(v) } } case types.ETJson: @@ -153,20 +107,8 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C result.AppendNull() } } else { - if sel == nil { - for i := 0; i < n; i++ { - result.AppendJSON(v) - } - } else { - pos := 0 - for _, i := range sel { - for pos < i { - result.AppendNull() - pos++ - } - result.AppendJSON(v) - pos++ - } + for i := 0; i < n; i++ { + result.AppendJSON(v) } } case types.ETString: @@ -180,20 +122,8 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C result.AppendNull() } } else { - if sel == nil { - for i := 0; i < n; i++ { - result.AppendString(v) - } - } else { - pos := 0 - for _, i := range sel { - for pos < i { - result.AppendNull() - pos++ - } - result.AppendString(v) - pos++ - } + for i := 0; i < n; i++ { + result.AppendString(v) } } default: diff --git a/util/chunk/chunk.go b/util/chunk/chunk.go index 914a88210d17a..453b57c71b85a 100644 --- a/util/chunk/chunk.go +++ b/util/chunk/chunk.go @@ -311,21 +311,6 @@ func (c *Chunk) NumRows() int { return c.columns[0].length } -// NumEffectiveRows returns the effective number of rows physically stored in this Chunk. -// It is different with NumRows when sel is not nil. -// For example: if sel is [2, 3, 5, 7, 9], then -// NumRow() returns 5 to indicate that 5 rows are selected logically in this Chunk, while -// NumEffectiveRows() returns 10(9+1) to indicate that at least 10 rows are stored in this Chunk physically. -func (c *Chunk) NumEffectiveRows() int { - if c.sel == nil { - return c.NumRows() - } - if len(c.sel) == 0 { - return 0 - } - return c.sel[len(c.sel)-1] + 1 -} - // GetRow gets the Row in the chunk with the row index. func (c *Chunk) GetRow(idx int) Row { if c.sel != nil { From 0dc91069d8b94300a449837bcd6d001af9117120 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 7 Aug 2019 10:16:48 +0800 Subject: [PATCH 167/196] table: avoid redundant encode in insert/update/delete. (#11532) --- table/tables/tables.go | 25 +++++++-------- types/mydecimal.go | 10 +++--- util/codec/codec.go | 61 +++++++++++++++++++++++++++++++++++++ util/codec/codec_test.go | 66 ++++++++++++++++++++++++++++++++++++++++ util/codec/decimal.go | 7 +++++ 5 files changed, 150 insertions(+), 19 deletions(-) diff --git a/table/tables/tables.go b/table/tables/tables.go index e6b04136e0e3c..74bb680c2f8dc 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -339,19 +339,18 @@ func (t *tableCommon) UpdateRecord(ctx sessionctx.Context, h int64, oldData, new return err } } - colSize := make(map[int64]int64) - encodedCol := make([]byte, 0, 16) + colSize := make(map[int64]int64, len(t.Cols())) for id, col := range t.Cols() { - encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], newData[id]) + size, err := codec.EstimateValueSize(sc, newData[id]) if err != nil { continue } - newLen := len(encodedCol) - 1 - encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], oldData[id]) + newLen := size - 1 + size, err = codec.EstimateValueSize(sc, oldData[id]) if err != nil { continue } - oldLen := len(encodedCol) - 1 + oldLen := size - 1 colSize[col.ID] = int64(newLen - oldLen) } sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 0, 1, colSize) @@ -544,14 +543,13 @@ func (t *tableCommon) AddRecord(ctx sessionctx.Context, r []types.Datum, opts .. } } sc.AddAffectedRows(1) - colSize := make(map[int64]int64) - encodedCol := make([]byte, 0, 16) + colSize := make(map[int64]int64, len(r)) for id, col := range t.Cols() { - encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], r[id]) + size, err := codec.EstimateValueSize(sc, r[id]) if err != nil { continue } - colSize[col.ID] = int64(len(encodedCol) - 1) + colSize[col.ID] = int64(size) - 1 } sessVars.TxnCtx.UpdateDeltaForTable(t.physicalTableID, 1, 1, colSize) return recordID, nil @@ -726,15 +724,14 @@ func (t *tableCommon) RemoveRecord(ctx sessionctx.Context, h int64, r []types.Da } err = t.addDeleteBinlog(ctx, binlogRow, colIDs) } - colSize := make(map[int64]int64) - encodedCol := make([]byte, 0, 16) + colSize := make(map[int64]int64, len(t.Cols())) sc := ctx.GetSessionVars().StmtCtx for id, col := range t.Cols() { - encodedCol, err = tablecodec.EncodeValue(sc, encodedCol[:0], r[id]) + size, err := codec.EstimateValueSize(sc, r[id]) if err != nil { continue } - colSize[col.ID] = -int64(len(encodedCol) - 1) + colSize[col.ID] = -int64(size - 1) } ctx.GetSessionVars().TxnCtx.UpdateDeltaForTable(t.physicalTableID, -1, 1, colSize) return err diff --git a/types/mydecimal.go b/types/mydecimal.go index 996323fa8706f..e6eba643ebfda 100644 --- a/types/mydecimal.go +++ b/types/mydecimal.go @@ -1088,7 +1088,7 @@ with the correct -1/0/+1 result then the encoded value is not memory comparable. NOTE - the buffer is assumed to be of the size decimalBinSize(precision, frac) + the buffer is assumed to be of the size DecimalBinSize(precision, frac) RETURN VALUE bin - binary value @@ -1334,7 +1334,7 @@ func (d *MyDecimal) FromBin(bin []byte, precision, frac int) (binSize int, err e if bin[binIdx]&0x80 > 0 { mask = 0 } - binSize = decimalBinSize(precision, frac) + binSize = DecimalBinSize(precision, frac) dCopy := make([]byte, 40) dCopy = dCopy[:binSize] copy(dCopy, bin) @@ -1409,8 +1409,8 @@ func (d *MyDecimal) FromBin(bin []byte, precision, frac int) (binSize int, err e return binSize, err } -// decimalBinSize returns the size of array to hold a binary representation of a decimal. -func decimalBinSize(precision, frac int) int { +// DecimalBinSize returns the size of array to hold a binary representation of a decimal. +func DecimalBinSize(precision, frac int) int { digitsInt := precision - frac wordsInt := digitsInt / digitsPerWord wordsFrac := frac / digitsPerWord @@ -2242,7 +2242,7 @@ func DecimalPeak(b []byte) (int, error) { } precision := int(b[0]) frac := int(b[1]) - return decimalBinSize(precision, frac) + 2, nil + return DecimalBinSize(precision, frac) + 2, nil } // NewDecFromInt creates a MyDecimal from int. diff --git a/util/codec/codec.go b/util/codec/codec.go index bec5fc74123ba..c9d3ffae0f135 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -147,6 +147,38 @@ func encode(sc *stmtctx.StatementContext, b []byte, vals []types.Datum, comparab return b, errors.Trace(err) } +// EstimateValueSize uses to estimate the value size of the encoded values. +func EstimateValueSize(sc *stmtctx.StatementContext, val types.Datum) (int, error) { + l := 0 + switch val.Kind() { + case types.KindInt64: + l = valueSizeOfSignedInt(val.GetInt64()) + case types.KindUint64: + l = valueSizeOfUnsignedInt(val.GetUint64()) + case types.KindFloat32, types.KindFloat64, types.KindMysqlTime, types.KindMysqlDuration: + l = 9 + case types.KindString, types.KindBytes: + l = valueSizeOfBytes(val.GetBytes()) + case types.KindMysqlDecimal: + l = valueSizeOfDecimal(val.GetMysqlDecimal(), val.Length(), val.Frac()) + 1 + case types.KindMysqlEnum: + l = valueSizeOfUnsignedInt(uint64(val.GetMysqlEnum().ToNumber())) + case types.KindMysqlSet: + l = valueSizeOfUnsignedInt(uint64(val.GetMysqlSet().ToNumber())) + case types.KindMysqlBit, types.KindBinaryLiteral: + val, err := val.GetBinaryLiteral().ToInt(sc) + terror.Log(errors.Trace(err)) + l = valueSizeOfUnsignedInt(val) + case types.KindMysqlJSON: + l = 2 + len(val.GetMysqlJSON().Value) + case types.KindNull, types.KindMinNotNull, types.KindMaxValue: + l = 1 + default: + return l, errors.Errorf("unsupported encode type %d", val.Kind()) + } + return l, nil +} + // EncodeMySQLTime encodes datum of `KindMysqlTime` to []byte. func EncodeMySQLTime(sc *stmtctx.StatementContext, d types.Datum, tp byte, b []byte) (_ []byte, err error) { t := d.GetMysqlTime() @@ -181,6 +213,10 @@ func encodeBytes(b []byte, v []byte, comparable bool) []byte { return b } +func valueSizeOfBytes(v []byte) int { + return valueSizeOfSignedInt(int64(len(v))) + len(v) +} + func sizeBytes(v []byte, comparable bool) int { if comparable { reallocSize := (len(v)/encGroupSize + 1) * (encGroupSize + 1) @@ -201,6 +237,20 @@ func encodeSignedInt(b []byte, v int64, comparable bool) []byte { return b } +func valueSizeOfSignedInt(v int64) int { + if v < 0 { + v = 0 - v - 1 + } + // Flag occupy 1 bit and at lease 1 bit. + size := 2 + v = v >> 6 + for v > 0 { + size++ + v = v >> 7 + } + return size +} + func encodeUnsignedInt(b []byte, v uint64, comparable bool) []byte { if comparable { b = append(b, uintFlag) @@ -212,6 +262,17 @@ func encodeUnsignedInt(b []byte, v uint64, comparable bool) []byte { return b } +func valueSizeOfUnsignedInt(v uint64) int { + // Flag occupy 1 bit and at lease 1 bit. + size := 2 + v = v >> 7 + for v > 0 { + size++ + v = v >> 7 + } + return size +} + func sizeInt(comparable bool) int { if comparable { return 9 diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index b54a6ae8adeb4..343046e4ed189 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -90,12 +90,27 @@ func (s *testCodecSuite) TestCodecKey(c *C) { b, err = EncodeValue(sc, nil, t.Input...) c.Assert(err, IsNil) + size, err := estimateValuesSize(sc, t.Input) + c.Assert(err, IsNil) + c.Assert(len(b), Equals, size) args, err = Decode(b, 1) c.Assert(err, IsNil) c.Assert(args, DeepEquals, t.Expect) } } +func estimateValuesSize(sc *stmtctx.StatementContext, vals []types.Datum) (int, error) { + size := 0 + for _, val := range vals { + length, err := EstimateValueSize(sc, val) + if err != nil { + return 0, err + } + size += length + } + return size, nil +} + func (s *testCodecSuite) TestCodecKeyCompare(c *C) { defer testleak.AfterTest(c)() table := []struct { @@ -729,6 +744,13 @@ func (s *testCodecSuite) TestDecimal(c *C) { ret := bytes.Compare(b1, b2) c.Assert(ret, Equals, t.Ret, Commentf("%v %x %x", t, b1, b2)) + + b1, err = EncodeValue(sc, b1[:0], d1) + c.Assert(err, IsNil) + size, err := EstimateValueSize(sc, d1) + c.Assert(err, IsNil) + c.Assert(len(b1), Equals, size) + } floats := []float64{-123.45, -123.40, -23.45, -1.43, -0.93, -0.4333, -0.068, @@ -743,6 +765,10 @@ func (s *testCodecSuite) TestDecimal(c *C) { b, err := EncodeDecimal(nil, d.GetMysqlDecimal(), d.Length(), d.Frac()) c.Assert(err, IsNil) decs = append(decs, b) + size, err := EstimateValueSize(sc, d) + c.Assert(err, IsNil) + // size - 1 because the flag occupy 1 bit. + c.Assert(len(b), Equals, size-1) } for i := 0; i < len(decs)-1; i++ { cmp := bytes.Compare(decs[i], decs[i+1]) @@ -1023,3 +1049,43 @@ func (s *testCodecSuite) TestHashChunkRow(c *C) { c.Assert(err2, IsNil) c.Assert(b1, BytesEquals, b2) } + +func (s *testCodecSuite) TestValueSizeOfSignedInt(c *C) { + testCase := []int64{64, 8192, 1048576, 134217728, 17179869184, 2199023255552, 281474976710656, 36028797018963968, 4611686018427387904} + var b []byte + for _, v := range testCase { + b := encodeSignedInt(b[:0], v-10, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(v-10)) + + b = encodeSignedInt(b[:0], v, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(v)) + + b = encodeSignedInt(b[:0], v+10, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(v+10)) + + // Test for negative value. + b = encodeSignedInt(b[:0], 0-v, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(0-v)) + + b = encodeSignedInt(b[:0], 0-v+10, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(0-v+10)) + + b = encodeSignedInt(b[:0], 0-v-10, false) + c.Assert(len(b), Equals, valueSizeOfSignedInt(0-v-10)) + } +} + +func (s *testCodecSuite) TestValueSizeOfUnsignedInt(c *C) { + testCase := []uint64{128, 16384, 2097152, 268435456, 34359738368, 4398046511104, 562949953421312, 72057594037927936, 9223372036854775808} + var b []byte + for _, v := range testCase { + b := encodeUnsignedInt(b[:0], v-10, false) + c.Assert(len(b), Equals, valueSizeOfUnsignedInt(v-10)) + + b = encodeUnsignedInt(b[:0], v, false) + c.Assert(len(b), Equals, valueSizeOfUnsignedInt(v)) + + b = encodeUnsignedInt(b[:0], v+10, false) + c.Assert(len(b), Equals, valueSizeOfUnsignedInt(v+10)) + } +} diff --git a/util/codec/decimal.go b/util/codec/decimal.go index abf7130a7f4b9..2c1668d2b8b5c 100644 --- a/util/codec/decimal.go +++ b/util/codec/decimal.go @@ -30,6 +30,13 @@ func EncodeDecimal(b []byte, dec *types.MyDecimal, precision, frac int) ([]byte, return b, errors.Trace(err) } +func valueSizeOfDecimal(dec *types.MyDecimal, precision, frac int) int { + if precision == 0 { + precision, frac = dec.PrecisionAndFrac() + } + return types.DecimalBinSize(precision, frac) + 2 +} + // DecodeDecimal decodes bytes to decimal. func DecodeDecimal(b []byte) ([]byte, *types.MyDecimal, int, int, error) { failpoint.Inject("errorInDecodeDecimal", func(val failpoint.Value) { From a530f87db7755e027200497b64dadc70f3dbdc2a Mon Sep 17 00:00:00 2001 From: Zijie Zhang Date: Wed, 7 Aug 2019 10:51:34 +0800 Subject: [PATCH 168/196] planner: add aggregation hints `TIDB_HASHAGG` and `TIDB_STREAMAGG` (#11364) --- cmd/explaintest/r/topn_push_down.result | 16 +-- executor/join_test.go | 24 ++-- planner/core/exhaust_physical_plans.go | 76 ++++++++++-- planner/core/expression_rewriter.go | 5 +- planner/core/logical_plan_builder.go | 65 +++++----- planner/core/logical_plans.go | 5 + planner/core/physical_plan_test.go | 158 +++++++++++++++++++++++- planner/core/planbuilder.go | 1 + 8 files changed, 285 insertions(+), 65 deletions(-) diff --git a/cmd/explaintest/r/topn_push_down.result b/cmd/explaintest/r/topn_push_down.result index 4d40a3b3b8caf..89813ab77c0c0 100644 --- a/cmd/explaintest/r/topn_push_down.result +++ b/cmd/explaintest/r/topn_push_down.result @@ -240,19 +240,19 @@ explain select /*+ TIDB_SMJ(t1, t2) */ * from t t1 join t t2 on t1.a = t2.a limi id count task operator info Limit_11 5.00 root offset:0, count:5 └─MergeJoin_12 5.00 root inner join, left key:test.t1.a, right key:test.t2.a - ├─IndexReader_14 4.00 root index:IndexScan_13 - │ └─IndexScan_13 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo - └─IndexReader_16 4.00 root index:IndexScan_15 - └─IndexScan_15 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + ├─IndexReader_15 4.00 root index:IndexScan_14 + │ └─IndexScan_14 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + └─IndexReader_17 4.00 root index:IndexScan_16 + └─IndexScan_16 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo explain select /*+ TIDB_SMJ(t1, t2) */ * from t t1 left join t t2 on t1.a = t2.a where t2.a is null limit 5; id count task operator info Limit_12 5.00 root offset:0, count:5 └─Selection_13 5.00 root isnull(test.t2.a) └─MergeJoin_14 5.00 root left outer join, left key:test.t1.a, right key:test.t2.a - ├─IndexReader_16 4.00 root index:IndexScan_15 - │ └─IndexScan_15 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo - └─IndexReader_18 4.00 root index:IndexScan_17 - └─IndexScan_17 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + ├─IndexReader_17 4.00 root index:IndexScan_16 + │ └─IndexScan_16 4.00 cop table:t1, index:a, range:[NULL,+inf], keep order:true, stats:pseudo + └─IndexReader_19 4.00 root index:IndexScan_18 + └─IndexScan_18 4.00 cop table:t2, index:a, range:[NULL,+inf], keep order:true, stats:pseudo explain select /*+ TIDB_HJ(t1, t2) */ * from t t1 join t t2 on t1.a = t2.a limit 5; id count task operator info Limit_11 5.00 root offset:0, count:5 diff --git a/executor/join_test.go b/executor/join_test.go index 0f0b66214bf9d..1fa8ba6f20c37 100644 --- a/executor/join_test.go +++ b/executor/join_test.go @@ -143,13 +143,13 @@ func (s *testSuite2) TestJoin(c *C) { tk.MustQuery("select /*+ TIDB_INLJ(t1) */ * from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1 1 1 2", "1 1 1 3", "1 1 1 4", "3 3 3 4", " 4 5")) tk.MustQuery("select /*+ TIDB_INLJ(t) */ avg(t.b) from t right outer join t1 on t.a=t1.a").Check(testkit.Rows("1.5000")) - // Test that two conflict hints will return error. - err := tk.ExecToErr("select /*+ TIDB_INLJ(t) TIDB_SMJ(t) */ * from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) - err = tk.ExecToErr("select /*+ TIDB_INLJ(t) TIDB_HJ(t) */ from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) - err = tk.ExecToErr("select /*+ TIDB_SMJ(t) TIDB_HJ(t) */ from t join t1 on t.a=t1.a") - c.Assert(err, NotNil) + // Test that two conflict hints will return warning. + tk.MustExec("select /*+ TIDB_INLJ(t) TIDB_SMJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + tk.MustExec("select /*+ TIDB_INLJ(t) TIDB_HJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) + tk.MustExec("select /*+ TIDB_SMJ(t) TIDB_HJ(t) */ * from t join t1 on t.a=t1.a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.GetWarnings(), HasLen, 1) tk.MustExec("drop table if exists t") tk.MustExec("create table t(a int)") @@ -888,11 +888,11 @@ func (s *testSuite2) TestMergejoinOrder(c *C) { tk.MustExec("insert into t2 select a*100, b*100 from t1;") tk.MustQuery("explain select /*+ TIDB_SMJ(t2) */ * from t1 left outer join t2 on t1.a=t2.a and t1.a!=3 order by t1.a;").Check(testkit.Rows( - "MergeJoin_15 10000.00 root left outer join, left key:test.t1.a, right key:test.t2.a, left cond:[ne(test.t1.a, 3)]", - "├─TableReader_11 10000.00 root data:TableScan_10", - "│ └─TableScan_10 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo", - "└─TableReader_13 6666.67 root data:TableScan_12", - " └─TableScan_12 6666.67 cop table:t2, range:[-inf,3), (3,+inf], keep order:true, stats:pseudo", + "MergeJoin_20 10000.00 root left outer join, left key:test.t1.a, right key:test.t2.a, left cond:[ne(test.t1.a, 3)]", + "├─TableReader_12 10000.00 root data:TableScan_11", + "│ └─TableScan_11 10000.00 cop table:t1, range:[-inf,+inf], keep order:true, stats:pseudo", + "└─TableReader_14 6666.67 root data:TableScan_13", + " └─TableScan_13 6666.67 cop table:t2, range:[-inf,3), (3,+inf], keep order:true, stats:pseudo", )) tk.MustExec("set @@tidb_init_chunk_size=1") diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 5ff411d6e5e7f..73145def95be6 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -117,7 +117,7 @@ func (p *PhysicalMergeJoin) tryToGetChildReqProp(prop *property.PhysicalProperty } func (p *LogicalJoin) getMergeJoin(prop *property.PhysicalProperty) []PhysicalPlan { - joins := make([]PhysicalPlan, 0, len(p.leftProperties)) + joins := make([]PhysicalPlan, 0, len(p.leftProperties)+1) // The leftProperties caches all the possible properties that are provided by its children. for _, lhsChildProperty := range p.leftProperties { offsets := getMaxSortPrefix(lhsChildProperty, p.LeftJoinKeys) @@ -158,10 +158,10 @@ func (p *LogicalJoin) getMergeJoin(prop *property.PhysicalProperty) []PhysicalPl joins = append(joins, mergeJoin) } } - // If TiDB_SMJ hint is existed && no join keys in children property, - // it should to enforce merge join. - if len(joins) == 0 && (p.preferJoinType&preferMergeJoin) > 0 { - return p.getEnforcedMergeJoin(prop) + // If TiDB_SMJ hint is existed, it should consider enforce merge join, + // because we can't trust lhsChildProperty completely. + if (p.preferJoinType & preferMergeJoin) > 0 { + joins = append(joins, p.getEnforcedMergeJoin(prop)...) } return joins @@ -1198,11 +1198,36 @@ func (p *baseLogicalPlan) exhaustPhysicalPlans(_ *property.PhysicalProperty) []P panic("baseLogicalPlan.exhaustPhysicalPlans() should never be called.") } +func (la *LogicalAggregation) getEnforcedStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { + _, desc := prop.AllSameOrder() + enforcedAggs := make([]PhysicalPlan, 0, len(wholeTaskTypes)) + childProp := &property.PhysicalProperty{ + ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), + Enforced: true, + Items: property.ItemsFromCols(la.groupByCols, desc), + } + + for _, taskTp := range wholeTaskTypes { + copiedChildProperty := new(property.PhysicalProperty) + *copiedChildProperty = *childProp // It's ok to not deep copy the "cols" field. + copiedChildProperty.TaskTp = taskTp + + agg := basePhysicalAgg{ + GroupByItems: la.GroupByItems, + AggFuncs: la.AggFuncs, + }.initForStream(la.ctx, la.stats.ScaleByExpectCnt(prop.ExpectedCnt), copiedChildProperty) + agg.SetSchema(la.schema.Clone()) + enforcedAggs = append(enforcedAggs, agg) + } + return enforcedAggs +} + func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []PhysicalPlan { all, desc := prop.AllSameOrder() - if len(la.possibleProperties) == 0 || !all { + if !all { return nil } + for _, aggFunc := range la.AggFuncs { if aggFunc.Mode == aggregation.FinalMode { return nil @@ -1213,7 +1238,7 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P return nil } - streamAggs := make([]PhysicalPlan, 0, len(la.possibleProperties)*(len(wholeTaskTypes)-1)) + streamAggs := make([]PhysicalPlan, 0, len(la.possibleProperties)*(len(wholeTaskTypes)-1)+len(wholeTaskTypes)) childProp := &property.PhysicalProperty{ ExpectedCnt: math.Max(prop.ExpectedCnt*la.inputCount/la.stats.RowCount, prop.ExpectedCnt), } @@ -1244,6 +1269,11 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P streamAggs = append(streamAggs, agg) } } + // If STREAM_AGG hint is existed, it should consider enforce stream aggregation, + // because we can't trust possibleChildProperty completely. + if (la.preferAggType & preferStreamAgg) > 0 { + streamAggs = append(streamAggs, la.getEnforcedStreamAggs(prop)...) + } return streamAggs } @@ -1264,9 +1294,35 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy } func (la *LogicalAggregation) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { - aggs := make([]PhysicalPlan, 0, len(la.possibleProperties)+1) - aggs = append(aggs, la.getHashAggs(prop)...) - aggs = append(aggs, la.getStreamAggs(prop)...) + preferHash := (la.preferAggType & preferHashAgg) > 0 + preferStream := (la.preferAggType & preferStreamAgg) > 0 + if preferHash && preferStream { + errMsg := "Optimizer aggregation hints are conflicted" + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + la.preferAggType = 0 + preferHash, preferStream = false, false + } + + hashAggs := la.getHashAggs(prop) + if hashAggs != nil && preferHash { + return hashAggs + } + + streamAggs := la.getStreamAggs(prop) + if streamAggs != nil && preferStream { + return streamAggs + } + + if streamAggs == nil && preferStream { + errMsg := "Optimizer Hint TIDB_STREAMAGG is inapplicable" + warning := ErrInternal.GenWithStack(errMsg) + la.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) + } + + aggs := make([]PhysicalPlan, 0, len(hashAggs)+len(streamAggs)) + aggs = append(aggs, hashAggs...) + aggs = append(aggs, streamAggs...) return aggs } diff --git a/planner/core/expression_rewriter.go b/planner/core/expression_rewriter.go index ef197daa5a63b..af1be06afc031 100644 --- a/planner/core/expression_rewriter.go +++ b/planner/core/expression_rewriter.go @@ -756,10 +756,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, v *ast.Patte join.attachOnConds(expression.SplitCNFItems(checkCondition)) // Set join hint for this join. if er.b.TableHints() != nil { - er.err = join.setPreferredJoinType(er.b.TableHints()) - if er.err != nil { - return v, true - } + join.setPreferredJoinType(er.b.TableHints()) } er.p = join } else { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 69e831a015009..a85a635a791af 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -54,6 +54,10 @@ const ( TiDBIndexNestedLoopJoin = "tidb_inlj" // TiDBHashJoin is hint enforce hash join. TiDBHashJoin = "tidb_hj" + // TiDBHashAgg is hint enforce hash aggregation. + TiDBHashAgg = "tidb_hashagg" + // TiDBStreamAgg is hint enforce stream aggregation. + TiDBStreamAgg = "tidb_streamagg" ) const ( @@ -137,6 +141,9 @@ func (b *PlanBuilder) buildAggregation(ctx context.Context, p LogicalPlan, aggFu plan4Agg.GroupByItems = gbyItems plan4Agg.SetSchema(schema4Agg) plan4Agg.collectGroupByColumns() + if hint := b.TableHints(); hint != nil { + plan4Agg.preferAggType = hint.preferAggType + } return plan4Agg, aggIndexMap, nil } @@ -327,9 +334,9 @@ func extractTableAlias(p LogicalPlan) *model.CIStr { return nil } -func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) error { +func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) { if hintInfo == nil { - return nil + return } lhsAlias := extractTableAlias(p.children[0]) @@ -355,9 +362,10 @@ func (p *LogicalJoin) setPreferredJoinType(hintInfo *tableHintInfo) error { // If there're multiple join types and one of them is not index join hint, // then there is a conflict of join types. if bits.OnesCount(p.preferJoinType) > 1 && (p.preferJoinType^preferRightAsIndexInner^preferLeftAsIndexInner) > 0 { - return errors.New("Join hints are conflict, you can only specify one type of join") + errMsg := "Join hints are conflict, you can only specify one type of join" + warning := ErrInternal.GenWithStack(errMsg) + p.ctx.GetSessionVars().StmtCtx.AppendWarning(warning) } - return nil } func resetNotNullFlag(schema *expression.Schema, start, end int) { @@ -424,10 +432,7 @@ func (b *PlanBuilder) buildJoin(ctx context.Context, joinNode *ast.Join) (Logica joinPlan.redundantSchema = expression.MergeSchema(lRedundant, rRedundant) // Set preferred join algorithm if some join hints is specified by user. - err = joinPlan.setPreferredJoinType(b.TableHints()) - if err != nil { - return nil, err - } + joinPlan.setPreferredJoinType(b.TableHints()) // "NATURAL JOIN" doesn't have "ON" or "USING" conditions. // @@ -1931,8 +1936,9 @@ func (b *PlanBuilder) unfoldWildStar(p LogicalPlan, selectFields []*ast.SelectFi return resultList, nil } -func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { +func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) { var sortMergeTables, INLJTables, hashJoinTables []hintTableInfo + var preferAggType uint for _, hint := range hints { switch hint.HintName.L { case TiDBMergeJoin: @@ -1941,19 +1947,20 @@ func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint) bool { INLJTables = tableNames2HintTableInfo(hint.Tables) case TiDBHashJoin: hashJoinTables = tableNames2HintTableInfo(hint.Tables) + case TiDBHashAgg: + preferAggType |= preferHashAgg + case TiDBStreamAgg: + preferAggType |= preferStreamAgg default: // ignore hints that not implemented } } - if len(sortMergeTables)+len(INLJTables)+len(hashJoinTables) > 0 { - b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ - sortMergeJoinTables: sortMergeTables, - indexNestedLoopJoinTables: INLJTables, - hashJoinTables: hashJoinTables, - }) - return true - } - return false + b.tableHintInfo = append(b.tableHintInfo, tableHintInfo{ + sortMergeJoinTables: sortMergeTables, + indexNestedLoopJoinTables: INLJTables, + hashJoinTables: hashJoinTables, + preferAggType: preferAggType, + }) } func (b *PlanBuilder) popTableHints() { @@ -1983,10 +1990,10 @@ func (b *PlanBuilder) TableHints() *tableHintInfo { } func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p LogicalPlan, err error) { - if b.pushTableHints(sel.TableHints) { - // table hints are only visible in the current SELECT statement. - defer b.popTableHints() - } + b.pushTableHints(sel.TableHints) + // table hints are only visible in the current SELECT statement. + defer b.popTableHints() + if sel.SelectStmtOpts != nil { origin := b.inStraightJoin b.inStraightJoin = sel.SelectStmtOpts.StraightJoin @@ -2602,10 +2609,9 @@ func buildColumns2Handle( } func (b *PlanBuilder) buildUpdate(ctx context.Context, update *ast.UpdateStmt) (Plan, error) { - if b.pushTableHints(update.TableHints) { - // table hints are only visible in the current UPDATE statement. - defer b.popTableHints() - } + b.pushTableHints(update.TableHints) + // table hints are only visible in the current UPDATE statement. + defer b.popTableHints() // update subquery table should be forbidden var asNameList []string @@ -2823,10 +2829,9 @@ func extractTableAsNameForUpdate(p LogicalPlan, asNames map[*model.TableInfo][]* } func (b *PlanBuilder) buildDelete(ctx context.Context, delete *ast.DeleteStmt) (Plan, error) { - if b.pushTableHints(delete.TableHints) { - // table hints are only visible in the current DELETE statement. - defer b.popTableHints() - } + b.pushTableHints(delete.TableHints) + // table hints are only visible in the current DELETE statement. + defer b.popTableHints() p, err := b.buildResultSetNode(ctx, delete.TableRefs.TableRefs) if err != nil { diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 25094e5d0028b..c459d0d75810f 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -97,6 +97,8 @@ const ( preferRightAsIndexInner preferHashJoin preferMergeJoin + preferHashAgg + preferStreamAgg ) // LogicalJoin is the logical join plan. @@ -246,6 +248,9 @@ type LogicalAggregation struct { // groupByCols stores the columns that are group-by items. groupByCols []*expression.Column + // preferAggType stores preferred aggregation algorithm type. + preferAggType uint + possibleProperties [][]*expression.Column inputCount float64 // inputCount is the input count of this plan. } diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 8fe8eeff128f7..7d7be7839e4f9 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1521,7 +1521,7 @@ func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { } } -func (s *testPlanSuite) TestIndexJoinHint(c *C) { +func (s *testPlanSuite) TestJoinHints(c *C) { defer testleak.AfterTest(c)() store, dom, err := newStoreWithBootstrap() c.Assert(err, IsNil) @@ -1570,3 +1570,159 @@ func (s *testPlanSuite) TestIndexJoinHint(c *C) { } } } + +func (s *testPlanSuite) TestAggregationHints(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + warning string + }{ + // without Aggregation hints + { + sql: "select count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", + warning: "", + }, + { + sql: "select count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->HashAgg", + warning: "", + }, + // with Aggregation hints + { + sql: "select /*+ TIDB_HASHAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->HashAgg", + warning: "", + }, + { + sql: "select /*+ TIDB_STREAMAGG() */ count(t1.a) from t t1, t t2 where t1.a = t2.a*2 group by t1.a", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))->Projection}(test.t1.a,mul(test.t2.a, 2))->Sort->StreamAgg", + warning: "", + }, + // test conflict warning + { + sql: "select /*+ TIDB_HASHAGG() TIDB_STREAMAGG() */ count(*) from t t1, t t2 where t1.a = t2.b", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->StreamAgg", + warning: "[planner:1815]Optimizer aggregation hints are conflicted", + }, + } + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + if test.warning == "" { + c.Assert(len(warnings), Equals, 0) + } else { + c.Assert(len(warnings), Equals, 1) + c.Assert(warnings[0].Level, Equals, stmtctx.WarnLevelWarning) + c.Assert(warnings[0].Err.Error(), Equals, test.warning) + } + } +} + +func (s *testPlanSuite) TestHintScope(c *C) { + defer testleak.AfterTest(c)() + store, dom, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + defer func() { + dom.Close() + store.Close() + }() + se, err := session.CreateSession4Test(store) + c.Assert(err, IsNil) + _, err = se.Execute(context.Background(), "use test") + c.Assert(err, IsNil) + + tests := []struct { + sql string + best string + }{ + // join hints + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t3) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)->Sort}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_HJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_SMJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select /*+ TIDB_INLJ(t2) */ t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t3.c,test.t2.a)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_SMJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "MergeInnerJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + { + sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "IndexJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t2.a,test.t1.a)->Projection", + }, + { + sql: "select /*+ TIDB_HJ(t1) */ t1.a, t1.b from t t1, (select t2.a from t t2, t t3 where t2.a = t3.c) s where t1.a=s.a", + best: "RightHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->IndexReader(Index(t.c_d_e)[[NULL,+inf]])}(test.t2.a,test.t3.c)}(test.t1.a,test.t2.a)->Projection", + }, + // aggregation hints + { + sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select /*+ TIDB_HASHAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", + }, + { + sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select /*+ TIDB_STREAMAGG() */ sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Sort->Projection->StreamAgg->HashAgg->Projection", + }, + { + sql: "select /*+ TIDB_HASHAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->HashAgg->Projection", + }, + { + sql: "select /*+ TIDB_STREAMAGG() */ s, count(s) from (select sum(t1.a) as s from t t1, t t2 where t1.a = t2.b group by t1.a) p group by s", + best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.b)->Projection->HashAgg->Sort->StreamAgg->Projection", + }, + } + + ctx := context.Background() + for i, test := range tests { + comment := Commentf("case:%v sql:%s", i, test) + stmt, err := s.ParseOneStmt(test.sql, "", "") + c.Assert(err, IsNil, comment) + + p, err := planner.Optimize(ctx, se, stmt, s.is) + c.Assert(err, IsNil) + c.Assert(core.ToString(p), Equals, test.best) + + warnings := se.GetSessionVars().StmtCtx.GetWarnings() + c.Assert(warnings, HasLen, 0, comment) + } +} diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 0b351cd360dfb..3134485529e87 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -57,6 +57,7 @@ type tableHintInfo struct { indexNestedLoopJoinTables []hintTableInfo sortMergeJoinTables []hintTableInfo hashJoinTables []hintTableInfo + preferAggType uint } type hintTableInfo struct { From 028f63cb4ab40ff1a7ebe47714f7fd9399a98da0 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Wed, 7 Aug 2019 15:04:56 +0800 Subject: [PATCH 169/196] *: rename tidb_back_off_weight (#11655) --- executor/set_test.go | 16 ++++++++-------- session/bootstrap.go | 11 +++++++++++ session/session.go | 2 +- session/session_test.go | 2 +- sessionctx/variable/tidb_vars.go | 4 ++-- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/executor/set_test.go b/executor/set_test.go index fb370b1e855da..6fefbdadc10b0 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -370,16 +370,16 @@ func (s *testSuite2) TestSetVar(c *C) { c.Assert(err.Error(), Equals, "tidb_wait_split_region_timeout(0) cannot be smaller than 1") tk.MustQuery(`select @@session.tidb_wait_split_region_timeout;`).Check(testkit.Rows("1")) - tk.MustExec("set session tidb_back_off_weight = 3") - tk.MustQuery("select @@session.tidb_back_off_weight;").Check(testkit.Rows("3")) - tk.MustExec("set session tidb_back_off_weight = 20") - tk.MustQuery("select @@session.tidb_back_off_weight;").Check(testkit.Rows("20")) - _, err = tk.Exec("set session tidb_back_off_weight = -1") + tk.MustExec("set session tidb_backoff_weight = 3") + tk.MustQuery("select @@session.tidb_backoff_weight;").Check(testkit.Rows("3")) + tk.MustExec("set session tidb_backoff_weight = 20") + tk.MustQuery("select @@session.tidb_backoff_weight;").Check(testkit.Rows("20")) + _, err = tk.Exec("set session tidb_backoff_weight = -1") c.Assert(err, NotNil) - _, err = tk.Exec("set global tidb_back_off_weight = 0") + _, err = tk.Exec("set global tidb_backoff_weight = 0") c.Assert(err, NotNil) - tk.MustExec("set global tidb_back_off_weight = 10") - tk.MustQuery("select @@global.tidb_back_off_weight;").Check(testkit.Rows("10")) + tk.MustExec("set global tidb_backoff_weight = 10") + tk.MustQuery("select @@global.tidb_backoff_weight;").Check(testkit.Rows("10")) tk.MustExec("set @@tidb_expensive_query_time_threshold=70") tk.MustQuery("select @@tidb_expensive_query_time_threshold;").Check(testkit.Rows("70")) diff --git a/session/bootstrap.go b/session/bootstrap.go index dca15445ed679..b9ed24428bf55 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -348,6 +348,7 @@ const ( version32 = 32 version33 = 33 version34 = 34 + version35 = 35 ) func checkBootstrapped(s Session) (bool, error) { @@ -543,6 +544,10 @@ func upgrade(s Session) { upgradeToVer34(s) } + if ver < version35 { + upgradeToVer35(s) + } + updateBootstrapVer(s) _, err = s.Execute(context.Background(), "COMMIT") @@ -853,6 +858,12 @@ func upgradeToVer34(s Session) { doReentrantDDL(s, CreateOptRuleBlacklist) } +func upgradeToVer35(s Session) { + sql := fmt.Sprintf("UPDATE HIGH_PRIORITY %s.%s SET VARIABLE_NAME = '%s' WHERE VARIABLE_NAME = 'tidb_back_off_weight'", + mysql.SystemDB, mysql.GlobalVariablesTable, variable.TiDBBackOffWeight) + mustExecute(s, sql) +} + // updateBootstrapVer updates bootstrap version variable in mysql.TiDB table. func updateBootstrapVer(s Session) { // Update bootstrap version. diff --git a/session/session.go b/session/session.go index 236c78955e645..4ab10550ed299 100644 --- a/session/session.go +++ b/session/session.go @@ -1641,7 +1641,7 @@ func createSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, er const ( notBootstrapped = 0 - currentBootstrapVersion = 34 + currentBootstrapVersion = 35 ) func getStoreBootstrapVersion(store kv.Storage) int64 { diff --git a/session/session_test.go b/session/session_test.go index e7c01d368c0ce..5973742d411c6 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -2419,7 +2419,7 @@ func (s *testSessionSuite) TestKVVars(c *C) { tk.MustExec("insert kvvars values (1, 1)") tk2 := testkit.NewTestKitWithInit(c, s.store) tk2.MustExec("set @@tidb_backoff_lock_fast = 1") - tk2.MustExec("set @@tidb_back_off_weight = 100") + tk2.MustExec("set @@tidb_backoff_weight = 100") backoffVal := new(int64) backOffWeightVal := new(int32) tk2.Se.GetSessionVars().KVVars.Hook = func(name string, vars *kv.Variables) { diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index f2b7998ab0341..ade964cb85966 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -225,11 +225,11 @@ const ( // tidb_backoff_lock_fast is used for tikv backoff base time in milliseconds. TiDBBackoffLockFast = "tidb_backoff_lock_fast" - // tidb_back_off_weight is used to control the max back off time in TiDB. + // tidb_backoff_weight is used to control the max back off time in TiDB. // The default maximum back off time is a small value. // BackOffWeight could multiply it to let the user adjust the maximum time for retrying. // Only positive integers can be accepted, which means that the maximum back off time can only grow. - TiDBBackOffWeight = "tidb_back_off_weight" + TiDBBackOffWeight = "tidb_backoff_weight" // tidb_ddl_reorg_worker_cnt defines the count of ddl reorg workers. TiDBDDLReorgWorkerCount = "tidb_ddl_reorg_worker_cnt" From 8fac5e23edd2dd1eadcfcccecb189da8facb1b59 Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 7 Aug 2019 16:42:10 +0800 Subject: [PATCH 170/196] expression: fix a bug in when comparing bit with string (#11654) --- expression/builtin_compare.go | 3 +++ expression/integration_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/expression/builtin_compare.go b/expression/builtin_compare.go index d9b8b45a86ec4..6e7cc3a2e4d0b 100644 --- a/expression/builtin_compare.go +++ b/expression/builtin_compare.go @@ -1117,6 +1117,9 @@ func RefineComparedConstant(ctx sessionctx.Context, targetFieldType types.FieldT } sc := ctx.GetSessionVars().StmtCtx + if targetFieldType.Tp == mysql.TypeBit { + targetFieldType = *types.NewFieldType(mysql.TypeLonglong) + } var intDatum types.Datum intDatum, err = dt.ConvertTo(sc, &targetFieldType) if err != nil { diff --git a/expression/integration_test.go b/expression/integration_test.go index c3578ad96d9f7..df87ece0ff292 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4630,6 +4630,22 @@ func (s *testIntegrationSuite) TestIssue10675(c *C) { tk.MustQuery(`select * from t where a < 184467440737095516167.1;`).Check( testkit.Rows("1")) tk.MustQuery(`select * from t where a > 184467440737095516167.1;`).Check(testkit.Rows()) + + // issue 11647 + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(b bit(1));`) + tk.MustExec(`insert into t values(b'1');`) + tk.MustQuery(`select count(*) from t where b = 1;`).Check(testkit.Rows("1")) + tk.MustQuery(`select count(*) from t where b = '1';`).Check(testkit.Rows("1")) + tk.MustQuery(`select count(*) from t where b = b'1';`).Check(testkit.Rows("1")) + + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(b bit(63));`) + // Not 64, because the behavior of mysql is amazing. I have no idea to fix it. + tk.MustExec(`insert into t values(b'111111111111111111111111111111111111111111111111111111111111111');`) + tk.MustQuery(`select count(*) from t where b = 9223372036854775807;`).Check(testkit.Rows("1")) + tk.MustQuery(`select count(*) from t where b = '9223372036854775807';`).Check(testkit.Rows("1")) + tk.MustQuery(`select count(*) from t where b = b'111111111111111111111111111111111111111111111111111111111111111';`).Check(testkit.Rows("1")) } func (s *testIntegrationSuite) TestDatetimeMicrosecond(c *C) { From fe038642e5c3f37730e9c55362a9651f6a16d1fd Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Wed, 7 Aug 2019 17:44:09 +0800 Subject: [PATCH 171/196] *: refactor cost model formulas and constants (#10581) --- .../r/access_path_selection.result | 6 +- cmd/explaintest/r/explain.result | 2 + .../r/explain_complex_stats.result | 11 +- cmd/explaintest/r/explain_easy.result | 100 ++-- cmd/explaintest/r/explain_easy_stats.result | 53 +- cmd/explaintest/r/generated_columns.result | 17 +- cmd/explaintest/r/index_join.result | 13 +- cmd/explaintest/r/partition_pruning.result | 30 +- cmd/explaintest/r/select.result | 2 + cmd/explaintest/r/subquery.result | 2 + cmd/explaintest/r/tpch.result | 39 +- cmd/explaintest/t/explain.test | 2 + cmd/explaintest/t/explain_easy.test | 2 + cmd/explaintest/t/explain_easy_stats.test | 2 + cmd/explaintest/t/index_join.test | 2 + cmd/explaintest/t/select.test | 2 + cmd/explaintest/t/subquery.test | 2 + executor/aggregate_test.go | 40 +- executor/builder.go | 17 +- executor/index_lookup_join_test.go | 46 -- expression/constant_propagation_test.go | 2 +- planner/core/cbo_test.go | 42 +- planner/core/exhaust_physical_plans.go | 78 +-- planner/core/find_best_task.go | 81 +-- planner/core/logical_plan_builder.go | 6 +- planner/core/logical_plan_test.go | 12 +- planner/core/logical_plans.go | 3 + planner/core/physical_plan_test.go | 60 ++- planner/core/physical_plans.go | 12 +- planner/core/planbuilder.go | 10 +- planner/core/resolve_indices.go | 2 + planner/core/stats.go | 69 ++- planner/core/task.go | 471 ++++++++++++++---- planner/property/stats_info.go | 7 +- statistics/selectivity_test.go | 6 +- statistics/table.go | 23 + util/ranger/ranger_test.go | 10 +- 37 files changed, 816 insertions(+), 468 deletions(-) diff --git a/cmd/explaintest/r/access_path_selection.result b/cmd/explaintest/r/access_path_selection.result index 3e857d0b1d028..58ea41eca95ca 100644 --- a/cmd/explaintest/r/access_path_selection.result +++ b/cmd/explaintest/r/access_path_selection.result @@ -15,9 +15,9 @@ IndexReader_6 3323.33 root index:IndexScan_5 └─IndexScan_5 3323.33 cop table:access_path_selection, index:a, b, range:[-inf,3), keep order:false, stats:pseudo explain select a, b from access_path_selection where b < 3; id count task operator info -IndexLookUp_10 3323.33 root -├─IndexScan_8 3323.33 cop table:access_path_selection, index:b, range:[-inf,3), keep order:false, stats:pseudo -└─TableScan_9 3323.33 cop table:access_path_selection, keep order:false, stats:pseudo +TableReader_7 3323.33 root data:Selection_6 +└─Selection_6 3323.33 cop lt(test.access_path_selection.b, 3) + └─TableScan_5 10000.00 cop table:access_path_selection, range:[-inf,+inf], keep order:false, stats:pseudo explain select a, b from access_path_selection where a < 3 and b < 3; id count task operator info IndexReader_11 1104.45 root index:Selection_10 diff --git a/cmd/explaintest/r/explain.result b/cmd/explaintest/r/explain.result index 0055b8834a554..a14ff14fe5276 100644 --- a/cmd/explaintest/r/explain.result +++ b/cmd/explaintest/r/explain.result @@ -24,6 +24,8 @@ Field Type Null Key Default Extra id int(11) YES NULL drop table if exists t; create table t(id int primary key, a int, b int); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; explain select group_concat(a) from t group by id; id count task operator info StreamAgg_8 8000.00 root group by:col_1, funcs:group_concat(col_0, ",") diff --git a/cmd/explaintest/r/explain_complex_stats.result b/cmd/explaintest/r/explain_complex_stats.result index 419d1ba3834d8..9f98b3f967a91 100644 --- a/cmd/explaintest/r/explain_complex_stats.result +++ b/cmd/explaintest/r/explain_complex_stats.result @@ -117,12 +117,11 @@ explain SELECT ds, p1, p2, p3, p4, p5, p6_md5, p7_md5, count(dic) as install_dev id count task operator info Projection_7 21.53 root test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, install_device └─Sort_8 21.53 root test.dt.ds2:desc - └─HashAgg_16 21.53 root group by:col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, funcs:count(col_0), firstrow(col_1), firstrow(col_2), firstrow(col_3), firstrow(col_4), firstrow(col_5), firstrow(col_6), firstrow(col_7), firstrow(col_8), firstrow(col_9) - └─IndexLookUp_17 21.53 root - ├─IndexScan_13 128.32 cop table:dt, index:cm, range:[1062,1062], [1086,1086], [1423,1423], [1424,1424], [1425,1425], [1426,1426], [1427,1427], [1428,1428], [1429,1429], [1430,1430], [1431,1431], [1432,1432], [1433,1433], [1434,1434], [1435,1435], [1436,1436], [1437,1437], [1438,1438], [1439,1439], [1440,1440], [1441,1441], [1442,1442], [1443,1443], [1444,1444], [1445,1445], [1446,1446], [1447,1447], [1448,1448], [1449,1449], [1450,1450], [1451,1451], [1452,1452], [1488,1488], [1489,1489], [1490,1490], [1491,1491], [1492,1492], [1493,1493], [1494,1494], [1495,1495], [1496,1496], [1497,1497], [1550,1550], [1551,1551], [1552,1552], [1553,1553], [1554,1554], [1555,1555], [1556,1556], [1557,1557], [1558,1558], [1559,1559], [1597,1597], [1598,1598], [1599,1599], [1600,1600], [1601,1601], [1602,1602], [1603,1603], [1604,1604], [1605,1605], [1606,1606], [1607,1607], [1608,1608], [1609,1609], [1610,1610], [1611,1611], [1612,1612], [1613,1613], [1614,1614], [1615,1615], [1616,1616], [1623,1623], [1624,1624], [1625,1625], [1626,1626], [1627,1627], [1628,1628], [1629,1629], [1630,1630], [1631,1631], [1632,1632], [1709,1709], [1719,1719], [1720,1720], [1843,1843], [2813,2813], [2814,2814], [2815,2815], [2816,2816], [2817,2817], [2818,2818], [2819,2819], [2820,2820], [2821,2821], [2822,2822], [2823,2823], [2824,2824], [2825,2825], [2826,2826], [2827,2827], [2828,2828], [2829,2829], [2830,2830], [2831,2831], [2832,2832], [2833,2833], [2834,2834], [2835,2835], [2836,2836], [2837,2837], [2838,2838], [2839,2839], [2840,2840], [2841,2841], [2842,2842], [2843,2843], [2844,2844], [2845,2845], [2846,2846], [2847,2847], [2848,2848], [2849,2849], [2850,2850], [2851,2851], [2852,2852], [2853,2853], [2854,2854], [2855,2855], [2856,2856], [2857,2857], [2858,2858], [2859,2859], [2860,2860], [2861,2861], [2862,2862], [2863,2863], [2864,2864], [2865,2865], [2866,2866], [2867,2867], [2868,2868], [2869,2869], [2870,2870], [2871,2871], [2872,2872], [3139,3139], [3140,3140], [3141,3141], [3142,3142], [3143,3143], [3144,3144], [3145,3145], [3146,3146], [3147,3147], [3148,3148], [3149,3149], [3150,3150], [3151,3151], [3152,3152], [3153,3153], [3154,3154], [3155,3155], [3156,3156], [3157,3157], [3158,3158], [3386,3386], [3387,3387], [3388,3388], [3389,3389], [3390,3390], [3391,3391], [3392,3392], [3393,3393], [3394,3394], [3395,3395], [3664,3664], [3665,3665], [3666,3666], [3667,3667], [3668,3668], [3670,3670], [3671,3671], [3672,3672], [3673,3673], [3674,3674], [3676,3676], [3677,3677], [3678,3678], [3679,3679], [3680,3680], [3681,3681], [3682,3682], [3683,3683], [3684,3684], [3685,3685], [3686,3686], [3687,3687], [3688,3688], [3689,3689], [3690,3690], [3691,3691], [3692,3692], [3693,3693], [3694,3694], [3695,3695], [3696,3696], [3697,3697], [3698,3698], [3699,3699], [3700,3700], [3701,3701], [3702,3702], [3703,3703], [3704,3704], [3705,3705], [3706,3706], [3707,3707], [3708,3708], [3709,3709], [3710,3710], [3711,3711], [3712,3712], [3713,3713], [3714,3714], [3715,3715], [3960,3960], [3961,3961], [3962,3962], [3963,3963], [3964,3964], [3965,3965], [3966,3966], [3967,3967], [3968,3968], [3978,3978], [3979,3979], [3980,3980], [3981,3981], [3982,3982], [3983,3983], [3984,3984], [3985,3985], [3986,3986], [3987,3987], [4208,4208], [4209,4209], [4210,4210], [4211,4211], [4212,4212], [4304,4304], [4305,4305], [4306,4306], [4307,4307], [4308,4308], [4866,4866], [4867,4867], [4868,4868], [4869,4869], [4870,4870], [4871,4871], [4872,4872], [4873,4873], [4874,4874], [4875,4875], keep order:false - └─HashAgg_11 21.53 cop group by:test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, funcs:count(test.dt.dic), firstrow(test.dt.ds), firstrow(test.dt.ds2), firstrow(test.dt.p1), firstrow(test.dt.p2), firstrow(test.dt.p3), firstrow(test.dt.p4), firstrow(test.dt.p5), firstrow(test.dt.p6_md5), firstrow(test.dt.p7_md5) - └─Selection_15 21.56 cop ge(test.dt.ds, 2016-09-01 00:00:00.000000), le(test.dt.ds, 2016-11-03 00:00:00.000000) - └─TableScan_14 128.32 cop table:dt, keep order:false + └─HashAgg_12 21.53 root group by:test.dt.ds, test.dt.p1, test.dt.p2, test.dt.p3, test.dt.p4, test.dt.p5, test.dt.p6_md5, test.dt.p7_md5, funcs:count(test.dt.dic), firstrow(test.dt.ds), firstrow(test.dt.ds2), firstrow(test.dt.p1), firstrow(test.dt.p2), firstrow(test.dt.p3), firstrow(test.dt.p4), firstrow(test.dt.p5), firstrow(test.dt.p6_md5), firstrow(test.dt.p7_md5) + └─IndexLookUp_21 21.56 root + ├─IndexScan_18 128.32 cop table:dt, index:cm, range:[1062,1062], [1086,1086], [1423,1423], [1424,1424], [1425,1425], [1426,1426], [1427,1427], [1428,1428], [1429,1429], [1430,1430], [1431,1431], [1432,1432], [1433,1433], [1434,1434], [1435,1435], [1436,1436], [1437,1437], [1438,1438], [1439,1439], [1440,1440], [1441,1441], [1442,1442], [1443,1443], [1444,1444], [1445,1445], [1446,1446], [1447,1447], [1448,1448], [1449,1449], [1450,1450], [1451,1451], [1452,1452], [1488,1488], [1489,1489], [1490,1490], [1491,1491], [1492,1492], [1493,1493], [1494,1494], [1495,1495], [1496,1496], [1497,1497], [1550,1550], [1551,1551], [1552,1552], [1553,1553], [1554,1554], [1555,1555], [1556,1556], [1557,1557], [1558,1558], [1559,1559], [1597,1597], [1598,1598], [1599,1599], [1600,1600], [1601,1601], [1602,1602], [1603,1603], [1604,1604], [1605,1605], [1606,1606], [1607,1607], [1608,1608], [1609,1609], [1610,1610], [1611,1611], [1612,1612], [1613,1613], [1614,1614], [1615,1615], [1616,1616], [1623,1623], [1624,1624], [1625,1625], [1626,1626], [1627,1627], [1628,1628], [1629,1629], [1630,1630], [1631,1631], [1632,1632], [1709,1709], [1719,1719], [1720,1720], [1843,1843], [2813,2813], [2814,2814], [2815,2815], [2816,2816], [2817,2817], [2818,2818], [2819,2819], [2820,2820], [2821,2821], [2822,2822], [2823,2823], [2824,2824], [2825,2825], [2826,2826], [2827,2827], [2828,2828], [2829,2829], [2830,2830], [2831,2831], [2832,2832], [2833,2833], [2834,2834], [2835,2835], [2836,2836], [2837,2837], [2838,2838], [2839,2839], [2840,2840], [2841,2841], [2842,2842], [2843,2843], [2844,2844], [2845,2845], [2846,2846], [2847,2847], [2848,2848], [2849,2849], [2850,2850], [2851,2851], [2852,2852], [2853,2853], [2854,2854], [2855,2855], [2856,2856], [2857,2857], [2858,2858], [2859,2859], [2860,2860], [2861,2861], [2862,2862], [2863,2863], [2864,2864], [2865,2865], [2866,2866], [2867,2867], [2868,2868], [2869,2869], [2870,2870], [2871,2871], [2872,2872], [3139,3139], [3140,3140], [3141,3141], [3142,3142], [3143,3143], [3144,3144], [3145,3145], [3146,3146], [3147,3147], [3148,3148], [3149,3149], [3150,3150], [3151,3151], [3152,3152], [3153,3153], [3154,3154], [3155,3155], [3156,3156], [3157,3157], [3158,3158], [3386,3386], [3387,3387], [3388,3388], [3389,3389], [3390,3390], [3391,3391], [3392,3392], [3393,3393], [3394,3394], [3395,3395], [3664,3664], [3665,3665], [3666,3666], [3667,3667], [3668,3668], [3670,3670], [3671,3671], [3672,3672], [3673,3673], [3674,3674], [3676,3676], [3677,3677], [3678,3678], [3679,3679], [3680,3680], [3681,3681], [3682,3682], [3683,3683], [3684,3684], [3685,3685], [3686,3686], [3687,3687], [3688,3688], [3689,3689], [3690,3690], [3691,3691], [3692,3692], [3693,3693], [3694,3694], [3695,3695], [3696,3696], [3697,3697], [3698,3698], [3699,3699], [3700,3700], [3701,3701], [3702,3702], [3703,3703], [3704,3704], [3705,3705], [3706,3706], [3707,3707], [3708,3708], [3709,3709], [3710,3710], [3711,3711], [3712,3712], [3713,3713], [3714,3714], [3715,3715], [3960,3960], [3961,3961], [3962,3962], [3963,3963], [3964,3964], [3965,3965], [3966,3966], [3967,3967], [3968,3968], [3978,3978], [3979,3979], [3980,3980], [3981,3981], [3982,3982], [3983,3983], [3984,3984], [3985,3985], [3986,3986], [3987,3987], [4208,4208], [4209,4209], [4210,4210], [4211,4211], [4212,4212], [4304,4304], [4305,4305], [4306,4306], [4307,4307], [4308,4308], [4866,4866], [4867,4867], [4868,4868], [4869,4869], [4870,4870], [4871,4871], [4872,4872], [4873,4873], [4874,4874], [4875,4875], keep order:false + └─Selection_20 21.56 cop ge(test.dt.ds, 2016-09-01 00:00:00.000000), le(test.dt.ds, 2016-11-03 00:00:00.000000) + └─TableScan_19 128.32 cop table:dt, keep order:false explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic as dic,sdk.ip as ip, sdk.t as t, gad.p1 as p1, gad.p2 as p2, gad.p3 as p3, gad.p4 as p4, gad.p5 as p5, gad.p6_md5 as p6, gad.p7_md5 as p7, gad.ext as ext, gad.t as gtime from st gad join (select id, aid, pt, dic, ip, t from dd where pt = 'android' and bm = 0 and t > 1478143908) sdk on gad.aid = sdk.aid and gad.ip = sdk.ip and sdk.t > gad.t where gad.t > 1478143908 and gad.bm = 0 and gad.pt = 'android' group by gad.aid, sdk.dic limit 2500; id count task operator info Projection_13 424.00 root test.gad.id, test.dd.id, test.gad.aid, test.gad.cm, test.dd.dic, test.dd.ip, test.dd.t, test.gad.p1, test.gad.p2, test.gad.p3, test.gad.p4, test.gad.p5, test.gad.p6_md5, test.gad.p7_md5, test.gad.ext, test.gad.t diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 5d33b516c4cb8..d06d685d8995f 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -7,6 +7,8 @@ create table t3 (a bigint, b bigint, c bigint, d bigint); create table t4 (a int, b int, c int, index idx(a, b), primary key(a)); set @@session.tidb_opt_agg_push_down = 1; set @@session.tidb_opt_insubq_to_join_and_agg=1; +set @@session.tidb_hashagg_partial_concurrency = 1; +set @@session.tidb_hashagg_final_concurrency = 1; explain select * from t3 where exists (select s.a from t3 s having sum(s.a) = t3.a ); id count task operator info Projection_12 8000.00 root test.t3.a, test.t3.b, test.t3.c, test.t3.d @@ -42,13 +44,12 @@ IndexReader_6 10.00 root index:IndexScan_5 └─IndexScan_5 10.00 cop table:t1, index:c2, range:[1,1], keep order:false, stats:pseudo explain select * from t1 left join t2 on t1.c2 = t2.c1 where t1.c1 > 1; id count task operator info -IndexJoin_12 4166.67 root left outer join, inner:IndexLookUp_11, outer key:test.t1.c2, inner key:test.t2.c1 +HashLeftJoin_13 4166.67 root left outer join, inner:TableReader_27, equal:[eq(test.t1.c2, test.t2.c1)] ├─TableReader_24 3333.33 root data:TableScan_23 │ └─TableScan_23 3333.33 cop table:t1, range:(1,+inf], keep order:false, stats:pseudo -└─IndexLookUp_11 9.99 root - ├─Selection_10 9.99 cop not(isnull(test.t2.c1)) - │ └─IndexScan_8 10.00 cop table:t2, index:c1, range: decided by [eq(test.t2.c1, test.t1.c2)], keep order:false, stats:pseudo - └─TableScan_9 9.99 cop table:t2, keep order:false, stats:pseudo +└─TableReader_27 9990.00 root data:Selection_26 + └─Selection_26 9990.00 cop not(isnull(test.t2.c1)) + └─TableScan_25 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo explain update t1 set t1.c2 = 2 where t1.c1 = 1; id count task operator info Point_Get_1 1.00 root table:t1, handle:1 @@ -60,14 +61,13 @@ IndexLookUp_9 10.00 root explain select count(b.c2) from t1 a, t2 b where a.c1 = b.c2 group by a.c1; id count task operator info Projection_11 9990.00 root cast(join_agg_0) -└─IndexJoin_14 9990.00 root inner join, inner:TableReader_13, outer key:test.b.c2, inner key:test.a.c1 - ├─TableReader_13 1.00 root data:TableScan_12 - │ └─TableScan_12 1.00 cop table:a, range: decided by [test.b.c2], keep order:false, stats:pseudo - └─HashAgg_22 7992.00 root group by:col_2, funcs:count(col_0), firstrow(col_1) - └─TableReader_23 7992.00 root data:HashAgg_17 - └─HashAgg_17 7992.00 cop group by:test.b.c2, funcs:count(test.b.c2), firstrow(test.b.c2) - └─Selection_21 9990.00 cop not(isnull(test.b.c2)) - └─TableScan_20 10000.00 cop table:b, range:[-inf,+inf], keep order:false, stats:pseudo +└─HashLeftJoin_15 9990.00 root inner join, inner:HashAgg_19, equal:[eq(test.a.c1, test.b.c2)] + ├─TableReader_28 10000.00 root data:TableScan_27 + │ └─TableScan_27 10000.00 cop table:a, range:[-inf,+inf], keep order:false, stats:pseudo + └─HashAgg_19 7992.00 root group by:test.b.c2, funcs:count(test.b.c2), firstrow(test.b.c2) + └─TableReader_26 9990.00 root data:Selection_25 + └─Selection_25 9990.00 cop not(isnull(test.b.c2)) + └─TableScan_24 10000.00 cop table:b, range:[-inf,+inf], keep order:false, stats:pseudo explain select * from t2 order by t2.c2 limit 0, 1; id count task operator info TopN_7 1.00 root test.t2.c2:asc, offset:0, count:1 @@ -97,9 +97,9 @@ StreamAgg_12 1.00 root funcs:sum(col_0) explain select c1 from t1 where c1 in (select c2 from t2); id count task operator info Projection_9 9990.00 root test.t1.c1 -└─IndexJoin_12 9990.00 root inner join, inner:TableReader_11, outer key:test.t2.c2, inner key:test.t1.c1 - ├─TableReader_11 1.00 root data:TableScan_10 - │ └─TableScan_10 1.00 cop table:t1, range: decided by [test.t2.c2], keep order:false, stats:pseudo +└─HashLeftJoin_13 9990.00 root inner join, inner:HashAgg_20, equal:[eq(test.t1.c1, test.t2.c2)] + ├─TableReader_26 10000.00 root data:TableScan_25 + │ └─TableScan_25 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo └─HashAgg_20 7992.00 root group by:col_1, funcs:firstrow(col_0) └─TableReader_21 7992.00 root data:HashAgg_15 └─HashAgg_15 7992.00 cop group by:test.t2.c2, funcs:firstrow(test.t2.c2) @@ -165,32 +165,27 @@ id count task operator info Union_17 26000.00 root ├─HashAgg_21 16000.00 root group by:c1, funcs:firstrow(join_agg_0) │ └─Union_22 16000.00 root -│ ├─StreamAgg_34 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1) -│ │ └─IndexReader_35 8000.00 root index:StreamAgg_26 -│ │ └─StreamAgg_26 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) -│ │ └─IndexScan_33 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo -│ └─StreamAgg_49 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1) -│ └─IndexReader_50 8000.00 root index:StreamAgg_41 -│ └─StreamAgg_41 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) -│ └─IndexScan_48 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo +│ ├─StreamAgg_27 8000.00 root group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) +│ │ └─IndexReader_37 10000.00 root index:IndexScan_36 +│ │ └─IndexScan_36 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo +│ └─StreamAgg_42 8000.00 root group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) +│ └─IndexReader_52 10000.00 root index:IndexScan_51 +│ └─IndexScan_51 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo └─TableReader_55 10000.00 root data:TableScan_54 └─TableScan_54 10000.00 cop table:t2, range:[-inf,+inf], keep order:false, stats:pseudo explain select c1 from t2 union all select c1 from t2 union select c1 from t2; id count task operator info HashAgg_18 24000.00 root group by:c1, funcs:firstrow(join_agg_0) └─Union_19 24000.00 root - ├─StreamAgg_31 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1) - │ └─IndexReader_32 8000.00 root index:StreamAgg_23 - │ └─StreamAgg_23 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) - │ └─IndexScan_30 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo - ├─StreamAgg_46 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1) - │ └─IndexReader_47 8000.00 root index:StreamAgg_38 - │ └─StreamAgg_38 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) - │ └─IndexScan_45 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo - └─StreamAgg_61 8000.00 root group by:col_2, funcs:firstrow(col_0), firstrow(col_1) - └─IndexReader_62 8000.00 root index:StreamAgg_53 - └─StreamAgg_53 8000.00 cop group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) - └─IndexScan_60 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo + ├─StreamAgg_24 8000.00 root group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) + │ └─IndexReader_34 10000.00 root index:IndexScan_33 + │ └─IndexScan_33 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo + ├─StreamAgg_39 8000.00 root group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) + │ └─IndexReader_49 10000.00 root index:IndexScan_48 + │ └─IndexScan_48 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo + └─StreamAgg_54 8000.00 root group by:test.t2.c1, funcs:firstrow(test.t2.c1), firstrow(test.t2.c1) + └─IndexReader_64 10000.00 root index:IndexScan_63 + └─IndexScan_63 10000.00 cop table:t2, index:c1, range:[NULL,+inf], keep order:true, stats:pseudo explain select count(1) from (select count(1) from (select * from t1 where c3 = 100) k) k2; id count task operator info StreamAgg_13 1.00 root funcs:count(1) @@ -316,12 +311,11 @@ Projection_11 10000.00 root 9_aux_0 ├─TableReader_15 10000.00 root data:TableScan_14 │ └─TableScan_14 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─StreamAgg_20 1.00 root funcs:count(1) - └─IndexJoin_44 12.50 root inner join, inner:TableReader_43, outer key:test.s.a, inner key:test.t1.a - ├─TableReader_37 1.00 root data:TableScan_36 - │ └─TableScan_36 1.00 cop table:s, range: decided by [eq(test.s.a, test.t.a)], keep order:false, stats:pseudo - └─TableReader_43 0.80 root data:Selection_42 - └─Selection_42 0.80 cop eq(test.t1.a, test.t.a) - └─TableScan_41 1.00 cop table:t1, range: decided by [test.s.a], keep order:false, stats:pseudo + └─MergeJoin_40 12.50 root inner join, left key:test.s.a, right key:test.t1.a + ├─TableReader_33 1.00 root data:TableScan_32 + │ └─TableScan_32 1.00 cop table:s, range: decided by [eq(test.s.a, test.t.a)], keep order:true, stats:pseudo + └─TableReader_35 1.00 root data:TableScan_34 + └─TableScan_34 1.00 cop table:t1, range: decided by [eq(test.t1.a, test.t.a)], keep order:true, stats:pseudo explain select t.c in (select count(*) from t s use index(idx), t t1 where s.b = t.a and s.a = t1.a) from t; id count task operator info Projection_11 10000.00 root 9_aux_0 @@ -563,15 +557,14 @@ Projection_12 10000.00 root 9_aux_0 ├─TableReader_16 10000.00 root data:TableScan_15 │ └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─HashAgg_19 1.00 root funcs:count(join_agg_0) - └─HashLeftJoin_20 9.99 root inner join, inner:HashAgg_30, equal:[eq(test.s.a, test.t1.a)] + └─HashLeftJoin_20 9.99 root inner join, inner:HashAgg_27, equal:[eq(test.s.a, test.t1.a)] ├─TableReader_24 9.99 root data:Selection_23 │ └─Selection_23 9.99 cop eq(test.s.a, test.t.a), not(isnull(test.s.a)) │ └─TableScan_22 10000.00 cop table:s, range:[-inf,+inf], keep order:false, stats:pseudo - └─HashAgg_30 7.99 root group by:col_2, funcs:count(col_0), firstrow(col_1) - └─TableReader_31 7.99 root data:HashAgg_25 - └─HashAgg_25 7.99 cop group by:test.t1.a, funcs:count(1), firstrow(test.t1.a) - └─Selection_29 9.99 cop eq(test.t1.a, test.t.a), not(isnull(test.t1.a)) - └─TableScan_28 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo + └─HashAgg_27 7.99 root group by:test.t1.a, funcs:count(1), firstrow(test.t1.a) + └─TableReader_34 9.99 root data:Selection_33 + └─Selection_33 9.99 cop eq(test.t1.a, test.t.a), not(isnull(test.t1.a)) + └─TableScan_32 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo explain select * from t ta left outer join t tb on ta.nb = tb.nb and ta.a > 1 where ifnull(tb.a, 1) or tb.a is null; id count task operator info Selection_7 10000.00 root or(ifnull(test.tb.a, 1), isnull(test.tb.a)) @@ -596,15 +589,14 @@ Projection_12 10000.00 root 9_aux_0 ├─TableReader_16 10000.00 root data:TableScan_15 │ └─TableScan_15 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo └─HashAgg_19 1.00 root funcs:count(join_agg_0) - └─HashLeftJoin_20 9.99 root inner join, inner:HashAgg_30, equal:[eq(test.s.a, test.t1.a)] + └─HashLeftJoin_20 9.99 root inner join, inner:HashAgg_27, equal:[eq(test.s.a, test.t1.a)] ├─TableReader_24 9.99 root data:Selection_23 │ └─Selection_23 9.99 cop eq(test.s.a, test.t.a), not(isnull(test.s.a)) │ └─TableScan_22 10000.00 cop table:s, range:[-inf,+inf], keep order:false, stats:pseudo - └─HashAgg_30 7.99 root group by:col_2, funcs:count(col_0), firstrow(col_1) - └─TableReader_31 7.99 root data:HashAgg_25 - └─HashAgg_25 7.99 cop group by:test.t1.a, funcs:count(1), firstrow(test.t1.a) - └─Selection_29 9.99 cop eq(test.t1.a, test.t.a), not(isnull(test.t1.a)) - └─TableScan_28 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo + └─HashAgg_27 7.99 root group by:test.t1.a, funcs:count(1), firstrow(test.t1.a) + └─TableReader_34 9.99 root data:Selection_33 + └─Selection_33 9.99 cop eq(test.t1.a, test.t.a), not(isnull(test.t1.a)) + └─TableScan_32 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo drop table if exists t; create table t(a int); explain select * from t where _tidb_rowid = 0; diff --git a/cmd/explaintest/r/explain_easy_stats.result b/cmd/explaintest/r/explain_easy_stats.result index 4945dd14c628d..756bf2eade4ec 100644 --- a/cmd/explaintest/r/explain_easy_stats.result +++ b/cmd/explaintest/r/explain_easy_stats.result @@ -10,6 +10,8 @@ create table index_prune(a bigint(20) NOT NULL, b bigint(20) NOT NULL, c tinyint load stats 's/explain_easy_stats_index_prune.json'; set @@session.tidb_opt_agg_push_down = 1; set @@session.tidb_opt_insubq_to_join_and_agg=1; +set @@session.tidb_hashagg_partial_concurrency = 1; +set @@session.tidb_hashagg_final_concurrency = 1; explain select * from t3 where exists (select s.a from t3 s having sum(s.a) = t3.a ); id count task operator info Projection_12 1600.00 root test.t3.a, test.t3.b, test.t3.c, test.t3.d @@ -45,15 +47,12 @@ IndexReader_6 0.00 root index:IndexScan_5 └─IndexScan_5 0.00 cop table:t1, index:c2, range:[1,1], keep order:false explain select * from t1 left join t2 on t1.c2 = t2.c1 where t1.c1 > 1; id count task operator info -MergeJoin_7 2481.25 root left outer join, left key:test.t1.c2, right key:test.t2.c1 -├─IndexLookUp_17 1998.00 root -│ ├─Selection_16 1998.00 cop gt(test.t1.c1, 1) -│ │ └─IndexScan_14 1999.00 cop table:t1, index:c2, range:[NULL,+inf], keep order:true -│ └─TableScan_15 1998.00 cop table:t1, keep order:false, stats:pseudo -└─Projection_22 1985.00 root test.t2.c1, test.t2.c2 - └─IndexLookUp_21 1985.00 root - ├─IndexScan_19 1985.00 cop table:t2, index:c1, range:[-inf,+inf], keep order:true - └─TableScan_20 1985.00 cop table:t2, keep order:false +HashLeftJoin_13 2481.25 root left outer join, inner:TableReader_27, equal:[eq(test.t1.c2, test.t2.c1)] +├─TableReader_24 1998.00 root data:TableScan_23 +│ └─TableScan_23 1998.00 cop table:t1, range:(1,+inf], keep order:false +└─TableReader_27 1985.00 root data:Selection_26 + └─Selection_26 1985.00 cop not(isnull(test.t2.c1)) + └─TableScan_25 1985.00 cop table:t2, range:[-inf,+inf], keep order:false explain update t1 set t1.c2 = 2 where t1.c1 = 1; id count task operator info Point_Get_1 1.00 root table:t1, handle:1 @@ -65,14 +64,13 @@ IndexLookUp_9 0.00 root explain select count(b.c2) from t1 a, t2 b where a.c1 = b.c2 group by a.c1; id count task operator info Projection_11 1985.00 root cast(join_agg_0) -└─IndexJoin_14 1985.00 root inner join, inner:TableReader_13, outer key:test.b.c2, inner key:test.a.c1 - ├─TableReader_13 1.00 root data:TableScan_12 - │ └─TableScan_12 1.00 cop table:a, range: decided by [test.b.c2], keep order:false - └─HashAgg_22 1985.00 root group by:col_2, funcs:count(col_0), firstrow(col_1) - └─TableReader_23 1985.00 root data:HashAgg_17 - └─HashAgg_17 1985.00 cop group by:test.b.c2, funcs:count(test.b.c2), firstrow(test.b.c2) - └─Selection_21 1985.00 cop not(isnull(test.b.c2)) - └─TableScan_20 1985.00 cop table:b, range:[-inf,+inf], keep order:false +└─HashLeftJoin_15 1985.00 root inner join, inner:HashAgg_19, equal:[eq(test.a.c1, test.b.c2)] + ├─TableReader_28 1999.00 root data:TableScan_27 + │ └─TableScan_27 1999.00 cop table:a, range:[-inf,+inf], keep order:false + └─HashAgg_19 1985.00 root group by:test.b.c2, funcs:count(test.b.c2), firstrow(test.b.c2) + └─TableReader_26 1985.00 root data:Selection_25 + └─Selection_25 1985.00 cop not(isnull(test.b.c2)) + └─TableScan_24 1985.00 cop table:b, range:[-inf,+inf], keep order:false explain select * from t2 order by t2.c2 limit 0, 1; id count task operator info TopN_7 1.00 root test.t2.c2:asc, offset:0, count:1 @@ -93,14 +91,13 @@ TableReader_7 0.50 root data:Selection_6 explain select c1 from t1 where c1 in (select c2 from t2); id count task operator info Projection_9 1985.00 root test.t1.c1 -└─IndexJoin_12 1985.00 root inner join, inner:TableReader_11, outer key:test.t2.c2, inner key:test.t1.c1 - ├─TableReader_11 1.00 root data:TableScan_10 - │ └─TableScan_10 1.00 cop table:t1, range: decided by [test.t2.c2], keep order:false - └─HashAgg_20 1985.00 root group by:col_1, funcs:firstrow(col_0) - └─TableReader_21 1985.00 root data:HashAgg_15 - └─HashAgg_15 1985.00 cop group by:test.t2.c2, funcs:firstrow(test.t2.c2) - └─Selection_19 1985.00 cop not(isnull(test.t2.c2)) - └─TableScan_18 1985.00 cop table:t2, range:[-inf,+inf], keep order:false +└─HashLeftJoin_13 1985.00 root inner join, inner:HashAgg_17, equal:[eq(test.t1.c1, test.t2.c2)] + ├─TableReader_26 1999.00 root data:TableScan_25 + │ └─TableScan_25 1999.00 cop table:t1, range:[-inf,+inf], keep order:false + └─HashAgg_17 1985.00 root group by:test.t2.c2, funcs:firstrow(test.t2.c2) + └─TableReader_24 1985.00 root data:Selection_23 + └─Selection_23 1985.00 cop not(isnull(test.t2.c2)) + └─TableScan_22 1985.00 cop table:t2, range:[-inf,+inf], keep order:false explain select * from information_schema.columns; id count task operator info MemTableScan_4 10000.00 root @@ -176,10 +173,10 @@ Limit_9 1.00 root offset:1, count:1 explain select * from index_prune WHERE a = 1010010404050976781 AND b = 26467085526790 LIMIT 1, 0; id count task operator info Limit_9 0.00 root offset:1, count:0 -└─IndexLookUp_14 0.00 root - ├─Limit_13 0.00 cop offset:0, count:1 +└─IndexLookUp_14 1.00 root + ├─Limit_13 1.00 cop offset:0, count:1 │ └─IndexScan_11 1.00 cop table:index_prune, index:a, b, range:[1010010404050976781 26467085526790,1010010404050976781 26467085526790], keep order:false - └─TableScan_12 0.00 cop table:index_prune, keep order:false, stats:pseudo + └─TableScan_12 1.00 cop table:index_prune, keep order:false, stats:pseudo explain select * from index_prune WHERE a = 1010010404050976781 AND b = 26467085526790 LIMIT 0, 1; id count task operator info Point_Get_1 1.00 root table:index_prune, index:a b diff --git a/cmd/explaintest/r/generated_columns.result b/cmd/explaintest/r/generated_columns.result index 0add5d3921876..d2a4bcfbbf634 100644 --- a/cmd/explaintest/r/generated_columns.result +++ b/cmd/explaintest/r/generated_columns.result @@ -32,9 +32,9 @@ IndexReader_6 3323.33 root index:IndexScan_5 └─IndexScan_5 3323.33 cop table:sgc, index:a, b, range:[-inf,3), keep order:false, stats:pseudo EXPLAIN SELECT a, b from sgc where b < 3; id count task operator info -IndexLookUp_10 3323.33 root -├─IndexScan_8 3323.33 cop table:sgc, index:b, range:[-inf,3), keep order:false, stats:pseudo -└─TableScan_9 3323.33 cop table:sgc, keep order:false, stats:pseudo +TableReader_7 3323.33 root data:Selection_6 +└─Selection_6 3323.33 cop lt(test.sgc.b, 3) + └─TableScan_5 10000.00 cop table:sgc, range:[-inf,+inf], keep order:false, stats:pseudo EXPLAIN SELECT a, b from sgc where a < 3 and b < 3; id count task operator info IndexReader_11 1104.45 root index:Selection_10 @@ -75,21 +75,20 @@ IndexJoin_17 5.00 root inner join, inner:IndexLookUp_16, outer key:test.sgc2.a, ├─IndexLookUp_16 5.00 root │ ├─Selection_15 5.00 cop not(isnull(test.sgc1.a)) │ │ └─IndexScan_13 5.00 cop table:sgc1, index:a, range: decided by [eq(test.sgc1.a, test.sgc2.a)], keep order:false -│ └─TableScan_14 5.00 cop table:sgc1, keep order:false, stats:pseudo +│ └─TableScan_14 5.00 cop table:sgc1, keep order:false └─TableReader_20 1.00 root data:Selection_19 └─Selection_19 1.00 cop not(isnull(test.sgc2.a)) └─TableScan_18 1.00 cop table:sgc2, range:[-inf,+inf], keep order:false EXPLAIN SELECT * from sgc1 join sgc2 on sgc1.a=sgc2.a; id count task operator info Projection_6 5.00 root test.sgc1.j1, test.sgc1.j2, test.sgc1.a, test.sgc1.b, test.sgc2.j1, test.sgc2.j2, test.sgc2.a, test.sgc2.b -└─IndexJoin_13 5.00 root inner join, inner:IndexLookUp_12, outer key:test.sgc2.a, inner key:test.sgc1.a +└─HashRightJoin_20 5.00 root inner join, inner:TableReader_39, equal:[eq(test.sgc2.a, test.sgc1.a)] ├─TableReader_39 1.00 root data:Selection_38 │ └─Selection_38 1.00 cop not(isnull(test.sgc2.a)) │ └─TableScan_37 1.00 cop table:sgc2, range:[-inf,+inf], keep order:false - └─IndexLookUp_12 5.00 root - ├─Selection_11 5.00 cop not(isnull(test.sgc1.a)) - │ └─IndexScan_9 5.00 cop table:sgc1, index:a, range: decided by [eq(test.sgc1.a, test.sgc2.a)], keep order:false - └─TableScan_10 5.00 cop table:sgc1, keep order:false, stats:pseudo + └─TableReader_48 5.00 root data:Selection_47 + └─Selection_47 5.00 cop not(isnull(test.sgc1.a)) + └─TableScan_46 5.00 cop table:sgc1, range:[-inf,+inf], keep order:false DROP TABLE IF EXISTS sgc3; CREATE TABLE sgc3 ( j JSON, diff --git a/cmd/explaintest/r/index_join.result b/cmd/explaintest/r/index_join.result index 6f52e88587d38..5ee4f6242f0bb 100644 --- a/cmd/explaintest/r/index_join.result +++ b/cmd/explaintest/r/index_join.result @@ -4,27 +4,28 @@ create table t2(a bigint, b bigint, index idx(a)); insert into t1 values(1, 1), (1, 1), (1, 1), (1, 1), (1, 1); insert into t2 values(1, 1); analyze table t1, t2; +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; explain select /*+ TIDB_INLJ(t1, t2) */ * from t1 join t2 on t1.a=t2.a; id count task operator info IndexJoin_16 5.00 root inner join, inner:IndexLookUp_15, outer key:test.t2.a, inner key:test.t1.a ├─IndexLookUp_15 5.00 root │ ├─Selection_14 5.00 cop not(isnull(test.t1.a)) │ │ └─IndexScan_12 5.00 cop table:t1, index:a, range: decided by [eq(test.t1.a, test.t2.a)], keep order:false -│ └─TableScan_13 5.00 cop table:t1, keep order:false, stats:pseudo +│ └─TableScan_13 5.00 cop table:t1, keep order:false └─TableReader_19 1.00 root data:Selection_18 └─Selection_18 1.00 cop not(isnull(test.t2.a)) └─TableScan_17 1.00 cop table:t2, range:[-inf,+inf], keep order:false explain select * from t1 join t2 on t1.a=t2.a; id count task operator info Projection_6 5.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b -└─IndexJoin_12 5.00 root inner join, inner:IndexLookUp_11, outer key:test.t2.a, inner key:test.t1.a +└─HashRightJoin_19 5.00 root inner join, inner:TableReader_30, equal:[eq(test.t2.a, test.t1.a)] ├─TableReader_30 1.00 root data:Selection_29 │ └─Selection_29 1.00 cop not(isnull(test.t2.a)) │ └─TableScan_28 1.00 cop table:t2, range:[-inf,+inf], keep order:false - └─IndexLookUp_11 5.00 root - ├─Selection_10 5.00 cop not(isnull(test.t1.a)) - │ └─IndexScan_8 5.00 cop table:t1, index:a, range: decided by [eq(test.t1.a, test.t2.a)], keep order:false - └─TableScan_9 5.00 cop table:t1, keep order:false, stats:pseudo + └─TableReader_36 5.00 root data:Selection_35 + └─Selection_35 5.00 cop not(isnull(test.t1.a)) + └─TableScan_34 5.00 cop table:t1, range:[-inf,+inf], keep order:false drop table if exists t1, t2; create table t1(a int not null, b int not null); create table t2(a int not null, b int not null, key a(a)); diff --git a/cmd/explaintest/r/partition_pruning.result b/cmd/explaintest/r/partition_pruning.result index d4430618a0342..216897925d8d4 100644 --- a/cmd/explaintest/r/partition_pruning.result +++ b/cmd/explaintest/r/partition_pruning.result @@ -3892,21 +3892,21 @@ Union_11 150.00 root explain select * from t2 where b > 5; id count task operator info Union_11 16666.67 root -├─IndexLookUp_17 3333.33 root -│ ├─IndexScan_15 3333.33 cop table:t2, partition:p0, index:b, range:(5,+inf], keep order:false, stats:pseudo -│ └─TableScan_16 3333.33 cop table:t2, partition:p0, keep order:false, stats:pseudo -├─IndexLookUp_23 3333.33 root -│ ├─IndexScan_21 3333.33 cop table:t2, partition:p1, index:b, range:(5,+inf], keep order:false, stats:pseudo -│ └─TableScan_22 3333.33 cop table:t2, partition:p1, keep order:false, stats:pseudo -├─IndexLookUp_29 3333.33 root -│ ├─IndexScan_27 3333.33 cop table:t2, partition:p2, index:b, range:(5,+inf], keep order:false, stats:pseudo -│ └─TableScan_28 3333.33 cop table:t2, partition:p2, keep order:false, stats:pseudo -├─IndexLookUp_35 3333.33 root -│ ├─IndexScan_33 3333.33 cop table:t2, partition:p3, index:b, range:(5,+inf], keep order:false, stats:pseudo -│ └─TableScan_34 3333.33 cop table:t2, partition:p3, keep order:false, stats:pseudo -└─IndexLookUp_41 3333.33 root - ├─IndexScan_39 3333.33 cop table:t2, partition:p4, index:b, range:(5,+inf], keep order:false, stats:pseudo - └─TableScan_40 3333.33 cop table:t2, partition:p4, keep order:false, stats:pseudo +├─TableReader_14 3333.33 root data:Selection_13 +│ └─Selection_13 3333.33 cop gt(test.t2.b, 5) +│ └─TableScan_12 10000.00 cop table:t2, partition:p0, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_20 3333.33 root data:Selection_19 +│ └─Selection_19 3333.33 cop gt(test.t2.b, 5) +│ └─TableScan_18 10000.00 cop table:t2, partition:p1, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_26 3333.33 root data:Selection_25 +│ └─Selection_25 3333.33 cop gt(test.t2.b, 5) +│ └─TableScan_24 10000.00 cop table:t2, partition:p2, range:[-inf,+inf], keep order:false, stats:pseudo +├─TableReader_32 3333.33 root data:Selection_31 +│ └─Selection_31 3333.33 cop gt(test.t2.b, 5) +│ └─TableScan_30 10000.00 cop table:t2, partition:p3, range:[-inf,+inf], keep order:false, stats:pseudo +└─TableReader_38 3333.33 root data:Selection_37 + └─Selection_37 3333.33 cop gt(test.t2.b, 5) + └─TableScan_36 10000.00 cop table:t2, partition:p4, range:[-inf,+inf], keep order:false, stats:pseudo explain select * from t2 where b > 5 and b < 8; id count task operator info Union_11 1250.00 root diff --git a/cmd/explaintest/r/select.result b/cmd/explaintest/r/select.result index 143cf7cc28a9f..3c780d0891fce 100644 --- a/cmd/explaintest/r/select.result +++ b/cmd/explaintest/r/select.result @@ -6,6 +6,8 @@ c3 int, PRIMARY KEY (c1) ); INSERT INTO t VALUES (1,2,3); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; SELECT * from t; c1 c2 c3 1 2 3 diff --git a/cmd/explaintest/r/subquery.result b/cmd/explaintest/r/subquery.result index 3d367f8bbeb07..127702a8b239a 100644 --- a/cmd/explaintest/r/subquery.result +++ b/cmd/explaintest/r/subquery.result @@ -2,6 +2,8 @@ drop table if exists t1; drop table if exists t2; create table t1(a bigint, b bigint); create table t2(a bigint, b bigint); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; explain select * from t1 where t1.a in (select t1.b + t2.b from t2); id count task operator info HashLeftJoin_8 8000.00 root CARTESIAN semi join, inner:TableReader_12, other cond:eq(test.t1.a, plus(test.t1.b, test.t2.b)) diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index 45f0cdc023f06..a0c9480b2f720 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -347,8 +347,8 @@ Sort_23 5.00 root revenue:desc └─Projection_25 5.00 root tpch.nation.n_name, 13_col_0 └─HashAgg_28 5.00 root group by:col_2, funcs:sum(col_0), firstrow(col_1) └─Projection_66 11822812.50 root mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), tpch.nation.n_name, tpch.nation.n_name - └─IndexJoin_31 11822812.50 root inner join, inner:TableReader_30, outer key:tpch.orders.o_custkey, inner key:tpch.customer.c_custkey, other cond:eq(tpch.supplier.s_nationkey, tpch.customer.c_nationkey) - ├─IndexJoin_38 11822812.50 root inner join, inner:TableReader_37, outer key:tpch.lineitem.l_orderkey, inner key:tpch.orders.o_orderkey + └─HashLeftJoin_32 11822812.50 root inner join, inner:TableReader_64, equal:[eq(tpch.supplier.s_nationkey, tpch.customer.c_nationkey) eq(tpch.orders.o_custkey, tpch.customer.c_custkey)] + ├─HashLeftJoin_39 11822812.50 root inner join, inner:TableReader_62, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)] │ ├─HashRightJoin_42 61163763.01 root inner join, inner:HashRightJoin_44, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] │ │ ├─HashRightJoin_44 100000.00 root inner join, inner:HashRightJoin_50, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] │ │ │ ├─HashRightJoin_50 5.00 root inner join, inner:TableReader_55, equal:[eq(tpch.region.r_regionkey, tpch.nation.n_regionkey)] @@ -361,11 +361,11 @@ Sort_23 5.00 root revenue:desc │ │ │ └─TableScan_56 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false │ │ └─TableReader_59 300005811.00 root data:TableScan_58 │ │ └─TableScan_58 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false - │ └─TableReader_37 0.80 root data:Selection_36 - │ └─Selection_36 0.80 cop ge(tpch.orders.o_orderdate, 1994-01-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1995-01-01) - │ └─TableScan_35 1.00 cop table:orders, range: decided by [tpch.lineitem.l_orderkey], keep order:false - └─TableReader_30 1.00 root data:TableScan_29 - └─TableScan_29 1.00 cop table:customer, range: decided by [tpch.supplier.s_nationkey tpch.orders.o_custkey], keep order:false + │ └─TableReader_62 11822812.50 root data:Selection_61 + │ └─Selection_61 11822812.50 cop ge(tpch.orders.o_orderdate, 1994-01-01 00:00:00.000000), lt(tpch.orders.o_orderdate, 1995-01-01) + │ └─TableScan_60 75000000.00 cop table:orders, range:[-inf,+inf], keep order:false + └─TableReader_64 7500000.00 root data:TableScan_63 + └─TableScan_63 7500000.00 cop table:customer, range:[-inf,+inf], keep order:false /* Q6 Forecasting Revenue Change Query This query quantifies the amount of revenue increase that would have resulted from eliminating certain companywide @@ -449,7 +449,7 @@ Sort_22 769.96 root tpch.shipping.supp_nation:asc, tpch.shipping.cust_nation:asc └─HashAgg_27 769.96 root group by:shipping.l_year, tpch.shipping.cust_nation, tpch.shipping.supp_nation, funcs:sum(shipping.volume), firstrow(tpch.shipping.supp_nation), firstrow(tpch.shipping.cust_nation), firstrow(shipping.l_year) └─Projection_28 1957240.42 root tpch.n1.n_name, tpch.n2.n_name, extract("YEAR", tpch.lineitem.l_shipdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)) └─HashLeftJoin_33 1957240.42 root inner join, inner:TableReader_68, equal:[eq(tpch.customer.c_nationkey, tpch.n2.n_nationkey)], other cond:or(and(eq(tpch.n1.n_name, "JAPAN"), eq(tpch.n2.n_name, "INDIA")), and(eq(tpch.n1.n_name, "INDIA"), eq(tpch.n2.n_name, "JAPAN"))) - ├─IndexJoin_37 24465505.20 root inner join, inner:TableReader_36, outer key:tpch.orders.o_custkey, inner key:tpch.customer.c_custkey + ├─HashLeftJoin_38 24465505.20 root inner join, inner:TableReader_65, equal:[eq(tpch.orders.o_custkey, tpch.customer.c_custkey)] │ ├─IndexJoin_43 24465505.20 root inner join, inner:TableReader_42, outer key:tpch.lineitem.l_orderkey, inner key:tpch.orders.o_orderkey │ │ ├─HashRightJoin_47 24465505.20 root inner join, inner:HashRightJoin_53, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] │ │ │ ├─HashRightJoin_53 40000.00 root inner join, inner:TableReader_58, equal:[eq(tpch.n1.n_nationkey, tpch.supplier.s_nationkey)] @@ -463,8 +463,8 @@ Sort_22 769.96 root tpch.shipping.supp_nation:asc, tpch.shipping.cust_nation:asc │ │ │ └─TableScan_59 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false │ │ └─TableReader_42 1.00 root data:TableScan_41 │ │ └─TableScan_41 1.00 cop table:orders, range: decided by [tpch.lineitem.l_orderkey], keep order:false - │ └─TableReader_36 1.00 root data:TableScan_35 - │ └─TableScan_35 1.00 cop table:customer, range: decided by [tpch.orders.o_custkey], keep order:false + │ └─TableReader_65 7500000.00 root data:TableScan_64 + │ └─TableScan_64 7500000.00 cop table:customer, range:[-inf,+inf], keep order:false └─TableReader_68 2.00 root data:Selection_67 └─Selection_67 2.00 cop or(eq(tpch.n2.n_name, "INDIA"), eq(tpch.n2.n_name, "JAPAN")) └─TableScan_66 25.00 cop table:n2, range:[-inf,+inf], keep order:false @@ -522,7 +522,7 @@ Sort_29 719.02 root all_nations.o_year:asc └─Projection_89 563136.02 root case(eq(tpch.all_nations.nation, "INDIA"), all_nations.volume, 0), all_nations.volume, all_nations.o_year, all_nations.o_year └─Projection_35 563136.02 root extract("YEAR", tpch.orders.o_orderdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), tpch.n2.n_name └─HashLeftJoin_39 563136.02 root inner join, inner:TableReader_87, equal:[eq(tpch.supplier.s_nationkey, tpch.n2.n_nationkey)] - ├─IndexJoin_43 563136.02 root inner join, inner:TableReader_42, outer key:tpch.lineitem.l_suppkey, inner key:tpch.supplier.s_suppkey + ├─HashLeftJoin_44 563136.02 root inner join, inner:TableReader_85, equal:[eq(tpch.lineitem.l_suppkey, tpch.supplier.s_suppkey)] │ ├─HashLeftJoin_50 563136.02 root inner join, inner:TableReader_83, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)] │ │ ├─IndexJoin_56 90788402.51 root inner join, inner:IndexLookUp_55, outer key:tpch.orders.o_orderkey, inner key:tpch.lineitem.l_orderkey │ │ │ ├─HashRightJoin_60 22413367.93 root inner join, inner:HashRightJoin_62, equal:[eq(tpch.customer.c_custkey, tpch.orders.o_custkey)] @@ -544,8 +544,8 @@ Sort_29 719.02 root all_nations.o_year:asc │ │ └─TableReader_83 61674.00 root data:Selection_82 │ │ └─Selection_82 61674.00 cop eq(tpch.part.p_type, "SMALL PLATED COPPER") │ │ └─TableScan_81 10000000.00 cop table:part, range:[-inf,+inf], keep order:false - │ └─TableReader_42 1.00 root data:TableScan_41 - │ └─TableScan_41 1.00 cop table:supplier, range: decided by [tpch.lineitem.l_suppkey], keep order:false + │ └─TableReader_85 500000.00 root data:TableScan_84 + │ └─TableScan_84 500000.00 cop table:supplier, range:[-inf,+inf], keep order:false └─TableReader_87 25.00 root data:TableScan_86 └─TableScan_86 25.00 cop table:n2, range:[-inf,+inf], keep order:false /* @@ -596,8 +596,8 @@ Sort_25 2406.00 root tpch.profit.nation:asc, profit.o_year:desc └─Projection_27 2406.00 root tpch.profit.nation, profit.o_year, 14_col_0 └─HashAgg_30 2406.00 root group by:profit.o_year, tpch.profit.nation, funcs:sum(profit.amount), firstrow(tpch.profit.nation), firstrow(profit.o_year) └─Projection_31 971049283.51 root tpch.nation.n_name, extract("YEAR", tpch.orders.o_orderdate), minus(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)), mul(tpch.partsupp.ps_supplycost, tpch.lineitem.l_quantity)) - └─IndexJoin_35 971049283.51 root inner join, inner:IndexLookUp_34, outer key:tpch.lineitem.l_suppkey, tpch.lineitem.l_partkey, inner key:tpch.partsupp.ps_suppkey, tpch.partsupp.ps_partkey - ├─IndexJoin_41 241379546.70 root inner join, inner:TableReader_40, outer key:tpch.lineitem.l_orderkey, inner key:tpch.orders.o_orderkey + └─HashLeftJoin_36 971049283.51 root inner join, inner:TableReader_73, equal:[eq(tpch.lineitem.l_suppkey, tpch.partsupp.ps_suppkey) eq(tpch.lineitem.l_partkey, tpch.partsupp.ps_partkey)] + ├─HashLeftJoin_42 241379546.70 root inner join, inner:TableReader_71, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)] │ ├─HashLeftJoin_52 241379546.70 root inner join, inner:TableReader_69, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)] │ │ ├─HashRightJoin_55 300005811.00 root inner join, inner:HashRightJoin_60, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] │ │ │ ├─HashRightJoin_60 500000.00 root inner join, inner:TableReader_64, equal:[eq(tpch.nation.n_nationkey, tpch.supplier.s_nationkey)] @@ -610,11 +610,10 @@ Sort_25 2406.00 root tpch.profit.nation:asc, profit.o_year:desc │ │ └─TableReader_69 8000000.00 root data:Selection_68 │ │ └─Selection_68 8000000.00 cop like(tpch.part.p_name, "%dim%", 92) │ │ └─TableScan_67 10000000.00 cop table:part, range:[-inf,+inf], keep order:false - │ └─TableReader_40 1.00 root data:TableScan_39 - │ └─TableScan_39 1.00 cop table:orders, range: decided by [tpch.lineitem.l_orderkey], keep order:false - └─IndexLookUp_34 1.00 root - ├─IndexScan_32 1.00 cop table:partsupp, index:PS_PARTKEY, PS_SUPPKEY, range: decided by [eq(tpch.partsupp.ps_partkey, tpch.lineitem.l_partkey) eq(tpch.partsupp.ps_suppkey, tpch.lineitem.l_suppkey)], keep order:false - └─TableScan_33 1.00 cop table:partsupp, keep order:false + │ └─TableReader_71 75000000.00 root data:TableScan_70 + │ └─TableScan_70 75000000.00 cop table:orders, range:[-inf,+inf], keep order:false + └─TableReader_73 40000000.00 root data:TableScan_72 + └─TableScan_72 40000000.00 cop table:partsupp, range:[-inf,+inf], keep order:false /* Q10 Returned Item Reporting Query The query identifies customers who might be having problems with the parts that are shipped to them. diff --git a/cmd/explaintest/t/explain.test b/cmd/explaintest/t/explain.test index 91bd0aa23034d..3bde3f5d7d5ca 100644 --- a/cmd/explaintest/t/explain.test +++ b/cmd/explaintest/t/explain.test @@ -9,6 +9,8 @@ desc t id; drop table if exists t; create table t(id int primary key, a int, b int); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; explain select group_concat(a) from t group by id; explain select group_concat(a, b) from t group by id; drop table t; diff --git a/cmd/explaintest/t/explain_easy.test b/cmd/explaintest/t/explain_easy.test index 8c1ff5e5502f7..e5800a502a222 100644 --- a/cmd/explaintest/t/explain_easy.test +++ b/cmd/explaintest/t/explain_easy.test @@ -8,6 +8,8 @@ create table t4 (a int, b int, c int, index idx(a, b), primary key(a)); set @@session.tidb_opt_agg_push_down = 1; set @@session.tidb_opt_insubq_to_join_and_agg=1; +set @@session.tidb_hashagg_partial_concurrency = 1; +set @@session.tidb_hashagg_final_concurrency = 1; explain select * from t3 where exists (select s.a from t3 s having sum(s.a) = t3.a ); explain select * from t1; diff --git a/cmd/explaintest/t/explain_easy_stats.test b/cmd/explaintest/t/explain_easy_stats.test index f175e5d0ec021..dc2be5d47f89c 100644 --- a/cmd/explaintest/t/explain_easy_stats.test +++ b/cmd/explaintest/t/explain_easy_stats.test @@ -10,6 +10,8 @@ create table index_prune(a bigint(20) NOT NULL, b bigint(20) NOT NULL, c tinyint load stats 's/explain_easy_stats_index_prune.json'; set @@session.tidb_opt_agg_push_down = 1; set @@session.tidb_opt_insubq_to_join_and_agg=1; +set @@session.tidb_hashagg_partial_concurrency = 1; +set @@session.tidb_hashagg_final_concurrency = 1; explain select * from t3 where exists (select s.a from t3 s having sum(s.a) = t3.a ); diff --git a/cmd/explaintest/t/index_join.test b/cmd/explaintest/t/index_join.test index b635bbb4a2f06..8ed32104a0697 100644 --- a/cmd/explaintest/t/index_join.test +++ b/cmd/explaintest/t/index_join.test @@ -5,6 +5,8 @@ insert into t1 values(1, 1), (1, 1), (1, 1), (1, 1), (1, 1); insert into t2 values(1, 1); analyze table t1, t2; +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; -- Test https://github.com/pingcap/tidb/issues/9577 -- we expect the following two SQL chose t2 as the outer table diff --git a/cmd/explaintest/t/select.test b/cmd/explaintest/t/select.test index 758bfb7466970..a5b04f20f463b 100644 --- a/cmd/explaintest/t/select.test +++ b/cmd/explaintest/t/select.test @@ -8,6 +8,8 @@ CREATE TABLE t ( ); INSERT INTO t VALUES (1,2,3); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; SELECT * from t; SELECT c1, c2, c3 from t; diff --git a/cmd/explaintest/t/subquery.test b/cmd/explaintest/t/subquery.test index be52346acac74..d0fa322f52ba0 100644 --- a/cmd/explaintest/t/subquery.test +++ b/cmd/explaintest/t/subquery.test @@ -2,6 +2,8 @@ drop table if exists t1; drop table if exists t2; create table t1(a bigint, b bigint); create table t2(a bigint, b bigint); +set session tidb_hashagg_partial_concurrency = 1; +set session tidb_hashagg_final_concurrency = 1; explain select * from t1 where t1.a in (select t1.b + t2.b from t2); drop table if exists t; diff --git a/executor/aggregate_test.go b/executor/aggregate_test.go index 9332b22985aa8..43329e425a396 100644 --- a/executor/aggregate_test.go +++ b/executor/aggregate_test.go @@ -273,7 +273,7 @@ func (s *testSuite1) TestAggregation(c *C) { tk.MustQuery("select 11 from idx_agg group by a").Check(testkit.Rows("11", "11")) tk.MustExec("set @@tidb_init_chunk_size=1;") - tk.MustQuery("select group_concat(b) from idx_agg group by b;").Check(testkit.Rows("1", "2,2")) + tk.MustQuery("select group_concat(b) from idx_agg group by b;").Sort().Check(testkit.Rows("1", "2,2")) tk.MustExec("set @@tidb_init_chunk_size=2;") tk.MustExec("drop table if exists t") @@ -353,26 +353,6 @@ func (s *testSuite1) TestAggregation(c *C) { c.Assert(errors.Cause(err).Error(), Equals, "unsupported agg function: var_samp") } -func (s *testSuite1) TestStreamAggPushDown(c *C) { - tk := testkit.NewTestKit(c, s.store) - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t (a int, b int, c int)") - tk.MustExec("alter table t add index idx(a, b, c)") - // test for empty table - tk.MustQuery("select count(a) from t group by a;").Check(testkit.Rows()) - tk.MustQuery("select count(a) from t;").Check(testkit.Rows("0")) - // test for one row - tk.MustExec("insert t values(0,0,0)") - tk.MustQuery("select distinct b from t").Check(testkit.Rows("0")) - tk.MustQuery("select count(b) from t group by a;").Check(testkit.Rows("1")) - // test for rows - tk.MustExec("insert t values(1,1,1),(3,3,6),(3,2,5),(2,1,4),(1,1,3),(1,1,2);") - tk.MustQuery("select count(a) from t where b>0 group by a, b;").Check(testkit.Rows("3", "1", "1", "1")) - tk.MustQuery("select count(a) from t where b>0 group by a, b order by a;").Check(testkit.Rows("3", "1", "1", "1")) - tk.MustQuery("select count(a) from t where b>0 group by a, b order by a limit 1;").Check(testkit.Rows("3")) -} - func (s *testSuite1) TestAggPrune(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -445,6 +425,22 @@ func (s *testSuite) TestSelectDistinct(c *C) { func (s *testSuite1) TestAggPushDown(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b int, c int)") + tk.MustExec("alter table t add index idx(a, b, c)") + // test for empty table + tk.MustQuery("select count(a) from t group by a;").Check(testkit.Rows()) + tk.MustQuery("select count(a) from t;").Check(testkit.Rows("0")) + // test for one row + tk.MustExec("insert t values(0,0,0)") + tk.MustQuery("select distinct b from t").Check(testkit.Rows("0")) + tk.MustQuery("select count(b) from t group by a;").Check(testkit.Rows("1")) + // test for rows + tk.MustExec("insert t values(1,1,1),(3,3,6),(3,2,5),(2,1,4),(1,1,3),(1,1,2);") + tk.MustQuery("select count(a) from t where b>0 group by a, b;").Sort().Check(testkit.Rows("1", "1", "1", "3")) + tk.MustQuery("select count(a) from t where b>0 group by a, b order by a;").Check(testkit.Rows("3", "1", "1", "1")) + tk.MustQuery("select count(a) from t where b>0 group by a, b order by a limit 1;").Check(testkit.Rows("3")) + tk.MustExec("drop table if exists t, tt") tk.MustExec("create table t(a int primary key, b int, c int)") tk.MustExec("create table tt(a int primary key, b int, c int)") @@ -601,7 +597,7 @@ func (s *testSuite1) TestAggEliminator(c *C) { tk.MustQuery("select min(b) from t").Check(testkit.Rows("-2")) tk.MustQuery("select max(b*b) from t").Check(testkit.Rows("4")) tk.MustQuery("select min(b*b) from t").Check(testkit.Rows("1")) - tk.MustQuery("select group_concat(b, b) from t group by a").Check(testkit.Rows("-1-1", "-2-2", "11", "")) + tk.MustQuery("select group_concat(b, b) from t group by a").Sort().Check(testkit.Rows("-1-1", "-2-2", "11", "")) } func (s *testSuite1) TestMaxMinFloatScalaFunc(c *C) { diff --git a/executor/builder.go b/executor/builder.go index ffa4a8318e4de..11de9888cac88 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -966,15 +966,6 @@ func (b *executorBuilder) buildMergeJoin(v *plannercore.PhysicalMergeJoin) Execu } func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executor { - leftHashKey := make([]*expression.Column, 0, len(v.EqualConditions)) - rightHashKey := make([]*expression.Column, 0, len(v.EqualConditions)) - for _, eqCond := range v.EqualConditions { - ln, _ := eqCond.GetArgs()[0].(*expression.Column) - rn, _ := eqCond.GetArgs()[1].(*expression.Column) - leftHashKey = append(leftHashKey, ln) - rightHashKey = append(rightHashKey, rn) - } - leftExec := b.build(v.Children()[0]) if b.err != nil { return nil @@ -1002,8 +993,8 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo e.innerExec = leftExec e.outerExec = rightExec e.outerFilter = v.RightConditions - e.innerKeys = leftHashKey - e.outerKeys = rightHashKey + e.innerKeys = v.LeftJoinKeys + e.outerKeys = v.RightJoinKeys if defaultValues == nil { defaultValues = make([]types.Datum, e.innerExec.Schema().Len()) } @@ -1015,8 +1006,8 @@ func (b *executorBuilder) buildHashJoin(v *plannercore.PhysicalHashJoin) Executo e.innerExec = rightExec e.outerExec = leftExec e.outerFilter = v.LeftConditions - e.innerKeys = rightHashKey - e.outerKeys = leftHashKey + e.innerKeys = v.RightJoinKeys + e.outerKeys = v.LeftJoinKeys if defaultValues == nil { defaultValues = make([]types.Datum, e.innerExec.Schema().Len()) } diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index faa4230e22590..c20ff9c461ca0 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -47,49 +47,15 @@ func (s *testSuite1) TestIndexJoinUnionScan(c *C) { tk.MustExec("insert into t1 values(2,2)") tk.MustExec("insert into t2 values(2,2,2), (3,3,3)") // TableScan below UnionScan - tk.MustQuery("explain select /*+ TIDB_INLJ(t1, t2)*/ * from t1 join t2 on t1.a = t2.id").Check(testkit.Rows( - "IndexJoin_11 12487.50 root inner join, inner:UnionScan_10, outer key:test.t1.a, inner key:test.t2.id", - "├─UnionScan_12 9990.00 root not(isnull(test.t1.a))", - "│ └─TableReader_15 9990.00 root data:Selection_14", - "│ └─Selection_14 9990.00 cop not(isnull(test.t1.a))", - "│ └─TableScan_13 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", - "└─UnionScan_10 1.00 root ", - " └─TableReader_9 1.00 root data:TableScan_8", - " └─TableScan_8 1.00 cop table:t2, range: decided by [test.t1.a], keep order:false, stats:pseudo", - )) tk.MustQuery("select /*+ TIDB_INLJ(t1, t2)*/ * from t1 join t2 on t1.a = t2.id").Check(testkit.Rows( "2 2 2 2 2", )) // IndexLookUp below UnionScan - tk.MustQuery("explain select /*+ TIDB_INLJ(t1, t2)*/ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows( - "IndexJoin_13 12487.50 root inner join, inner:UnionScan_12, outer key:test.t1.a, inner key:test.t2.a", - "├─UnionScan_14 9990.00 root not(isnull(test.t1.a))", - "│ └─TableReader_17 9990.00 root data:Selection_16", - "│ └─Selection_16 9990.00 cop not(isnull(test.t1.a))", - "│ └─TableScan_15 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", - "└─UnionScan_12 9.99 root not(isnull(test.t2.a))", - " └─IndexLookUp_11 9.99 root ", - " ├─Selection_10 9.99 cop not(isnull(test.t2.a))", - " │ └─IndexScan_8 10.00 cop table:t2, index:a, range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo", - " └─TableScan_9 9.99 cop table:t2, keep order:false, stats:pseudo", - )) tk.MustQuery("select /*+ TIDB_INLJ(t1, t2)*/ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows( "2 2 2 2 2", "2 2 4 2 4", )) // IndexScan below UnionScan - tk.MustQuery("explain select /*+ TIDB_INLJ(t1, t2)*/ t1.a, t2.a from t1 join t2 on t1.a = t2.a").Check(testkit.Rows( - "Projection_7 12487.50 root test.t1.a, test.t2.a", - "└─IndexJoin_12 12487.50 root inner join, inner:UnionScan_11, outer key:test.t1.a, inner key:test.t2.a", - " ├─UnionScan_13 9990.00 root not(isnull(test.t1.a))", - " │ └─TableReader_16 9990.00 root data:Selection_15", - " │ └─Selection_15 9990.00 cop not(isnull(test.t1.a))", - " │ └─TableScan_14 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", - " └─UnionScan_11 9.99 root not(isnull(test.t2.a))", - " └─IndexReader_10 9.99 root index:Selection_9", - " └─Selection_9 9.99 cop not(isnull(test.t2.a))", - " └─IndexScan_8 10.00 cop table:t2, index:a, range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo", - )) tk.MustQuery("select /*+ TIDB_INLJ(t1, t2)*/ t1.a, t2.a from t1 join t2 on t1.a = t2.a").Check(testkit.Rows( "2 2", "2 2", @@ -107,18 +73,6 @@ func (s *testSuite1) TestBatchIndexJoinUnionScan(c *C) { tk.MustExec("begin") tk.MustExec("insert into t1 values(1,1),(2,1),(3,1),(4,1)") tk.MustExec("insert into t2 values(1,1)") - tk.MustQuery("explain select /*+ TIDB_INLJ(t1, t2)*/ count(*) from t1 join t2 on t1.a = t2.a").Check(testkit.Rows( - "StreamAgg_13 1.00 root funcs:count(1)", - "└─IndexJoin_27 12487.50 root inner join, inner:UnionScan_26, outer key:test.t1.a, inner key:test.t2.a", - " ├─UnionScan_19 9990.00 root not(isnull(test.t1.a))", - " │ └─TableReader_22 9990.00 root data:Selection_21", - " │ └─Selection_21 9990.00 cop not(isnull(test.t1.a))", - " │ └─TableScan_20 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", - " └─UnionScan_26 9.99 root not(isnull(test.t2.a))", - " └─IndexReader_25 9.99 root index:Selection_24", - " └─Selection_24 9.99 cop not(isnull(test.t2.a))", - " └─IndexScan_23 10.00 cop table:t2, index:a, range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo", - )) tk.MustQuery("select /*+ TIDB_INLJ(t1, t2)*/ count(*) from t1 join t2 on t1.a = t2.id").Check(testkit.Rows( "4", )) diff --git a/expression/constant_propagation_test.go b/expression/constant_propagation_test.go index f8a9ab352dca2..93307045a8b18 100644 --- a/expression/constant_propagation_test.go +++ b/expression/constant_propagation_test.go @@ -228,7 +228,7 @@ func (s *testSuite) TestOuterJoinPropConst(c *C) { "└─TableDual_11 8000.00 root rows:0", )) tk.MustQuery("explain select * from t1 left join t2 on t2.a = 1 and t2.a = 2;").Check(testkit.Rows( - "HashLeftJoin_6 0.00 root CARTESIAN left outer join, inner:TableReader_11", + "HashLeftJoin_6 10000.00 root CARTESIAN left outer join, inner:TableReader_11", "├─TableReader_8 10000.00 root data:TableScan_7", "│ └─TableScan_7 10000.00 cop table:t1, range:[-inf,+inf], keep order:false, stats:pseudo", "└─TableReader_11 0.00 root data:Selection_10", diff --git a/planner/core/cbo_test.go b/planner/core/cbo_test.go index 69f8fbe284ece..2225347dc2f57 100644 --- a/planner/core/cbo_test.go +++ b/planner/core/cbo_test.go @@ -280,6 +280,10 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { testKit.MustExec(fmt.Sprintf("insert into t1 values(%v, %v)", i, i)) } testKit.MustExec("analyze table t1") + ctx := testKit.Se.(sessionctx.Context) + sessionVars := ctx.GetSessionVars() + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 tests := []struct { sql string best string @@ -302,11 +306,11 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { }, { sql: "select count(*) from t where e = 1 group by b", - best: "IndexLookUp(Index(t.e)[[1,1]], Table(t)->HashAgg)->HashAgg", + best: "IndexLookUp(Index(t.e)[[1,1]], Table(t))->HashAgg", }, { sql: "select count(*) from t where e > 1 group by b", - best: "IndexLookUp(Index(t.b)[[NULL,+inf]], Table(t)->Sel([gt(test.t.e, 1)]))->Projection->StreamAgg", + best: "TableReader(Table(t)->Sel([gt(test.t.e, 1)])->HashAgg)->HashAgg", }, { sql: "select count(e) from t where t.b <= 20", @@ -314,15 +318,15 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { }, { sql: "select count(e) from t where t.b <= 30", - best: "IndexLookUp(Index(t.b)[[-inf,30]], Table(t)->HashAgg)->HashAgg", + best: "TableReader(Table(t)->Sel([le(test.t.b, 30)])->StreamAgg)->StreamAgg", }, { sql: "select count(e) from t where t.b <= 40", - best: "IndexLookUp(Index(t.b)[[-inf,40]], Table(t)->HashAgg)->HashAgg", + best: "TableReader(Table(t)->Sel([le(test.t.b, 40)])->StreamAgg)->StreamAgg", }, { sql: "select count(e) from t where t.b <= 50", - best: "IndexLookUp(Index(t.b)[[-inf,50]], Table(t)->HashAgg)->HashAgg", + best: "TableReader(Table(t)->Sel([le(test.t.b, 50)])->StreamAgg)->StreamAgg", }, { sql: "select count(e) from t where t.b <= 100000000000", @@ -330,11 +334,11 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { }, { sql: "select * from t where t.b <= 40", - best: "IndexLookUp(Index(t.b)[[-inf,40]], Table(t))", + best: "TableReader(Table(t)->Sel([le(test.t.b, 40)]))", }, { sql: "select * from t where t.b <= 50", - best: "IndexLookUp(Index(t.b)[[-inf,50]], Table(t))", + best: "TableReader(Table(t)->Sel([le(test.t.b, 50)]))", }, { sql: "select * from t where t.b <= 10000000000", @@ -343,7 +347,7 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { // test panic { sql: "select * from t where 1 and t.b <= 50", - best: "IndexLookUp(Index(t.b)[[-inf,50]], Table(t))", + best: "TableReader(Table(t)->Sel([le(test.t.b, 50)]))", }, { sql: "select * from t where t.b <= 100 order by t.a limit 1", @@ -373,7 +377,6 @@ func (s *testAnalyzeSuite) TestIndexRead(c *C) { }, } for _, tt := range tests { - ctx := testKit.Se.(sessionctx.Context) stmts, err := session.Parse(ctx, tt.sql) c.Assert(err, IsNil) c.Assert(stmts, HasLen, 1) @@ -411,7 +414,7 @@ func (s *testAnalyzeSuite) TestEmptyTable(c *C) { }, { sql: "select * from t where c1 in (select c1 from t1)", - best: "LeftHashJoin{TableReader(Table(t)->Sel([not(isnull(test.t.c1))]))->TableReader(Table(t1)->Sel([not(isnull(test.t1.c1))])->HashAgg)->HashAgg}(test.t.c1,test.t1.c1)->Projection", + best: "LeftHashJoin{TableReader(Table(t)->Sel([not(isnull(test.t.c1))]))->TableReader(Table(t1)->Sel([not(isnull(test.t1.c1))]))->HashAgg}(test.t.c1,test.t1.c1)->Projection", }, { sql: "select * from t, t1 where t.c1 = t1.c1", @@ -498,12 +501,12 @@ func (s *testAnalyzeSuite) TestAnalyze(c *C) { }, { sql: "select * from t where t.a = 1 and t.b <= 2", - best: "IndexLookUp(Index(t.b)[[-inf,2]], Table(t)->Sel([eq(test.t.a, 1)]))", + best: "TableReader(Table(t)->Sel([eq(test.t.a, 1) le(test.t.b, 2)]))", }, // Test not analyzed table. { sql: "select * from t1 where t1.a <= 2", - best: "IndexLookUp(Index(t1.a)[[-inf,2]], Table(t1))", + best: "TableReader(Table(t1)->Sel([le(test.t1.a, 2)]))", }, { sql: "select * from t1 where t1.a = 1 and t1.b <= 2", @@ -584,10 +587,9 @@ func (s *testAnalyzeSuite) TestOutdatedAnalyze(c *C) { )) statistics.RatioOfPseudoEstimate.Store(0.7) testKit.MustQuery("explain select * from t where a <= 5 and b <= 5").Check(testkit.Rows( - "IndexLookUp_11 8.84 root ", - "├─IndexScan_8 26.59 cop table:t, index:a, range:[-inf,5], keep order:false, stats:pseudo", - "└─Selection_10 8.84 cop le(test.t.b, 5)", - " └─TableScan_9 26.59 cop table:t, keep order:false, stats:pseudo", + "TableReader_7 8.84 root data:Selection_6", + "└─Selection_6 8.84 cop le(test.t.a, 5), le(test.t.b, 5)", + " └─TableScan_5 80.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo", )) } @@ -686,6 +688,10 @@ func (s *testAnalyzeSuite) TestCorrelatedEstimation(c *C) { tk.MustExec("create table t(a int, b int, c int, index idx(c))") tk.MustExec("insert into t values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5), (6,6,6), (7,7,7), (8,8,8), (9,9,9),(10,10,10)") tk.MustExec("analyze table t") + ctx := tk.Se.(sessionctx.Context) + sessionVars := ctx.GetSessionVars() + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 tk.MustQuery("explain select t.c in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t;"). Check(testkit.Rows( "Projection_11 10.00 root 9_aux_0", @@ -1079,8 +1085,8 @@ func (s *testAnalyzeSuite) TestLimitCrossEstimation(c *C) { )) // Multi-column filter. tk.MustExec("drop table t") - tk.MustExec("create table t(a int primary key, b int, c int, index idx_b(b))") - tk.MustExec("insert into t values (1, 1, 1),(2, 1, 2),(3, 1, 1),(4, 1, 2),(5, 1, 1),(6, 1, 2),(7, 1, 1),(8, 1, 2),(9, 1, 1),(10, 1, 2),(11, 1, 1),(12, 1, 2),(13, 1, 1),(14, 1, 2),(15, 1, 1),(16, 1, 2),(17, 1, 1),(18, 1, 2),(19, 1, 1),(20, 2, 2),(21, 2, 1),(22, 2, 2),(23, 2, 1),(24, 2, 2),(25, 2, 1)") + tk.MustExec("create table t(a int primary key, b int, c int, d bigint default 2147483648, e bigint default 2147483648, f bigint default 2147483648, index idx_b(b))") + tk.MustExec("insert into t(a, b, c) values (1, 1, 1),(2, 1, 2),(3, 1, 1),(4, 1, 2),(5, 1, 1),(6, 1, 2),(7, 1, 1),(8, 1, 2),(9, 1, 1),(10, 1, 2),(11, 1, 1),(12, 1, 2),(13, 1, 1),(14, 1, 2),(15, 1, 1),(16, 1, 2),(17, 1, 1),(18, 1, 2),(19, 1, 1),(20, 2, 2),(21, 2, 1),(22, 2, 2),(23, 2, 1),(24, 2, 2),(25, 2, 1)") tk.MustExec("analyze table t") tk.MustQuery("EXPLAIN SELECT * FROM t WHERE b = 2 and c > 0 ORDER BY a limit 1").Check(testkit.Rows( "TopN_9 1.00 root test.t.a:asc, offset:0, count:1", diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 73145def95be6..ab94db70d850b 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tidb/expression/aggregation" "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" @@ -282,6 +281,8 @@ func (p *LogicalJoin) getHashJoin(prop *property.PhysicalProperty, innerIdx int) LeftConditions: p.LeftConditions, RightConditions: p.RightConditions, OtherConditions: p.OtherConditions, + LeftJoinKeys: p.LeftJoinKeys, + RightJoinKeys: p.RightJoinKeys, JoinType: p.JoinType, Concurrency: uint(p.ctx.GetSessionVars().HashJoinConcurrency), DefaultValues: p.DefaultValues, @@ -324,7 +325,7 @@ func joinKeysMatchIndex(keys, indexCols []*expression.Column, colLengths []int) func (p *LogicalJoin) constructIndexJoin( prop *property.PhysicalProperty, outerIdx int, - innerPlan PhysicalPlan, + innerTask task, ranges []*ranger.Range, keyOff2IdxOff []int, lens []int, @@ -377,7 +378,7 @@ func (p *LogicalJoin) constructIndexJoin( OuterJoinKeys: newOuterKeys, InnerJoinKeys: newInnerKeys, DefaultValues: p.DefaultValues, - innerPlan: innerPlan, + innerTask: innerTask, KeyOff2IdxOff: newKeyOff, IdxColLens: lens, Ranges: ranges, @@ -435,10 +436,10 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou keyOff2IdxOff[i] = 0 } if pkMatched { - innerPlan := p.constructInnerTableScan(ds, pkCol, outerJoinKeys, us) + innerTask := p.constructInnerTableScanTask(ds, pkCol, outerJoinKeys, us) // Since the primary key means one value corresponding to exact one row, this will always be a no worse one // comparing to other index. - return p.constructIndexJoin(prop, outerIdx, innerPlan, nil, keyOff2IdxOff, nil, nil) + return p.constructIndexJoin(prop, outerIdx, innerTask, nil, keyOff2IdxOff, nil, nil) } } helper := &indexJoinBuildHelper{join: p} @@ -464,8 +465,8 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *property.PhysicalProperty, ou } idxCols, lens := expression.IndexInfo2Cols(ds.schema.Columns, helper.chosenIndexInfo) rangeInfo := helper.buildRangeDecidedByInformation(idxCols, outerJoinKeys) - innerPlan := p.constructInnerIndexScan(ds, helper.chosenIndexInfo, helper.chosenRemained, outerJoinKeys, us, rangeInfo) - return p.constructIndexJoin(prop, outerIdx, innerPlan, helper.chosenRanges, keyOff2IdxOff, lens, helper.lastColManager) + innerTask := p.constructInnerIndexScanTask(ds, helper.chosenIndexInfo, helper.chosenRemained, outerJoinKeys, us, rangeInfo) + return p.constructIndexJoin(prop, outerIdx, innerTask, helper.chosenRanges, keyOff2IdxOff, lens, helper.lastColManager) } return nil } @@ -513,8 +514,8 @@ func (ijHelper *indexJoinBuildHelper) buildRangeDecidedByInformation(idxCols []* return buffer.String() } -// constructInnerTableScan is specially used to construct the inner plan for PhysicalIndexJoin. -func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Column, outerJoinKeys []*expression.Column, us *LogicalUnionScan) PhysicalPlan { +// constructInnerTableScanTask is specially used to construct the inner plan for PhysicalIndexJoin. +func (p *LogicalJoin) constructInnerTableScanTask(ds *DataSource, pk *expression.Column, outerJoinKeys []*expression.Column, us *LogicalUnionScan) task { ranges := ranger.FullIntRange(mysql.HasUnsignedFlag(pk.RetType.Flag)) ts := PhysicalTableScan{ Table: ds.tableInfo, @@ -526,22 +527,28 @@ func (p *LogicalJoin) constructInnerTableScan(ds *DataSource, pk *expression.Col rangeDecidedBy: outerJoinKeys, }.Init(ds.ctx) ts.SetSchema(ds.schema) - - ts.stats = property.NewSimpleStats(1) - ts.stats.StatsVersion = ds.statisticTable.Version - if ds.statisticTable.Pseudo { - ts.stats.StatsVersion = statistics.PseudoVersion + ts.stats = &property.StatsInfo{ + // TableScan as inner child of IndexJoin can return at most 1 tuple for each outer row. + RowCount: 1, + StatsVersion: ds.stats.StatsVersion, + // Cardinality would not be used in cost computation of IndexJoin, set leave it as default nil. } - + for i := range ds.stats.Cardinality { + ds.stats.Cardinality[i] = 1 + } + rowSize := ds.tableStats.HistColl.GetAvgRowSize(ds.tableCols, false) copTask := &copTask{ tablePlan: ts, indexPlanFinished: true, + cst: scanFactor * rowSize * ts.stats.RowCount, + tableStats: ds.tableStats, } selStats := ts.stats.Scale(selectionFactor) ts.addPushedDownSelection(copTask, selStats) - t := finishCopTask(ds.ctx, copTask) - reader := t.plan() - return p.constructInnerUnionScan(us, reader) + t := finishCopTask(ds.ctx, copTask).(*rootTask) + reader := t.p + t.p = p.constructInnerUnionScan(us, reader) + return t } func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader PhysicalPlan) PhysicalPlan { @@ -555,9 +562,9 @@ func (p *LogicalJoin) constructInnerUnionScan(us *LogicalUnionScan, reader Physi return physicalUnionScan } -// constructInnerIndexScan is specially used to construct the inner plan for PhysicalIndexJoin. -func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexInfo, filterConds []expression.Expression, - outerJoinKeys []*expression.Column, us *LogicalUnionScan, rangeInfo string) PhysicalPlan { +// constructInnerIndexScanTask is specially used to construct the inner plan for PhysicalIndexJoin. +func (p *LogicalJoin) constructInnerIndexScanTask(ds *DataSource, idx *model.IndexInfo, filterConds []expression.Expression, + outerJoinKeys []*expression.Column, us *LogicalUnionScan, rangeInfo string) task { is := PhysicalIndexScan{ Table: ds.tableInfo, TableAsName: ds.TableAsName, @@ -572,19 +579,24 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn var rowCount float64 idxHist, ok := ds.statisticTable.Indices[idx.ID] + // TODO: we are assuming that: + // - rows are uniformly distributed among the distinct values; + // - every outer row can find matches in inner table; + // The second assumption is too strong, we'd better analyze histograms + // of both sides to compute a factor we should multiply to the current + // estimated `rowCount`. + // We can improve this after https://github.com/pingcap/tidb/pull/8097 + // is merged, since it provides some utilities we need. if ok && !ds.statisticTable.Pseudo { rowCount = idxHist.AvgCountPerNotNullValue(ds.statisticTable.Count) } else { rowCount = ds.statisticTable.PseudoAvgCountPerValue() } - is.stats = property.NewSimpleStats(rowCount) - is.stats.StatsVersion = ds.statisticTable.Version - if ds.statisticTable.Pseudo { - is.stats.StatsVersion = statistics.PseudoVersion - } - + is.stats = ds.tableStats.ScaleByExpectCnt(rowCount) cop := &copTask{ - indexPlan: is, + indexPlan: is, + tableStats: ds.tableStats, + tableCols: ds.tableCols, } if !isCoveringIndex(ds.schema.Columns, is.Index.Columns, is.Table.PKIsHandle) { // On this way, it's double read case. @@ -592,8 +604,9 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn ts.SetSchema(is.dataSourceSchema) cop.tablePlan = ts } - is.initSchema(ds.id, idx, cop.tablePlan != nil) + rowSize := is.indexScanRowSize(idx, ds) + cop.cst = rowCount * rowSize * scanFactor indexConds, tblConds := splitIndexFilterConditions(filterConds, idx.Columns, ds.tableInfo) path := &accessPath{ indexFilters: indexConds, @@ -612,9 +625,10 @@ func (p *LogicalJoin) constructInnerIndexScan(ds *DataSource, idx *model.IndexIn selectivity := ds.stats.RowCount / ds.tableStats.RowCount finalStats := ds.stats.ScaleByExpectCnt(selectivity * rowCount) is.addPushedDownSelection(cop, ds, path, finalStats) - t := finishCopTask(ds.ctx, cop) - reader := t.plan() - return p.constructInnerUnionScan(us, reader) + t := finishCopTask(ds.ctx, cop).(*rootTask) + reader := t.p + t.p = p.constructInnerUnionScan(us, reader) + return t } var symmetricOp = map[string]string{ diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 40e20ea43287d..268d93f971d7b 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -31,18 +31,16 @@ import ( ) const ( - netWorkFactor = 1.5 - netWorkStartFactor = 20.0 - scanFactor = 2.0 - descScanFactor = 2 * scanFactor - memoryFactor = 5.0 - // 0.5 is the looking up agg context factor. - hashAggFactor = 1.2 + 0.5 - selectionFactor = 0.8 - distinctFactor = 0.8 - cpuFactor = 0.9 - distinctAggFactor = 1.6 - createAggCtxFactor = 6 + netWorkFactor = 1.0 + cpuFactor = 3 * netWorkFactor + copCPUFactor = 3 * netWorkFactor + scanFactor = 1.5 * netWorkFactor + descScanFactor = 2 * scanFactor + memoryFactor = 0.001 + concurrencyFactor = 0.001 + + selectionFactor = 0.8 + distinctFactor = 0.8 ) // wholeTaskTypes records all possible kinds of task that a plan can return. For Agg, TopN and Limit, we will try to get @@ -490,7 +488,11 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid is.Hist = &statsTbl.Indices[idx.ID].Histogram } rowCount := path.countAfterAccess - cop := &copTask{indexPlan: is} + cop := &copTask{ + indexPlan: is, + tableStats: ds.tableStats, + tableCols: ds.tableCols, + } if !candidate.isSingleScan { // On this way, it's double read case. ts := PhysicalTableScan{ @@ -510,18 +512,14 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid selectivity := ds.stats.RowCount / path.countAfterAccess rowCount = math.Min(prop.ExpectedCnt/selectivity, rowCount) } - is.stats = property.NewSimpleStats(rowCount) - is.stats.StatsVersion = ds.statisticTable.Version - if ds.statisticTable.Pseudo { - is.stats.StatsVersion = statistics.PseudoVersion - } - - cop.cst = rowCount * scanFactor + is.stats = ds.tableStats.ScaleByExpectCnt(rowCount) + rowSize := is.indexScanRowSize(idx, ds) + cop.cst = rowCount * rowSize * scanFactor task = cop if candidate.isMatchProp { if prop.Items[0].Desc { is.Desc = true - cop.cst = rowCount * descScanFactor + cop.cst = rowCount * rowSize * descScanFactor } if cop.tablePlan != nil { cop.tablePlan.(*PhysicalTableScan).appendExtraHandleCol(ds) @@ -542,6 +540,21 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid return task, nil } +func (is *PhysicalIndexScan) indexScanRowSize(idx *model.IndexInfo, ds *DataSource) float64 { + scanCols := make([]*expression.Column, 0, len(idx.Columns)+1) + // If `initSchema` has already appended the handle column in schema, just use schema columns, otherwise, add extra handle column. + if len(idx.Columns) == len(is.schema.Columns) { + scanCols = append(scanCols, is.schema.Columns...) + handleCols, ok := ds.schema.TblID2Handle[ds.tableInfo.ID] + if ok { + scanCols = append(scanCols, handleCols[0]) + } + } else { + scanCols = is.schema.Columns + } + return ds.tableStats.HistColl.GetAvgRowSize(scanCols, true) +} + // TODO: refactor this part, we should not call Clone in fact. func (is *PhysicalIndexScan) initSchema(id int, idx *model.IndexInfo, isDoubleRead bool) { indexCols := make([]*expression.Column, 0, len(idx.Columns)) @@ -578,20 +591,20 @@ func (is *PhysicalIndexScan) addPushedDownSelection(copTask *copTask, p *DataSou // Add filter condition to table plan now. indexConds, tableConds := path.indexFilters, path.tableFilters if indexConds != nil { - copTask.cst += copTask.count() * cpuFactor + copTask.cst += copTask.count() * copCPUFactor var selectivity float64 if path.countAfterAccess > 0 { selectivity = path.countAfterIndex / path.countAfterAccess } count := is.stats.RowCount * selectivity - stats := &property.StatsInfo{RowCount: count} + stats := p.tableStats.ScaleByExpectCnt(count) indexSel := PhysicalSelection{Conditions: indexConds}.Init(is.ctx, stats) indexSel.SetChildren(is) copTask.indexPlan = indexSel } if tableConds != nil { copTask.finishIndexPlan() - copTask.cst += copTask.count() * cpuFactor + copTask.cst += copTask.count() * copCPUFactor tableSel := PhysicalSelection{Conditions: tableConds}.Init(is.ctx, finalStats) tableSel.SetChildren(copTask.tablePlan) copTask.tablePlan = tableSel @@ -805,6 +818,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid copTask := &copTask{ tablePlan: ts, indexPlanFinished: true, + tableStats: ds.tableStats, } task = copTask // Adjust number of rows we actually need to scan if prop.ExpectedCnt is smaller than the count we calculated. @@ -825,17 +839,18 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid rowCount = math.Min(prop.ExpectedCnt/selectivity/correlationFactor, rowCount) } } - ts.stats = property.NewSimpleStats(rowCount) - ts.stats.StatsVersion = ds.statisticTable.Version - if ds.statisticTable.Pseudo { - ts.stats.StatsVersion = statistics.PseudoVersion - } - - copTask.cst = rowCount * scanFactor + // We need NDV of columns since it may be used in cost estimation of join. Precisely speaking, + // we should track NDV of each histogram bucket, and sum up the NDV of buckets we actually need + // to scan, but this would only help improve accuracy of NDV for one column, for other columns, + // we still need to assume values are uniformly distributed. For simplicity, we use uniform-assumption + // for all columns now, as we do in `deriveStatsByFilter`. + ts.stats = ds.tableStats.ScaleByExpectCnt(rowCount) + rowSize := ds.tableStats.HistColl.GetAvgRowSize(ds.tableCols, false) + copTask.cst = rowCount * rowSize * scanFactor if candidate.isMatchProp { if prop.Items[0].Desc { ts.Desc = true - copTask.cst = rowCount * descScanFactor + copTask.cst = rowCount * rowSize * descScanFactor } ts.KeepOrder = true copTask.keepOrder = true @@ -852,7 +867,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid func (ts *PhysicalTableScan) addPushedDownSelection(copTask *copTask, stats *property.StatsInfo) { // Add filter condition to table plan now. if len(ts.filterCondition) > 0 { - copTask.cst += copTask.count() * cpuFactor + copTask.cst += copTask.count() * copCPUFactor sel := PhysicalSelection{Conditions: ts.filterCondition}.Init(ts.ctx, stats) sel.SetChildren(ts) copTask.tablePlan = sel diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index a85a635a791af..0b671e6ee70a2 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2267,6 +2267,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L possibleAccessPaths: possiblePaths, Columns: make([]*model.ColumnInfo, 0, len(columns)), partitionNames: tn.PartitionNames, + tableCols: make([]*expression.Column, 0, len(columns)), }.Init(b.ctx) var handleCol *expression.Column @@ -2287,9 +2288,8 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L handleCol = newCol } schema.Append(newCol) + ds.tableCols = append(ds.tableCols, newCol) } - ds.SetSchema(schema) - // We append an extra handle column to the schema when "ds" is not a memory // table e.g. table in the "INFORMATION_SCHEMA" database, and the handle // column is not the primary key of "ds". @@ -2298,10 +2298,12 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L ds.Columns = append(ds.Columns, model.NewExtraHandleColInfo()) handleCol = ds.newExtraHandleSchemaCol() schema.Append(handleCol) + ds.tableCols = append(ds.tableCols, handleCol) } if handleCol != nil { schema.TblID2Handle[tableInfo.ID] = []*expression.Column{handleCol} } + ds.SetSchema(schema) var result LogicalPlan = ds diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index e9334b16ee206..c79b630efb2b1 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2249,7 +2249,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { }, { sql: "select sum(avg(a)) over() from t", - result: "TableReader(Table(t)->StreamAgg)->StreamAgg->Window(sum(sel_agg_2) over())->Projection", + result: "TableReader(Table(t)->HashAgg)->HashAgg->Window(sum(sel_agg_2) over())->Projection", }, { sql: "select b from t order by(sum(a) over())", @@ -2261,7 +2261,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { }, { sql: "select b from t order by(sum(avg(a)) over())", - result: "TableReader(Table(t)->StreamAgg)->StreamAgg->Window(sum(sel_agg_2) over())->Sort->Projection", + result: "TableReader(Table(t)->HashAgg)->HashAgg->Window(sum(sel_agg_2) over())->Sort->Projection", }, { sql: "select a from t having (select sum(a) over() as w from t tt where a > t.a)", @@ -2373,7 +2373,7 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { }, { sql: "select avg(b), max(avg(b)) over(rows between 1 preceding and 1 following) max from t group by c", - result: "IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->Projection->StreamAgg->Window(max(sel_agg_3) over(rows between 1 preceding and 1 following))->Projection", + result: "TableReader(Table(t))->HashAgg->Window(max(sel_agg_3) over(rows between 1 preceding and 1 following))->Projection", }, { sql: "select nth_value(a, 1.0) over() from t", @@ -2472,8 +2472,14 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { } s.Parser.EnableWindowFunc(true) + sessionVars := s.ctx.GetSessionVars() + finalCon, partialCon := sessionVars.HashAggFinalConcurrency, sessionVars.HashAggPartialConcurrency + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 defer func() { s.Parser.EnableWindowFunc(false) + sessionVars.HashAggFinalConcurrency = finalCon + sessionVars.HashAggPartialConcurrency = partialCon }() ctx := context.TODO() for i, tt := range tests { diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index c459d0d75810f..37f2d15494fa7 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -358,6 +358,9 @@ type DataSource struct { // handleCol represents the handle column for the datasource, either the // int primary key column or extra handle column. handleCol *expression.Column + // tableCols contains the original columns of table before being pruned, and it + // is used for estimating table scan cost. + tableCols []*expression.Column } // accessPath indicates the way we access a table: by using single index, or by using multiple indexes, diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 7d7be7839e4f9..22bddc2b88b0f 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -259,7 +259,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderJoin(c *C) { }, { sql: "select * from t t1 join t t2 on t1.a = t2.a join t t3 on t1.a = t3.a", - best: "MergeInnerJoin{MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.a)->TableReader(Table(t))}(test.t1.a,test.t3.a)", + best: "LeftHashJoin{MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.a,test.t2.a)->TableReader(Table(t))}(test.t1.a,test.t3.a)", }, { sql: "select * from t t1 join t t2 on t1.a = t2.a join t t3 on t1.b = t3.a", @@ -267,7 +267,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderJoin(c *C) { }, { sql: "select * from t t1 join t t2 on t1.b = t2.a order by t1.a", - best: "LeftHashJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.b,test.t2.a)->Sort", + best: "IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t1.b,test.t2.a)", }, { sql: "select * from t t1 join t t2 on t1.b = t2.a order by t1.a limit 1", @@ -486,6 +486,10 @@ func (s *testPlanSuite) TestDAGPlanBuilderSubquery(c *C) { _, err = se.Execute(context.Background(), "use test") c.Assert(err, IsNil) se.Execute(context.Background(), "set sql_mode='STRICT_TRANS_TABLES'") // disable only full group by + ctx := se.(sessionctx.Context) + sessionVars := ctx.GetSessionVars() + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 tests := []struct { sql string best string @@ -520,15 +524,15 @@ func (s *testPlanSuite) TestDAGPlanBuilderSubquery(c *C) { }, // Test Apply. { - sql: "select t.c in (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t", - best: "Apply{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t)->Sel([eq(test.t1.a, test.t.a)]))}(test.s.a,test.t1.a)->StreamAgg}->Projection", + sql: "select t.c in (select count(*) from t s, t t1 where s.a = t.a and s.a = t1.a) from t", + best: "Apply{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.s.a,test.t1.a)->StreamAgg}->Projection", }, { - sql: "select (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t", + sql: "select (select count(*) from t s, t t1 where s.a = t.a and s.a = t1.a) from t", best: "LeftHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.s.a,test.t1.a)->StreamAgg}(test.t.a,test.s.a)->Projection->Projection", }, { - sql: "select (select count(*) from t s , t t1 where s.a = t.a and s.a = t1.a) from t order by t.a", + sql: "select (select count(*) from t s, t t1 where s.a = t.a and s.a = t1.a) from t order by t.a", best: "LeftHashJoin{TableReader(Table(t))->MergeInnerJoin{TableReader(Table(t))->TableReader(Table(t))}(test.s.a,test.t1.a)->StreamAgg}(test.t.a,test.s.a)->Projection->Sort->Projection", }, } @@ -835,6 +839,10 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) { se.Execute(context.Background(), "use test") se.Execute(context.Background(), "set sql_mode='STRICT_TRANS_TABLES'") // disable only full group by c.Assert(err, IsNil) + ctx := se.(sessionctx.Context) + sessionVars := ctx.GetSessionVars() + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 tests := []struct { sql string @@ -874,18 +882,18 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) { }, // Test hash agg + index single. { - sql: "select sum(e), avg(e + c) from t where c = 1 group by d", - best: "IndexReader(Index(t.c_d_e)[[1,1]]->StreamAgg)->StreamAgg", + sql: "select sum(e), avg(e + c) from t where c = 1 group by e", + best: "IndexReader(Index(t.c_d_e)[[1,1]]->HashAgg)->HashAgg", }, // Test hash agg + index double. { sql: "select sum(e), avg(b + c) from t where c = 1 and e = 1 group by d", - best: "IndexLookUp(Index(t.c_d_e)[[1,1]]->Sel([eq(test.t.e, 1)]), Table(t))->Projection->Projection->StreamAgg", + best: "IndexLookUp(Index(t.c_d_e)[[1,1]]->Sel([eq(test.t.e, 1)]), Table(t))->Projection->HashAgg", }, // Test stream agg + index double. { - sql: "select sum(e), avg(b + c) from t where c = 1 and b = 1 group by c", - best: "IndexLookUp(Index(t.c_d_e)[[1,1]], Table(t)->Sel([eq(test.t.b, 1)]))->Projection->Projection->StreamAgg", + sql: "select sum(e), avg(b + c) from t where c = 1 and b = 1", + best: "IndexLookUp(Index(t.c_d_e)[[1,1]], Table(t)->Sel([eq(test.t.b, 1)]))->Projection->StreamAgg", }, // Test hash agg + order. { @@ -926,7 +934,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) { // Test stream agg + limit or sort { sql: "select count(*) from t group by g order by g limit 10", - best: "IndexReader(Index(t.g)[[NULL,+inf]]->StreamAgg)->StreamAgg->Limit->Projection", + best: "IndexReader(Index(t.g)[[NULL,+inf]])->StreamAgg->Limit->Projection", }, { sql: "select count(*) from t group by g limit 10", @@ -934,20 +942,20 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) { }, { sql: "select count(*) from t group by g order by g", - best: "IndexReader(Index(t.g)[[NULL,+inf]]->StreamAgg)->StreamAgg->Projection", + best: "IndexReader(Index(t.g)[[NULL,+inf]])->StreamAgg->Projection", }, { sql: "select count(*) from t group by g order by g desc limit 1", - best: "IndexReader(Index(t.g)[[NULL,+inf]]->StreamAgg)->StreamAgg->Limit->Projection", + best: "IndexReader(Index(t.g)[[NULL,+inf]])->StreamAgg->Limit->Projection", }, // Test hash agg + limit or sort { sql: "select count(*) from t group by b order by b limit 10", - best: "TableReader(Table(t)->HashAgg)->HashAgg->TopN([test.t.b],0,10)->Projection", + best: "TableReader(Table(t))->HashAgg->TopN([test.t.b],0,10)->Projection", }, { sql: "select count(*) from t group by b order by b", - best: "TableReader(Table(t)->HashAgg)->HashAgg->Sort->Projection", + best: "TableReader(Table(t))->HashAgg->Sort->Projection", }, { sql: "select count(*) from t group by b limit 10", @@ -965,7 +973,7 @@ func (s *testPlanSuite) TestDAGPlanBuilderAgg(c *C) { }, { sql: "select sum(a.g), sum(b.g) from t a join t b on a.g = b.g and a.a>5 group by a.g order by a.g limit 1", - best: "IndexJoin{IndexReader(Index(t.g)[[NULL,+inf]]->Sel([gt(test.a.a, 5)]))->IndexReader(Index(t.g)[[NULL,+inf]])}(test.a.g,test.b.g)->Projection->StreamAgg->Limit->Projection", + best: "MergeInnerJoin{IndexReader(Index(t.g)[[NULL,+inf]]->Sel([gt(test.a.a, 5)]))->IndexReader(Index(t.g)[[NULL,+inf]])}(test.a.g,test.b.g)->Projection->StreamAgg->Limit->Projection", }, { sql: "select sum(d) from t", @@ -1224,6 +1232,8 @@ func (s *testPlanSuite) TestAggEliminater(c *C) { _, err = se.Execute(context.Background(), "use test") c.Assert(err, IsNil) se.Execute(context.Background(), "set sql_mode='STRICT_TRANS_TABLES'") // disable only full group by + se.GetSessionVars().HashAggFinalConcurrency = 1 + se.GetSessionVars().HashAggPartialConcurrency = 1 tests := []struct { sql string best string @@ -1352,25 +1362,25 @@ func (s *testPlanSuite) TestIndexJoinUnionScan(c *C) { }{ // Test Index Join + UnionScan + TableScan. { - sql: "select /*+ TIDB_INLJ(t1, t2) */ * from t t1, t t2 where t1.a = t2.a", + sql: "select /*+ TIDB_INLJ(t2) */ * from t t1, t t2 where t1.a = t2.a", best: "IndexJoin{TableReader(Table(t))->UnionScan([])->TableReader(Table(t))->UnionScan([])}(test.t1.a,test.t2.a)", is: s.is, }, // Test Index Join + UnionScan + DoubleRead. { - sql: "select /*+ TIDB_INLJ(t1, t2) */ * from t t1, t t2 where t1.a = t2.c", + sql: "select /*+ TIDB_INLJ(t2) */ * from t t1, t t2 where t1.a = t2.c", best: "IndexJoin{TableReader(Table(t))->UnionScan([])->IndexLookUp(Index(t.c_d_e)[[NULL,+inf]], Table(t))->UnionScan([])}(test.t1.a,test.t2.c)", is: s.is, }, // Test Index Join + UnionScan + IndexScan. { - sql: "select /*+ TIDB_INLJ(t1, t2) */ t1.a , t2.c from t t1, t t2 where t1.a = t2.c", + sql: "select /*+ TIDB_INLJ(t2) */ t1.a , t2.c from t t1, t t2 where t1.a = t2.c", best: "IndexJoin{TableReader(Table(t))->UnionScan([])->IndexReader(Index(t.c_d_e)[[NULL,+inf]])->UnionScan([])}(test.t1.a,test.t2.c)->Projection", is: s.is, }, // Index Join + Union Scan + Union All is not supported now. { - sql: "select /*+ TIDB_INLJ(t1, t2) */ * from t t1, t t2 where t1.a = t2.a", + sql: "select /*+ TIDB_INLJ(t2) */ * from t t1, t t2 where t1.a = t2.a", best: "LeftHashJoin{UnionAll{TableReader(Table(t))->UnionScan([])->TableReader(Table(t))->UnionScan([])}->UnionAll{TableReader(Table(t))->UnionScan([])->TableReader(Table(t))->UnionScan([])}}(test.t1.a,test.t2.a)", is: pis, }, @@ -1464,7 +1474,7 @@ func (s *testPlanSuite) TestSemiJoinToInner(c *C) { c.Assert(err, IsNil) p, err := planner.Optimize(context.TODO(), se, stmt, s.is) c.Assert(err, IsNil) - c.Assert(core.ToString(p), Equals, "Apply{TableReader(Table(t))->IndexJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]]->HashAgg)->HashAgg->IndexReader(Index(t.g)[[NULL,+inf]])}(test.t3.d,test.t2.g)}->StreamAgg") + c.Assert(core.ToString(p), Equals, "Apply{TableReader(Table(t))->IndexJoin{IndexReader(Index(t.c_d_e)[[NULL,+inf]]->HashAgg)->HashAgg->IndexReader(Index(t.g)[[NULL,+inf]])}(test.t3.d,test.t2.g)}->HashAgg") } func (s *testPlanSuite) TestUnmatchedTableInHint(c *C) { @@ -1541,7 +1551,7 @@ func (s *testPlanSuite) TestJoinHints(c *C) { }{ { sql: "select /*+ TIDB_INLJ(t1) */ t1.a, t2.a, t3.a from t t1, t t2, t t3 where t1.a = t2.a and t2.a = t3.a;", - best: "MergeInnerJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t1.a)}(test.t3.a,test.t2.a)->Projection", + best: "RightHashJoin{TableReader(Table(t))->IndexJoin{TableReader(Table(t))->TableReader(Table(t))}(test.t2.a,test.t1.a)}(test.t3.a,test.t2.a)->Projection", warning: "", }, { @@ -1584,6 +1594,10 @@ func (s *testPlanSuite) TestAggregationHints(c *C) { _, err = se.Execute(context.Background(), "use test") c.Assert(err, IsNil) + sessionVars := se.(sessionctx.Context).GetSessionVars() + sessionVars.HashAggFinalConcurrency = 1 + sessionVars.HashAggPartialConcurrency = 1 + tests := []struct { sql string best string diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index a43f3fa03f340..dcda164e98854 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -208,6 +208,10 @@ type PhysicalHashJoin struct { LeftConditions []expression.Expression RightConditions []expression.Expression OtherConditions []expression.Expression + + LeftJoinKeys []*expression.Column + RightJoinKeys []*expression.Column + // InnerChildIdx indicates which child is to build the hash table. // For inner join, the smaller one will be chosen. // For outer join or semi join, it's exactly the inner one. @@ -229,7 +233,7 @@ type PhysicalIndexJoin struct { OtherConditions expression.CNFExprs OuterIndex int outerSchema *expression.Schema - innerPlan PhysicalPlan + innerTask task DefaultValues []types.Datum @@ -316,13 +320,13 @@ type basePhysicalAgg struct { GroupByItems []expression.Expression } -func (p *basePhysicalAgg) hasDistinctFunc() bool { +func (p *basePhysicalAgg) numDistinctFunc() (num int) { for _, fun := range p.AggFuncs { if fun.HasDistinct { - return true + num++ } } - return false + return } // PhysicalHashAgg is hash operator of aggregate. diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 3134485529e87..a4083c05460b0 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" @@ -782,11 +783,16 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(ctx context.Context, dbName Ranges: ranger.FullRange(), GenExprs: genExprsMap, }.Init(b.ctx) - is.stats = property.NewSimpleStats(0) + // There is no alternative plan choices, so just use pseudo stats to avoid panic. + is.stats = &property.StatsInfo{HistColl: &(statistics.PseudoTable(tblInfo)).HistColl} // It's double read case. ts := PhysicalTableScan{Columns: tblReaderCols, Table: is.Table}.Init(b.ctx) ts.SetSchema(tblSchema) - cop := &copTask{indexPlan: is, tablePlan: ts} + cop := &copTask{ + indexPlan: is, + tablePlan: ts, + tableStats: is.stats, + } ts.HandleIdx = pkOffset is.initSchema(id, idx, true) rootT := finishCopTask(b.ctx, cop).(*rootTask) diff --git a/planner/core/resolve_indices.go b/planner/core/resolve_indices.go index 2340e747bb731..419b4b631845e 100644 --- a/planner/core/resolve_indices.go +++ b/planner/core/resolve_indices.go @@ -84,10 +84,12 @@ func (p *PhysicalHashJoin) ResolveIndices() (err error) { if err != nil { return err } + p.LeftJoinKeys[i] = lArg.(*expression.Column) rArg, err := fun.GetArgs()[1].ResolveIndices(rSchema) if err != nil { return err } + p.RightJoinKeys[i] = rArg.(*expression.Column) p.EqualConditions[i] = expression.NewFunctionInternal(fun.GetCtx(), fun.FuncName.L, fun.GetType(), lArg, rArg).(*expression.ScalarFunction) } for i, expr := range p.LeftConditions { diff --git a/planner/core/stats.go b/planner/core/stats.go index 530add46af499..4b977e5e168b7 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -292,41 +292,38 @@ func (p *LogicalUnionAll) DeriveStats(childStats []*property.StatsInfo) (*proper return p.stats, nil } -// DeriveStats implement LogicalPlan DeriveStats interface. -func (p *LogicalLimit) DeriveStats(childStats []*property.StatsInfo) (*property.StatsInfo, error) { - childProfile := childStats[0] - p.stats = &property.StatsInfo{ - RowCount: math.Min(float64(p.Count), childProfile.RowCount), +func deriveLimitStats(childProfile *property.StatsInfo, limitCount float64) *property.StatsInfo { + stats := &property.StatsInfo{ + RowCount: math.Min(limitCount, childProfile.RowCount), Cardinality: make([]float64, len(childProfile.Cardinality)), } - for i := range p.stats.Cardinality { - p.stats.Cardinality[i] = math.Min(childProfile.Cardinality[i], p.stats.RowCount) + for i := range stats.Cardinality { + stats.Cardinality[i] = math.Min(childProfile.Cardinality[i], stats.RowCount) } + return stats +} + +// DeriveStats implement LogicalPlan DeriveStats interface. +func (p *LogicalLimit) DeriveStats(childStats []*property.StatsInfo) (*property.StatsInfo, error) { + p.stats = deriveLimitStats(childStats[0], float64(p.Count)) return p.stats, nil } // DeriveStats implement LogicalPlan DeriveStats interface. func (lt *LogicalTopN) DeriveStats(childStats []*property.StatsInfo) (*property.StatsInfo, error) { - childProfile := childStats[0] - lt.stats = &property.StatsInfo{ - RowCount: math.Min(float64(lt.Count), childProfile.RowCount), - Cardinality: make([]float64, len(childProfile.Cardinality)), - } - for i := range lt.stats.Cardinality { - lt.stats.Cardinality[i] = math.Min(childProfile.Cardinality[i], lt.stats.RowCount) - } + lt.stats = deriveLimitStats(childStats[0], float64(lt.Count)) return lt.stats, nil } // getCardinality will return the Cardinality of a couple of columns. We simply return the max one, because we cannot know // the Cardinality for multi-dimension attributes properly. This is a simple and naive scheme of Cardinality estimation. func getCardinality(cols []*expression.Column, schema *expression.Schema, profile *property.StatsInfo) float64 { + cardinality := 1.0 indices := schema.ColumnsIndices(cols) if indices == nil { logutil.BgLogger().Error("column not found in schema", zap.Any("columns", cols), zap.String("schema", schema.String())) - return 0 + return cardinality } - var cardinality = 1.0 for _, idx := range indices { // It is a very elementary estimation. cardinality = math.Max(cardinality, profile.Cardinality[idx]) @@ -397,16 +394,16 @@ func (p *LogicalJoin) DeriveStats(childStats []*property.StatsInfo) (*property.S p.stats.Cardinality[len(p.stats.Cardinality)-1] = 2.0 return p.stats, nil } - if 0 == len(p.EqualConditions) { - p.stats = &property.StatsInfo{ - RowCount: leftProfile.RowCount * rightProfile.RowCount, - Cardinality: append(leftProfile.Cardinality, rightProfile.Cardinality...), - } - return p.stats, nil + helper := &fullJoinRowCountHelper{ + cartesian: 0 == len(p.EqualConditions), + leftProfile: leftProfile, + rightProfile: rightProfile, + leftJoinKeys: p.LeftJoinKeys, + rightJoinKeys: p.RightJoinKeys, + leftSchema: p.children[0].Schema(), + rightSchema: p.children[1].Schema(), } - leftKeyCardinality := getCardinality(p.LeftJoinKeys, p.children[0].Schema(), leftProfile) - rightKeyCardinality := getCardinality(p.RightJoinKeys, p.children[1].Schema(), rightProfile) - count := leftProfile.RowCount * rightProfile.RowCount / math.Max(leftKeyCardinality, rightKeyCardinality) + count := helper.estimate() if p.JoinType == LeftOuterJoin { count = math.Max(count, leftProfile.RowCount) } else if p.JoinType == RightOuterJoin { @@ -425,6 +422,26 @@ func (p *LogicalJoin) DeriveStats(childStats []*property.StatsInfo) (*property.S return p.stats, nil } +type fullJoinRowCountHelper struct { + cartesian bool + leftProfile *property.StatsInfo + rightProfile *property.StatsInfo + leftJoinKeys []*expression.Column + rightJoinKeys []*expression.Column + leftSchema *expression.Schema + rightSchema *expression.Schema +} + +func (h *fullJoinRowCountHelper) estimate() float64 { + if h.cartesian { + return h.leftProfile.RowCount * h.rightProfile.RowCount + } + leftKeyCardinality := getCardinality(h.leftJoinKeys, h.leftSchema, h.leftProfile) + rightKeyCardinality := getCardinality(h.rightJoinKeys, h.rightSchema, h.rightProfile) + count := h.leftProfile.RowCount * h.rightProfile.RowCount / math.Max(leftKeyCardinality, rightKeyCardinality) + return count +} + // DeriveStats implement LogicalPlan DeriveStats interface. func (la *LogicalApply) DeriveStats(childStats []*property.StatsInfo) (*property.StatsInfo, error) { leftProfile := childStats[0] diff --git a/planner/core/task.go b/planner/core/task.go index a1dabd6bc4ef3..91b344b0ac34f 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" + "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" ) @@ -50,6 +51,12 @@ type copTask struct { // In double read case, it may output one more column for handle(row id). // We need to prune it, so we add a project do this. doubleReadNeedProj bool + // tableStats stores the original stats of DataSource, it is used to get + // average row width when computing network cost. + tableStats *property.StatsInfo + // tableCols store the original columns of DataSource before being pruned, it + // is used to compute average row width when computing scan cost. + tableCols []*expression.Column } func (t *copTask) invalid() bool { @@ -106,14 +113,20 @@ func attachPlan2Task(p PhysicalPlan, t task) task { // finishIndexPlan means we no longer add plan to index plan, and compute the network cost for it. func (t *copTask) finishIndexPlan() { - if !t.indexPlanFinished { - t.cst += t.count() * netWorkFactor - t.indexPlanFinished = true - if t.tablePlan != nil { - t.tablePlan.(*PhysicalTableScan).stats = t.indexPlan.statsInfo() - t.cst += t.count() * scanFactor - } + if t.indexPlanFinished { + return + } + cnt := t.count() + t.indexPlanFinished = true + // Network cost of transferring rows of index scan to TiDB. + t.cst += cnt * netWorkFactor * t.tableStats.HistColl.GetAvgRowSize(t.indexPlan.Schema().Columns, true) + if t.tablePlan == nil { + return } + // Calculate the IO cost of table scan here because we cannot know its stats until we finish index plan. + t.tablePlan.(*PhysicalTableScan).stats = t.indexPlan.statsInfo() + rowSize := t.tableStats.HistColl.GetAvgRowSize(t.tableCols, false) + t.cst += cnt * rowSize * scanFactor } func (p *basePhysicalPlan) attach2Task(tasks ...task) task { @@ -126,46 +139,139 @@ func (p *PhysicalApply) attach2Task(tasks ...task) task { rTask := finishCopTask(p.ctx, tasks[1].copy()) p.SetChildren(lTask.plan(), rTask.plan()) p.schema = buildPhysicalJoinSchema(p.JoinType, p) + var cpuCost float64 + lCount := lTask.count() + if len(p.LeftConditions) > 0 { + cpuCost += lCount * cpuFactor + lCount *= selectionFactor + } + rCount := rTask.count() + if len(p.RightConditions) > 0 { + cpuCost += lCount * rCount * cpuFactor + rCount *= selectionFactor + } + if len(p.EqualConditions)+len(p.OtherConditions) > 0 { + cpuCost += lCount * rCount * cpuFactor + } return &rootTask{ p: p, - cst: lTask.cost() + lTask.count()*rTask.cost(), + cst: cpuCost + lTask.cost(), } } func (p *PhysicalIndexJoin) attach2Task(tasks ...task) task { + innerTask := p.innerTask outerTask := finishCopTask(p.ctx, tasks[p.OuterIndex].copy()) if p.OuterIndex == 0 { - p.SetChildren(outerTask.plan(), p.innerPlan) + p.SetChildren(outerTask.plan(), innerTask.plan()) } else { - p.SetChildren(p.innerPlan, outerTask.plan()) + p.SetChildren(innerTask.plan(), outerTask.plan()) } p.schema = buildPhysicalJoinSchema(p.JoinType, p) return &rootTask{ p: p, - cst: outerTask.cost() + p.getCost(outerTask.count()), - } -} - -func (p *PhysicalIndexJoin) getCost(lCnt float64) float64 { - if lCnt < 1 { - lCnt = 1 + cst: p.GetCost(outerTask, innerTask), + } +} + +// GetCost computes the cost of index join operator and its children. +func (p *PhysicalIndexJoin) GetCost(outerTask, innerTask task) float64 { + var cpuCost float64 + outerCnt, innerCnt := outerTask.count(), innerTask.count() + // Add the cost of evaluating outer filter, since inner filter of index join + // is always empty, we can simply tell whether outer filter is empty using the + // summed length of left/right conditions. + if len(p.LeftConditions)+len(p.RightConditions) > 0 { + cpuCost += cpuFactor * outerCnt + outerCnt *= selectionFactor + } + // Cost of extracting lookup keys. + innerCPUCost := cpuFactor * outerCnt + // Cost of sorting and removing duplicate lookup keys: + // (outerCnt / batchSize) * (batchSize * Log2(batchSize) + batchSize) * cpuFactor + batchSize := math.Min(float64(p.ctx.GetSessionVars().IndexJoinBatchSize), outerCnt) + if batchSize > 2 { + innerCPUCost += outerCnt * (math.Log2(float64(batchSize)) + 1) * cpuFactor + } + // Add cost of building inner executors. CPU cost of building copTasks: + // (outerCnt / batchSize) * (batchSize * distinctFactor) * cpuFactor + // Since we don't know the number of copTasks built, ignore these network cost now. + innerCPUCost += outerCnt * distinctFactor * cpuFactor + // CPU cost of building hash table for inner results: + // (outerCnt / batchSize) * (batchSize * distinctFactor) * innerCnt * cpuFactor + innerCPUCost += outerCnt * distinctFactor * innerCnt * cpuFactor + innerConcurrency := float64(p.ctx.GetSessionVars().IndexLookupJoinConcurrency) + cpuCost += innerCPUCost / innerConcurrency + // Cost of probing hash table in main thread. + numPairs := outerCnt * innerCnt + if p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin || + p.JoinType == LeftOuterSemiJoin || p.JoinType == AntiLeftOuterSemiJoin { + if len(p.OtherConditions) > 0 { + numPairs *= 0.5 + } else { + numPairs = 0 + } } - cst := lCnt * netWorkFactor - batchSize := p.ctx.GetSessionVars().IndexJoinBatchSize - cst += lCnt * math.Log2(math.Min(float64(batchSize), lCnt)) * 2 - cst += lCnt / float64(batchSize) * netWorkStartFactor - return cst + probeCost := numPairs * cpuFactor + // Cost of additional concurrent goroutines. + cpuCost += probeCost + (innerConcurrency+1.0)*concurrencyFactor + // Memory cost of hash tables for inner rows. The computed result is the upper bound, + // since the executor is pipelined and not all workers are always in full load. + memoryCost := innerConcurrency * (batchSize * distinctFactor) * innerCnt * memoryFactor + // Cost of inner child plan, i.e, mainly I/O and network cost. + innerPlanCost := outerCnt * innerTask.cost() + return outerTask.cost() + innerPlanCost + cpuCost + memoryCost } -func (p *PhysicalHashJoin) getCost(lCnt, rCnt float64) float64 { - smallTableCnt := lCnt +// GetCost computes cost of hash join operator itself. +func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64) float64 { + innerCnt, outerCnt := lCnt, rCnt if p.InnerChildIdx == 1 { - smallTableCnt = rCnt + innerCnt, outerCnt = rCnt, lCnt + } + // Cost of building hash table. + cpuCost := innerCnt * cpuFactor + memoryCost := innerCnt * memoryFactor + // Number of matched row pairs regarding the equal join conditions. + helper := &fullJoinRowCountHelper{ + cartesian: false, + leftProfile: p.children[0].statsInfo(), + rightProfile: p.children[1].statsInfo(), + leftJoinKeys: p.LeftJoinKeys, + rightJoinKeys: p.RightJoinKeys, + leftSchema: p.children[0].Schema(), + rightSchema: p.children[1].Schema(), + } + numPairs := helper.estimate() + // For semi-join class, if `OtherConditions` is empty, we already know + // the join results after querying hash table, otherwise, we have to + // evaluate those resulted row pairs after querying hash table; if we + // find one pair satisfying the `OtherConditions`, we then know the + // join result for this given outer row, otherwise we have to iterate + // to the end of those pairs; since we have no idea about when we can + // terminate the iteration, we assume that we need to iterate half of + // those pairs in average. + if p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin || + p.JoinType == LeftOuterSemiJoin || p.JoinType == AntiLeftOuterSemiJoin { + if len(p.OtherConditions) > 0 { + numPairs *= 0.5 + } else { + numPairs = 0 + } } - if smallTableCnt <= 1 { - smallTableCnt = 1 + // Cost of quering hash table is cheap actually, so we just compute the cost of + // evaluating `OtherConditions` and joining row pairs. + probeCost := numPairs * cpuFactor + // Cost of evaluating outer filter. + if len(p.LeftConditions)+len(p.RightConditions) > 0 { + // Input outer count for the above compution should be adjusted by selectionFactor. + probeCost *= selectionFactor + probeCost += outerCnt * cpuFactor } - return (lCnt + rCnt) * (1 + math.Log2(smallTableCnt)/float64(p.Concurrency)) + probeCost /= float64(p.Concurrency) + // Cost of additional concurrent goroutines. + cpuCost += probeCost + float64(p.Concurrency+1)*concurrencyFactor + return cpuCost + memoryCost } func (p *PhysicalHashJoin) attach2Task(tasks ...task) task { @@ -175,12 +281,53 @@ func (p *PhysicalHashJoin) attach2Task(tasks ...task) task { p.schema = buildPhysicalJoinSchema(p.JoinType, p) return &rootTask{ p: p, - cst: lTask.cost() + rTask.cost() + p.getCost(lTask.count(), rTask.count()), + cst: lTask.cost() + rTask.cost() + p.GetCost(lTask.count(), rTask.count()), + } +} + +// GetCost computes cost of merge join operator itself. +func (p *PhysicalMergeJoin) GetCost(lCnt, rCnt float64) float64 { + outerCnt := lCnt + innerKeys := p.RightKeys + innerSchema := p.children[1].Schema() + innerStats := p.children[1].statsInfo() + if p.JoinType == RightOuterJoin { + outerCnt = rCnt + innerKeys = p.LeftKeys + innerSchema = p.children[0].Schema() + innerStats = p.children[0].statsInfo() + } + helper := &fullJoinRowCountHelper{ + cartesian: false, + leftProfile: p.children[0].statsInfo(), + rightProfile: p.children[1].statsInfo(), + leftJoinKeys: p.LeftKeys, + rightJoinKeys: p.RightKeys, + leftSchema: p.children[0].Schema(), + rightSchema: p.children[1].Schema(), + } + numPairs := helper.estimate() + if p.JoinType == SemiJoin || p.JoinType == AntiSemiJoin || + p.JoinType == LeftOuterSemiJoin || p.JoinType == AntiLeftOuterSemiJoin { + if len(p.OtherConditions) > 0 { + numPairs *= 0.5 + } else { + numPairs = 0 + } } -} - -func (p *PhysicalMergeJoin) getCost(lCnt, rCnt float64) float64 { - return lCnt + rCnt + probeCost := numPairs * cpuFactor + // Cost of evaluating outer filters. + var cpuCost float64 + if len(p.LeftConditions)+len(p.RightConditions) > 0 { + probeCost *= selectionFactor + cpuCost += outerCnt * cpuFactor + } + cpuCost += probeCost + // For merge join, only one group of rows with same join key(not null) are cached, + // we compute averge memory cost using estimated group size. + NDV := getCardinality(innerKeys, innerSchema, innerStats) + memoryCost := (innerStats.RowCount / NDV) * memoryFactor + return cpuCost + memoryCost } func (p *PhysicalMergeJoin) attach2Task(tasks ...task) task { @@ -190,7 +337,7 @@ func (p *PhysicalMergeJoin) attach2Task(tasks ...task) task { p.schema = buildPhysicalJoinSchema(p.JoinType, p) return &rootTask{ p: p, - cst: lTask.cost() + rTask.cost() + p.getCost(lTask.count(), rTask.count()), + cst: lTask.cost() + rTask.cost() + p.GetCost(lTask.count(), rTask.count()), } } @@ -200,18 +347,51 @@ func finishCopTask(ctx sessionctx.Context, task task) task { if !ok { return task } - // FIXME: When it is a double reading. The cost should be more expensive. The right cost should add the - // `NetWorkStartCost` * (totalCount / perCountIndexRead) + // copTasks are run in parallel, to make the estimated cost closer to execution time, we amortize + // the cost to cop iterator workers. According to `CopClient::Send`, the concurrency + // is Min(DistSQLScanConcurrency, numRegionsInvolvedInScan), since we cannot infer + // the number of regions involved, we simply use DistSQLScanConcurrency. + copIterWorkers := float64(t.plan().context().GetSessionVars().DistSQLScanConcurrency) t.finishIndexPlan() + // Network cost of transferring rows of table scan to TiDB. if t.tablePlan != nil { - t.cst += t.count() * netWorkFactor + t.cst += t.count() * netWorkFactor * t.tableStats.HistColl.GetAvgRowSize(t.tablePlan.Schema().Columns, false) } + t.cst /= copIterWorkers newTask := &rootTask{ cst: t.cst, } if t.indexPlan != nil && t.tablePlan != nil { p := PhysicalIndexLookUpReader{tablePlan: t.tablePlan, indexPlan: t.indexPlan}.Init(ctx) p.stats = t.tablePlan.statsInfo() + // Add cost of building table reader executors. Handles are extracted in batch style, + // each handle is a range, the CPU cost of building copTasks should be: + // (indexRows / batchSize) * batchSize * cpuFactor + // Since we don't know the number of copTasks built, ignore these network cost now. + indexRows := t.indexPlan.statsInfo().RowCount + newTask.cst += indexRows * cpuFactor + // Add cost of worker goroutines in index lookup. + numTblWorkers := float64(t.indexPlan.context().GetSessionVars().IndexLookupConcurrency) + newTask.cst += (numTblWorkers + 1) * concurrencyFactor + // When building table reader executor for each batch, we would sort the handles. CPU + // cost of sort is: + // cpuFactor * batchSize * Log2(batchSize) * (indexRows / batchSize) + indexLookupSize := float64(t.indexPlan.context().GetSessionVars().IndexLookupSize) + batchSize := math.Min(indexLookupSize, indexRows) + if batchSize > 2 { + sortCPUCost := (indexRows * math.Log2(batchSize) * cpuFactor) / numTblWorkers + newTask.cst += sortCPUCost + } + // Also, we need to sort the retrieved rows if index lookup reader is expected to return + // ordered results. Note that row count of these two sorts can be different, if there are + // operators above table scan. + tableRows := t.tablePlan.statsInfo().RowCount + selectivity := tableRows / indexRows + batchSize = math.Min(indexLookupSize*selectivity, tableRows) + if t.keepOrder && batchSize > 2 { + sortCPUCost := (tableRows * math.Log2(batchSize) * cpuFactor) / numTblWorkers + newTask.cst += sortCPUCost + } if t.doubleReadNeedProj { schema := p.IndexPlans[0].(*PhysicalIndexScan).dataSourceSchema proj := PhysicalProjection{Exprs: expression.Column2Exprs(schema.Columns)}.Init(ctx, p.stats, nil) @@ -265,10 +445,16 @@ func (t *rootTask) plan() PhysicalPlan { func (p *PhysicalLimit) attach2Task(tasks ...task) task { t := tasks[0].copy() if cop, ok := t.(*copTask); ok { - // If the table/index scans data by order and applies a double read, the limit cannot be pushed to the table side. + // For double read which requires order being kept, the limit cannot be pushed down to the table side, + // because handles would be reordered before being sent to table scan. if !cop.keepOrder || !cop.indexPlanFinished || cop.indexPlan == nil { - // When limit be pushed down, it should remove its offset. - pushedDownLimit := PhysicalLimit{Count: p.Offset + p.Count}.Init(p.ctx, p.stats) + // When limit is pushed down, we should remove its offset. + newCount := p.Offset + p.Count + childProfile := cop.plan().statsInfo() + // Strictly speaking, for the row count of stats, we should multiply newCount with "regionNum", + // but "regionNum" is unknown since the copTask can be a double read, so we ignore it now. + stats := deriveLimitStats(childProfile, float64(newCount)) + pushedDownLimit := PhysicalLimit{Count: newCount}.Init(p.ctx, stats) cop = attachPlan2Task(pushedDownLimit, cop).(*copTask) } t = finishCopTask(p.ctx, cop) @@ -277,16 +463,26 @@ func (p *PhysicalLimit) attach2Task(tasks ...task) task { return t } -// GetCost computes the cost of in memory sort. -func (p *PhysicalSort) GetCost(count float64) float64 { - if count < 2.0 { - count = 2.0 +// GetCost computes cost of TopN operator itself. +func (p *PhysicalTopN) GetCost(count float64, isRoot bool) float64 { + heapSize := float64(p.Offset + p.Count) + if heapSize < 2.0 { + heapSize = 2.0 + } + // Ignore the cost of `doCompaction` in current implementation of `TopNExec`, since it is the + // special side-effect of our Chunk format in TiDB layer, which may not exist in coprocessor's + // implementation, or may be removed in the future if we change data format. + // Note that we are using worst complexity to compute CPU cost, because it is simpler compared with + // considering probabilities of average complexity, i.e, we may not need adjust heap for each input + // row. + var cpuCost float64 + if isRoot { + cpuCost = count * math.Log2(heapSize) * cpuFactor + } else { + cpuCost = count * math.Log2(heapSize) * copCPUFactor } - return count*cpuFactor + count*memoryFactor -} - -func (p *PhysicalTopN) getCost(count float64) float64 { - return count*cpuFactor + float64(p.Count)*memoryFactor + memoryCost := heapSize * memoryFactor + return cpuCost + memoryCost } // canPushDown checks if this topN can be pushed down. If each of the expression can be converted to pb, it can be pushed. @@ -307,6 +503,14 @@ func (p *PhysicalTopN) allColsFromSchema(schema *expression.Schema) bool { return len(schema.ColumnsIndices(cols)) > 0 } +// GetCost computes the cost of in memory sort. +func (p *PhysicalSort) GetCost(count float64) float64 { + if count < 2.0 { + count = 2.0 + } + return count*math.Log2(count)*cpuFactor + count*memoryFactor +} + func (p *PhysicalSort) attach2Task(tasks ...task) task { t := tasks[0].copy() t = attachPlan2Task(p, t) @@ -318,67 +522,86 @@ func (p *NominalSort) attach2Task(tasks ...task) task { return tasks[0] } -func (p *PhysicalTopN) getPushedDownTopN() *PhysicalTopN { +func (p *PhysicalTopN) getPushedDownTopN(childPlan PhysicalPlan) *PhysicalTopN { newByItems := make([]*ByItems, 0, len(p.ByItems)) for _, expr := range p.ByItems { newByItems = append(newByItems, expr.Clone()) } + newCount := p.Offset + p.Count + childProfile := childPlan.statsInfo() + // Strictly speaking, for the row count of pushed down TopN, we should multiply newCount with "regionNum", + // but "regionNum" is unknown since the copTask can be a double read, so we ignore it now. + stats := deriveLimitStats(childProfile, float64(newCount)) topN := PhysicalTopN{ ByItems: newByItems, - Count: p.Offset + p.Count, - }.Init(p.ctx, p.stats) + Count: newCount, + }.Init(p.ctx, stats) + topN.SetChildren(childPlan) return topN } func (p *PhysicalTopN) attach2Task(tasks ...task) task { t := tasks[0].copy() - // This is a topN plan. + inputCount := t.count() if copTask, ok := t.(*copTask); ok && p.canPushDown() { - pushedDownTopN := p.getPushedDownTopN() - // If all columns in topN are from index plan, we can push it to index plan. Or we finish the index plan and + // If all columns in topN are from index plan, we push it to index plan, otherwise we finish the index plan and // push it to table plan. + var pushedDownTopN *PhysicalTopN if !copTask.indexPlanFinished && p.allColsFromSchema(copTask.indexPlan.Schema()) { - pushedDownTopN.SetChildren(copTask.indexPlan) + pushedDownTopN = p.getPushedDownTopN(copTask.indexPlan) copTask.indexPlan = pushedDownTopN } else { - // FIXME: When we pushed down a top-N plan to table plan branch in case of double reading. The cost should - // be more expensive in case of single reading, because we may execute table scan multi times. copTask.finishIndexPlan() - pushedDownTopN.SetChildren(copTask.tablePlan) + pushedDownTopN = p.getPushedDownTopN(copTask.tablePlan) copTask.tablePlan = pushedDownTopN } - copTask.addCost(pushedDownTopN.getCost(t.count())) + copTask.addCost(pushedDownTopN.GetCost(inputCount, false)) } - t = finishCopTask(p.ctx, t) - t = attachPlan2Task(p, t) - t.addCost(p.getCost(t.count())) - return t + rootTask := finishCopTask(p.ctx, t) + rootTask.addCost(p.GetCost(rootTask.count(), true)) + rootTask = attachPlan2Task(p, rootTask) + return rootTask +} + +// GetCost computes the cost of projection operator itself. +func (p *PhysicalProjection) GetCost(count float64) float64 { + cpuCost := count * cpuFactor + concurrency := float64(p.ctx.GetSessionVars().ProjectionConcurrency) + if concurrency <= 0 { + return cpuCost + } + cpuCost /= concurrency + concurrencyCost := (1 + concurrency) * concurrencyFactor + return cpuCost + concurrencyCost } func (p *PhysicalProjection) attach2Task(tasks ...task) task { t := tasks[0].copy() - switch tp := t.(type) { - case *copTask: - // TODO: Support projection push down. - t = finishCopTask(p.ctx, t) - t = attachPlan2Task(p, t) - return t - case *rootTask: - return attachPlan2Task(p, tp) + if copTask, ok := t.(*copTask); ok { + // TODO: support projection push down. + t = finishCopTask(p.ctx, copTask) } - return nil + t = attachPlan2Task(p, t) + t.addCost(p.GetCost(t.count())) + return t } func (p *PhysicalUnionAll) attach2Task(tasks ...task) task { - newTask := &rootTask{p: p} - newChildren := make([]PhysicalPlan, 0, len(p.children)) + t := &rootTask{p: p} + childPlans := make([]PhysicalPlan, 0, len(tasks)) + var childMaxCost float64 for _, task := range tasks { task = finishCopTask(p.ctx, task) - newTask.cst += task.cost() - newChildren = append(newChildren, task.plan()) + childCost := task.cost() + if childCost > childMaxCost { + childMaxCost = childCost + } + childPlans = append(childPlans, task.plan()) } - p.SetChildren(newChildren...) - return newTask + p.SetChildren(childPlans...) + // Children of UnionExec are executed in parallel. + t.cst = childMaxCost + float64((1+len(tasks)))*concurrencyFactor + return t } func (sel *PhysicalSelection) attach2Task(tasks ...task) task { @@ -473,32 +696,64 @@ func (p *basePhysicalAgg) newPartialAggregate() (partial, final PhysicalPlan) { func (p *PhysicalStreamAgg) attach2Task(tasks ...task) task { t := tasks[0].copy() + inputRows := t.count() if cop, ok := t.(*copTask); ok { partialAgg, finalAgg := p.newPartialAggregate() if partialAgg != nil { if cop.tablePlan != nil { + cop.finishIndexPlan() partialAgg.SetChildren(cop.tablePlan) cop.tablePlan = partialAgg } else { partialAgg.SetChildren(cop.indexPlan) cop.indexPlan = partialAgg } + cop.addCost(p.GetCost(inputRows, false)) } t = finishCopTask(p.ctx, cop) + inputRows = t.count() attachPlan2Task(finalAgg, t) } else { attachPlan2Task(p, t) } - t.addCost(t.count() * cpuFactor) - if p.hasDistinctFunc() { - t.addCost(t.count() * cpuFactor * distinctAggFactor) - } + t.addCost(p.GetCost(inputRows, true)) return t } +// GetCost computes cost of stream aggregation considering CPU/memory. +func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot bool) float64 { + var cpuCost float64 + if isRoot { + cpuCost = inputRows * cpuFactor * float64(len(p.AggFuncs)) + } else { + cpuCost = inputRows * copCPUFactor * float64(len(p.AggFuncs)) + } + rowsPerGroup := inputRows / p.statsInfo().RowCount + memoryCost := rowsPerGroup * distinctFactor * memoryFactor * float64(p.numDistinctFunc()) + return cpuCost + memoryCost +} + +// cpuCostDivisor computes the concurrency to which we would amortize CPU cost +// for hash aggregation. +func (p *PhysicalHashAgg) cpuCostDivisor(hasDistinct bool) (float64, float64) { + if hasDistinct { + return 0, 0 + } + sessionVars := p.ctx.GetSessionVars() + finalCon, partialCon := sessionVars.HashAggFinalConcurrency, sessionVars.HashAggPartialConcurrency + // According to `ValidateSetSystemVar`, `finalCon` and `partialCon` cannot be less than or equal to 0. + if finalCon == 1 && partialCon == 1 { + return 0, 0 + } + // It is tricky to decide which concurrency we should use to amortize CPU cost. Since cost of hash + // aggregation is tend to be under-estimated as explained in `attach2Task`, we choose the smaller + // concurrecy to make some compensation. + return math.Min(float64(finalCon), float64(partialCon)), float64(finalCon + partialCon) +} + func (p *PhysicalHashAgg) attach2Task(tasks ...task) task { - cardinality := p.statsInfo().RowCount t := tasks[0].copy() + inputRows := t.count() if cop, ok := t.(*copTask); ok { partialAgg, finalAgg := p.newPartialAggregate() if partialAgg != nil { @@ -510,15 +765,55 @@ func (p *PhysicalHashAgg) attach2Task(tasks ...task) task { partialAgg.SetChildren(cop.indexPlan) cop.indexPlan = partialAgg } + cop.addCost(p.GetCost(inputRows, false)) } + // In `newPartialAggregate`, we are using stats of final aggregation as stats + // of `partialAgg`, so the network cost of transferring result rows of `partialAgg` + // to TiDB is normally under-estimated for hash aggregation, since the group-by + // column may be independent of the column used for region distribution, so a closer + // estimation of network cost for hash aggregation may multiply the number of + // regions involved in the `partialAgg`, which is unknown however. t = finishCopTask(p.ctx, cop) + inputRows = t.count() attachPlan2Task(finalAgg, t) } else { attachPlan2Task(p, t) } - t.addCost(t.count()*cpuFactor*hashAggFactor + cardinality*createAggCtxFactor) - if p.hasDistinctFunc() { - t.addCost(t.count() * cpuFactor * distinctAggFactor) - } + // We may have 3-phase hash aggregation actually, strictly speaking, we'd better + // calculate cost of each phase and sum the results up, but in fact we don't have + // region level table stats, and the concurrency of the `partialAgg`, + // i.e, max(number_of_regions, DistSQLScanConcurrency) is unknown either, so it is hard + // to compute costs separately. We ignore region level parallelism for both hash + // aggregation and stream aggregation when calculating cost, though this would lead to inaccuracy, + // hopefully this inaccuracy would be imposed on both aggregation implementations, + // so they are still comparable horizontally. + // Also, we use the stats of `partialAgg` as the input of cost computing for TiDB layer + // hash aggregation, it would cause under-estimation as the reason mentioned in comment above. + // To make it simple, we also treat 2-phase parallel hash aggregation in TiDB layer as + // 1-phase when computing cost. + t.addCost(p.GetCost(inputRows, true)) return t } + +// GetCost computes the cost of hash aggregation considering CPU/memory. +func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot bool) float64 { + cardinality := p.statsInfo().RowCount + numDistinctFunc := p.numDistinctFunc() + var cpuCost float64 + if isRoot { + cpuCost = inputRows * cpuFactor * float64(len(p.AggFuncs)) + divisor, con := p.cpuCostDivisor(numDistinctFunc > 0) + if divisor > 0 { + cpuCost /= divisor + // Cost of additional goroutines. + cpuCost += (con + 1) * concurrencyFactor + } + } else { + cpuCost = inputRows * copCPUFactor * float64(len(p.AggFuncs)) + } + memoryCost := cardinality * memoryFactor * float64(len(p.AggFuncs)) + // When aggregation has distinct flag, we would allocate a map for each group to + // check duplication. + memoryCost += inputRows * distinctFactor * memoryFactor * float64(numDistinctFunc) + return cpuCost + memoryCost +} diff --git a/planner/property/stats_info.go b/planner/property/stats_info.go index 21894731c2cab..372e3c0dff68b 100644 --- a/planner/property/stats_info.go +++ b/planner/property/stats_info.go @@ -30,11 +30,6 @@ type StatsInfo struct { StatsVersion uint64 } -// NewSimpleStats creates a simple StatsInfo with rowCount. -func NewSimpleStats(rowCount float64) *StatsInfo { - return &StatsInfo{RowCount: rowCount} -} - // String implements fmt.Stringer interface. func (s *StatsInfo) String() string { return fmt.Sprintf("count %v, Cardinality %v", s.RowCount, s.Cardinality) @@ -63,7 +58,7 @@ func (s *StatsInfo) Scale(factor float64) *StatsInfo { // smaller than the derived cnt. // TODO: try to use a better way to do this. func (s *StatsInfo) ScaleByExpectCnt(expectCnt float64) *StatsInfo { - if expectCnt > s.RowCount { + if expectCnt >= s.RowCount { return s } if s.RowCount > 1.0 { // if s.RowCount is too small, it will cause overflow diff --git a/statistics/selectivity_test.go b/statistics/selectivity_test.go index 41b03aaaafa6a..170282998610a 100644 --- a/statistics/selectivity_test.go +++ b/statistics/selectivity_test.go @@ -431,9 +431,9 @@ func (s *testStatsSuite) TestPrimaryKeySelectivity(c *C) { testKit.MustExec("drop table if exists t") testKit.MustExec("create table t(a char(10) primary key, b int)") testKit.MustQuery(`explain select * from t where a > "t"`).Check(testkit.Rows( - "IndexLookUp_10 3333.33 root ", - "├─IndexScan_8 3333.33 cop table:t, index:a, range:(\"t\",+inf], keep order:false, stats:pseudo", - "└─TableScan_9 3333.33 cop table:t, keep order:false, stats:pseudo")) + "TableReader_7 3333.33 root data:Selection_6", + "└─Selection_6 3333.33 cop gt(test.t.a, \"t\")", + " └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) testKit.MustExec("drop table t") testKit.MustExec("create table t(a int primary key, b int)") diff --git a/statistics/table.go b/statistics/table.go index 06ee867413903..5e3f4db8ee7eb 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -39,6 +39,7 @@ const ( pseudoEqualRate = 1000 pseudoLessRate = 3 pseudoBetweenRate = 40 + pseudoColSize = 8.0 outOfRangeBetweenRate = 100 ) @@ -643,3 +644,25 @@ func getPseudoRowCountByUnsignedIntRanges(intRanges []*ranger.Range, tableRowCou } return rowCount } + +// GetAvgRowSize computes average row size for given columns. +func (coll *HistColl) GetAvgRowSize(cols []*expression.Column, isEncodedKey bool) (size float64) { + if coll.Pseudo || len(coll.Columns) == 0 || coll.Count == 0 { + size = pseudoColSize * float64(len(cols)) + } else { + for _, col := range cols { + colHist, ok := coll.Columns[col.UniqueID] + // Normally this would not happen, it is for compatibility with old version stats which + // does not include TotColSize. + if !ok || (colHist.TotColSize == 0 && (colHist.NullCount != coll.Count)) { + size += pseudoColSize + continue + } + // We differentiate if the column is encoded as key or value, because the resulted size + // is different. + size += colHist.AvgColSize(coll.Count, isEncodedKey) + } + } + // Add 1 byte for each column's flag byte. See `encode` for details. + return size + float64(len(cols)) +} diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index 4fe376d2b5697..9dcbb659f9e86 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1093,15 +1093,15 @@ func (s *testRangerSuite) TestCompIndexInExprCorrCol(c *C) { testKit.MustExec("analyze table t") testKit.MustQuery("explain select t.e in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c in (1, 2) and s.d = t.a and s.a = t1.a) from t").Check(testkit.Rows( "Projection_11 2.00 root 9_aux_0", - "└─Apply_13 2.00 root CARTESIAN left outer semi join, inner:StreamAgg_20, other cond:eq(test.t.e, 7_col_0)", + "└─Apply_13 2.00 root CARTESIAN left outer semi join, inner:HashAgg_18, other cond:eq(test.t.e, 7_col_0)", " ├─TableReader_15 2.00 root data:TableScan_14", " │ └─TableScan_14 2.00 cop table:t, range:[-inf,+inf], keep order:false", - " └─StreamAgg_20 1.00 root funcs:count(1)", - " └─IndexJoin_32 2.00 root inner join, inner:TableReader_31, outer key:test.s.a, inner key:test.t1.a", + " └─HashAgg_18 1.00 root funcs:count(1)", + " └─HashLeftJoin_24 2.00 root inner join, inner:TableReader_29, equal:[eq(test.s.a, test.t1.a)]", " ├─IndexReader_27 2.00 root index:IndexScan_26", " │ └─IndexScan_26 2.00 cop table:s, index:b, c, d, range: decided by [eq(test.s.b, 1) in(test.s.c, 1, 2) eq(test.s.d, test.t.a)], keep order:false", - " └─TableReader_31 1.00 root data:TableScan_30", - " └─TableScan_30 1.00 cop table:t1, range: decided by [test.s.a], keep order:false", + " └─TableReader_29 2.00 root data:TableScan_28", + " └─TableScan_28 2.00 cop table:t1, range:[-inf,+inf], keep order:false", )) testKit.MustQuery("select t.e in (select count(*) from t s use index(idx), t t1 where s.b = 1 and s.c in (1, 2) and s.d = t.a and s.a = t1.a) from t").Check(testkit.Rows( "1", From 700bf2f3d802a46954c16678cab34088661ad7cf Mon Sep 17 00:00:00 2001 From: sduzh Date: Wed, 7 Aug 2019 05:08:37 -0500 Subject: [PATCH 172/196] planner: support PointGet plan when table has alias name (#11149) (#11270) --- executor/point_get_test.go | 195 +++++++++++++++++++++++++++++---- planner/core/point_get_plan.go | 69 +++++++----- 2 files changed, 212 insertions(+), 52 deletions(-) diff --git a/executor/point_get_test.go b/executor/point_get_test.go index e4df3c7a2c1bc..cfbba81dd6a88 100644 --- a/executor/point_get_test.go +++ b/executor/point_get_test.go @@ -162,7 +162,7 @@ func (s *testPointGetSuite) TestPointGetCharPK(c *C) { tk.MustPointGet(`select * from t where a = " ";`).Check(testkit.Rows(` `)) } -func (s *testPointGetSuite) TestIndexLookupCharPK(c *C) { +func (s *testPointGetSuite) TestPointGetAliasTableCharPK(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec(`use test;`) tk.MustExec(`drop table if exists t;`) @@ -171,46 +171,150 @@ func (s *testPointGetSuite) TestIndexLookupCharPK(c *C) { // Test truncate without sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="";`) - tk.MustIndexLookup(`select * from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) - tk.MustIndexLookup(`select * from t tmp where a = "aab";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select * from t tmp where a = "aab";`).Check(testkit.Rows()) // Test truncate with sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustPointGet(`select * from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select * from t tmp where a = "aab";`).Check(testkit.Rows()) + + tk.MustExec(`truncate table t;`) + tk.MustExec(`insert into t values("a ", "b ");`) + + // Test trailing spaces without sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="";`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + + // Test trailing spaces with sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + + // Test CHAR BINARY. + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(a char(2) binary primary key, b char(2));`) + tk.MustExec(`insert into t values(" ", " ");`) + tk.MustExec(`insert into t values("a ", "b ");`) + + // Test trailing spaces without sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="";`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + + // Test trailing spaces with sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + tk.MustPointGet(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + + // Test both wildcard and column name exist in select field list + tk.MustExec(`set @@sql_mode="";`) + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(a char(2) primary key, b char(2));`) + tk.MustExec(`insert into t values("aa", "bb");`) + tk.MustPointGet(`select *, a from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb aa`)) + + // Test using table alias in field list + tk.MustPointGet(`select tmp.* from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select tmp.a, tmp.b from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select tmp.*, tmp.a, tmp.b from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb aa bb`)) + tk.MustPointGet(`select tmp.* from t tmp where a = "aab";`).Check(testkit.Rows()) + tk.MustPointGet(`select tmp.a, tmp.b from t tmp where a = "aab";`).Check(testkit.Rows()) + tk.MustPointGet(`select tmp.*, tmp.a, tmp.b from t tmp where a = "aab";`).Check(testkit.Rows()) + + // Test using table alias in where clause + tk.MustPointGet(`select * from t tmp where tmp.a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select a, b from t tmp where tmp.a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustPointGet(`select *, a, b from t tmp where tmp.a = "aa";`).Check(testkit.Rows(`aa bb aa bb`)) + + // Unknown table name in where clause and field list + err := tk.ExecToErr(`select a from t where xxxxx.a = "aa"`) + c.Assert(err, ErrorMatches, ".*Unknown column 'xxxxx.a' in 'where clause'") + err = tk.ExecToErr(`select xxxxx.a from t where a = "aa"`) + c.Assert(err, ErrorMatches, ".*Unknown column 'xxxxx.a' in 'field list'") + + // When an alias is provided, it completely hides the actual name of the table. + err = tk.ExecToErr(`select a from t tmp where t.a = "aa"`) + c.Assert(err, ErrorMatches, ".*Unknown column 't.a' in 'where clause'") + err = tk.ExecToErr(`select t.a from t tmp where a = "aa"`) + c.Assert(err, ErrorMatches, ".*Unknown column 't.a' in 'field list'") + err = tk.ExecToErr(`select t.* from t tmp where a = "aa"`) + c.Assert(err, ErrorMatches, ".*Unknown table 't'") +} + +func (s *testPointGetSuite) TestIndexLookupChar(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test;`) + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(a char(2), b char(2), index idx_1(a));`) + tk.MustExec(`insert into t values("aa", "bb");`) + + // Test truncate without sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="";`) + tk.MustIndexLookup(`select * from t where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustIndexLookup(`select * from t where a = "aab";`).Check(testkit.Rows()) + + // Test query with table alias tk.MustIndexLookup(`select * from t tmp where a = "aa";`).Check(testkit.Rows(`aa bb`)) tk.MustIndexLookup(`select * from t tmp where a = "aab";`).Check(testkit.Rows()) + // Test truncate with sql mode `PAD_CHAR_TO_FULL_LENGTH`. + tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustIndexLookup(`select * from t where a = "aa";`).Check(testkit.Rows(`aa bb`)) + tk.MustIndexLookup(`select * from t where a = "aab";`).Check(testkit.Rows()) + tk.MustExec(`truncate table t;`) tk.MustExec(`insert into t values("a ", "b ");`) // Test trailing spaces without sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="";`) - tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) // Test trailing spaces with sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) - tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) // Test CHAR BINARY. tk.MustExec(`drop table if exists t;`) - tk.MustExec(`create table t(a char(2) binary primary key, b char(2));`) + tk.MustExec(`create table t(a char(2) binary, b char(2), index idx_1(a));`) tk.MustExec(`insert into t values(" ", " ");`) tk.MustExec(`insert into t values("a ", "b ");`) // Test trailing spaces without sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="";`) - tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) - tk.MustIndexLookup(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) - tk.MustIndexLookup(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) - tk.MustIndexLookup(`select * from t tmp where a = " ";`).Check(testkit.Rows(` `)) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) // Test trailing spaces with sql mode `PAD_CHAR_TO_FULL_LENGTH`. tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b`)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) + tk.MustIndexLookup(`select * from t where a = " ";`).Check(testkit.Rows(` `)) + + // Test query with table alias in `PAD_CHAR_TO_FULL_LENGTH` mode + tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows(`a b`)) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b`)) @@ -313,7 +417,7 @@ func (s *testPointGetSuite) TestPointGetBinaryPK(c *C) { tk.MustPointGet(`select * from t where a = "a ";`).Check(testkit.Rows()) } -func (s *testPointGetSuite) TestIndexLookupBinaryPK(c *C) { +func (s *testPointGetSuite) TestPointGetAliasTableBinaryPK(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec(`use test;`) tk.MustExec(`drop table if exists t;`) @@ -321,24 +425,67 @@ func (s *testPointGetSuite) TestIndexLookupBinaryPK(c *C) { tk.MustExec(`insert into t values("a", "b");`) tk.MustExec(`set @@sql_mode="";`) - tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) // `PAD_CHAR_TO_FULL_LENGTH` should not affect the result. tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) + + tk.MustExec(`insert into t values("a ", "b ");`) + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b `)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + + // `PAD_CHAR_TO_FULL_LENGTH` should not affect the result. + tk.MustPointGet(`select * from t tmp where a = "a";`).Check(testkit.Rows()) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b `)) + tk.MustPointGet(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) +} + +func (s *testPointGetSuite) TestIndexLookupBinary(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec(`use test;`) + tk.MustExec(`drop table if exists t;`) + tk.MustExec(`create table t(a binary(2), b binary(2), index idx_1(a));`) + tk.MustExec(`insert into t values("a", "b");`) + + tk.MustExec(`set @@sql_mode="";`) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) + + // Test query with table alias + tk.MustExec(`set @@sql_mode="";`) tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows()) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) tk.MustIndexLookup(`select * from t tmp where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) + // `PAD_CHAR_TO_FULL_LENGTH` should not affect the result. + tk.MustExec(`set @@sql_mode="PAD_CHAR_TO_FULL_LENGTH";`) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a\0";`).Check(testkit.Rows("a\x00 b\x00")) + tk.MustExec(`insert into t values("a ", "b ");`) - tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows()) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b `)) - tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b `)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) // `PAD_CHAR_TO_FULL_LENGTH` should not affect the result. + tk.MustIndexLookup(`select * from t where a = "a";`).Check(testkit.Rows()) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows(`a b `)) + tk.MustIndexLookup(`select * from t where a = "a ";`).Check(testkit.Rows()) + + // Test query with table alias in `PAD_CHAR_TO_FULL_LENGTH` mode tk.MustIndexLookup(`select * from t tmp where a = "a";`).Check(testkit.Rows()) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows(`a b `)) tk.MustIndexLookup(`select * from t tmp where a = "a ";`).Check(testkit.Rows()) diff --git a/planner/core/point_get_plan.go b/planner/core/point_get_plan.go index b7243ac4c9a05..5d48f282fbc60 100644 --- a/planner/core/point_get_plan.go +++ b/planner/core/point_get_plan.go @@ -177,7 +177,7 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt) *PointGetP return nil } } - tblName := getSingleTableName(selStmt.From) + tblName, tblAlias := getSingleTableNameAndAlias(selStmt.From) if tblName == nil { return nil } @@ -203,13 +203,13 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt) *PointGetP } } pairs := make([]nameValuePair, 0, 4) - pairs = getNameValuePairs(pairs, selStmt.Where) + pairs = getNameValuePairs(pairs, tblAlias, selStmt.Where) if pairs == nil { return nil } handlePair, fieldType := findPKHandle(tbl, pairs) if handlePair.value.Kind() != types.KindNull && len(pairs) == 1 { - schema := buildSchemaFromFields(ctx, tblName.Schema, tbl, selStmt.Fields.Fields) + schema := buildSchemaFromFields(ctx, tblName.Schema, tbl, tblAlias, selStmt.Fields.Fields) if schema == nil { return nil } @@ -249,7 +249,7 @@ func tryPointGetPlan(ctx sessionctx.Context, selStmt *ast.SelectStmt) *PointGetP if idxValues == nil { continue } - schema := buildSchemaFromFields(ctx, tblName.Schema, tbl, selStmt.Fields.Fields) + schema := buildSchemaFromFields(ctx, tblName.Schema, tbl, tblAlias, selStmt.Fields.Fields) if schema == nil { return nil } @@ -286,23 +286,29 @@ func checkFastPlanPrivilege(ctx sessionctx.Context, fastPlan *PointGetPlan, chec return nil } -func buildSchemaFromFields(ctx sessionctx.Context, dbName model.CIStr, tbl *model.TableInfo, fields []*ast.SelectField) *expression.Schema { +func buildSchemaFromFields(ctx sessionctx.Context, dbName model.CIStr, tbl *model.TableInfo, tblName model.CIStr, fields []*ast.SelectField) *expression.Schema { if dbName.L == "" { dbName = model.NewCIStr(ctx.GetSessionVars().CurrentDB) } columns := make([]*expression.Column, 0, len(tbl.Columns)+1) - if len(fields) == 1 && fields[0].WildCard != nil { - for _, col := range tbl.Columns { - columns = append(columns, colInfoToColumn(dbName, tbl.Name, col.Name, col, len(columns))) - } - return expression.NewSchema(columns...) - } if len(fields) > 0 { for _, field := range fields { + if field.WildCard != nil { + if field.WildCard.Table.L != "" && field.WildCard.Table.L != tblName.L { + return nil + } + for _, col := range tbl.Columns { + columns = append(columns, colInfoToColumn(dbName, tbl.Name, tblName, col.Name, col, len(columns))) + } + continue + } colNameExpr, ok := field.Expr.(*ast.ColumnNameExpr) if !ok { return nil } + if colNameExpr.Name.Table.L != "" && colNameExpr.Name.Table.L != tblName.L { + return nil + } col := findCol(tbl, colNameExpr.Name) if col == nil { return nil @@ -311,21 +317,21 @@ func buildSchemaFromFields(ctx sessionctx.Context, dbName model.CIStr, tbl *mode if field.AsName.L != "" { asName = field.AsName } - columns = append(columns, colInfoToColumn(dbName, tbl.Name, asName, col, len(columns))) + columns = append(columns, colInfoToColumn(dbName, tbl.Name, tblName, asName, col, len(columns))) } return expression.NewSchema(columns...) } // fields len is 0 for update and delete. var handleCol *expression.Column for _, col := range tbl.Columns { - column := colInfoToColumn(dbName, tbl.Name, col.Name, col, len(columns)) + column := colInfoToColumn(dbName, tbl.Name, tblName, col.Name, col, len(columns)) if tbl.PKIsHandle && mysql.HasPriKeyFlag(col.Flag) { handleCol = column } columns = append(columns, column) } if handleCol == nil { - handleCol = colInfoToColumn(dbName, tbl.Name, model.ExtraHandleName, model.NewExtraHandleColInfo(), len(columns)) + handleCol = colInfoToColumn(dbName, tbl.Name, tblName, model.ExtraHandleName, model.NewExtraHandleColInfo(), len(columns)) columns = append(columns, handleCol) } schema := expression.NewSchema(columns...) @@ -334,36 +340,40 @@ func buildSchemaFromFields(ctx sessionctx.Context, dbName model.CIStr, tbl *mode return schema } -func getSingleTableName(tableRefs *ast.TableRefsClause) *ast.TableName { +// getSingleTableNameAndAlias return the ast node of queried table name and the alias string. +// `tblName` is `nil` if there are multiple tables in the query. +// `tblAlias` will be the real table name if there is no table alias in the query. +func getSingleTableNameAndAlias(tableRefs *ast.TableRefsClause) (tblName *ast.TableName, tblAlias model.CIStr) { if tableRefs == nil || tableRefs.TableRefs == nil || tableRefs.TableRefs.Right != nil { - return nil + return nil, tblAlias } tblSrc, ok := tableRefs.TableRefs.Left.(*ast.TableSource) if !ok { - return nil - } - if tblSrc.AsName.L != "" { - return nil + return nil, tblAlias } - tblName, ok := tblSrc.Source.(*ast.TableName) + tblName, ok = tblSrc.Source.(*ast.TableName) if !ok { - return nil + return nil, tblAlias } - return tblName + tblAlias = tblSrc.AsName + if tblSrc.AsName.L == "" { + tblAlias = tblName.Name + } + return tblName, tblAlias } // getNameValuePairs extracts `column = constant/paramMarker` conditions from expr as name value pairs. -func getNameValuePairs(nvPairs []nameValuePair, expr ast.ExprNode) []nameValuePair { +func getNameValuePairs(nvPairs []nameValuePair, tblName model.CIStr, expr ast.ExprNode) []nameValuePair { binOp, ok := expr.(*ast.BinaryOperationExpr) if !ok { return nil } if binOp.Op == opcode.LogicAnd { - nvPairs = getNameValuePairs(nvPairs, binOp.L) + nvPairs = getNameValuePairs(nvPairs, tblName, binOp.L) if nvPairs == nil { return nil } - nvPairs = getNameValuePairs(nvPairs, binOp.R) + nvPairs = getNameValuePairs(nvPairs, tblName, binOp.R) if nvPairs == nil { return nil } @@ -395,6 +405,9 @@ func getNameValuePairs(nvPairs []nameValuePair, expr ast.ExprNode) []nameValuePa if d.IsNull() { return nil } + if colName.Name.Table.L != "" && colName.Name.Table.L != tblName.L { + return nil + } return append(nvPairs, nameValuePair{colName: colName.Name.Name.L, value: d, param: param}) } return nil @@ -572,10 +585,10 @@ func findCol(tbl *model.TableInfo, colName *ast.ColumnName) *model.ColumnInfo { return nil } -func colInfoToColumn(db model.CIStr, tblName model.CIStr, asName model.CIStr, col *model.ColumnInfo, idx int) *expression.Column { +func colInfoToColumn(db model.CIStr, origTblName model.CIStr, tblName model.CIStr, asName model.CIStr, col *model.ColumnInfo, idx int) *expression.Column { return &expression.Column{ ColName: asName, - OrigTblName: tblName, + OrigTblName: origTblName, DBName: db, TblName: tblName, RetType: &col.FieldType, From 497a9b83e988e0b9691ab0e3c617d947f5df1779 Mon Sep 17 00:00:00 2001 From: crazycs Date: Wed, 7 Aug 2019 21:15:42 +0800 Subject: [PATCH 173/196] ddl: make ddl test more stable (#11657) --- ddl/db_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 575b6017b4aca..50dc06010dcea 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -207,6 +207,17 @@ func (s *testDBSuite) testGetTable(c *C, name string) table.Table { return testGetTableByName(c, ctx, s.schemaName, name) } +func (s *testDBSuite) testGetDB(c *C, dbName string) *model.DBInfo { + ctx := s.s.(sessionctx.Context) + dom := domain.GetDomain(ctx) + // Make sure the table schema is the new schema. + err := dom.Reload() + c.Assert(err, IsNil) + db, ok := dom.InfoSchema().SchemaByName(model.NewCIStr(dbName)) + c.Assert(ok, IsTrue) + return db +} + func backgroundExec(s kv.Storage, sql string, done chan error) { se, err := session.CreateSession4Test(s) if err != nil { @@ -628,8 +639,11 @@ func (s *testDBSuite2) TestCancelDropTableAndSchema(c *C) { hook := &ddl.TestDDLCallback{} var jobID int64 testCase := &testCases[0] + s.mustExec(c, "create database if not exists test_drop_db") + dbInfo := s.testGetDB(c, "test_drop_db") + hook.OnJobRunBeforeExported = func(job *model.Job) { - if job.Type == testCase.action && job.State == testCase.jobState && job.SchemaState == testCase.JobSchemaState { + if job.Type == testCase.action && job.State == testCase.jobState && job.SchemaState == testCase.JobSchemaState && job.SchemaID == dbInfo.ID { jobIDs := []int64{job.ID} jobID = job.ID hookCtx := mock.NewContext() @@ -669,6 +683,8 @@ func (s *testDBSuite2) TestCancelDropTableAndSchema(c *C) { s.mustExec(c, "create table if not exists t(c1 int, c2 int)") } + dbInfo = s.testGetDB(c, "test_drop_db") + if testCase.action == model.ActionDropTable { sql = "drop table t;" } else if testCase.action == model.ActionDropSchema { From 28c0b1d22627ab6b07825c0f6a84cac762da6692 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 7 Aug 2019 21:28:44 +0800 Subject: [PATCH 174/196] executor: tiny refactor the `insertRows` function (#11643) --- executor/insert.go | 4 ++-- executor/insert_common.go | 27 +++++++++++++++++++++------ executor/replace.go | 4 ++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/executor/insert.go b/executor/insert.go index c541ddce08a1b..efe5e643c721c 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -136,9 +136,9 @@ func (e *InsertExec) batchUpdateDupRows(newRows [][]types.Datum) error { func (e *InsertExec) Next(ctx context.Context, req *chunk.Chunk) error { req.Reset() if len(e.children) > 0 && e.children[0] != nil { - return e.insertRowsFromSelect(ctx, e.exec) + return insertRowsFromSelect(ctx, e) } - return e.insertRows(ctx, e.exec) + return insertRows(ctx, e) } // Close implements the Executor Close interface. diff --git a/executor/insert_common.go b/executor/insert_common.go index 77c703532acaa..bbd033ad6279e 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -67,6 +67,19 @@ type defaultVal struct { valid bool } +type insertCommon interface { + insertCommon() *InsertValues + exec(ctx context.Context, rows [][]types.Datum) error +} + +func (e *InsertValues) insertCommon() *InsertValues { + return e +} + +func (e *InsertValues) exec(ctx context.Context, rows [][]types.Datum) error { + panic("derived should overload exec function") +} + // initInsertColumns sets the explicitly specified columns of an insert statement. There are three cases: // There are three types of insert statements: // 1 insert ... values(...) --> name type column @@ -176,7 +189,8 @@ func (e *InsertValues) processSetList() error { } // insertRows processes `insert|replace into values ()` or `insert|replace into set x=y` -func (e *InsertValues) insertRows(ctx context.Context, exec func(ctx context.Context, rows [][]types.Datum) error) (err error) { +func insertRows(ctx context.Context, base insertCommon) (err error) { + e := base.insertCommon() // For `insert|replace into set x=y`, process the set list here. if err = e.processSetList(); err != nil { return err @@ -194,7 +208,7 @@ func (e *InsertValues) insertRows(ctx context.Context, exec func(ctx context.Con } rows = append(rows, row) if batchInsert && e.rowCount%uint64(batchSize) == 0 { - if err = exec(ctx, rows); err != nil { + if err = base.exec(ctx, rows); err != nil { return err } rows = rows[:0] @@ -203,7 +217,7 @@ func (e *InsertValues) insertRows(ctx context.Context, exec func(ctx context.Con } } } - return exec(ctx, rows) + return base.exec(ctx, rows) } func (e *InsertValues) handleErr(col *table.Column, val *types.Datum, rowIdx int, err error) error { @@ -294,8 +308,9 @@ func (e *InsertValues) setValueForRefColumn(row []types.Datum, hasValue []bool) return nil } -func (e *InsertValues) insertRowsFromSelect(ctx context.Context, exec func(ctx context.Context, rows [][]types.Datum) error) error { +func insertRowsFromSelect(ctx context.Context, base insertCommon) error { // process `insert|replace into ... select ... from ...` + e := base.insertCommon() selectExec := e.children[0] fields := retTypes(selectExec) chk := newFirstChunk(selectExec) @@ -328,7 +343,7 @@ func (e *InsertValues) insertRowsFromSelect(ctx context.Context, exec func(ctx c } rows = append(rows, row) if batchInsert && e.rowCount%uint64(batchSize) == 0 { - if err = exec(ctx, rows); err != nil { + if err = base.exec(ctx, rows); err != nil { return err } rows = rows[:0] @@ -338,7 +353,7 @@ func (e *InsertValues) insertRowsFromSelect(ctx context.Context, exec func(ctx c } } } - return exec(ctx, rows) + return base.exec(ctx, rows) } func (e *InsertValues) doBatchInsert(ctx context.Context) error { diff --git a/executor/replace.go b/executor/replace.go index ab350cdf43fa0..c228e09440b72 100644 --- a/executor/replace.go +++ b/executor/replace.go @@ -184,9 +184,9 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error { func (e *ReplaceExec) Next(ctx context.Context, req *chunk.Chunk) error { req.Reset() if len(e.children) > 0 && e.children[0] != nil { - return e.insertRowsFromSelect(ctx, e.exec) + return insertRowsFromSelect(ctx, e) } - return e.insertRows(ctx, e.exec) + return insertRows(ctx, e) } // setMessage sets info message(ERR_INSERT_INFO) generated by REPLACE statement From 5fff4e9deac5c1c6ea06f554b63b9d6f1734a13c Mon Sep 17 00:00:00 2001 From: SunRunAway Date: Wed, 7 Aug 2019 21:40:42 +0800 Subject: [PATCH 175/196] distsql: control the sending rate of copIteratorTaskSender if keepOrder is true (#11578) --- executor/table_reader.go | 1 + store/tikv/coprocessor.go | 55 +++++++++++++++++++++++++++++++--- store/tikv/coprocessor_test.go | 27 +++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/executor/table_reader.go b/executor/table_reader.go index d6272e8b8bec6..b26001b836988 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -16,6 +16,7 @@ package executor import ( "context" "fmt" + "github.com/pingcap/parser/model" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/kv" diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 7b7b04ea2112a..9ea2f31fded9d 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -71,7 +71,9 @@ func (c *CopClient) Send(ctx context.Context, req *kv.Request, vars *kv.Variable // Make sure that there is at least one worker. it.concurrency = 1 } - if !it.req.KeepOrder { + if it.req.KeepOrder { + it.sendRate = newRateLimit(2 * it.concurrency) + } else { it.respChan = make(chan *copResponse, it.concurrency) } it.open(ctx) @@ -222,9 +224,11 @@ func buildCopTasks(bo *Backoffer, cache *RegionCache, ranges *copRanges, desc bo for i := 0; i < rLen; { nextI := mathutil.Min(i+rangesPerTask, rLen) tasks = append(tasks, &copTask{ - region: region, - ranges: ranges.slice(i, nextI), - respChan: make(chan *copResponse, 1), + region: region, + ranges: ranges.slice(i, nextI), + // Channel buffer is 2 for handling region split. + // In a common case, two region split tasks will not be blocked. + respChan: make(chan *copResponse, 2), cmdType: cmdType, }) i = nextI @@ -334,6 +338,9 @@ type copIterator struct { // If keepOrder, results are stored in copTask.respChan, read them out one by one. tasks []*copTask curr int + // sendRate controls the sending rate of copIteratorTaskSender, if keepOrder, + // to prevent all tasks being done (aka. all of the responses are buffered) + sendRate *rateLimit // Otherwise, results are stored in respChan. respChan chan *copResponse @@ -364,6 +371,7 @@ type copIteratorTaskSender struct { tasks []*copTask finishCh <-chan struct{} respChan chan<- *copResponse + sendRate *rateLimit } type copResponse struct { @@ -464,6 +472,7 @@ func (it *copIterator) open(ctx context.Context) { wg: &it.wg, tasks: it.tasks, finishCh: it.finishCh, + sendRate: it.sendRate, } taskSender.respChan = it.respChan go taskSender.run() @@ -472,6 +481,16 @@ func (it *copIterator) open(ctx context.Context) { func (sender *copIteratorTaskSender) run() { // Send tasks to feed the worker goroutines. for _, t := range sender.tasks { + // If keepOrder, we must control the sending rate to prevent all tasks + // being done (aka. all of the responses are buffered) by copIteratorWorker. + // We keep the number of inflight tasks within the number of concurrency * 2. + // It sends one more task if a task has been finished in copIterator.Next. + if sender.sendRate != nil { + exit := sender.sendRate.getToken(sender.finishCh) + if exit { + break + } + } exit := sender.sendToTaskCh(t) if exit { break @@ -559,6 +578,7 @@ func (it *copIterator) Next(ctx context.Context) (kv.ResultSubset, error) { // Switch to next task. it.tasks[it.curr] = nil it.curr++ + it.sendRate.putToken() } } @@ -832,6 +852,33 @@ func (it *copIterator) Close() error { return nil } +type rateLimit struct { + token chan struct{} +} + +func newRateLimit(n int) *rateLimit { + return &rateLimit{ + token: make(chan struct{}, n), + } +} + +func (r *rateLimit) getToken(done <-chan struct{}) (exit bool) { + select { + case <-done: + return true + case r.token <- struct{}{}: + return false + } +} + +func (r *rateLimit) putToken() { + select { + case <-r.token: + default: + panic("put a redundant token") + } +} + // copErrorResponse returns error when calling Next() type copErrorResponse struct{ error } diff --git a/store/tikv/coprocessor_test.go b/store/tikv/coprocessor_test.go index 6daec69cf5c0c..404434894036f 100644 --- a/store/tikv/coprocessor_test.go +++ b/store/tikv/coprocessor_test.go @@ -15,6 +15,7 @@ package tikv import ( "context" + "time" . "github.com/pingcap/check" "github.com/pingcap/tidb/kv" @@ -294,6 +295,32 @@ func (s *testCoprocessorSuite) TestCopRangeSplit(c *C) { ) } +func (s *testCoprocessorSuite) TestRateLimit(c *C) { + done := make(chan struct{}, 1) + rl := newRateLimit(1) + c.Assert(rl.putToken, PanicMatches, "put a redundant token") + exit := rl.getToken(done) + c.Assert(exit, Equals, false) + rl.putToken() + c.Assert(rl.putToken, PanicMatches, "put a redundant token") + + exit = rl.getToken(done) + c.Assert(exit, Equals, false) + done <- struct{}{} + exit = rl.getToken(done) // blocked but exit + c.Assert(exit, Equals, true) + + sig := make(chan int, 1) + go func() { + exit = rl.getToken(done) // blocked + c.Assert(exit, Equals, false) + close(sig) + }() + time.Sleep(200 * time.Millisecond) + rl.putToken() + <-sig +} + type splitCase struct { key string *copRanges From 110b073aa7ab27bfc7120c062c43d910188ca2ea Mon Sep 17 00:00:00 2001 From: Arenatlx Date: Wed, 7 Aug 2019 22:58:33 -0500 Subject: [PATCH 176/196] ddl: Fix generated column can refer only to generated columns defined prior to it. (#11549) --- ddl/db_integration_test.go | 24 +++++++++++++++++++ ddl/ddl_api.go | 16 +++++++++---- ddl/generated_column.go | 48 +++++++++++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index ce8fde13972f8..8e8124b558d21 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -586,6 +586,30 @@ func (s *testIntegrationSuite2) TestNullGeneratedColumn(c *C) { tk.MustExec("drop table t") } +func (s *testIntegrationSuite2) TestDependedGeneratedColumnPrior2GeneratedColumn(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("CREATE TABLE `t` (" + + "`a` int(11) DEFAULT NULL," + + "`b` int(11) GENERATED ALWAYS AS (`a` + 1) VIRTUAL," + + "`c` int(11) GENERATED ALWAYS AS (`b` + 1) VIRTUAL" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin") + // should check unknown column first, then the prior ones. + sql := "alter table t add column d int as (c + f + 1) first" + assertErrorCode(c, tk, sql, mysql.ErrBadField) + + // depended generated column should be prior to generated column self + sql = "alter table t add column d int as (c+1) first" + assertErrorCode(c, tk, sql, mysql.ErrGeneratedColumnNonPrior) + + // correct case + tk.MustExec("alter table t add column d int as (c+1) after c") + + // check position nil case + tk.MustExec("alter table t add column(e int as (c+1))") +} + func (s *testIntegrationSuite3) TestChangingCharsetToUtf8(c *C) { tk := testkit.NewTestKit(c, s.store) diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index c82e8c3599995..d9b84cf3d7c2a 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -2093,9 +2093,8 @@ func (d *ddl) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTab } // If new column is a generated column, do validation. - // NOTE: Because now we can only append columns to table, - // we don't need check whether the column refers other - // generated columns occurring later in table. + // NOTE: we do check whether the column refers other generated + // columns occurring later in a table, but we don't handle the col offset. for _, option := range specNewColumn.Options { if option.Tp == ast.ColumnOptionGenerated { if err := checkIllegalFn4GeneratedColumn(specNewColumn.Name.Name.L, option.Expr); err != nil { @@ -2110,8 +2109,17 @@ func (d *ddl) AddColumn(ctx sessionctx.Context, ti ast.Ident, spec *ast.AlterTab if err = checkAutoIncrementRef(specNewColumn.Name.Name.L, dependColNames, t.Meta()); err != nil { return errors.Trace(err) } + duplicateColNames := make(map[string]struct{}, len(dependColNames)) + for k := range dependColNames { + duplicateColNames[k] = struct{}{} + } + cols := t.Cols() + + if err = checkDependedColExist(dependColNames, cols); err != nil { + return errors.Trace(err) + } - if err = checkDependedColExist(dependColNames, t.Cols()); err != nil { + if err = verifyColumnGenerationSingle(duplicateColNames, cols, spec.Position); err != nil { return errors.Trace(err) } } diff --git a/ddl/generated_column.go b/ddl/generated_column.go index c0550a292efc3..dab4e0b3f50a1 100644 --- a/ddl/generated_column.go +++ b/ddl/generated_column.go @@ -50,8 +50,26 @@ func verifyColumnGeneration(colName2Generation map[string]columnGenerationInDDL, return nil } +// verifyColumnGenerationSingle is for ADD GENERATED COLUMN, we just need verify one column itself. +func verifyColumnGenerationSingle(dependColNames map[string]struct{}, cols []*table.Column, position *ast.ColumnPosition) error { + // Since the added column does not exist yet, we should derive it's offset from ColumnPosition. + pos, err := findPositionRelativeColumn(cols, position) + if err != nil { + return errors.Trace(err) + } + // should check unknown column first, then the prior ones. + for _, col := range cols { + if _, ok := dependColNames[col.Name.L]; ok { + if col.IsGenerated() && col.Offset >= pos { + // Generated column can refer only to generated columns defined prior to it. + return errGeneratedColumnNonPrior.GenWithStackByArgs() + } + } + } + return nil +} + // checkDependedColExist ensure all depended columns exist. -// // NOTE: this will MODIFY parameter `dependCols`. func checkDependedColExist(dependCols map[string]struct{}, cols []*table.Column) error { for _, col := range cols { @@ -65,6 +83,34 @@ func checkDependedColExist(dependCols map[string]struct{}, cols []*table.Column) return nil } +// findPositionRelativeColumn returns a pos relative to added generated column position. +func findPositionRelativeColumn(cols []*table.Column, pos *ast.ColumnPosition) (int, error) { + position := len(cols) + // Get the column position, default is cols's length means appending. + // For "alter table ... add column(...)", the position will be nil. + // For "alter table ... add column ... ", the position will be default one. + if pos == nil { + return position, nil + } + if pos.Tp == ast.ColumnPositionFirst { + position = 0 + } else if pos.Tp == ast.ColumnPositionAfter { + var col *table.Column + for _, c := range cols { + if c.Name.L == pos.RelativeColumn.Name.L { + col = c + break + } + } + if col == nil { + return -1, ErrBadField.GenWithStackByArgs(pos.RelativeColumn, "generated column function") + } + // Inserted position is after the mentioned column. + position = col.Offset + 1 + } + return position, nil +} + // findDependedColumnNames returns a set of string, which indicates // the names of the columns that are depended by colDef. func findDependedColumnNames(colDef *ast.ColumnDef) (generated bool, colsMap map[string]struct{}) { From f74b9e9f6c64e4ac424916bb5fe8ed832a6f0e6e Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Thu, 8 Aug 2019 14:21:28 +0800 Subject: [PATCH 177/196] planner: implement projection and table dual in cascades (#11664) --- planner/cascades/implementation_rules.go | 56 +++++++++++++++++- planner/cascades/integration_test.go | 60 ++++++++++++++++++++ planner/cascades/optimize.go | 2 +- planner/core/common_plans.go | 4 +- planner/core/exhaust_physical_plans.go | 10 ++-- planner/core/logical_plan_builder.go | 2 +- planner/core/logical_plans.go | 8 +-- planner/core/plan.go | 5 +- planner/core/planbuilder.go | 2 +- planner/core/rule_aggregation_push_down.go | 2 +- planner/core/rule_inject_extra_projection.go | 12 ++-- planner/core/rule_join_reorder.go | 2 +- planner/core/rule_max_min_eliminate.go | 2 +- planner/core/rule_partition_processor.go | 10 ++-- planner/core/rule_predicate_push_down.go | 8 +-- planner/core/task.go | 6 +- planner/implementation/datasource.go | 34 +++++++++++ planner/implementation/simple_plans.go | 28 +++++++++ planner/memo/group.go | 2 + planner/memo/group_expr.go | 1 + 20 files changed, 218 insertions(+), 38 deletions(-) create mode 100644 planner/cascades/integration_test.go create mode 100644 planner/implementation/datasource.go create mode 100644 planner/implementation/simple_plans.go diff --git a/planner/cascades/implementation_rules.go b/planner/cascades/implementation_rules.go index 8ab4bcf4726ef..167ccf5f118de 100644 --- a/planner/cascades/implementation_rules.go +++ b/planner/cascades/implementation_rules.go @@ -15,6 +15,7 @@ package cascades import ( plannercore "github.com/pingcap/tidb/planner/core" + impl "github.com/pingcap/tidb/planner/implementation" "github.com/pingcap/tidb/planner/memo" "github.com/pingcap/tidb/planner/property" ) @@ -25,7 +26,7 @@ type ImplementationRule interface { Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) // OnImplement generates physical plan using this rule for current GroupExpr. Note that // childrenReqProps of generated physical plan should be set correspondingly in this function. - OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (impl memo.Implementation, err error) + OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) } // GetImplementationRules gets the all the candidate implementation rules based @@ -44,4 +45,57 @@ var implementationMap = map[memo.Operand][]ImplementationRule{ nil, }, */ + memo.OperandTableDual: { + &ImplTableDual{}, + }, + memo.OperandProjection: { + &ImplProjection{}, + }, +} + +// ImplTableDual implements LogicalTableDual as PhysicalTableDual. +type ImplTableDual struct { +} + +// Match implements ImplementationRule Match interface. +func (r *ImplTableDual) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { + if !prop.IsEmpty() { + return false + } + return true +} + +// OnImplement implements ImplementationRule OnImplement interface. +func (r *ImplTableDual) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { + logicProp := expr.Group.Prop + logicDual := expr.ExprNode.(*plannercore.LogicalTableDual) + dual := plannercore.PhysicalTableDual{RowCount: logicDual.RowCount}.Init(logicDual.SCtx(), logicProp.Stats) + dual.SetSchema(logicProp.Schema) + return impl.NewTableDualImpl(dual), nil +} + +// ImplProjection implements LogicalProjection as PhysicalProjection. +type ImplProjection struct { +} + +// Match implements ImplementationRule Match interface. +func (r *ImplProjection) Match(expr *memo.GroupExpr, prop *property.PhysicalProperty) (matched bool) { + return true +} + +// OnImplement implements ImplementationRule OnImplement interface. +func (r *ImplProjection) OnImplement(expr *memo.GroupExpr, reqProp *property.PhysicalProperty) (memo.Implementation, error) { + logicProp := expr.Group.Prop + logicProj := expr.ExprNode.(*plannercore.LogicalProjection) + childProp, ok := logicProj.TryToGetChildProp(reqProp) + if !ok { + return nil, nil + } + proj := plannercore.PhysicalProjection{ + Exprs: logicProj.Exprs, + CalculateNoDelay: logicProj.CalculateNoDelay, + AvoidColumnEvaluator: logicProj.AvoidColumnEvaluator, + }.Init(logicProj.SCtx(), logicProp.Stats.ScaleByExpectCnt(reqProp.ExpectedCnt), childProp) + proj.SetSchema(logicProp.Schema) + return impl.NewProjectionImpl(proj), nil } diff --git a/planner/cascades/integration_test.go b/planner/cascades/integration_test.go new file mode 100644 index 0000000000000..4230aefe5e6b7 --- /dev/null +++ b/planner/cascades/integration_test.go @@ -0,0 +1,60 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cascades_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/util/testkit" +) + +var _ = Suite(&testIntegrationSuite{}) + +type testIntegrationSuite struct { + store kv.Storage + tk *testkit.TestKit +} + +func newStoreWithBootstrap() (kv.Storage, error) { + store, err := mockstore.NewMockTikvStore() + if err != nil { + return nil, err + } + _, err = session.BootstrapSession(store) + return store, err +} + +func (s *testIntegrationSuite) SetUpSuite(c *C) { + var err error + s.store, err = newStoreWithBootstrap() + c.Assert(err, IsNil) + s.tk = testkit.NewTestKitWithInit(c, s.store) + s.tk.MustExec("set session tidb_enable_cascades_planner = 1") +} + +func (s *testIntegrationSuite) TearDownSuite(c *C) { + s.store.Close() +} + +func (s *testIntegrationSuite) TestSimpleProjDual(c *C) { + s.tk.MustQuery("explain select 1").Check(testkit.Rows( + "Projection_3 1.00 root 1", + "└─TableDual_4 1.00 root rows:1", + )) + s.tk.MustQuery("select 1").Check(testkit.Rows( + "1", + )) +} diff --git a/planner/cascades/optimize.go b/planner/cascades/optimize.go index 226d0196e92b7..2d51b54e3fd3a 100644 --- a/planner/cascades/optimize.go +++ b/planner/cascades/optimize.go @@ -65,7 +65,7 @@ func exploreGroup(g *memo.Group) error { } g.Explored = true - for elem := g.Equivalents.Front(); elem != nil; elem.Next() { + for elem := g.Equivalents.Front(); elem != nil; elem = elem.Next() { curExpr := elem.Value.(*memo.GroupExpr) if curExpr.Explored { continue diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 725519de0322c..529f9c55a13fb 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -274,8 +274,8 @@ func (e *Execute) getPhysicalPlan(ctx context.Context, sctx sessionctx.Context, } func (e *Execute) rebuildRange(p Plan) error { - sctx := p.context() - sc := p.context().GetSessionVars().StmtCtx + sctx := p.SCtx() + sc := p.SCtx().GetSessionVars().StmtCtx var err error switch x := p.(type) { case *PhysicalTableReader: diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index ab94db70d850b..f9d15c23a008a 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -1087,10 +1087,10 @@ func (p *LogicalJoin) exhaustPhysicalPlans(prop *property.PhysicalProperty) []Ph return joins } -// tryToGetChildProp will check if this sort property can be pushed or not. +// TryToGetChildProp will check if this sort property can be pushed or not. // When a sort column will be replaced by scalar function, we refuse it. // When a sort column will be replaced by a constant, we just remove it. -func (p *LogicalProjection) tryToGetChildProp(prop *property.PhysicalProperty) (*property.PhysicalProperty, bool) { +func (p *LogicalProjection) TryToGetChildProp(prop *property.PhysicalProperty) (*property.PhysicalProperty, bool) { newProp := &property.PhysicalProperty{TaskTp: property.RootTaskType, ExpectedCnt: prop.ExpectedCnt} newCols := make([]property.Item, 0, len(prop.Items)) for _, col := range prop.Items { @@ -1107,14 +1107,14 @@ func (p *LogicalProjection) tryToGetChildProp(prop *property.PhysicalProperty) ( } func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty) []PhysicalPlan { - newProp, ok := p.tryToGetChildProp(prop) + newProp, ok := p.TryToGetChildProp(prop) if !ok { return nil } proj := PhysicalProjection{ Exprs: p.Exprs, - CalculateNoDelay: p.calculateNoDelay, - AvoidColumnEvaluator: p.avoidColumnEvaluator, + CalculateNoDelay: p.CalculateNoDelay, + AvoidColumnEvaluator: p.AvoidColumnEvaluator, }.Init(p.ctx, p.stats.ScaleByExpectCnt(prop.ExpectedCnt), newProp) proj.SetSchema(p.schema) return []PhysicalPlan{proj} diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 0b671e6ee70a2..b25ec243d38a1 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -857,7 +857,7 @@ func (b *PlanBuilder) buildProjection4Union(ctx context.Context, u *LogicalUnion } } b.optFlag |= flagEliminateProjection - proj := LogicalProjection{Exprs: exprs, avoidColumnEvaluator: true}.Init(b.ctx) + proj := LogicalProjection{Exprs: exprs, AvoidColumnEvaluator: true}.Init(b.ctx) proj.SetSchema(u.schema.Clone()) proj.SetChildren(child) u.children[childID] = proj diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 37f2d15494fa7..4612d4b5fc7d1 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -217,18 +217,18 @@ type LogicalProjection struct { // In *UPDATE*, we should know this to tell different projections. calculateGenCols bool - // calculateNoDelay indicates this Projection is the root Plan and should be + // CalculateNoDelay indicates this Projection is the root Plan and should be // calculated without delay and will not return any result to client. // Currently it is "true" only when the current sql query is a "DO" statement. // See "https://dev.mysql.com/doc/refman/5.7/en/do.html" for more detail. - calculateNoDelay bool + CalculateNoDelay bool - // avoidColumnRef is a temporary variable which is ONLY used to avoid + // AvoidColumnEvaluator is a temporary variable which is ONLY used to avoid // building columnEvaluator for the expressions of Projection which is // built by buildProjection4Union. // This can be removed after column pool being supported. // Related issue: TiDB#8141(https://github.com/pingcap/tidb/issues/8141) - avoidColumnEvaluator bool + AvoidColumnEvaluator bool } func (p *LogicalProjection) extractCorrelatedCols() []*expression.CorrelatedColumn { diff --git a/planner/core/plan.go b/planner/core/plan.go index f2d90ddf648d6..2bf9ab6e10ade 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -40,7 +40,7 @@ type Plan interface { // replaceExprColumns replace all the column reference in the plan's expression node. replaceExprColumns(replace map[string]*expression.Column) - context() sessionctx.Context + SCtx() sessionctx.Context // property.StatsInfo will return the property.StatsInfo for this plan. statsInfo() *property.StatsInfo @@ -312,7 +312,8 @@ func (p *basePhysicalPlan) SetChild(i int, child PhysicalPlan) { p.children[i] = child } -func (p *basePlan) context() sessionctx.Context { +// Context implements Plan Context interface. +func (p *basePlan) SCtx() sessionctx.Context { return p.ctx } diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index a4083c05460b0..070060777789b 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -338,7 +338,7 @@ func (b *PlanBuilder) buildDo(ctx context.Context, v *ast.DoStmt) (Plan, error) proj.SetChildren(p) proj.self = proj proj.SetSchema(schema) - proj.calculateNoDelay = true + proj.CalculateNoDelay = true return proj, nil } diff --git a/planner/core/rule_aggregation_push_down.go b/planner/core/rule_aggregation_push_down.go index 6f7488667b0ac..91bb6fe0d9067 100644 --- a/planner/core/rule_aggregation_push_down.go +++ b/planner/core/rule_aggregation_push_down.go @@ -316,7 +316,7 @@ func (a *aggregationPushDownSolver) pushAggCrossUnion(agg *LogicalAggregation, u } func (a *aggregationPushDownSolver) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { - if !p.context().GetSessionVars().AllowAggPushDown { + if !p.SCtx().GetSessionVars().AllowAggPushDown { return p, nil } return a.aggPushDown(p) diff --git a/planner/core/rule_inject_extra_projection.go b/planner/core/rule_inject_extra_projection.go index 9b629b894040f..73b3949ce7a49 100644 --- a/planner/core/rule_inject_extra_projection.go +++ b/planner/core/rule_inject_extra_projection.go @@ -73,7 +73,7 @@ func wrapCastForAggFuncs(sctx sessionctx.Context, aggFuncs []*aggregation.AggFun func injectProjBelowAgg(aggPlan PhysicalPlan, aggFuncs []*aggregation.AggFuncDesc, groupByItems []expression.Expression) PhysicalPlan { hasScalarFunc := false - wrapCastForAggFuncs(aggPlan.context(), aggFuncs) + wrapCastForAggFuncs(aggPlan.SCtx(), aggFuncs) for i := 0; !hasScalarFunc && i < len(aggFuncs); i++ { for _, arg := range aggFuncs[i].Args { _, isScalarFunc := arg.(*expression.ScalarFunction) @@ -115,7 +115,7 @@ func injectProjBelowAgg(aggPlan PhysicalPlan, aggFuncs []*aggregation.AggFuncDes } projExprs = append(projExprs, item) newArg := &expression.Column{ - UniqueID: aggPlan.context().GetSessionVars().AllocPlanColumnID(), + UniqueID: aggPlan.SCtx().GetSessionVars().AllocPlanColumnID(), RetType: item.GetType(), ColName: model.NewCIStr(fmt.Sprintf("col_%d", len(projSchemaCols))), Index: cursor, @@ -130,7 +130,7 @@ func injectProjBelowAgg(aggPlan PhysicalPlan, aggFuncs []*aggregation.AggFuncDes proj := PhysicalProjection{ Exprs: projExprs, AvoidColumnEvaluator: false, - }.Init(aggPlan.context(), child.statsInfo().ScaleByExpectCnt(prop.ExpectedCnt), prop) + }.Init(aggPlan.SCtx(), child.statsInfo().ScaleByExpectCnt(prop.ExpectedCnt), prop) proj.SetSchema(expression.NewSchema(projSchemaCols...)) proj.SetChildren(child) @@ -164,7 +164,7 @@ func injectProjBelowSort(p PhysicalPlan, orderByItems []*ByItems) PhysicalPlan { topProj := PhysicalProjection{ Exprs: topProjExprs, AvoidColumnEvaluator: false, - }.Init(p.context(), p.statsInfo(), nil) + }.Init(p.SCtx(), p.statsInfo(), nil) topProj.SetSchema(p.Schema().Clone()) topProj.SetChildren(p) @@ -185,7 +185,7 @@ func injectProjBelowSort(p PhysicalPlan, orderByItems []*ByItems) PhysicalPlan { } bottomProjExprs = append(bottomProjExprs, itemExpr) newArg := &expression.Column{ - UniqueID: p.context().GetSessionVars().AllocPlanColumnID(), + UniqueID: p.SCtx().GetSessionVars().AllocPlanColumnID(), RetType: itemExpr.GetType(), ColName: model.NewCIStr(fmt.Sprintf("col_%d", len(bottomProjSchemaCols))), Index: len(bottomProjSchemaCols), @@ -198,7 +198,7 @@ func injectProjBelowSort(p PhysicalPlan, orderByItems []*ByItems) PhysicalPlan { bottomProj := PhysicalProjection{ Exprs: bottomProjExprs, AvoidColumnEvaluator: false, - }.Init(p.context(), childPlan.statsInfo().ScaleByExpectCnt(childProp.ExpectedCnt), childProp) + }.Init(p.SCtx(), childPlan.statsInfo().ScaleByExpectCnt(childProp.ExpectedCnt), childProp) bottomProj.SetSchema(expression.NewSchema(bottomProjSchemaCols...)) bottomProj.SetChildren(childPlan) p.SetChildren(bottomProj) diff --git a/planner/core/rule_join_reorder.go b/planner/core/rule_join_reorder.go index 562ab9bf5a438..32fe3204a0bf8 100644 --- a/planner/core/rule_join_reorder.go +++ b/planner/core/rule_join_reorder.go @@ -55,7 +55,7 @@ type jrNode struct { } func (s *joinReOrderSolver) optimize(ctx context.Context, p LogicalPlan) (LogicalPlan, error) { - return s.optimizeRecursive(p.context(), p) + return s.optimizeRecursive(p.SCtx(), p) } // optimizeRecursive recursively collects join groups and applies join reorder algorithm for each group. diff --git a/planner/core/rule_max_min_eliminate.go b/planner/core/rule_max_min_eliminate.go index fddedf0440b70..51c28c02dd067 100644 --- a/planner/core/rule_max_min_eliminate.go +++ b/planner/core/rule_max_min_eliminate.go @@ -46,7 +46,7 @@ func (a *maxMinEliminator) eliminateMaxMin(p LogicalPlan) { } child := p.Children()[0] - ctx := p.context() + ctx := p.SCtx() // If there's no column in f.GetArgs()[0], we still need limit and read data from real table because the result should NULL if the below is empty. if len(expression.ExtractColumns(f.Args[0])) > 0 { diff --git a/planner/core/rule_partition_processor.go b/planner/core/rule_partition_processor.go index 38335b9ce9804..51bbe3bc6291d 100644 --- a/planner/core/rule_partition_processor.go +++ b/planner/core/rule_partition_processor.go @@ -112,7 +112,7 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { children := make([]LogicalPlan, 0, len(pi.Definitions)) for i, expr := range partitionExprs { // If the select condition would never be satisified, prune that partition. - pruned, err := s.canBePruned(ds.context(), col, expr, ds.allConds) + pruned, err := s.canBePruned(ds.SCtx(), col, expr, ds.allConds) if err != nil { return nil, err } @@ -128,19 +128,19 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { // Not a deep copy. newDataSource := *ds - newDataSource.baseLogicalPlan = newBaseLogicalPlan(ds.context(), TypeTableScan, &newDataSource) + newDataSource.baseLogicalPlan = newBaseLogicalPlan(ds.SCtx(), TypeTableScan, &newDataSource) newDataSource.isPartition = true newDataSource.physicalTableID = pi.Definitions[i].ID // There are many expression nodes in the plan tree use the original datasource // id as FromID. So we set the id of the newDataSource with the original one to // avoid traversing the whole plan tree to update the references. newDataSource.id = ds.id - newDataSource.statisticTable = getStatsTable(ds.context(), ds.table.Meta(), pi.Definitions[i].ID) + newDataSource.statisticTable = getStatsTable(ds.SCtx(), ds.table.Meta(), pi.Definitions[i].ID) children = append(children, &newDataSource) } if len(children) == 0 { // No result after table pruning. - tableDual := LogicalTableDual{RowCount: 0}.Init(ds.context()) + tableDual := LogicalTableDual{RowCount: 0}.Init(ds.SCtx()) tableDual.schema = ds.Schema() return tableDual, nil } @@ -148,7 +148,7 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) { // No need for the union all. return children[0], nil } - unionAll := LogicalUnionAll{}.Init(ds.context()) + unionAll := LogicalUnionAll{}.Init(ds.SCtx()) unionAll.SetChildren(children...) unionAll.SetSchema(ds.schema) return unionAll, nil diff --git a/planner/core/rule_predicate_push_down.go b/planner/core/rule_predicate_push_down.go index aa909538abadc..1b14b3eba9513 100644 --- a/planner/core/rule_predicate_push_down.go +++ b/planner/core/rule_predicate_push_down.go @@ -35,14 +35,14 @@ func addSelection(p LogicalPlan, child LogicalPlan, conditions []expression.Expr p.Children()[chIdx] = child return } - conditions = expression.PropagateConstant(p.context(), conditions) + conditions = expression.PropagateConstant(p.SCtx(), conditions) // Return table dual when filter is constant false or null. dual := conds2TableDual(child, conditions) if dual != nil { p.Children()[chIdx] = dual return } - selection := LogicalSelection{Conditions: conditions}.Init(p.context()) + selection := LogicalSelection{Conditions: conditions}.Init(p.SCtx()) selection.SetChildren(child) p.Children()[chIdx] = selection } @@ -480,9 +480,9 @@ func conds2TableDual(p LogicalPlan, conds []expression.Expression) LogicalPlan { if !ok { return nil } - sc := p.context().GetSessionVars().StmtCtx + sc := p.SCtx().GetSessionVars().StmtCtx if isTrue, err := con.Value.ToBool(sc); (err == nil && isTrue == 0) || con.Value.IsNull() { - dual := LogicalTableDual{}.Init(p.context()) + dual := LogicalTableDual{}.Init(p.SCtx()) dual.SetSchema(p.Schema()) return dual } diff --git a/planner/core/task.go b/planner/core/task.go index 91b344b0ac34f..9d545955307ca 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -351,7 +351,7 @@ func finishCopTask(ctx sessionctx.Context, task task) task { // the cost to cop iterator workers. According to `CopClient::Send`, the concurrency // is Min(DistSQLScanConcurrency, numRegionsInvolvedInScan), since we cannot infer // the number of regions involved, we simply use DistSQLScanConcurrency. - copIterWorkers := float64(t.plan().context().GetSessionVars().DistSQLScanConcurrency) + copIterWorkers := float64(t.plan().SCtx().GetSessionVars().DistSQLScanConcurrency) t.finishIndexPlan() // Network cost of transferring rows of table scan to TiDB. if t.tablePlan != nil { @@ -371,12 +371,12 @@ func finishCopTask(ctx sessionctx.Context, task task) task { indexRows := t.indexPlan.statsInfo().RowCount newTask.cst += indexRows * cpuFactor // Add cost of worker goroutines in index lookup. - numTblWorkers := float64(t.indexPlan.context().GetSessionVars().IndexLookupConcurrency) + numTblWorkers := float64(t.indexPlan.SCtx().GetSessionVars().IndexLookupConcurrency) newTask.cst += (numTblWorkers + 1) * concurrencyFactor // When building table reader executor for each batch, we would sort the handles. CPU // cost of sort is: // cpuFactor * batchSize * Log2(batchSize) * (indexRows / batchSize) - indexLookupSize := float64(t.indexPlan.context().GetSessionVars().IndexLookupSize) + indexLookupSize := float64(t.indexPlan.SCtx().GetSessionVars().IndexLookupSize) batchSize := math.Min(indexLookupSize, indexRows) if batchSize > 2 { sortCPUCost := (indexRows * math.Log2(batchSize) * cpuFactor) / numTblWorkers diff --git a/planner/implementation/datasource.go b/planner/implementation/datasource.go new file mode 100644 index 0000000000000..320a6f5e78a86 --- /dev/null +++ b/planner/implementation/datasource.go @@ -0,0 +1,34 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package implementation + +import ( + plannercore "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/planner/memo" +) + +// TableDualImpl implementation of PhysicalTableDual. +type TableDualImpl struct { + baseImpl +} + +// NewTableDualImpl creates a new table dual Implementation. +func NewTableDualImpl(dual *plannercore.PhysicalTableDual) *TableDualImpl { + return &TableDualImpl{baseImpl{plan: dual}} +} + +// CalcCost calculates the cost of the table dual Implementation. +func (impl *TableDualImpl) CalcCost(outCount float64, childCosts []float64, children ...*memo.Group) float64 { + return 0 +} diff --git a/planner/implementation/simple_plans.go b/planner/implementation/simple_plans.go new file mode 100644 index 0000000000000..e09b0f35721fb --- /dev/null +++ b/planner/implementation/simple_plans.go @@ -0,0 +1,28 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package implementation + +import ( + plannercore "github.com/pingcap/tidb/planner/core" +) + +// ProjectionImpl implementation of PhysicalProjection. +type ProjectionImpl struct { + baseImpl +} + +// NewProjectionImpl creates a new projection Implementation. +func NewProjectionImpl(proj *plannercore.PhysicalProjection) *ProjectionImpl { + return &ProjectionImpl{baseImpl{plan: proj}} +} diff --git a/planner/memo/group.go b/planner/memo/group.go index d805448241f98..4b10b71b77a18 100644 --- a/planner/memo/group.go +++ b/planner/memo/group.go @@ -71,6 +71,7 @@ func (g *Group) Insert(e *GroupExpr) bool { g.FirstExpr[operand] = newEquiv } g.Fingerprints[e.FingerPrint()] = newEquiv + e.Group = g return true } @@ -84,6 +85,7 @@ func (g *Group) Delete(e *GroupExpr) { g.Equivalents.Remove(equiv) delete(g.Fingerprints, fingerprint) + e.Group = nil operand := GetOperand(equiv.Value.(*GroupExpr).ExprNode) if g.FirstExpr[operand] != equiv { diff --git a/planner/memo/group_expr.go b/planner/memo/group_expr.go index c69ee946d0823..6d461c583a8b5 100644 --- a/planner/memo/group_expr.go +++ b/planner/memo/group_expr.go @@ -28,6 +28,7 @@ type GroupExpr struct { ExprNode plannercore.LogicalPlan Children []*Group Explored bool + Group *Group selfFingerprint string } From 554594b0bbcf8a762ca7ce60d24b4f3ac4d762b9 Mon Sep 17 00:00:00 2001 From: SunRunAway Date: Thu, 8 Aug 2019 15:23:28 +0800 Subject: [PATCH 178/196] build: use `make testSuite` to ensure all testSuites enabled (#11627) --- Makefile | 6 +++++- executor/executor_test.go | 1 + store/mockstore/mocktikv/mock_tikv_test.go | 1 + store/tikv/backoff_test.go | 4 ++-- tools/check/check_testSuite.sh | 17 +++++++++++++++++ 5 files changed, 26 insertions(+), 3 deletions(-) create mode 100755 tools/check/check_testSuite.sh diff --git a/Makefile b/Makefile index a4d0195433e3c..9029562c4be4a 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ build: # Install the check tools. check-setup:tools/bin/revive tools/bin/goword tools/bin/gometalinter tools/bin/gosec -check: fmt errcheck lint tidy check-static vet +check: fmt errcheck lint tidy testSuite check-static vet # These need to be fixed before they can be ran regularly check-fail: goword check-slow @@ -106,6 +106,10 @@ tidy: @echo "go mod tidy" ./tools/check/check-tidy.sh +testSuite: + @echo "testSuite" + ./tools/check/check_testSuite.sh + clean: $(GO) clean -i ./... rm -rf *.out diff --git a/executor/executor_test.go b/executor/executor_test.go index fba261cb887b3..d72a124902508 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -93,6 +93,7 @@ var _ = Suite(&testSuite1{}) var _ = Suite(&testSuite2{}) var _ = Suite(&testSuite3{}) var _ = Suite(&testSuite4{}) +var _ = SerialSuites(&testShowStatsSuite{testSuite{&baseTestSuite{}}}) var _ = Suite(&testBypassSuite{}) var _ = Suite(&testUpdateSuite{}) var _ = Suite(&testOOMSuite{}) diff --git a/store/mockstore/mocktikv/mock_tikv_test.go b/store/mockstore/mocktikv/mock_tikv_test.go index d36700f8efd75..ddb71d775d46e 100644 --- a/store/mockstore/mocktikv/mock_tikv_test.go +++ b/store/mockstore/mocktikv/mock_tikv_test.go @@ -39,6 +39,7 @@ type testMVCCLevelDB struct { } var ( + _ = Suite(&testMockTiKVSuite{}) _ = Suite(&testMVCCLevelDB{}) _ = Suite(testMarshal{}) ) diff --git a/store/tikv/backoff_test.go b/store/tikv/backoff_test.go index 13aa5ef4c8fd2..ddf7d1fcf86f6 100644 --- a/store/tikv/backoff_test.go +++ b/store/tikv/backoff_test.go @@ -24,7 +24,7 @@ type testBackoffSuite struct { store *tikvStore } -var _ = Suite(&testLockSuite{}) +var _ = Suite(&testBackoffSuite{}) func (s *testBackoffSuite) SetUpTest(c *C) { s.store = NewTestStore(c).(*tikvStore) @@ -37,6 +37,6 @@ func (s *testBackoffSuite) TearDownTest(c *C) { func (s *testBackoffSuite) TestBackoffWithMax(c *C) { b := NewBackoffer(context.TODO(), 2000) err := b.BackoffWithMaxSleep(boTxnLockFast, 30, errors.New("test")) - c.Assert(err, NotNil) + c.Assert(err, IsNil) c.Assert(b.totalSleep, Equals, 30) } diff --git a/tools/check/check_testSuite.sh b/tools/check/check_testSuite.sh new file mode 100755 index 0000000000000..dd743df3830bf --- /dev/null +++ b/tools/check/check_testSuite.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +exitCode=0 +for testSuite in $(find . -name "*_test.go" -print0 | xargs -0 grep -P "type test(.*)Suite" | awk '{print $2}'); do + # TODO: ugly regex + # TODO: check code comment + if ! find . -name "*_test.go" -print0 | xargs -0 grep -P "_ = (check\.)?(Suite|SerialSuites)\((&?${testSuite}{|new\(${testSuite}\))" > /dev/null + then + if find . -name "*_test.go" -print0 | xargs -0 grep -P "func \((.* )?\*?${testSuite}\) Test" > /dev/null + then + echo "${testSuite} is not enabled" && exitCode=1 + fi + fi +done +exit ${exitCode} From 6098ed66e0135cc61260603ba7b512279b5a0bdf Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 8 Aug 2019 16:00:37 +0800 Subject: [PATCH 179/196] executor: add vectorized access methods for `time.Duration` stored in `Column`(#11677) --- util/chunk/column.go | 24 +++++++++--- util/chunk/column_test.go | 79 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/util/chunk/column.go b/util/chunk/column.go index 66d1778ced7ea..09a2fb7c9058d 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -234,11 +234,12 @@ func (c *Column) AppendEnum(enum types.Enum) { } const ( - sizeInt64 = int(unsafe.Sizeof(int64(0))) - sizeUint64 = int(unsafe.Sizeof(uint64(0))) - sizeFloat32 = int(unsafe.Sizeof(float32(0))) - sizeFloat64 = int(unsafe.Sizeof(float64(0))) - sizeMyDecimal = int(unsafe.Sizeof(types.MyDecimal{})) + sizeInt64 = int(unsafe.Sizeof(int64(0))) + sizeUint64 = int(unsafe.Sizeof(uint64(0))) + sizeFloat32 = int(unsafe.Sizeof(float32(0))) + sizeFloat64 = int(unsafe.Sizeof(float64(0))) + sizeMyDecimal = int(unsafe.Sizeof(types.MyDecimal{})) + sizeGoDuration = int(unsafe.Sizeof(time.Duration(0))) ) // preAlloc allocates space for a fixed-length-type slice and resets all slots to null. @@ -337,6 +338,11 @@ func (c *Column) PreAllocDecimal(length int) { c.preAlloc(length, sizeMyDecimal) } +// PreAllocDuration allocates space for a duration slice and resets all slots to null. +func (c *Column) PreAllocDuration(length int) { + c.preAlloc(length, sizeGoDuration) +} + func (c *Column) castSliceHeader(header *reflect.SliceHeader, typeSize int) { header.Data = uintptr(unsafe.Pointer(&c.data[0])) header.Len = c.length @@ -371,6 +377,14 @@ func (c *Column) Float64s() []float64 { return res } +// GoDurations returns a Golang time.Duration slice stored in this Column. +// Different from the Row.GetDuration method, the argument Fsp is ignored, so the user should handle it outside. +func (c *Column) GoDurations() []time.Duration { + var res []time.Duration + c.castSliceHeader((*reflect.SliceHeader)(unsafe.Pointer(&res)), sizeGoDuration) + return res +} + // Decimals returns a MyDecimal slice stored in this Column. func (c *Column) Decimals() []types.MyDecimal { var res []types.MyDecimal diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 277cde93664b2..5cc7a0dc0766b 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -16,6 +16,7 @@ package chunk import ( "fmt" "math/rand" + "testing" "time" "github.com/pingcap/check" @@ -257,6 +258,30 @@ func (s *testChunkSuite) TestF32Column(c *check.C) { } } +func (s *testChunkSuite) TestDurationSliceColumn(c *check.C) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDuration)}, 1024) + col := chk.Column(0) + for i := 0; i < 1024; i++ { + col.AppendDuration(types.Duration{Duration: time.Duration(i)}) + } + + ds := col.GoDurations() + for i := 0; i < 1024; i++ { + c.Assert(ds[i], check.Equals, time.Duration(i)) + d := types.Duration{Duration: ds[i]} + d, _ = d.Add(d) + ds[i] = d.Duration + } + + it := NewIterator4Chunk(chk) + var i int64 + for row := it.Begin(); row != it.End(); row = it.Next() { + c.Assert(row.GetDuration(0, 0).Duration, check.Equals, time.Duration(i)*2) + c.Assert(col.GetDuration(int(i), 0).Duration, check.Equals, time.Duration(i)*2) + i++ + } +} + func (s *testChunkSuite) TestMyDecimal(c *check.C) { chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeNewDecimal)}, 1024) col := chk.Column(0) @@ -681,3 +706,57 @@ func (s *testChunkSuite) TestSetNulls(c *check.C) { } } } + +func BenchmarkDurationRow(b *testing.B) { + chk1 := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDuration)}, 1024) + col1 := chk1.Column(0) + for i := 0; i < 1024; i++ { + col1.AppendDuration(types.Duration{Duration: time.Second * time.Duration(i)}) + } + chk2 := chk1.CopyConstruct() + result := chk1.CopyConstruct() + + b.ResetTimer() + for k := 0; k < b.N; k++ { + result.Reset() + it1 := NewIterator4Chunk(chk1) + it2 := NewIterator4Chunk(chk2) + for r1, r2 := it1.Begin(), it2.Begin(); r1 != it1.End() && r2 != it2.End(); r1, r2 = it1.Next(), it2.Next() { + d1 := r1.GetDuration(0, 0) + d2 := r2.GetDuration(0, 0) + r, err := d1.Add(d2) + if err != nil { + b.Fatal(err) + } + result.AppendDuration(0, r) + } + } +} + +func BenchmarkDurationVec(b *testing.B) { + chk := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDuration)}, 1024) + col1 := chk.Column(0) + for i := 0; i < 1024; i++ { + col1.AppendDuration(types.Duration{Duration: time.Second * time.Duration(i)}) + } + col2 := col1.CopyConstruct(nil) + result := col1.CopyConstruct(nil) + + ds1 := col1.GoDurations() + ds2 := col2.GoDurations() + rs := result.GoDurations() + + b.ResetTimer() + for k := 0; k < b.N; k++ { + result.PreAllocDuration(1024) + for i := 0; i < 1024; i++ { + d1 := types.Duration{Duration: ds1[i]} + d2 := types.Duration{Duration: ds2[i]} + r, err := d1.Add(d2) + if err != nil { + b.Fatal(err) + } + rs[i] = r.Duration + } + } +} From 036e7e207deb4b58a98bdea2c3bf500a2188d3f2 Mon Sep 17 00:00:00 2001 From: lysu Date: Fri, 9 Aug 2019 12:43:33 +0800 Subject: [PATCH 180/196] *: support new 'log' format for trace (#11681) --- ddl/index.go | 2 +- executor/admin.go | 4 +- executor/batch_checker.go | 24 +++---- executor/builder.go | 15 ++++- executor/executor_test.go | 2 +- executor/insert.go | 18 ++++-- executor/insert_common.go | 6 +- executor/load_data.go | 4 +- executor/replace.go | 4 +- executor/table_reader.go | 9 +++ executor/trace.go | 118 ++++++++++++++++++++++++++++------- executor/trace_test.go | 6 +- executor/write_test.go | 4 +- kv/fault_injection.go | 8 +-- kv/fault_injection_test.go | 8 +-- kv/kv.go | 4 +- kv/mock.go | 4 +- kv/mock_test.go | 2 +- planner/core/planbuilder.go | 23 ++++++- server/conn.go | 4 +- session/session.go | 6 ++ session/txn.go | 4 +- store/tikv/2pc.go | 17 +++-- store/tikv/client_batch.go | 3 + store/tikv/lock_test.go | 2 +- store/tikv/region_cache.go | 2 + store/tikv/region_request.go | 1 + store/tikv/safepoint_test.go | 2 +- store/tikv/snapshot.go | 4 +- store/tikv/snapshot_test.go | 2 +- store/tikv/txn.go | 6 +- util/logutil/log.go | 26 ++++++++ 32 files changed, 256 insertions(+), 88 deletions(-) diff --git a/ddl/index.go b/ddl/index.go index 3c58ffb79e560..73e603e1f9fc4 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -764,7 +764,7 @@ func (w *addIndexWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*i w.distinctCheckFlags = append(w.distinctCheckFlags, distinct) } - batchVals, err := txn.BatchGet(w.batchCheckKeys) + batchVals, err := txn.BatchGet(context.Background(), w.batchCheckKeys) if err != nil { return errors.Trace(err) } diff --git a/executor/admin.go b/executor/admin.go index 5f8ca71e2680c..01c1fc8ca8556 100644 --- a/executor/admin.go +++ b/executor/admin.go @@ -377,7 +377,7 @@ func (e *RecoverIndexExec) batchMarkDup(txn kv.Transaction, rows []recoverRows) distinctFlags[i] = distinct } - values, err := txn.BatchGet(e.batchKeys) + values, err := txn.BatchGet(context.Background(), e.batchKeys) if err != nil { return err } @@ -502,7 +502,7 @@ func (e *CleanupIndexExec) batchGetRecord(txn kv.Transaction) (map[string][]byte for handle := range e.idxValues { e.batchKeys = append(e.batchKeys, e.table.RecordKey(handle)) } - values, err := txn.BatchGet(e.batchKeys) + values, err := txn.BatchGet(context.Background(), e.batchKeys) if err != nil { return nil, err } diff --git a/executor/batch_checker.go b/executor/batch_checker.go index 12d2b7c63a834..be604ceca480b 100644 --- a/executor/batch_checker.go +++ b/executor/batch_checker.go @@ -14,6 +14,8 @@ package executor import ( + "context" + "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" @@ -53,12 +55,12 @@ type batchChecker struct { } // batchGetOldValues gets the values of storage in batch. -func (b *batchChecker) batchGetOldValues(ctx sessionctx.Context, batchKeys []kv.Key) error { - txn, err := ctx.Txn(true) +func (b *batchChecker) batchGetOldValues(ctx context.Context, sctx sessionctx.Context, batchKeys []kv.Key) error { + txn, err := sctx.Txn(true) if err != nil { return err } - values, err := txn.BatchGet(batchKeys) + values, err := txn.BatchGet(ctx, batchKeys) if err != nil { return err } @@ -187,9 +189,9 @@ func (b *batchChecker) getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Ta } // batchGetInsertKeys uses batch-get to fetch all key-value pairs to be checked for ignore or duplicate key update. -func (b *batchChecker) batchGetInsertKeys(ctx sessionctx.Context, t table.Table, newRows [][]types.Datum) (err error) { +func (b *batchChecker) batchGetInsertKeys(ctx context.Context, sctx sessionctx.Context, t table.Table, newRows [][]types.Datum) (err error) { // Get keys need to be checked. - b.toBeCheckedRows, err = b.getKeysNeedCheck(ctx, t, newRows) + b.toBeCheckedRows, err = b.getKeysNeedCheck(sctx, t, newRows) if err != nil { return err } @@ -211,11 +213,11 @@ func (b *batchChecker) batchGetInsertKeys(ctx sessionctx.Context, t table.Table, batchKeys = append(batchKeys, k.newKV.key) } } - txn, err := ctx.Txn(true) + txn, err := sctx.Txn(true) if err != nil { return err } - b.dupKVs, err = txn.BatchGet(batchKeys) + b.dupKVs, err = txn.BatchGet(ctx, batchKeys) return err } @@ -231,7 +233,7 @@ func (b *batchChecker) initDupOldRowFromHandleKey() { } } -func (b *batchChecker) initDupOldRowFromUniqueKey(ctx sessionctx.Context, newRows [][]types.Datum) error { +func (b *batchChecker) initDupOldRowFromUniqueKey(ctx context.Context, sctx sessionctx.Context, newRows [][]types.Datum) error { batchKeys := make([]kv.Key, 0, len(newRows)) for _, r := range b.toBeCheckedRows { for _, uk := range r.uniqueKeys { @@ -244,14 +246,14 @@ func (b *batchChecker) initDupOldRowFromUniqueKey(ctx sessionctx.Context, newRow } } } - return b.batchGetOldValues(ctx, batchKeys) + return b.batchGetOldValues(ctx, sctx, batchKeys) } // initDupOldRowValue initializes dupOldRowValues which contain the to-be-updated rows from storage. -func (b *batchChecker) initDupOldRowValue(ctx sessionctx.Context, t table.Table, newRows [][]types.Datum) error { +func (b *batchChecker) initDupOldRowValue(ctx context.Context, sctx sessionctx.Context, t table.Table, newRows [][]types.Datum) error { b.dupOldRowValues = make(map[string][]byte, len(newRows)) b.initDupOldRowFromHandleKey() - return b.initDupOldRowFromUniqueKey(ctx, newRows) + return b.initDupOldRowFromUniqueKey(ctx, sctx, newRows) } // fillBackKeys fills the updated key-value pair to the dupKeyValues for further check. diff --git a/executor/builder.go b/executor/builder.go index 11de9888cac88..b18d6ca87f1c7 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -798,12 +798,25 @@ func (b *executorBuilder) buildDDL(v *plannercore.DDL) Executor { // buildTrace builds a TraceExec for future executing. This method will be called // at build(). func (b *executorBuilder) buildTrace(v *plannercore.Trace) Executor { - return &TraceExec{ + t := &TraceExec{ baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()), stmtNode: v.StmtNode, builder: b, format: v.Format, } + if t.format == plannercore.TraceFormatLog { + return &SortExec{ + baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID(), t), + ByItems: []*plannercore.ByItems{ + {Expr: &expression.Column{ + Index: 0, + RetType: types.NewFieldType(mysql.TypeTimestamp), + }}, + }, + schema: v.Schema(), + } + } + return t } // buildExplain builds a explain executor. `e.rows` collects final result to `ExplainExec`. diff --git a/executor/executor_test.go b/executor/executor_test.go index d72a124902508..348c7c896d1de 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -501,7 +501,7 @@ func checkCases(tests []testCase, ld *executor.LoadDataInfo, data, reachLimit, err1 := ld.InsertData(context.Background(), tt.data1, tt.data2) c.Assert(err1, IsNil) c.Assert(reachLimit, IsFalse) - err1 = ld.CheckAndInsertOneBatch() + err1 = ld.CheckAndInsertOneBatch(context.Background()) c.Assert(err1, IsNil) if tt.restData == nil { c.Assert(data, HasLen, 0, diff --git a/executor/insert.go b/executor/insert.go index efe5e643c721c..c905e7c554f60 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/stringutil" "go.uber.org/zap" ) @@ -35,6 +36,13 @@ type InsertExec struct { } func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { + logutil.Eventf(ctx, "insert %d rows into table `%s`", len(rows), stringutil.MemoizeStr(func() string { + var tblName string + if meta := e.Table.Meta(); meta != nil { + tblName = meta.Name.L + } + return tblName + })) // If tidb_batch_insert is ON and not in a transaction, we could use BatchInsert mode. sessVars := e.ctx.GetSessionVars() defer sessVars.CleanBuffers() @@ -57,12 +65,12 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { // If `ON DUPLICATE KEY UPDATE` is specified, and no `IGNORE` keyword, // the to-be-insert rows will be check on duplicate keys and update to the new rows. if len(e.OnDuplicate) > 0 { - err := e.batchUpdateDupRows(rows) + err := e.batchUpdateDupRows(ctx, rows) if err != nil { return err } } else if ignoreErr { - err := e.batchCheckAndInsert(rows, e.addRecord) + err := e.batchCheckAndInsert(ctx, rows, e.addRecord) if err != nil { return err } @@ -77,14 +85,14 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { } // batchUpdateDupRows updates multi-rows in batch if they are duplicate with rows in table. -func (e *InsertExec) batchUpdateDupRows(newRows [][]types.Datum) error { - err := e.batchGetInsertKeys(e.ctx, e.Table, newRows) +func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.Datum) error { + err := e.batchGetInsertKeys(ctx, e.ctx, e.Table, newRows) if err != nil { return err } // Batch get the to-be-updated rows in storage. - err = e.initDupOldRowValue(e.ctx, e.Table, newRows) + err = e.initDupOldRowValue(ctx, e.ctx, e.Table, newRows) if err != nil { return err } diff --git a/executor/insert_common.go b/executor/insert_common.go index bbd033ad6279e..f467603c7d18c 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -326,7 +326,7 @@ func insertRowsFromSelect(ctx context.Context, base insertCommon) error { batchSize := sessVars.DMLBatchSize for { - err := selectExec.Next(ctx, chk) + err := Next(ctx, selectExec, chk) if err != nil { return err } @@ -582,10 +582,10 @@ func (e *InsertValues) handleWarning(err error) { // batchCheckAndInsert checks rows with duplicate errors. // All duplicate rows will be ignored and appended as duplicate warnings. -func (e *InsertValues) batchCheckAndInsert(rows [][]types.Datum, addRecord func(row []types.Datum) (int64, error)) error { +func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.Datum, addRecord func(row []types.Datum) (int64, error)) error { // all the rows will be checked, so it is safe to set BatchCheck = true e.ctx.GetSessionVars().StmtCtx.BatchCheck = true - err := e.batchGetInsertKeys(e.ctx, e.Table, rows) + err := e.batchGetInsertKeys(ctx, e.ctx, e.Table, rows) if err != nil { return err } diff --git a/executor/load_data.go b/executor/load_data.go index 05cd1fe8d11d1..3a525d8f94cea 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -278,13 +278,13 @@ func (e *LoadDataInfo) InsertData(ctx context.Context, prevData, curData []byte) } // CheckAndInsertOneBatch is used to commit one transaction batch full filled data -func (e *LoadDataInfo) CheckAndInsertOneBatch() error { +func (e *LoadDataInfo) CheckAndInsertOneBatch(ctx context.Context) error { var err error if e.curBatchCnt == 0 { return err } e.ctx.GetSessionVars().StmtCtx.AddRecordRows(e.curBatchCnt) - err = e.batchCheckAndInsert(e.rows[0:e.curBatchCnt], e.addRecordLD) + err = e.batchCheckAndInsert(ctx, e.rows[0:e.curBatchCnt], e.addRecordLD) if err != nil { return err } diff --git a/executor/replace.go b/executor/replace.go index c228e09440b72..829ecf3eb8744 100644 --- a/executor/replace.go +++ b/executor/replace.go @@ -160,13 +160,13 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error { * because in this case, one row was inserted after the duplicate was deleted. * See http://dev.mysql.com/doc/refman/5.7/en/mysql-affected-rows.html */ - err := e.batchGetInsertKeys(e.ctx, e.Table, newRows) + err := e.batchGetInsertKeys(ctx, e.ctx, e.Table, newRows) if err != nil { return err } // Batch get the to-be-replaced rows in storage. - err = e.initDupOldRowValue(e.ctx, e.Table, newRows) + err = e.initDupOldRowValue(ctx, e.ctx, e.Table, newRows) if err != nil { return err } diff --git a/executor/table_reader.go b/executor/table_reader.go index b26001b836988..9bc481bca6c4e 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -26,8 +26,10 @@ import ( "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/memory" "github.com/pingcap/tidb/util/ranger" + "github.com/pingcap/tidb/util/stringutil" "github.com/pingcap/tipb/go-tipb" ) @@ -137,6 +139,13 @@ func (e *TableReaderExecutor) Open(ctx context.Context) error { // Next fills data into the chunk passed by its caller. // The task was actually done by tableReaderHandler. func (e *TableReaderExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + logutil.Eventf(ctx, "table scan table: %s, range: %v", stringutil.MemoizeStr(func() string { + var tableName string + if meta := e.table.Meta(); meta != nil { + tableName = meta.Name.L + } + return tableName + }), e.ranges) if err := e.resultHandler.nextChunk(ctx, req); err != nil { e.feedback.Invalidate() return err diff --git a/executor/trace.go b/executor/trace.go index 92644c28015f5..91acb341928c5 100644 --- a/executor/trace.go +++ b/executor/trace.go @@ -16,15 +16,22 @@ package executor import ( "context" "encoding/json" + "fmt" "time" "github.com/opentracing/basictracer-go" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/sqlexec" + "go.uber.org/zap" "sourcegraph.com/sourcegraph/appdash" traceImpl "sourcegraph.com/sourcegraph/appdash/opentracing" ) @@ -58,25 +65,36 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { return nil } + switch e.format { + case core.TraceFormatLog: + return e.nextTraceLog(ctx, se, req) + default: + return e.nextRowJSON(ctx, se, req) + } +} + +func (e *TraceExec) nextTraceLog(ctx context.Context, se sqlexec.SQLExecutor, req *chunk.Chunk) error { + recorder := basictracer.NewInMemoryRecorder() + tracer := basictracer.New(recorder) + span := tracer.StartSpan("trace") + ctx = opentracing.ContextWithSpan(ctx, span) + + e.executeChild(ctx, se) + span.Finish() + + generateLogResult(recorder.GetSpans(), req) + e.exhausted = true + return nil +} + +func (e *TraceExec) nextRowJSON(ctx context.Context, se sqlexec.SQLExecutor, req *chunk.Chunk) error { store := appdash.NewMemoryStore() tracer := traceImpl.NewTracer(store) span := tracer.StartSpan("trace") - defer span.Finish() ctx = opentracing.ContextWithSpan(ctx, span) - recordSets, err := se.Execute(ctx, e.stmtNode.Text()) - if err != nil { - return errors.Trace(err) - } - for _, rs := range recordSets { - _, err = drainRecordSet(ctx, e.ctx, rs) - if err != nil { - return errors.Trace(err) - } - if err = rs.Close(); err != nil { - return errors.Trace(err) - } - } + e.executeChild(ctx, se) + span.Finish() traces, err := store.Traces(appdash.TracesOpts{}) if err != nil { @@ -84,7 +102,7 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { } // Row format. - if e.format != "json" { + if e.format != core.TraceFormatJSON { if len(traces) < 1 { e.exhausted = true return nil @@ -112,20 +130,46 @@ func (e *TraceExec) Next(ctx context.Context, req *chunk.Chunk) error { return nil } -func drainRecordSet(ctx context.Context, sctx sessionctx.Context, rs sqlexec.RecordSet) ([]chunk.Row, error) { - var rows []chunk.Row - req := rs.NewChunk() +func (e *TraceExec) executeChild(ctx context.Context, se sqlexec.SQLExecutor) { + recordSets, err := se.Execute(ctx, e.stmtNode.Text()) + if len(recordSets) == 0 { + if err != nil { + var errCode uint16 + if te, ok := err.(*terror.Error); ok { + errCode = te.ToSQLError().Code + } + logutil.Eventf(ctx, "execute with error(%d): %s", errCode, err.Error()) + } else { + logutil.Eventf(ctx, "execute done, modify row: %d", e.ctx.GetSessionVars().StmtCtx.AffectedRows()) + } + } + for _, rs := range recordSets { + drainRecordSet(ctx, e.ctx, rs) + if err = rs.Close(); err != nil { + logutil.Logger(ctx).Error("run trace close result with error", zap.Error(err)) + } + } +} +func drainRecordSet(ctx context.Context, sctx sessionctx.Context, rs sqlexec.RecordSet) { + req := rs.NewChunk() + var rowCount int for { err := rs.Next(ctx, req) if err != nil || req.NumRows() == 0 { - return rows, errors.Trace(err) - } - iter := chunk.NewIterator4Chunk(req) - for r := iter.Begin(); r != iter.End(); r = iter.Next() { - rows = append(rows, r) + if err != nil { + var errCode uint16 + if te, ok := err.(*terror.Error); ok { + errCode = te.ToSQLError().Code + } + logutil.Eventf(ctx, "execute with error(%d): %s", errCode, err.Error()) + } else { + logutil.Eventf(ctx, "execute done, ReturnRow: %d, ModifyRow: %d", rowCount, sctx.GetSessionVars().StmtCtx.AffectedRows()) + } + return } - req = chunk.Renew(req, sctx.GetSessionVars().MaxChunkSize) + rowCount += req.NumRows() + req.Reset() } } @@ -159,3 +203,29 @@ func dfsTree(t *appdash.Trace, prefix string, isLast bool, chk *chunk.Chunk) { dfsTree(sp, newPrefix, i == (len(t.Sub))-1 /*last element of array*/, chk) } } + +func generateLogResult(allSpans []basictracer.RawSpan, chk *chunk.Chunk) { + for rIdx := range allSpans { + span := &allSpans[rIdx] + + chk.AppendTime(0, types.Time{Time: types.FromGoTime(span.Start), Type: mysql.TypeTimestamp, Fsp: 6}) + chk.AppendString(1, "--- start span "+span.Operation+" ----") + chk.AppendString(2, "") + chk.AppendString(3, span.Operation) + + var tags string + if len(span.Tags) > 0 { + tags = fmt.Sprintf("%v", span.Tags) + } + for _, l := range span.Logs { + for _, field := range l.Fields { + if field.Key() == logutil.TraceEventKey { + chk.AppendTime(0, types.Time{Time: types.FromGoTime(l.Timestamp), Type: mysql.TypeTimestamp, Fsp: 6}) + chk.AppendString(1, field.Value().(string)) + chk.AppendString(2, tags) + chk.AppendString(3, span.Operation) + } + } + } + } +} diff --git a/executor/trace_test.go b/executor/trace_test.go index 15d7477653036..fb2edfa687958 100644 --- a/executor/trace_test.go +++ b/executor/trace_test.go @@ -25,7 +25,7 @@ func (s *testSuite1) TestTraceExec(c *C) { tk.MustExec(testSQL) tk.MustExec("trace insert into trace (c1, c2, c3) values (1, 2, 3)") rows := tk.MustQuery("trace select * from trace where id = 0;").Rows() - c.Assert(rows, HasLen, 1) + c.Assert(len(rows), GreaterEqual, 1) // +---------------------------+-----------------+------------+ // | operation | startTS | duration | @@ -43,4 +43,8 @@ func (s *testSuite1) TestTraceExec(c *C) { rows = tk.MustQuery("trace format='row' delete from trace where id = 0").Rows() c.Assert(len(rows) > 1, IsTrue) + + tk.MustExec("trace format='log' insert into trace (c1, c2, c3) values (1, 2, 3)") + rows = tk.MustQuery("trace format='log' select * from trace where id = 0;").Rows() + c.Assert(len(rows), GreaterEqual, 1) } diff --git a/executor/write_test.go b/executor/write_test.go index 14e86b09a3c05..9f15c278ce3ac 100644 --- a/executor/write_test.go +++ b/executor/write_test.go @@ -1845,7 +1845,7 @@ func (s *testSuite4) TestLoadData(c *C) { _, reachLimit, err := ld.InsertData(context.Background(), nil, nil) c.Assert(err, IsNil) c.Assert(reachLimit, IsFalse) - err = ld.CheckAndInsertOneBatch() + err = ld.CheckAndInsertOneBatch(context.Background()) c.Assert(err, IsNil) r := tk.MustQuery(selectSQL) r.Check(nil) @@ -2096,7 +2096,7 @@ func (s *testSuite4) TestLoadDataIntoPartitionedTable(c *C) { _, _, err := ld.InsertData(context.Background(), nil, []byte("1,2\n3,4\n5,6\n7,8\n9,10\n")) c.Assert(err, IsNil) - err = ld.CheckAndInsertOneBatch() + err = ld.CheckAndInsertOneBatch(context.Background()) c.Assert(err, IsNil) ld.SetMessage() err = ctx.StmtCommit() diff --git a/kv/fault_injection.go b/kv/fault_injection.go index 6760516c3e97b..c2815e0030038 100644 --- a/kv/fault_injection.go +++ b/kv/fault_injection.go @@ -98,13 +98,13 @@ func (t *InjectedTransaction) Get(k Key) ([]byte, error) { } // BatchGet returns an error if cfg.getError is set. -func (t *InjectedTransaction) BatchGet(keys []Key) (map[string][]byte, error) { +func (t *InjectedTransaction) BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) { t.cfg.RLock() defer t.cfg.RUnlock() if t.cfg.getError != nil { return nil, t.cfg.getError } - return t.Transaction.BatchGet(keys) + return t.Transaction.BatchGet(ctx, keys) } // Commit returns an error if cfg.commitError is set. @@ -134,11 +134,11 @@ func (t *InjectedSnapshot) Get(k Key) ([]byte, error) { } // BatchGet returns an error if cfg.getError is set. -func (t *InjectedSnapshot) BatchGet(keys []Key) (map[string][]byte, error) { +func (t *InjectedSnapshot) BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) { t.cfg.RLock() defer t.cfg.RUnlock() if t.cfg.getError != nil { return nil, t.cfg.getError } - return t.Snapshot.BatchGet(keys) + return t.Snapshot.BatchGet(ctx, keys) } diff --git a/kv/fault_injection_test.go b/kv/fault_injection_test.go index afcfd74c2edd0..362d6d04dec4a 100644 --- a/kv/fault_injection_test.go +++ b/kv/fault_injection_test.go @@ -47,11 +47,11 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { c.Assert(err.Error(), Equals, err1.Error()) c.Assert(b, IsNil) - bs, err := snap.BatchGet(nil) + bs, err := snap.BatchGet(context.Background(), nil) c.Assert(err.Error(), Equals, err1.Error()) c.Assert(bs, IsNil) - bs, err = txn.BatchGet(nil) + bs, err = txn.BatchGet(context.Background(), nil) c.Assert(err.Error(), Equals, err1.Error()) c.Assert(bs, IsNil) @@ -71,7 +71,7 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { c.Assert(err, IsNil) c.Assert(b, IsNil) - bs, err = txn.BatchGet(nil) + bs, err = txn.BatchGet(context.Background(), nil) c.Assert(err, IsNil) c.Assert(bs, IsNil) @@ -79,7 +79,7 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { c.Assert(terror.ErrorEqual(kv.ErrNotExist, err), IsTrue) c.Assert(b, IsNil) - bs, err = snap.BatchGet([]kv.Key{[]byte("a")}) + bs, err = snap.BatchGet(context.Background(), []kv.Key{[]byte("a")}) c.Assert(err, IsNil) c.Assert(len(bs), Equals, 0) diff --git a/kv/kv.go b/kv/kv.go index 67daa736110ef..50b65cefb6b9d 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -156,7 +156,7 @@ type Transaction interface { // SetVars sets variables to the transaction. SetVars(vars *Variables) // BatchGet gets kv from the memory buffer of statement and transaction, and the kv storage. - BatchGet(keys []Key) (map[string][]byte, error) + BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) IsPessimistic() bool } @@ -250,7 +250,7 @@ type Response interface { type Snapshot interface { Retriever // BatchGet gets a batch of values from snapshot. - BatchGet(keys []Key) (map[string][]byte, error) + BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) // SetPriority snapshot set the priority SetPriority(priority int) } diff --git a/kv/mock.go b/kv/mock.go index 9fc2c44dd982d..7223b2be8800d 100644 --- a/kv/mock.go +++ b/kv/mock.go @@ -66,7 +66,7 @@ func (t *mockTxn) Get(k Key) ([]byte, error) { return nil, nil } -func (t *mockTxn) BatchGet(keys []Key) (map[string][]byte, error) { +func (t *mockTxn) BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) { return nil, nil } @@ -207,7 +207,7 @@ func (s *mockSnapshot) SetPriority(priority int) { } -func (s *mockSnapshot) BatchGet(keys []Key) (map[string][]byte, error) { +func (s *mockSnapshot) BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) { m := make(map[string][]byte) for _, k := range keys { v, err := s.store.Get(k) diff --git a/kv/mock_test.go b/kv/mock_test.go index 74be6c61db31d..329171c8e5ccc 100644 --- a/kv/mock_test.go +++ b/kv/mock_test.go @@ -32,7 +32,7 @@ func (s testMockSuite) TestInterface(c *C) { c.Check(err, IsNil) snapshot, err := storage.GetSnapshot(version) c.Check(err, IsNil) - _, err = snapshot.BatchGet([]Key{Key("abc"), Key("def")}) + _, err = snapshot.BatchGet(context.Background(), []Key{Key("abc"), Key("def")}) c.Check(err, IsNil) snapshot.SetPriority(0) diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 070060777789b..84baf9e28e162 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -2184,26 +2184,43 @@ func (b *PlanBuilder) buildDDL(ctx context.Context, node ast.DDLNode) (Plan, err return p, nil } +const ( + // TraceFormatRow indicates row tracing format. + TraceFormatRow = "row" + // TraceFormatJSON indicates json tracing format. + TraceFormatJSON = "json" + // TraceFormatLog indicates log tracing format. + TraceFormatLog = "log" +) + // buildTrace builds a trace plan. Inside this method, it first optimize the // underlying query and then constructs a schema, which will be used to constructs // rows result. func (b *PlanBuilder) buildTrace(trace *ast.TraceStmt) (Plan, error) { p := &Trace{StmtNode: trace.Stmt, Format: trace.Format} switch trace.Format { - case "row": + case TraceFormatRow: retFields := []string{"operation", "duration", "spanID"} schema := expression.NewSchema(make([]*expression.Column, 0, len(retFields))...) schema.Append(buildColumn("", "operation", mysql.TypeString, mysql.MaxBlobWidth)) schema.Append(buildColumn("", "startTS", mysql.TypeString, mysql.MaxBlobWidth)) schema.Append(buildColumn("", "duration", mysql.TypeString, mysql.MaxBlobWidth)) p.SetSchema(schema) - case "json": + case TraceFormatJSON: retFields := []string{"json"} schema := expression.NewSchema(make([]*expression.Column, 0, len(retFields))...) schema.Append(buildColumn("", "operation", mysql.TypeString, mysql.MaxBlobWidth)) p.SetSchema(schema) + case TraceFormatLog: + retFields := []string{"time", "event", "tags", "spanName", "spanID"} + schema := expression.NewSchema(make([]*expression.Column, 0, len(retFields))...) + schema.Append(buildColumn("", "time", mysql.TypeTimestamp, mysql.MaxBlobWidth)) + schema.Append(buildColumn("", "event", mysql.TypeString, mysql.MaxBlobWidth)) + schema.Append(buildColumn("", "tags", mysql.TypeString, mysql.MaxBlobWidth)) + schema.Append(buildColumn("", "spanName", mysql.TypeString, mysql.MaxBlobWidth)) + p.SetSchema(schema) default: - return nil, errors.New("trace format should be one of 'row' or 'json'") + return nil, errors.New("trace format should be one of 'row', 'log' or 'json'") } return p, nil } diff --git a/server/conn.go b/server/conn.go index 6b3767d3156a9..c3d9da6e0abcf 100644 --- a/server/conn.go +++ b/server/conn.go @@ -1055,7 +1055,7 @@ func insertDataWithCommit(ctx context.Context, prevData, curData []byte, loadDat if !reachLimit { break } - err := loadDataInfo.CheckAndInsertOneBatch() + err := loadDataInfo.CheckAndInsertOneBatch(ctx) if err != nil { return nil, err } @@ -1123,7 +1123,7 @@ func (cc *clientConn) handleLoadData(ctx context.Context, loadDataInfo *executor if err != nil { loadDataInfo.Ctx.StmtRollback() } else { - err = loadDataInfo.CheckAndInsertOneBatch() + err = loadDataInfo.CheckAndInsertOneBatch(ctx) if err == nil { err = loadDataInfo.Ctx.StmtCommit() } diff --git a/session/session.go b/session/session.go index 4ab10550ed299..a383398dffd45 100644 --- a/session/session.go +++ b/session/session.go @@ -444,6 +444,11 @@ func (s *session) doCommitWithRetry(ctx context.Context) error { txnSize = s.txn.Size() isPessimistic = s.txn.IsPessimistic() } + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("session.doCommitWitRetry", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } err := s.doCommit(ctx) if err != nil { commitRetryLimit := s.sessionVars.RetryLimit @@ -1029,6 +1034,7 @@ func (s *session) Execute(ctx context.Context, sql string) (recordSets []sqlexec span1 := span.Tracer().StartSpan("session.Execute", opentracing.ChildOf(span.Context())) defer span1.Finish() ctx = opentracing.ContextWithSpan(ctx, span1) + logutil.Eventf(ctx, "execute: %s", sql) } if recordSets, err = s.execute(ctx, sql); err != nil { s.sessionVars.StmtCtx.AppendError(err) diff --git a/session/txn.go b/session/txn.go index fca3be468b7f4..11a1d17e0dbb3 100644 --- a/session/txn.go +++ b/session/txn.go @@ -228,7 +228,7 @@ func (st *TxnState) Get(k kv.Key) ([]byte, error) { } // BatchGet overrides the Transaction interface. -func (st *TxnState) BatchGet(keys []kv.Key) (map[string][]byte, error) { +func (st *TxnState) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]byte, error) { bufferValues := make([][]byte, len(keys)) shrinkKeys := make([]kv.Key, 0, len(keys)) for i, key := range keys { @@ -244,7 +244,7 @@ func (st *TxnState) BatchGet(keys []kv.Key) (map[string][]byte, error) { bufferValues[i] = val } } - storageValues, err := st.Transaction.BatchGet(shrinkKeys) + storageValues, err := st.Transaction.BatchGet(ctx, shrinkKeys) if err != nil { return nil, err } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 9c750a2678abf..edceb74cbc48a 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -819,10 +819,10 @@ func (c *twoPhaseCommitter) pessimisticRollbackKeys(bo *Backoffer, keys [][]byte func (c *twoPhaseCommitter) executeAndWriteFinishBinlog(ctx context.Context) error { err := c.execute(ctx) if err != nil { - c.writeFinishBinlog(binlog.BinlogType_Rollback, 0) + c.writeFinishBinlog(ctx, binlog.BinlogType_Rollback, 0) } else { c.txn.commitTS = c.commitTS - c.writeFinishBinlog(binlog.BinlogType_Commit, int64(c.commitTS)) + c.writeFinishBinlog(ctx, binlog.BinlogType_Commit, int64(c.commitTS)) } return errors.Trace(err) } @@ -854,7 +854,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) error { } }() - binlogChan := c.prewriteBinlog() + binlogChan := c.prewriteBinlog(ctx) prewriteBo := NewBackoffer(ctx, prewriteMaxBackoff).WithVars(c.txn.vars) start := time.Now() err := c.prewriteKeys(prewriteBo, c.keys) @@ -874,6 +874,7 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) error { } start = time.Now() + logutil.Event(ctx, "start get commit ts") commitTS, err := c.store.getTimestampWithRetry(NewBackoffer(ctx, tsoMaxBackoff).WithVars(c.txn.vars)) if err != nil { logutil.Logger(ctx).Warn("2PC get commitTS failed", @@ -882,6 +883,8 @@ func (c *twoPhaseCommitter) execute(ctx context.Context) error { return errors.Trace(err) } c.detail.GetCommitTsTime = time.Since(start) + logutil.Event(ctx, "finish get commit ts") + logutil.SetTag(ctx, "commitTs", commitTS) // check commitTS if commitTS <= c.startTS { @@ -948,12 +951,13 @@ func (c *twoPhaseCommitter) checkSchemaValid() error { return nil } -func (c *twoPhaseCommitter) prewriteBinlog() chan error { +func (c *twoPhaseCommitter) prewriteBinlog(ctx context.Context) chan error { if !c.shouldWriteBinlog() { return nil } ch := make(chan error, 1) go func() { + logutil.Eventf(ctx, "start prewrite binlog") binInfo := c.txn.us.GetOption(kv.BinlogInfo).(*binloginfo.BinlogInfo) bin := binInfo.Data bin.StartTs = int64(c.startTS) @@ -961,12 +965,13 @@ func (c *twoPhaseCommitter) prewriteBinlog() chan error { bin.PrewriteKey = c.keys[0] } err := binInfo.WriteBinlog(c.store.clusterID) + logutil.Eventf(ctx, "finish prewrite binlog") ch <- errors.Trace(err) }() return ch } -func (c *twoPhaseCommitter) writeFinishBinlog(tp binlog.BinlogType, commitTS int64) { +func (c *twoPhaseCommitter) writeFinishBinlog(ctx context.Context, tp binlog.BinlogType, commitTS int64) { if !c.shouldWriteBinlog() { return } @@ -975,11 +980,13 @@ func (c *twoPhaseCommitter) writeFinishBinlog(tp binlog.BinlogType, commitTS int binInfo.Data.CommitTs = commitTS binInfo.Data.PrewriteValue = nil go func() { + logutil.Eventf(ctx, "start write finish binlog") err := binInfo.WriteBinlog(c.store.clusterID) if err != nil { logutil.BgLogger().Error("failed to write binlog", zap.Error(err)) } + logutil.Eventf(ctx, "finish write finish binlog") }() } diff --git a/store/tikv/client_batch.go b/store/tikv/client_batch.go index 2721c34e99586..4fda6bb2083fd 100644 --- a/store/tikv/client_batch.go +++ b/store/tikv/client_batch.go @@ -299,6 +299,7 @@ func (c *batchCommandsClient) batchRecvLoop(cfg config.TiKVClient, tikvTransport panic("batchRecvLoop receives a unknown response") } entry := value.(*batchCommandsEntry) + logutil.Eventf(entry.ctx, "receive %T response with other %d batched requests from %s", responses[i].GetCmd(), len(responses), c.target) if atomic.LoadInt32(&entry.canceled) == 0 { // Put the response only if the request is not canceled. entry.res <- responses[i] @@ -338,6 +339,7 @@ func (c *batchCommandsClient) reCreateStreamingClient(err error) (stopped bool) } type batchCommandsEntry struct { + ctx context.Context req *tikvpb.BatchCommandsRequest_Request res chan *tikvpb.BatchCommandsResponse_Response @@ -478,6 +480,7 @@ func sendBatchRequest( timeout time.Duration, ) (*tikvrpc.Response, error) { entry := &batchCommandsEntry{ + ctx: ctx, req: req, res: make(chan *tikvpb.BatchCommandsResponse_Response, 1), canceled: 0, diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index f2379cd142f87..c729b031f1081 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -159,7 +159,7 @@ func (s *testLockSuite) TestScanLockResolveWithBatchGet(c *C) { ver, err := s.store.CurrentVersion() c.Assert(err, IsNil) snapshot := newTiKVSnapshot(s.store, ver) - m, err := snapshot.BatchGet(keys) + m, err := snapshot.BatchGet(context.Background(), keys) c.Assert(err, IsNil) c.Assert(len(m), Equals, int('z'-'a'+1)) for ch := byte('a'); ch <= byte('z'); ch++ { diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index 7f398af45a97e..36827778095e9 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -360,6 +360,7 @@ func (c *RegionCache) findRegionByKey(bo *Backoffer, key []byte, isEndKey bool) // no region data, return error if failure. return nil, err } + logutil.Eventf(bo.ctx, "load region %d from pd, due to cache-miss", lr.GetID()) r = lr c.mu.Lock() c.insertRegionToCache(r) @@ -372,6 +373,7 @@ func (c *RegionCache) findRegionByKey(bo *Backoffer, key []byte, isEndKey bool) logutil.Logger(bo.ctx).Error("load region failure", zap.ByteString("key", key), zap.Error(err)) } else { + logutil.Eventf(bo.ctx, "load region %d from pd, due to need-reload", lr.GetID()) r = lr c.mu.Lock() c.insertRegionToCache(r) diff --git a/store/tikv/region_request.go b/store/tikv/region_request.go index 45c0d53667e44..95ffd4d239974 100644 --- a/store/tikv/region_request.go +++ b/store/tikv/region_request.go @@ -110,6 +110,7 @@ func (s *RegionRequestSender) SendReqCtx(bo *Backoffer, req *tikvrpc.Request, re return resp, nil, err } + logutil.Eventf(bo.ctx, "send %s request to region %d at %s", req.Type, regionID.id, ctx.Addr) s.storeAddr = ctx.Addr resp, retry, err := s.sendReqToRegion(bo, ctx, req, timeout) if err != nil { diff --git a/store/tikv/safepoint_test.go b/store/tikv/safepoint_test.go index fecacd642ad47..f05b6de85c420 100644 --- a/store/tikv/safepoint_test.go +++ b/store/tikv/safepoint_test.go @@ -114,7 +114,7 @@ func (s *testSafePointSuite) TestSafePoint(c *C) { s.waitUntilErrorPlugIn(txn4.startTS) snapshot := newTiKVSnapshot(s.store, kv.Version{Ver: txn4.StartTS()}) - _, batchgeterr := snapshot.BatchGet(keys) + _, batchgeterr := snapshot.BatchGet(context.Background(), keys) c.Assert(batchgeterr, NotNil) isFallBehind = terror.ErrorEqual(errors.Cause(geterr2), ErrGCTooEarly) isMayFallBehind = terror.ErrorEqual(errors.Cause(geterr2), ErrPDServerTimeout.GenWithStackByArgs("start timestamp may fall behind safe point")) diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index 258856fb6eb3b..a05a05fb69428 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -74,7 +74,7 @@ func (s *tikvSnapshot) SetPriority(priority int) { // BatchGet gets all the keys' value from kv-server and returns a map contains key/value pairs. // The map will not contain nonexistent keys. -func (s *tikvSnapshot) BatchGet(keys []kv.Key) (map[string][]byte, error) { +func (s *tikvSnapshot) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]byte, error) { m := make(map[string][]byte) if len(keys) == 0 { return m, nil @@ -85,7 +85,7 @@ func (s *tikvSnapshot) BatchGet(keys []kv.Key) (map[string][]byte, error) { // We want [][]byte instead of []kv.Key, use some magic to save memory. bytesKeys := *(*[][]byte)(unsafe.Pointer(&keys)) - ctx := context.WithValue(context.Background(), txnStartKey, s.version.Ver) + ctx = context.WithValue(ctx, txnStartKey, s.version.Ver) bo := NewBackoffer(ctx, batchGetMaxBackoff).WithVars(s.vars) // Create a map to collect key-values from region servers. diff --git a/store/tikv/snapshot_test.go b/store/tikv/snapshot_test.go index 770b43e7351ce..6c15cf1ac8249 100644 --- a/store/tikv/snapshot_test.go +++ b/store/tikv/snapshot_test.go @@ -68,7 +68,7 @@ func (s *testSnapshotSuite) beginTxn(c *C) *tikvTxn { func (s *testSnapshotSuite) checkAll(keys []kv.Key, c *C) { txn := s.beginTxn(c) snapshot := newTiKVSnapshot(s.store, kv.Version{Ver: txn.StartTS()}) - m, err := snapshot.BatchGet(keys) + m, err := snapshot.BatchGet(context.Background(), keys) c.Assert(err, IsNil) scan, err := txn.Iter(encodeKey(s.prefix, ""), nil) diff --git a/store/tikv/txn.go b/store/tikv/txn.go index 8e14c0980993f..ce7ec609850b5 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -160,9 +160,9 @@ func (txn *tikvTxn) Get(k kv.Key) ([]byte, error) { return ret, nil } -func (txn *tikvTxn) BatchGet(keys []kv.Key) (map[string][]byte, error) { +func (txn *tikvTxn) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]byte, error) { if txn.IsReadOnly() { - return txn.snapshot.BatchGet(keys) + return txn.snapshot.BatchGet(ctx, keys) } bufferValues := make([][]byte, len(keys)) shrinkKeys := make([]kv.Key, 0, len(keys)) @@ -179,7 +179,7 @@ func (txn *tikvTxn) BatchGet(keys []kv.Key) (map[string][]byte, error) { bufferValues[i] = val } } - storageValues, err := txn.snapshot.BatchGet(shrinkKeys) + storageValues, err := txn.snapshot.BatchGet(ctx, shrinkKeys) if err != nil { return nil, errors.Trace(err) } diff --git a/util/logutil/log.go b/util/logutil/log.go index b2392282e1e1a..dc674e666d34c 100644 --- a/util/logutil/log.go +++ b/util/logutil/log.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "github.com/opentracing/opentracing-go" + tlog "github.com/opentracing/opentracing-go/log" "github.com/pingcap/errors" zaplog "github.com/pingcap/log" log "github.com/sirupsen/logrus" @@ -356,3 +358,27 @@ func WithKeyValue(ctx context.Context, key, value string) context.Context { } return context.WithValue(ctx, ctxLogKey, logger.With(zap.String(key, value))) } + +// TraceEventKey presents the TraceEventKey in span log. +const TraceEventKey = "event" + +// Event records event in current tracing span. +func Event(ctx context.Context, event string) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span.LogFields(tlog.String(TraceEventKey, event)) + } +} + +// Eventf records event in current tracing span with format support. +func Eventf(ctx context.Context, format string, args ...interface{}) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span.LogFields(tlog.String(TraceEventKey, fmt.Sprintf(format, args...))) + } +} + +// SetTag sets tag kv-pair in current tracing span +func SetTag(ctx context.Context, key string, value interface{}) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span.SetTag(key, value) + } +} From 1595c0101ef61d6c94ce094f2a54bb2812e79b1b Mon Sep 17 00:00:00 2001 From: lauhg <582215603@qq.com> Date: Fri, 9 Aug 2019 01:11:48 -0500 Subject: [PATCH 181/196] *: replace `context.WithValue` string key to typed `struct{}` key (#11675) --- kv/txn.go | 3 --- plugin/audit.go | 8 ++++---- sessionctx/context.go | 4 +++- store/tikv/backoff.go | 4 +++- util/execdetails/execdetails.go | 4 +++- util/logutil/log.go | 4 ++-- util/testkit/ctestkit.go | 4 ++-- 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/kv/txn.go b/kv/txn.go index cde53cf6c37d4..f62fa4d5a54bd 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -25,9 +25,6 @@ import ( "go.uber.org/zap" ) -// ContextKey is the type of context's key -type ContextKey string - // RunInNewTxn will run the f in a new transaction environment. func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error { var ( diff --git a/plugin/audit.go b/plugin/audit.go index f1471562fc657..644ea591b349e 100644 --- a/plugin/audit.go +++ b/plugin/audit.go @@ -85,7 +85,7 @@ type AuditManifest struct { OnParseEvent func(ctx context.Context, sctx *variable.SessionVars, event ParseEvent) error } -const ( - // ExecStartTimeCtxKey indicates stmt start execution time. - ExecStartTimeCtxKey = "ExecStartTime" -) +type execStartTimeCtxKeyType struct{} + +// ExecStartTimeCtxKey indicates stmt start execution time. +var ExecStartTimeCtxKey = execStartTimeCtxKeyType{} diff --git a/sessionctx/context.go b/sessionctx/context.go index 5b430dc938352..5589832918b70 100644 --- a/sessionctx/context.go +++ b/sessionctx/context.go @@ -124,8 +124,10 @@ const ( LastExecuteDDL basicCtxType = 3 ) +type connIDCtxKeyType struct{} + // ConnID is the key in context. -const ConnID kv.ContextKey = "conn ID" +var ConnID = connIDCtxKeyType{} // SetCommitCtx sets the variables for context before commit a transaction. func SetCommitCtx(ctx context.Context, sessCtx Context) context.Context { diff --git a/store/tikv/backoff.go b/store/tikv/backoff.go index 8bc5da9b5bc01..4a5bf3b97a643 100644 --- a/store/tikv/backoff.go +++ b/store/tikv/backoff.go @@ -233,8 +233,10 @@ type Backoffer struct { vars *kv.Variables } +type txnStartCtxKeyType struct{} + // txnStartKey is a key for transaction start_ts info in context.Context. -const txnStartKey = "_txn_start_key" +var txnStartKey = txnStartCtxKeyType{} // NewBackoffer creates a Backoffer with maximum sleep time(in ms). func NewBackoffer(ctx context.Context, maxSleep int) *Backoffer { diff --git a/util/execdetails/execdetails.go b/util/execdetails/execdetails.go index 9fa7a6d05ae4e..e707023895d73 100644 --- a/util/execdetails/execdetails.go +++ b/util/execdetails/execdetails.go @@ -26,8 +26,10 @@ import ( "go.uber.org/zap" ) +type commitDetailCtxKeyType struct{} + // CommitDetailCtxKey presents CommitDetail info key in context. -const CommitDetailCtxKey = "commitDetail" +var CommitDetailCtxKey = commitDetailCtxKeyType{} // ExecDetails contains execution detail information. type ExecDetails struct { diff --git a/util/logutil/log.go b/util/logutil/log.go index dc674e666d34c..d7c020a5896a1 100644 --- a/util/logutil/log.go +++ b/util/logutil/log.go @@ -319,9 +319,9 @@ func SetLevel(level string) error { return nil } -type ctxKeyType int +type ctxLogKeyType struct{} -const ctxLogKey ctxKeyType = iota +var ctxLogKey = ctxLogKeyType{} // Logger gets a contextual logger from current context. // contextual logger will output common fields from context. diff --git a/util/testkit/ctestkit.go b/util/testkit/ctestkit.go index d32091aff44ef..5be49f330d492 100644 --- a/util/testkit/ctestkit.go +++ b/util/testkit/ctestkit.go @@ -29,9 +29,9 @@ import ( "github.com/pingcap/tidb/util/sqlexec" ) -type contextKeyType int +type sessionCtxKeyType struct{} -const sessionKey contextKeyType = iota +var sessionKey = sessionCtxKeyType{} func getSession(ctx context.Context) session.Session { s := ctx.Value(sessionKey) From d210889a6c928ef5a21c440de28c8a0a93fbccd3 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Fri, 9 Aug 2019 14:22:03 +0800 Subject: [PATCH 182/196] *: trace the execution of the insert operation (#11667) --- ddl/column_test.go | 2 +- distsql/distsql.go | 7 ++++++ executor/batch_checker.go | 16 +++++++++----- executor/insert.go | 32 +++++++++++++++++---------- executor/insert_common.go | 8 +++---- executor/load_data.go | 4 ++-- executor/point_get.go | 10 ++++----- executor/replace.go | 18 +++++++-------- executor/table_reader.go | 7 ++++++ executor/union_scan.go | 22 +++++++++--------- executor/update.go | 6 ++--- executor/write.go | 30 ++++++++++++++++--------- kv/buffer_store.go | 10 ++++++--- kv/buffer_store_test.go | 9 ++++---- kv/fault_injection.go | 8 +++---- kv/fault_injection_test.go | 8 +++---- kv/kv.go | 2 +- kv/mem_buffer_test.go | 5 +++-- kv/memdb_buffer.go | 3 ++- kv/mock.go | 8 +++---- kv/mock_test.go | 2 +- kv/union_store.go | 12 +++++----- kv/union_store_test.go | 14 +++++++----- kv/utils.go | 7 +++--- kv/utils_test.go | 6 +++-- session/txn.go | 8 +++---- store/store_test.go | 10 ++++----- store/tikv/2pc.go | 13 +++++++++++ store/tikv/2pc_fail_test.go | 4 ++-- store/tikv/2pc_test.go | 20 ++++++++--------- store/tikv/client.go | 7 ++++++ store/tikv/gcworker/gc_worker_test.go | 4 ++-- store/tikv/isolation_test.go | 2 +- store/tikv/kv.go | 7 ++++++ store/tikv/lock_test.go | 2 +- store/tikv/safepoint_test.go | 4 ++-- store/tikv/scan_test.go | 2 +- store/tikv/snapshot.go | 4 ++-- store/tikv/split_test.go | 4 ++-- store/tikv/store_fail_test.go | 2 +- store/tikv/store_test.go | 4 ++-- store/tikv/ticlient_test.go | 6 ++--- store/tikv/txn.go | 19 +++++++++++++--- structure/hash.go | 7 +++--- structure/list.go | 9 ++++---- structure/string.go | 3 ++- table/index.go | 11 +++++++++ table/tables/index.go | 25 ++++++++++++++++----- table/tables/partition_test.go | 12 +++++----- table/tables/tables.go | 29 +++++++++++++++--------- util/admin/admin.go | 2 +- util/prefix_helper_test.go | 2 +- 52 files changed, 301 insertions(+), 177 deletions(-) diff --git a/ddl/column_test.go b/ddl/column_test.go index 2ec1e2a5d3584..b5a7dab8d7690 100644 --- a/ddl/column_test.go +++ b/ddl/column_test.go @@ -280,7 +280,7 @@ func (s *testColumnSuite) checkColumnKVExist(ctx sessionctx.Context, t table.Tab if err != nil { return errors.Trace(err) } - data, err := txn.Get(key) + data, err := txn.Get(context.TODO(), key) if !isExist { if terror.ErrorEqual(err, kv.ErrNotExist) { return nil diff --git a/distsql/distsql.go b/distsql/distsql.go index e34449f9e5b4f..57909b5457c2b 100644 --- a/distsql/distsql.go +++ b/distsql/distsql.go @@ -17,6 +17,7 @@ import ( "context" "fmt" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/metrics" @@ -33,6 +34,12 @@ const ( // Select sends a DAG request, returns SelectResult. // In kvReq, KeyRanges is required, Concurrency/KeepOrder/Desc/IsolationLevel/Priority are optional. func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fieldTypes []*types.FieldType, fb *statistics.QueryFeedback) (SelectResult, error) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("distsql.Select", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + // For testing purpose. if hook := ctx.Value("CheckSelectRequestHook"); hook != nil { hook.(func(*kv.Request))(kvReq) diff --git a/executor/batch_checker.go b/executor/batch_checker.go index be604ceca480b..65272865b8daf 100644 --- a/executor/batch_checker.go +++ b/executor/batch_checker.go @@ -16,6 +16,7 @@ package executor import ( "context" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/expression" @@ -89,7 +90,7 @@ func (b *batchChecker) encodeNewRow(ctx sessionctx.Context, t table.Table, row [ // getKeysNeedCheck gets keys converted from to-be-insert rows to record keys and unique index keys, // which need to be checked whether they are duplicate keys. -func (b *batchChecker) getKeysNeedCheck(ctx sessionctx.Context, t table.Table, rows [][]types.Datum) ([]toBeCheckedRow, error) { +func (b *batchChecker) getKeysNeedCheck(ctx context.Context, sctx sessionctx.Context, t table.Table, rows [][]types.Datum) ([]toBeCheckedRow, error) { nUnique := 0 for _, v := range t.WritableIndices() { if v.Meta().Unique { @@ -111,7 +112,7 @@ func (b *batchChecker) getKeysNeedCheck(ctx sessionctx.Context, t table.Table, r var err error for _, row := range rows { - toBeCheckRows, err = b.getKeysNeedCheckOneRow(ctx, t, row, nUnique, handleCol, toBeCheckRows) + toBeCheckRows, err = b.getKeysNeedCheckOneRow(sctx, t, row, nUnique, handleCol, toBeCheckRows) if err != nil { return nil, err } @@ -191,7 +192,7 @@ func (b *batchChecker) getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Ta // batchGetInsertKeys uses batch-get to fetch all key-value pairs to be checked for ignore or duplicate key update. func (b *batchChecker) batchGetInsertKeys(ctx context.Context, sctx sessionctx.Context, t table.Table, newRows [][]types.Datum) (err error) { // Get keys need to be checked. - b.toBeCheckedRows, err = b.getKeysNeedCheck(sctx, t, newRows) + b.toBeCheckedRows, err = b.getKeysNeedCheck(ctx, sctx, t, newRows) if err != nil { return err } @@ -251,6 +252,11 @@ func (b *batchChecker) initDupOldRowFromUniqueKey(ctx context.Context, sctx sess // initDupOldRowValue initializes dupOldRowValues which contain the to-be-updated rows from storage. func (b *batchChecker) initDupOldRowValue(ctx context.Context, sctx sessionctx.Context, t table.Table, newRows [][]types.Datum) error { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("batchCheck.initDupOldRowValue", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } b.dupOldRowValues = make(map[string][]byte, len(newRows)) b.initDupOldRowFromHandleKey() return b.initDupOldRowFromUniqueKey(ctx, sctx, newRows) @@ -270,8 +276,8 @@ func (b *batchChecker) fillBackKeys(t table.Table, row toBeCheckedRow, handle in } // deleteDupKeys picks primary/unique key-value pairs from rows and remove them from the dupKVs -func (b *batchChecker) deleteDupKeys(ctx sessionctx.Context, t table.Table, rows [][]types.Datum) error { - cleanupRows, err := b.getKeysNeedCheck(ctx, t, rows) +func (b *batchChecker) deleteDupKeys(ctx context.Context, sctx sessionctx.Context, t table.Table, rows [][]types.Datum) error { + cleanupRows, err := b.getKeysNeedCheck(ctx, sctx, t, rows) if err != nil { return err } diff --git a/executor/insert.go b/executor/insert.go index c905e7c554f60..5a13a82bd9c76 100644 --- a/executor/insert.go +++ b/executor/insert.go @@ -16,6 +16,8 @@ package executor import ( "context" "fmt" + + "github.com/opentracing/opentracing-go" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/kv" @@ -76,7 +78,7 @@ func (e *InsertExec) exec(ctx context.Context, rows [][]types.Datum) error { } } else { for _, row := range rows { - if _, err := e.addRecord(row); err != nil { + if _, err := e.addRecord(ctx, row); err != nil { return err } } @@ -104,7 +106,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D if err != nil { return err } - err = e.updateDupRow(r, handle, e.OnDuplicate) + err = e.updateDupRow(ctx, r, handle, e.OnDuplicate) if err != nil { return err } @@ -117,7 +119,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D if err != nil { return err } - err = e.updateDupRow(r, handle, e.OnDuplicate) + err = e.updateDupRow(ctx, r, handle, e.OnDuplicate) if err != nil { return err } @@ -130,7 +132,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D // and key-values should be filled back to dupOldRowValues for the further row check, // due to there may be duplicate keys inside the insert statement. if newRows[i] != nil { - newHandle, err := e.addRecord(newRows[i]) + newHandle, err := e.addRecord(ctx, newRows[i]) if err != nil { return err } @@ -169,14 +171,20 @@ func (e *InsertExec) Open(ctx context.Context) error { } // updateDupRow updates a duplicate row to a new row. -func (e *InsertExec) updateDupRow(row toBeCheckedRow, handle int64, onDuplicate []*expression.Assignment) error { +func (e *InsertExec) updateDupRow(ctx context.Context, row toBeCheckedRow, handle int64, onDuplicate []*expression.Assignment) error { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("InsertExec.updateDupRow", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + oldRow, err := e.getOldRow(e.ctx, row.t, handle, e.GenExprs) if err != nil { logutil.BgLogger().Error("get old row failed when insert on dup", zap.Int64("handle", handle), zap.String("toBeInsertedRow", types.DatumsToStrNoErr(row.row))) return err } // Do update row. - updatedRow, handleChanged, newHandle, err := e.doDupRowUpdate(handle, oldRow, row.row, onDuplicate) + updatedRow, handleChanged, newHandle, err := e.doDupRowUpdate(ctx, handle, oldRow, row.row, onDuplicate) if e.ctx.GetSessionVars().StmtCtx.DupKeyAsWarning && kv.ErrKeyExists.Equal(err) { e.ctx.GetSessionVars().StmtCtx.AppendWarning(err) return nil @@ -184,11 +192,11 @@ func (e *InsertExec) updateDupRow(row toBeCheckedRow, handle int64, onDuplicate if err != nil { return err } - return e.updateDupKeyValues(handle, newHandle, handleChanged, oldRow, updatedRow) + return e.updateDupKeyValues(ctx, handle, newHandle, handleChanged, oldRow, updatedRow) } // doDupRowUpdate updates the duplicate row. -func (e *InsertExec) doDupRowUpdate(handle int64, oldRow []types.Datum, newRow []types.Datum, +func (e *InsertExec) doDupRowUpdate(ctx context.Context, handle int64, oldRow []types.Datum, newRow []types.Datum, cols []*expression.Assignment) ([]types.Datum, bool, int64, error) { assignFlag := make([]bool, len(e.Table.WritableCols())) // See http://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_values @@ -212,7 +220,7 @@ func (e *InsertExec) doDupRowUpdate(handle int64, oldRow []types.Datum, newRow [ } newData := row4Update[:len(oldRow)] - _, handleChanged, newHandle, err := updateRecord(e.ctx, handle, oldRow, newData, assignFlag, e.Table, true) + _, handleChanged, newHandle, err := updateRecord(ctx, e.ctx, handle, oldRow, newData, assignFlag, e.Table, true) if err != nil { return nil, false, 0, err } @@ -220,15 +228,15 @@ func (e *InsertExec) doDupRowUpdate(handle int64, oldRow []types.Datum, newRow [ } // updateDupKeyValues updates the dupKeyValues for further duplicate key check. -func (e *InsertExec) updateDupKeyValues(oldHandle int64, newHandle int64, +func (e *InsertExec) updateDupKeyValues(ctx context.Context, oldHandle int64, newHandle int64, handleChanged bool, oldRow []types.Datum, updatedRow []types.Datum) error { // There is only one row per update. - fillBackKeysInRows, err := e.getKeysNeedCheck(e.ctx, e.Table, [][]types.Datum{updatedRow}) + fillBackKeysInRows, err := e.getKeysNeedCheck(ctx, e.ctx, e.Table, [][]types.Datum{updatedRow}) if err != nil { return err } // Delete old keys and fill back new key-values of the updated row. - err = e.deleteDupKeys(e.ctx, e.Table, [][]types.Datum{oldRow}) + err = e.deleteDupKeys(ctx, e.ctx, e.Table, [][]types.Datum{oldRow}) if err != nil { return err } diff --git a/executor/insert_common.go b/executor/insert_common.go index f467603c7d18c..c976315442ed3 100644 --- a/executor/insert_common.go +++ b/executor/insert_common.go @@ -582,7 +582,7 @@ func (e *InsertValues) handleWarning(err error) { // batchCheckAndInsert checks rows with duplicate errors. // All duplicate rows will be ignored and appended as duplicate warnings. -func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.Datum, addRecord func(row []types.Datum) (int64, error)) error { +func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.Datum, addRecord func(ctx context.Context, row []types.Datum) (int64, error)) error { // all the rows will be checked, so it is safe to set BatchCheck = true e.ctx.GetSessionVars().StmtCtx.BatchCheck = true err := e.batchGetInsertKeys(ctx, e.ctx, e.Table, rows) @@ -611,7 +611,7 @@ func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.D // There may be duplicate keys inside the insert statement. if !skip { e.ctx.GetSessionVars().StmtCtx.AddCopiedRows(1) - _, err = addRecord(rows[i]) + _, err = addRecord(ctx, rows[i]) if err != nil { return err } @@ -626,7 +626,7 @@ func (e *InsertValues) batchCheckAndInsert(ctx context.Context, rows [][]types.D return nil } -func (e *InsertValues) addRecord(row []types.Datum) (int64, error) { +func (e *InsertValues) addRecord(ctx context.Context, row []types.Datum) (int64, error) { txn, err := e.ctx.Txn(true) if err != nil { return 0, err @@ -634,7 +634,7 @@ func (e *InsertValues) addRecord(row []types.Datum) (int64, error) { if !e.ctx.GetSessionVars().ConstraintCheckInPlace { txn.SetOption(kv.PresumeKeyNotExists, nil) } - h, err := e.Table.AddRecord(e.ctx, row) + h, err := e.Table.AddRecord(e.ctx, row, table.WithCtx(ctx)) txn.DelOption(kv.PresumeKeyNotExists) if err != nil { return 0, err diff --git a/executor/load_data.go b/executor/load_data.go index 3a525d8f94cea..0f055b33bd396 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -332,11 +332,11 @@ func (e *LoadDataInfo) colsToRow(ctx context.Context, cols []field) []types.Datu return row } -func (e *LoadDataInfo) addRecordLD(row []types.Datum) (int64, error) { +func (e *LoadDataInfo) addRecordLD(ctx context.Context, row []types.Datum) (int64, error) { if row == nil { return 0, nil } - h, err := e.addRecord(row) + h, err := e.addRecord(ctx, row) if err != nil { e.handleWarning(err) } diff --git a/executor/point_get.go b/executor/point_get.go index 6e5f78d97e226..e34c74f30c5d1 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -97,7 +97,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { return err1 } - handleVal, err1 := e.get(idxKey) + handleVal, err1 := e.get(ctx, idxKey) if err1 != nil && !kv.ErrNotExist.Equal(err1) { return err1 } @@ -125,7 +125,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { } key := tablecodec.EncodeRowKeyWithHandle(e.tblInfo.ID, e.handle) - val, err := e.get(key) + val, err := e.get(ctx, key) if err != nil && !kv.ErrNotExist.Equal(err) { return err } @@ -175,7 +175,7 @@ func (e *PointGetExecutor) encodeIndexKey() (_ []byte, err error) { return tablecodec.EncodeIndexSeekKey(e.tblInfo.ID, e.idxInfo.ID, encodedIdxVals), nil } -func (e *PointGetExecutor) get(key kv.Key) (val []byte, err error) { +func (e *PointGetExecutor) get(ctx context.Context, key kv.Key) (val []byte, err error) { txn, err := e.ctx.Txn(true) if err != nil { return nil, err @@ -183,7 +183,7 @@ func (e *PointGetExecutor) get(key kv.Key) (val []byte, err error) { if txn != nil && txn.Valid() && !txn.IsReadOnly() { // We cannot use txn.Get directly here because the snapshot in txn and the snapshot of e.snapshot may be // different for pessimistic transaction. - val, err = txn.GetMemBuffer().Get(key) + val, err = txn.GetMemBuffer().Get(ctx, key) if err == nil { return val, err } @@ -192,7 +192,7 @@ func (e *PointGetExecutor) get(key kv.Key) (val []byte, err error) { } // fallthrough to snapshot get. } - return e.snapshot.Get(key) + return e.snapshot.Get(ctx, key) } func (e *PointGetExecutor) decodeRowValToChunk(rowVal []byte, chk *chunk.Chunk) error { diff --git a/executor/replace.go b/executor/replace.go index 829ecf3eb8744..9fc306ddbf923 100644 --- a/executor/replace.go +++ b/executor/replace.go @@ -52,7 +52,7 @@ func (e *ReplaceExec) Open(ctx context.Context) error { // removeRow removes the duplicate row and cleanup its keys in the key-value map, // but if the to-be-removed row equals to the to-be-added row, no remove or add things to do. -func (e *ReplaceExec) removeRow(handle int64, r toBeCheckedRow) (bool, error) { +func (e *ReplaceExec) removeRow(ctx context.Context, handle int64, r toBeCheckedRow) (bool, error) { newRow := r.row oldRow, err := e.batchChecker.getOldRow(e.ctx, r.t, handle, e.GenExprs) if err != nil { @@ -75,7 +75,7 @@ func (e *ReplaceExec) removeRow(handle int64, r toBeCheckedRow) (bool, error) { e.ctx.GetSessionVars().StmtCtx.AddAffectedRows(1) // Cleanup keys map, because the record was removed. - err = e.deleteDupKeys(e.ctx, r.t, [][]types.Datum{oldRow}) + err = e.deleteDupKeys(ctx, e.ctx, r.t, [][]types.Datum{oldRow}) if err != nil { return false, err } @@ -83,14 +83,14 @@ func (e *ReplaceExec) removeRow(handle int64, r toBeCheckedRow) (bool, error) { } // replaceRow removes all duplicate rows for one row, then inserts it. -func (e *ReplaceExec) replaceRow(r toBeCheckedRow) error { +func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow) error { if r.handleKey != nil { if _, found := e.dupKVs[string(r.handleKey.newKV.key)]; found { handle, err := tablecodec.DecodeRowKey(r.handleKey.newKV.key) if err != nil { return err } - rowUnchanged, err := e.removeRow(handle, r) + rowUnchanged, err := e.removeRow(ctx, handle, r) if err != nil { return err } @@ -102,7 +102,7 @@ func (e *ReplaceExec) replaceRow(r toBeCheckedRow) error { // Keep on removing duplicated rows. for { - rowUnchanged, foundDupKey, err := e.removeIndexRow(r) + rowUnchanged, foundDupKey, err := e.removeIndexRow(ctx, r) if err != nil { return err } @@ -116,7 +116,7 @@ func (e *ReplaceExec) replaceRow(r toBeCheckedRow) error { } // No duplicated rows now, insert the row. - newHandle, err := e.addRecord(r.row) + newHandle, err := e.addRecord(ctx, r.row) if err != nil { return err } @@ -130,14 +130,14 @@ func (e *ReplaceExec) replaceRow(r toBeCheckedRow) error { // 2. bool: true when found the duplicated key. This only means that duplicated key was found, // and the row was removed. // 3. error: the error. -func (e *ReplaceExec) removeIndexRow(r toBeCheckedRow) (bool, bool, error) { +func (e *ReplaceExec) removeIndexRow(ctx context.Context, r toBeCheckedRow) (bool, bool, error) { for _, uk := range r.uniqueKeys { if val, found := e.dupKVs[string(uk.newKV.key)]; found { handle, err := tables.DecodeHandle(val) if err != nil { return false, found, err } - rowUnchanged, err := e.removeRow(handle, r) + rowUnchanged, err := e.removeRow(ctx, handle, r) if err != nil { return false, found, err } @@ -172,7 +172,7 @@ func (e *ReplaceExec) exec(ctx context.Context, newRows [][]types.Datum) error { } e.ctx.GetSessionVars().StmtCtx.AddRecordRows(uint64(len(newRows))) for _, r := range e.toBeCheckedRows { - err = e.replaceRow(r) + err = e.replaceRow(ctx, r) if err != nil { return err } diff --git a/executor/table_reader.go b/executor/table_reader.go index 9bc481bca6c4e..ec61cbf30dc8c 100644 --- a/executor/table_reader.go +++ b/executor/table_reader.go @@ -17,6 +17,7 @@ import ( "context" "fmt" + "github.com/opentracing/opentracing-go" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/kv" @@ -83,6 +84,12 @@ type TableReaderExecutor struct { // Open initialzes necessary variables for using this executor. func (e *TableReaderExecutor) Open(ctx context.Context) error { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("TableReaderExecutor.Open", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + e.memTracker = memory.NewTracker(e.id, e.ctx.GetSessionVars().MemQuotaDistSQL) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) diff --git a/executor/union_scan.go b/executor/union_scan.go index c9377cbd5e49c..9594697a72ef5 100644 --- a/executor/union_scan.go +++ b/executor/union_scan.go @@ -133,7 +133,7 @@ func (us *UnionScanExec) open(ctx context.Context) error { us.memIdxHandles = mIdxReader.memIdxHandles case *IndexLookUpExecutor: us.memIdxHandles = set.NewInt64Set() - err = us.buildAndSortAddedRows(x.table) + err = us.buildAndSortAddedRows(ctx, x.table) } if err != nil { return err @@ -219,7 +219,7 @@ func (us *UnionScanExec) getSnapshotRow(ctx context.Context) ([]types.Datum, err for row := iter.Begin(); row != iter.End(); row = iter.Next() { snapshotHandle := row.GetInt64(us.belowHandleIndex) if _, ok := us.dirty.deletedRows[snapshotHandle]; ok { - err = us.getMissIndexRowsByHandle(snapshotHandle) + err = us.getMissIndexRowsByHandle(ctx, snapshotHandle) if err != nil { return nil, err } @@ -228,7 +228,7 @@ func (us *UnionScanExec) getSnapshotRow(ctx context.Context) ([]types.Datum, err if _, ok := us.dirty.addedRows[snapshotHandle]; ok { // If src handle appears in added rows, it means there is conflict and the transaction will fail to // commit, but for simplicity, we don't handle it here. - err = us.getMissIndexRowsByHandle(snapshotHandle) + err = us.getMissIndexRowsByHandle(ctx, snapshotHandle) if err != nil { return nil, err } @@ -243,7 +243,7 @@ func (us *UnionScanExec) getSnapshotRow(ctx context.Context) ([]types.Datum, err // For index reader and index look up reader, update doesn't write index to txn memBuffer when the idx column // is unchanged. So the `memIndexReader` and `memIndexLookUpReader` can't read the index from txn memBuffer. // This function is used to get the missing row by the handle if the handle is in dirtyTable.addedRows. -func (us *UnionScanExec) getMissIndexRowsByHandle(handle int64) error { +func (us *UnionScanExec) getMissIndexRowsByHandle(ctx context.Context, handle int64) error { reader := us.children[0] switch reader.(type) { case *TableReaderExecutor: @@ -256,7 +256,7 @@ func (us *UnionScanExec) getMissIndexRowsByHandle(handle int64) error { if us.memIdxHandles.Exist(handle) { return nil } - memRow, err := us.getMemRow(handle) + memRow, err := us.getMemRow(ctx, handle) if memRow == nil || err != nil { return err } @@ -320,13 +320,13 @@ func (us *UnionScanExec) compare(a, b []types.Datum) (int, error) { } // rowWithColsInTxn gets the row from the transaction buffer. -func (us *UnionScanExec) rowWithColsInTxn(t table.Table, h int64) ([]types.Datum, error) { +func (us *UnionScanExec) rowWithColsInTxn(ctx context.Context, t table.Table, h int64) ([]types.Datum, error) { key := t.RecordKey(h) txn, err := us.ctx.Txn(true) if err != nil { return nil, err } - value, err := txn.GetMemBuffer().Get(key) + value, err := txn.GetMemBuffer().Get(ctx, key) if err != nil { return nil, err } @@ -337,8 +337,8 @@ func (us *UnionScanExec) rowWithColsInTxn(t table.Table, h int64) ([]types.Datum return decodeRowData(us.ctx, us.table.Meta(), us.columns, colIDs, h, []byte{}, value) } -func (us *UnionScanExec) getMemRow(h int64) ([]types.Datum, error) { - data, err := us.rowWithColsInTxn(us.table, h) +func (us *UnionScanExec) getMemRow(ctx context.Context, h int64) ([]types.Datum, error) { + data, err := us.rowWithColsInTxn(ctx, us.table, h) if err != nil { return nil, err } @@ -354,12 +354,12 @@ func (us *UnionScanExec) getMemRow(h int64) ([]types.Datum, error) { } // TODO: remove `buildAndSortAddedRows` functions and `DirtyTable`. -func (us *UnionScanExec) buildAndSortAddedRows(t table.Table) error { +func (us *UnionScanExec) buildAndSortAddedRows(ctx context.Context, t table.Table) error { us.addedRows = make([][]types.Datum, 0, len(us.dirty.addedRows)) mutableRow := chunk.MutRowFromTypes(retTypes(us)) for h := range us.dirty.addedRows { us.memIdxHandles.Insert(h) - newData, err := us.rowWithColsInTxn(t, h) + newData, err := us.rowWithColsInTxn(ctx, t, h) if err != nil { return err } diff --git a/executor/update.go b/executor/update.go index 11538c1c8df12..deac8b587ef19 100644 --- a/executor/update.go +++ b/executor/update.go @@ -51,7 +51,7 @@ type UpdateExec struct { evalBuffer chunk.MutRow } -func (e *UpdateExec) exec(schema *expression.Schema) ([]types.Datum, error) { +func (e *UpdateExec) exec(ctx context.Context, schema *expression.Schema) ([]types.Datum, error) { assignFlag, err := e.getUpdateColumns(e.ctx, schema.Len()) if err != nil { return nil, err @@ -99,7 +99,7 @@ func (e *UpdateExec) exec(schema *expression.Schema) ([]types.Datum, error) { } // Update row - changed, _, _, err1 := updateRecord(e.ctx, handle, oldData, newTableData, flags, tbl, false) + changed, _, _, err1 := updateRecord(ctx, e.ctx, handle, oldData, newTableData, flags, tbl, false) if err1 == nil { e.updatedRowKeys[content.TblID][handle] = changed continue @@ -138,7 +138,7 @@ func (e *UpdateExec) Next(ctx context.Context, req *chunk.Chunk) error { e.ctx.GetSessionVars().StmtCtx.AddRecordRows(uint64(len(e.rows))) for { - row, err := e.exec(e.children[0].Schema()) + row, err := e.exec(ctx, e.children[0].Schema()) if err != nil { return err } diff --git a/executor/write.go b/executor/write.go index e551ad558dc38..a09fb33849bc8 100644 --- a/executor/write.go +++ b/executor/write.go @@ -14,8 +14,10 @@ package executor import ( + "context" "strings" + "github.com/opentracing/opentracing-go" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" @@ -43,9 +45,15 @@ var ( // 2. handleChanged (bool) : is the handle changed after the update. // 3. newHandle (int64) : if handleChanged == true, the newHandle means the new handle after update. // 4. err (error) : error in the update. -func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datum, modified []bool, t table.Table, +func updateRecord(ctx context.Context, sctx sessionctx.Context, h int64, oldData, newData []types.Datum, modified []bool, t table.Table, onDup bool) (bool, bool, int64, error) { - sc := ctx.GetSessionVars().StmtCtx + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("executor.updateRecord", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + + sc := sctx.GetSessionVars().StmtCtx changed, handleChanged := false, false // onUpdateSpecified is for "UPDATE SET ts_field = old_value", the // timestamp field is explicitly set, but not changed in fact. @@ -60,7 +68,7 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu for i, col := range t.Cols() { if modified[i] { // Cast changed fields with respective columns. - v, err := table.CastValue(ctx, newData[i], col.ToInfo()) + v, err := table.CastValue(sctx, newData[i], col.ToInfo()) if err != nil { return false, false, 0, err } @@ -91,7 +99,7 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu if err != nil { return false, false, 0, err } - if err = t.RebaseAutoID(ctx, recordID, true); err != nil { + if err = t.RebaseAutoID(sctx, recordID, true); err != nil { return false, false, 0, err } } @@ -112,7 +120,7 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu // If no changes, nothing to do, return directly. if !changed { // See https://dev.mysql.com/doc/refman/5.7/en/mysql-real-connect.html CLIENT_FOUND_ROWS - if ctx.GetSessionVars().ClientCapability&mysql.ClientFoundRows > 0 { + if sctx.GetSessionVars().ClientCapability&mysql.ClientFoundRows > 0 { sc.AddAffectedRows(1) } return false, false, 0, nil @@ -121,7 +129,7 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu // 4. Fill values into on-update-now fields, only if they are really changed. for i, col := range t.Cols() { if mysql.HasOnUpdateNowFlag(col.Flag) && !modified[i] && !onUpdateSpecified[i] { - if v, err := expression.GetTimeValue(ctx, strings.ToUpper(ast.CurrentTimestamp), col.Tp, col.Decimal); err == nil { + if v, err := expression.GetTimeValue(sctx, strings.ToUpper(ast.CurrentTimestamp), col.Tp, col.Decimal); err == nil { newData[i] = v modified[i] = true } else { @@ -136,19 +144,19 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu if sc.DupKeyAsWarning { // For `UPDATE IGNORE`/`INSERT IGNORE ON DUPLICATE KEY UPDATE` // If the new handle exists, this will avoid to remove the record. - err = tables.CheckHandleExists(ctx, t, newHandle, newData) + err = tables.CheckHandleExists(ctx, sctx, t, newHandle, newData) if err != nil { return false, handleChanged, newHandle, err } } - if err = t.RemoveRecord(ctx, h, oldData); err != nil { + if err = t.RemoveRecord(sctx, h, oldData); err != nil { return false, false, 0, err } // the `affectedRows` is increased when adding new record. if sc.DupKeyAsWarning { - newHandle, err = t.AddRecord(ctx, newData, table.IsUpdate, table.SkipHandleCheck) + newHandle, err = t.AddRecord(sctx, newData, table.IsUpdate, table.SkipHandleCheck, table.WithCtx(ctx)) } else { - newHandle, err = t.AddRecord(ctx, newData, table.IsUpdate) + newHandle, err = t.AddRecord(sctx, newData, table.IsUpdate, table.WithCtx(ctx)) } if err != nil { @@ -159,7 +167,7 @@ func updateRecord(ctx sessionctx.Context, h int64, oldData, newData []types.Datu } } else { // Update record to new value and update index. - if err = t.UpdateRecord(ctx, h, oldData, newData, modified); err != nil { + if err = t.UpdateRecord(sctx, h, oldData, newData, modified); err != nil { return false, false, 0, err } if onDup { diff --git a/kv/buffer_store.go b/kv/buffer_store.go index 7b48e1e476273..3a7054193a9d7 100644 --- a/kv/buffer_store.go +++ b/kv/buffer_store.go @@ -13,6 +13,10 @@ package kv +import ( + "context" +) + var ( // DefaultTxnMembufCap is the default transaction membuf capability. DefaultTxnMembufCap = 4 * 1024 @@ -56,10 +60,10 @@ func (s *BufferStore) SetCap(cap int) { } // Get implements the Retriever interface. -func (s *BufferStore) Get(k Key) ([]byte, error) { - val, err := s.MemBuffer.Get(k) +func (s *BufferStore) Get(ctx context.Context, k Key) ([]byte, error) { + val, err := s.MemBuffer.Get(ctx, k) if IsErrNotFound(err) { - val, err = s.r.Get(k) + val, err = s.r.Get(ctx, k) } if err != nil { return nil, err diff --git a/kv/buffer_store_test.go b/kv/buffer_store_test.go index bc3bc4eb7f16d..1de50aef3f328 100644 --- a/kv/buffer_store_test.go +++ b/kv/buffer_store_test.go @@ -15,6 +15,7 @@ package kv import ( "bytes" + "context" "fmt" . "github.com/pingcap/check" @@ -28,13 +29,13 @@ var _ = Suite(testBufferStoreSuite{}) func (s testBufferStoreSuite) TestGetSet(c *C) { bs := NewBufferStore(&mockSnapshot{NewMemDbBuffer(DefaultTxnMembufCap)}, DefaultTxnMembufCap) key := Key("key") - _, err := bs.Get(key) + _, err := bs.Get(context.TODO(), key) c.Check(err, NotNil) err = bs.Set(key, []byte("value")) c.Check(err, IsNil) - value, err := bs.Get(key) + value, err := bs.Get(context.TODO(), key) c.Check(err, IsNil) c.Check(bytes.Compare(value, []byte("value")), Equals, 0) } @@ -78,11 +79,11 @@ func (s testBufferStoreSuite) TestBufferStore(c *C) { err = bs.Delete(key) c.Check(err, IsNil) - _, err = bs.Get(key) + _, err = bs.Get(context.TODO(), key) c.Check(terror.ErrorEqual(err, ErrNotExist), IsTrue) bs.Reset() - _, err = bs.Get(key) + _, err = bs.Get(context.TODO(), key) c.Check(terror.ErrorEqual(err, ErrNotExist), IsTrue) } diff --git a/kv/fault_injection.go b/kv/fault_injection.go index c2815e0030038..ca187d36bb190 100644 --- a/kv/fault_injection.go +++ b/kv/fault_injection.go @@ -88,13 +88,13 @@ type InjectedTransaction struct { } // Get returns an error if cfg.getError is set. -func (t *InjectedTransaction) Get(k Key) ([]byte, error) { +func (t *InjectedTransaction) Get(ctx context.Context, k Key) ([]byte, error) { t.cfg.RLock() defer t.cfg.RUnlock() if t.cfg.getError != nil { return nil, t.cfg.getError } - return t.Transaction.Get(k) + return t.Transaction.Get(ctx, k) } // BatchGet returns an error if cfg.getError is set. @@ -124,13 +124,13 @@ type InjectedSnapshot struct { } // Get returns an error if cfg.getError is set. -func (t *InjectedSnapshot) Get(k Key) ([]byte, error) { +func (t *InjectedSnapshot) Get(ctx context.Context, k Key) ([]byte, error) { t.cfg.RLock() defer t.cfg.RUnlock() if t.cfg.getError != nil { return nil, t.cfg.getError } - return t.Snapshot.Get(k) + return t.Snapshot.Get(ctx, k) } // BatchGet returns an error if cfg.getError is set. diff --git a/kv/fault_injection_test.go b/kv/fault_injection_test.go index 362d6d04dec4a..87ed8b5ad62b2 100644 --- a/kv/fault_injection_test.go +++ b/kv/fault_injection_test.go @@ -40,10 +40,10 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { ver := kv.Version{Ver: 1} snap, err := storage.GetSnapshot(ver) c.Assert(err, IsNil) - b, err := txn.Get([]byte{'a'}) + b, err := txn.Get(context.TODO(), []byte{'a'}) c.Assert(err.Error(), Equals, err1.Error()) c.Assert(b, IsNil) - b, err = snap.Get([]byte{'a'}) + b, err = snap.Get(context.TODO(), []byte{'a'}) c.Assert(err.Error(), Equals, err1.Error()) c.Assert(b, IsNil) @@ -67,7 +67,7 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { snap, err = storage.GetSnapshot(ver) c.Assert(err, IsNil) - b, err = txn.Get([]byte{'a'}) + b, err = txn.Get(context.TODO(), []byte{'a'}) c.Assert(err, IsNil) c.Assert(b, IsNil) @@ -75,7 +75,7 @@ func (s testFaultInjectionSuite) TestFaultInjectionBasic(c *C) { c.Assert(err, IsNil) c.Assert(bs, IsNil) - b, err = snap.Get([]byte{'a'}) + b, err = snap.Get(context.TODO(), []byte{'a'}) c.Assert(terror.ErrorEqual(kv.ErrNotExist, err), IsTrue) c.Assert(b, IsNil) diff --git a/kv/kv.go b/kv/kv.go index 50b65cefb6b9d..a826a9756b63b 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -83,7 +83,7 @@ var ( type Retriever interface { // Get gets the value for key k from kv store. // If corresponding kv pair does not exist, it returns nil and ErrNotExist. - Get(k Key) ([]byte, error) + Get(ctx context.Context, k Key) ([]byte, error) // Iter creates an Iterator positioned on the first entry that k <= entry's key. // If such entry is not found, it returns an invalid Iterator with no error. // It yields only keys that < upperBound. If upperBound is nil, it means the upperBound is unbounded. diff --git a/kv/mem_buffer_test.go b/kv/mem_buffer_test.go index 4032fe8f6cc63..767fb064e01f7 100644 --- a/kv/mem_buffer_test.go +++ b/kv/mem_buffer_test.go @@ -16,6 +16,7 @@ package kv import ( + "context" "fmt" "math/rand" "testing" @@ -121,7 +122,7 @@ func checkNewIterator(c *C, buffer MemBuffer) { func mustGet(c *C, buffer MemBuffer) { for i := startIndex; i < testCount; i++ { s := encodeInt(i * indexStep) - val, err := buffer.Get(s) + val, err := buffer.Get(context.TODO(), s) c.Assert(err, IsNil) c.Assert(string(val), Equals, string(s)) } @@ -284,7 +285,7 @@ func benchmarkSetGet(b *testing.B, buffer MemBuffer, data [][]byte) { buffer.Set(k, k) } for _, k := range data { - buffer.Get(k) + buffer.Get(context.TODO(), k) } } } diff --git a/kv/memdb_buffer.go b/kv/memdb_buffer.go index 6fc48a3ce922d..f4fff437433a6 100644 --- a/kv/memdb_buffer.go +++ b/kv/memdb_buffer.go @@ -16,6 +16,7 @@ package kv import ( + "context" "sync/atomic" "github.com/pingcap/errors" @@ -77,7 +78,7 @@ func (m *memDbBuffer) IterReverse(k Key) (Iterator, error) { } // Get returns the value associated with key. -func (m *memDbBuffer) Get(k Key) ([]byte, error) { +func (m *memDbBuffer) Get(ctx context.Context, k Key) ([]byte, error) { v, err := m.db.Get(k) if terror.ErrorEqual(err, leveldb.ErrNotFound) { return nil, ErrNotExist diff --git a/kv/mock.go b/kv/mock.go index 7223b2be8800d..ca50a069ec813 100644 --- a/kv/mock.go +++ b/kv/mock.go @@ -62,7 +62,7 @@ func (t *mockTxn) IsReadOnly() bool { func (t *mockTxn) StartTS() uint64 { return uint64(0) } -func (t *mockTxn) Get(k Key) ([]byte, error) { +func (t *mockTxn) Get(ctx context.Context, k Key) ([]byte, error) { return nil, nil } @@ -199,8 +199,8 @@ type mockSnapshot struct { store MemBuffer } -func (s *mockSnapshot) Get(k Key) ([]byte, error) { - return s.store.Get(k) +func (s *mockSnapshot) Get(ctx context.Context, k Key) ([]byte, error) { + return s.store.Get(ctx, k) } func (s *mockSnapshot) SetPriority(priority int) { @@ -210,7 +210,7 @@ func (s *mockSnapshot) SetPriority(priority int) { func (s *mockSnapshot) BatchGet(ctx context.Context, keys []Key) (map[string][]byte, error) { m := make(map[string][]byte) for _, k := range keys { - v, err := s.store.Get(k) + v, err := s.store.Get(ctx, k) if IsErrNotFound(err) { continue } diff --git a/kv/mock_test.go b/kv/mock_test.go index 329171c8e5ccc..0555feb38f018 100644 --- a/kv/mock_test.go +++ b/kv/mock_test.go @@ -47,7 +47,7 @@ func (s testMockSuite) TestInterface(c *C) { transaction.StartTS() transaction.DelOption(Option(23)) if transaction.IsReadOnly() { - _, err = transaction.Get(Key("lock")) + _, err = transaction.Get(context.TODO(), Key("lock")) c.Check(err, IsNil) err = transaction.Set(Key("lock"), []byte{}) c.Check(err, IsNil) diff --git a/kv/union_store.go b/kv/union_store.go index 3fb233b755d35..846308771f701 100644 --- a/kv/union_store.go +++ b/kv/union_store.go @@ -13,6 +13,8 @@ package kv +import "context" + // UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. // Also, it provides some transaction related utilities. type UnionStore interface { @@ -114,12 +116,12 @@ type lazyMemBuffer struct { cap int } -func (lmb *lazyMemBuffer) Get(k Key) ([]byte, error) { +func (lmb *lazyMemBuffer) Get(ctx context.Context, k Key) ([]byte, error) { if lmb.mb == nil { return nil, ErrNotExist } - return lmb.mb.Get(k) + return lmb.mb.Get(ctx, k) } func (lmb *lazyMemBuffer) Set(key Key, value []byte) error { @@ -177,8 +179,8 @@ func (lmb *lazyMemBuffer) SetCap(cap int) { } // Get implements the Retriever interface. -func (us *unionStore) Get(k Key) ([]byte, error) { - v, err := us.MemBuffer.Get(k) +func (us *unionStore) Get(ctx context.Context, k Key) ([]byte, error) { + v, err := us.MemBuffer.Get(ctx, k) if IsErrNotFound(err) { if _, ok := us.opts.Get(PresumeKeyNotExists); ok { e, ok := us.opts.Get(PresumeKeyNotExistsError) @@ -189,7 +191,7 @@ func (us *unionStore) Get(k Key) ([]byte, error) { } return nil, ErrNotExist } - v, err = us.BufferStore.r.Get(k) + v, err = us.BufferStore.r.Get(ctx, k) } if err != nil { return v, err diff --git a/kv/union_store_test.go b/kv/union_store_test.go index fa33ea8a57bc7..0f1ffa0705997 100644 --- a/kv/union_store_test.go +++ b/kv/union_store_test.go @@ -14,6 +14,8 @@ package kv import ( + "context" + . "github.com/pingcap/check" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/util/testleak" @@ -35,12 +37,12 @@ func (s *testUnionStoreSuite) TestGetSet(c *C) { defer testleak.AfterTest(c)() err := s.store.Set([]byte("1"), []byte("1")) c.Assert(err, IsNil) - v, err := s.us.Get([]byte("1")) + v, err := s.us.Get(context.TODO(), []byte("1")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("1")) err = s.us.Set([]byte("1"), []byte("2")) c.Assert(err, IsNil) - v, err = s.us.Get([]byte("1")) + v, err = s.us.Get(context.TODO(), []byte("1")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("2")) c.Assert(s.us.Size(), Equals, 2) @@ -53,12 +55,12 @@ func (s *testUnionStoreSuite) TestDelete(c *C) { c.Assert(err, IsNil) err = s.us.Delete([]byte("1")) c.Assert(err, IsNil) - _, err = s.us.Get([]byte("1")) + _, err = s.us.Get(context.TODO(), []byte("1")) c.Assert(IsErrNotFound(err), IsTrue) err = s.us.Set([]byte("1"), []byte("2")) c.Assert(err, IsNil) - v, err := s.us.Get([]byte("1")) + v, err := s.us.Get(context.TODO(), []byte("1")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("2")) } @@ -130,13 +132,13 @@ func (s *testUnionStoreSuite) TestLazyConditionCheck(c *C) { err = s.store.Set([]byte("2"), []byte("2")) c.Assert(err, IsNil) - v, err := s.us.Get([]byte("1")) + v, err := s.us.Get(context.TODO(), []byte("1")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("1")) s.us.SetOption(PresumeKeyNotExists, nil) s.us.SetOption(PresumeKeyNotExistsError, ErrNotExist) - _, err = s.us.Get([]byte("2")) + _, err = s.us.Get(context.TODO(), []byte("2")) c.Assert(terror.ErrorEqual(err, ErrNotExist), IsTrue, Commentf("err %v", err)) condionPair1 := s.us.LookupConditionPair([]byte("1")) diff --git a/kv/utils.go b/kv/utils.go index 0c503081805dc..e759bcff762c3 100644 --- a/kv/utils.go +++ b/kv/utils.go @@ -14,6 +14,7 @@ package kv import ( + "context" "strconv" "github.com/pingcap/errors" @@ -21,7 +22,7 @@ import ( // IncInt64 increases the value for key k in kv store by step. func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error) { - val, err := rm.Get(k) + val, err := rm.Get(context.TODO(), k) if IsErrNotFound(err) { err = rm.Set(k, []byte(strconv.FormatInt(step, 10))) if err != nil { @@ -47,8 +48,8 @@ func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error) { } // GetInt64 get int64 value which created by IncInt64 method. -func GetInt64(r Retriever, k Key) (int64, error) { - val, err := r.Get(k) +func GetInt64(ctx context.Context, r Retriever, k Key) (int64, error) { + val, err := r.Get(ctx, k) if IsErrNotFound(err) { return 0, nil } diff --git a/kv/utils_test.go b/kv/utils_test.go index 1c189c4a5c57c..1d3cab0ac24dc 100644 --- a/kv/utils_test.go +++ b/kv/utils_test.go @@ -14,6 +14,8 @@ package kv import ( + "context" + . "github.com/pingcap/check" ) @@ -41,13 +43,13 @@ func (s testUtilsSuite) TestIncInt64(c *C) { func (s testUtilsSuite) TestGetInt64(c *C) { mb := NewMemDbBuffer(DefaultTxnMembufCap) key := Key("key") - v, err := GetInt64(mb, key) + v, err := GetInt64(context.TODO(), mb, key) c.Check(v, Equals, int64(0)) c.Check(err, IsNil) _, err = IncInt64(mb, key, 15) c.Check(err, IsNil) - v, err = GetInt64(mb, key) + v, err = GetInt64(context.TODO(), mb, key) c.Check(v, Equals, int64(15)) c.Check(err, IsNil) } diff --git a/session/txn.go b/session/txn.go index 11a1d17e0dbb3..75d914ae7313c 100644 --- a/session/txn.go +++ b/session/txn.go @@ -210,10 +210,10 @@ func (st *TxnState) reset() { } // Get overrides the Transaction interface. -func (st *TxnState) Get(k kv.Key) ([]byte, error) { - val, err := st.buf.Get(k) +func (st *TxnState) Get(ctx context.Context, k kv.Key) ([]byte, error) { + val, err := st.buf.Get(ctx, k) if kv.IsErrNotFound(err) { - val, err = st.Transaction.Get(k) + val, err = st.Transaction.Get(ctx, k) if kv.IsErrNotFound(err) { return nil, err } @@ -232,7 +232,7 @@ func (st *TxnState) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]b bufferValues := make([][]byte, len(keys)) shrinkKeys := make([]kv.Key, 0, len(keys)) for i, key := range keys { - val, err := st.buf.Get(key) + val, err := st.buf.Get(ctx, key) if kv.IsErrNotFound(err) { shrinkKeys = append(shrinkKeys, key) continue diff --git a/store/store_test.go b/store/store_test.go index 65219b13ec9da..50340d2c2c342 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -147,7 +147,7 @@ func checkSeek(c *C, txn kv.Transaction) { func mustNotGet(c *C, txn kv.Transaction) { for i := startIndex; i < testCount; i++ { s := encodeInt(i * indexStep) - _, err := txn.Get(s) + _, err := txn.Get(context.TODO(), s) c.Assert(err, NotNil) } } @@ -155,7 +155,7 @@ func mustNotGet(c *C, txn kv.Transaction) { func mustGet(c *C, txn kv.Transaction) { for i := startIndex; i < testCount; i++ { s := encodeInt(i * indexStep) - val, err := txn.Get(s) + val, err := txn.Get(context.TODO(), s) c.Assert(err, IsNil) c.Assert(string(val), Equals, string(s)) } @@ -407,7 +407,7 @@ func (s *testKVSuite) TestRollback(c *C) { defer txn.Commit(context.Background()) for i := startIndex; i < testCount; i++ { - _, err := txn.Get([]byte(strconv.Itoa(i))) + _, err := txn.Get(context.TODO(), []byte(strconv.Itoa(i))) c.Assert(err, NotNil) } } @@ -550,7 +550,7 @@ func (s *testKVSuite) TestDBClose(c *C) { snap, err := store.GetSnapshot(kv.MaxVersion) c.Assert(err, IsNil) - _, err = snap.Get([]byte("a")) + _, err = snap.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) txn, err = store.Begin() @@ -646,7 +646,7 @@ func (s *testKVSuite) TestIsolationMultiInc(c *C) { err := kv.RunInNewTxn(s.s, false, func(txn kv.Transaction) error { for _, key := range keys { - id, err1 := kv.GetInt64(txn, key) + id, err1 := kv.GetInt64(context.TODO(), txn, key) if err1 != nil { return err1 } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index edceb74cbc48a..70086747225fc 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -22,6 +22,7 @@ import ( "sync/atomic" "time" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" pb "github.com/pingcap/kvproto/pkg/kvrpcpb" @@ -797,10 +798,22 @@ func (c *twoPhaseCommitter) cleanupSingleBatch(bo *Backoffer, batch batchKeys) e } func (c *twoPhaseCommitter) prewriteKeys(bo *Backoffer, keys [][]byte) error { + if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("twoPhaseCommitter.prewriteKeys", opentracing.ChildOf(span.Context())) + defer span1.Finish() + bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + } + return c.doActionOnKeys(bo, actionPrewrite, keys) } func (c *twoPhaseCommitter) commitKeys(bo *Backoffer, keys [][]byte) error { + if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("twoPhaseCommitter.commitKeys", opentracing.ChildOf(span.Context())) + defer span1.Finish() + bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + } + return c.doActionOnKeys(bo, actionCommit, keys) } diff --git a/store/tikv/2pc_fail_test.go b/store/tikv/2pc_fail_test.go index 49624f5be5197..1ceb40d23f78d 100644 --- a/store/tikv/2pc_fail_test.go +++ b/store/tikv/2pc_fail_test.go @@ -109,10 +109,10 @@ func (s *testCommitterSuite) TestFailCommitTimeout(c *C) { c.Assert(err, NotNil) txn2 := s.begin(c) - value, err := txn2.Get([]byte("a")) + value, err := txn2.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(len(value), Greater, 0) - _, err = txn2.Get([]byte("b")) + _, err = txn2.Get(context.TODO(), []byte("b")) c.Assert(err, IsNil) c.Assert(len(value), Greater, 0) } diff --git a/store/tikv/2pc_test.go b/store/tikv/2pc_test.go index 8b173375138f2..1e33d185c5c64 100644 --- a/store/tikv/2pc_test.go +++ b/store/tikv/2pc_test.go @@ -66,7 +66,7 @@ func (s *testCommitterSuite) begin(c *C) *tikvTxn { func (s *testCommitterSuite) checkValues(c *C, m map[string]string) { txn := s.begin(c) for k, v := range m { - val, err := txn.Get([]byte(k)) + val, err := txn.Get(context.TODO(), []byte(k)) c.Assert(err, IsNil) c.Assert(string(val), Equals, v) } @@ -140,7 +140,7 @@ func (s *testCommitterSuite) TestPrewriteRollback(c *C) { c.Assert(err, IsNil) txn2 := s.begin(c) - v, err := txn2.Get([]byte("a")) + v, err := txn2.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("a0")) @@ -163,7 +163,7 @@ func (s *testCommitterSuite) TestPrewriteRollback(c *C) { c.Assert(err, IsNil) txn3 := s.begin(c) - v, err = txn3.Get([]byte("b")) + v, err = txn3.Get(context.TODO(), []byte("b")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("b1")) } @@ -359,7 +359,7 @@ func (s *testCommitterSuite) TestPrewritePrimaryKeyFailed(c *C) { // check a txn := s.begin(c) - v, err := txn.Get([]byte("a")) + v, err := txn.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("a1")) @@ -376,10 +376,10 @@ func (s *testCommitterSuite) TestPrewritePrimaryKeyFailed(c *C) { // txn2 failed with a rollback for record a. txn = s.begin(c) - v, err = txn.Get([]byte("a")) + v, err = txn.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("a1")) - _, err = txn.Get([]byte("b")) + _, err = txn.Get(context.TODO(), []byte("b")) errMsgMustContain(c, err, "key not exist") // clean again, shouldn't be failed when a rollback already exist. @@ -391,7 +391,7 @@ func (s *testCommitterSuite) TestPrewritePrimaryKeyFailed(c *C) { // check the data after rollback twice. txn = s.begin(c) - v, err = txn.Get([]byte("a")) + v, err = txn.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("a1")) @@ -402,7 +402,7 @@ func (s *testCommitterSuite) TestPrewritePrimaryKeyFailed(c *C) { c.Assert(err, IsNil) // check value txn = s.begin(c) - v, err = txn.Get([]byte("a")) + v, err = txn.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte("a3")) } @@ -432,7 +432,7 @@ func (s *testCommitterSuite) TestWrittenKeysOnConflict(c *C) { commiter1.cleanWg.Wait() txn3 := s.begin(c) start := time.Now() - txn3.Get([]byte("y1")) + txn3.Get(context.TODO(), []byte("y1")) totalTime += time.Since(start) txn3.Commit(context.Background()) } @@ -466,7 +466,7 @@ func (s *testCommitterSuite) TestUnsetPrimaryKey(c *C) { txn = s.begin(c) txn.SetOption(kv.Pessimistic, true) txn.SetOption(kv.PresumeKeyNotExists, nil) - _, _ = txn.us.Get(key) + _, _ = txn.us.Get(context.TODO(), key) c.Assert(txn.Set(key, key), IsNil) txn.DelOption(kv.PresumeKeyNotExists) err := txn.LockKeys(context.Background(), txn.startTS, key) diff --git a/store/tikv/client.go b/store/tikv/client.go index 102d8cf4db38d..c1585e0a24d81 100644 --- a/store/tikv/client.go +++ b/store/tikv/client.go @@ -24,6 +24,7 @@ import ( "time" "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/coprocessor" "github.com/pingcap/kvproto/pkg/debugpb" @@ -270,6 +271,12 @@ func (c *rpcClient) closeConns() { // SendRequest sends a Request to server and receives Response. func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("rpcClient.SendRequest", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + start := time.Now() reqType := req.Type.String() storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10) diff --git a/store/tikv/gcworker/gc_worker_test.go b/store/tikv/gcworker/gc_worker_test.go index 4f762242dcc94..28753cc669796 100644 --- a/store/tikv/gcworker/gc_worker_test.go +++ b/store/tikv/gcworker/gc_worker_test.go @@ -112,7 +112,7 @@ func (s *testGCWorkerSuite) mustPut(c *C, key, value string) { func (s *testGCWorkerSuite) mustGet(c *C, key string, ts uint64) string { snap, err := s.store.GetSnapshot(kv.Version{Ver: ts}) c.Assert(err, IsNil) - value, err := snap.Get([]byte(key)) + value, err := snap.Get(context.TODO(), []byte(key)) c.Assert(err, IsNil) return string(value) } @@ -120,7 +120,7 @@ func (s *testGCWorkerSuite) mustGet(c *C, key string, ts uint64) string { func (s *testGCWorkerSuite) mustGetNone(c *C, key string, ts uint64) { snap, err := s.store.GetSnapshot(kv.Version{Ver: ts}) c.Assert(err, IsNil) - _, err = snap.Get([]byte(key)) + _, err = snap.Get(context.TODO(), []byte(key)) c.Assert(err, Equals, kv.ErrNotExist) } diff --git a/store/tikv/isolation_test.go b/store/tikv/isolation_test.go index 8179668f2f444..a1adac43d56d2 100644 --- a/store/tikv/isolation_test.go +++ b/store/tikv/isolation_test.go @@ -92,7 +92,7 @@ func (s *testIsolationSuite) GetWithRetry(c *C, k []byte) readRecord { txn, err := s.store.Begin() c.Assert(err, IsNil) - val, err := txn.Get(k) + val, err := txn.Get(context.TODO(), k) if err == nil { return readRecord{ startTS: txn.StartTS(), diff --git a/store/tikv/kv.go b/store/tikv/kv.go index 1c488961b6d7f..8f2cbe68ea2d7 100644 --- a/store/tikv/kv.go +++ b/store/tikv/kv.go @@ -24,6 +24,7 @@ import ( "time" "github.com/coreos/etcd/clientv3" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" pd "github.com/pingcap/pd/client" @@ -314,6 +315,12 @@ func (s *tikvStore) CurrentVersion() (kv.Version, error) { } func (s *tikvStore) getTimestampWithRetry(bo *Backoffer) (uint64, error) { + if span := opentracing.SpanFromContext(bo.ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("tikvStore.getTimestampWithRetry", opentracing.ChildOf(span.Context())) + defer span1.Finish() + bo.ctx = opentracing.ContextWithSpan(bo.ctx, span1) + } + for { startTS, err := s.oracle.GetTimestamp(bo.ctx) // mockGetTSErrorInRetry should wait MockCommitErrorOnce first, then will run into retry() logic. diff --git a/store/tikv/lock_test.go b/store/tikv/lock_test.go index c729b031f1081..2fa2c8e12c20a 100644 --- a/store/tikv/lock_test.go +++ b/store/tikv/lock_test.go @@ -109,7 +109,7 @@ func (s *testLockSuite) TestScanLockResolveWithGet(c *C) { txn, err := s.store.Begin() c.Assert(err, IsNil) for ch := byte('a'); ch <= byte('z'); ch++ { - v, err := txn.Get([]byte{ch}) + v, err := txn.Get(context.TODO(), []byte{ch}) c.Assert(err, IsNil) c.Assert(v, BytesEquals, []byte{ch}) } diff --git a/store/tikv/safepoint_test.go b/store/tikv/safepoint_test.go index f05b6de85c420..03d84467d23ea 100644 --- a/store/tikv/safepoint_test.go +++ b/store/tikv/safepoint_test.go @@ -83,12 +83,12 @@ func (s *testSafePointSuite) TestSafePoint(c *C) { // for txn get txn2 := s.beginTxn(c) - _, err = txn2.Get(encodeKey(s.prefix, s08d("key", 0))) + _, err = txn2.Get(context.TODO(), encodeKey(s.prefix, s08d("key", 0))) c.Assert(err, IsNil) s.waitUntilErrorPlugIn(txn2.startTS) - _, geterr2 := txn2.Get(encodeKey(s.prefix, s08d("key", 0))) + _, geterr2 := txn2.Get(context.TODO(), encodeKey(s.prefix, s08d("key", 0))) c.Assert(geterr2, NotNil) isFallBehind := terror.ErrorEqual(errors.Cause(geterr2), ErrGCTooEarly) isMayFallBehind := terror.ErrorEqual(errors.Cause(geterr2), ErrPDServerTimeout.GenWithStackByArgs("start timestamp may fall behind safe point")) diff --git a/store/tikv/scan_test.go b/store/tikv/scan_test.go index 605e2c17d28d4..0deeb0751ef48 100644 --- a/store/tikv/scan_test.go +++ b/store/tikv/scan_test.go @@ -101,7 +101,7 @@ func (s *testScanSuite) TestScan(c *C) { } txn2 := s.beginTxn(c) - val, err := txn2.Get(encodeKey(s.prefix, s08d("key", 0))) + val, err := txn2.Get(context.TODO(), encodeKey(s.prefix, s08d("key", 0))) c.Assert(err, IsNil) c.Assert(val, BytesEquals, valueBytes(0)) // Test scan without upperBound diff --git a/store/tikv/snapshot.go b/store/tikv/snapshot.go index a05a05fb69428..98ce8eb8954b3 100644 --- a/store/tikv/snapshot.go +++ b/store/tikv/snapshot.go @@ -217,8 +217,8 @@ func (s *tikvSnapshot) batchGetSingleRegion(bo *Backoffer, batch batchKeys, coll } // Get gets the value for key k from snapshot. -func (s *tikvSnapshot) Get(k kv.Key) ([]byte, error) { - ctx := context.WithValue(context.Background(), txnStartKey, s.version.Ver) +func (s *tikvSnapshot) Get(ctx context.Context, k kv.Key) ([]byte, error) { + ctx = context.WithValue(ctx, txnStartKey, s.version.Ver) val, err := s.get(NewBackoffer(ctx, getMaxBackoff), k) if err != nil { return nil, errors.Trace(err) diff --git a/store/tikv/split_test.go b/store/tikv/split_test.go index 3a40c844b14e4..4ee508282e564 100644 --- a/store/tikv/split_test.go +++ b/store/tikv/split_test.go @@ -97,8 +97,8 @@ func (s *testSplitSuite) TestStaleEpoch(c *C) { mockPDClient.disable() txn = s.begin(c) - _, err = txn.Get([]byte("a")) + _, err = txn.Get(context.TODO(), []byte("a")) c.Assert(err, IsNil) - _, err = txn.Get([]byte("c")) + _, err = txn.Get(context.TODO(), []byte("c")) c.Assert(err, IsNil) } diff --git a/store/tikv/store_fail_test.go b/store/tikv/store_fail_test.go index 17b9038a82a99..bf0f190c71c91 100644 --- a/store/tikv/store_fail_test.go +++ b/store/tikv/store_fail_test.go @@ -44,7 +44,7 @@ func (s *testStoreSuite) TestFailBusyServerKV(c *C) { defer wg.Done() txn, err := s.store.Begin() c.Assert(err, IsNil) - val, err := txn.Get([]byte("key")) + val, err := txn.Get(context.TODO(), []byte("key")) c.Assert(err, IsNil) c.Assert(val, BytesEquals, []byte("value")) }() diff --git a/store/tikv/store_test.go b/store/tikv/store_test.go index 5fff7dd47ccc5..04e0caeaa7d78 100644 --- a/store/tikv/store_test.go +++ b/store/tikv/store_test.go @@ -250,13 +250,13 @@ func (s *testStoreSuite) TestRequestPriority(c *C) { c.Assert(err, IsNil) client.priority = pb.CommandPri_Low txn.SetOption(kv.Priority, kv.PriorityLow) - _, err = txn.Get([]byte("key")) + _, err = txn.Get(context.TODO(), []byte("key")) c.Assert(err, IsNil) // A counter example. client.priority = pb.CommandPri_Low txn.SetOption(kv.Priority, kv.PriorityNormal) - _, err = txn.Get([]byte("key")) + _, err = txn.Get(context.TODO(), []byte("key")) // err is translated to "try again later" by backoffer, so doesn't check error value here. c.Assert(err, NotNil) diff --git a/store/tikv/ticlient_test.go b/store/tikv/ticlient_test.go index 44ca2d40f7f41..e771a9bd61315 100644 --- a/store/tikv/ticlient_test.go +++ b/store/tikv/ticlient_test.go @@ -125,7 +125,7 @@ func (s *testTiclientSuite) TestSingleKey(c *C) { c.Assert(err, IsNil) txn = s.beginTxn(c) - val, err := txn.Get(encodeKey(s.prefix, "key")) + val, err := txn.Get(context.TODO(), encodeKey(s.prefix, "key")) c.Assert(err, IsNil) c.Assert(val, BytesEquals, []byte("value")) @@ -149,7 +149,7 @@ func (s *testTiclientSuite) TestMultiKeys(c *C) { txn = s.beginTxn(c) for i := 0; i < keyNum; i++ { - val, err1 := txn.Get(encodeKey(s.prefix, s08d("key", i))) + val, err1 := txn.Get(context.TODO(), encodeKey(s.prefix, s08d("key", i))) c.Assert(err1, IsNil) c.Assert(val, BytesEquals, valueBytes(i)) } @@ -165,7 +165,7 @@ func (s *testTiclientSuite) TestMultiKeys(c *C) { func (s *testTiclientSuite) TestNotExist(c *C) { txn := s.beginTxn(c) - _, err := txn.Get(encodeKey(s.prefix, "noSuchKey")) + _, err := txn.Get(context.TODO(), encodeKey(s.prefix, "noSuchKey")) c.Assert(err, NotNil) } diff --git a/store/tikv/txn.go b/store/tikv/txn.go index ce7ec609850b5..4c9ad48742a45 100644 --- a/store/tikv/txn.go +++ b/store/tikv/txn.go @@ -20,6 +20,7 @@ import ( "time" "github.com/dgryski/go-farm" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/kv" @@ -139,12 +140,12 @@ func (txn *tikvTxn) Reset() { } // Get implements transaction interface. -func (txn *tikvTxn) Get(k kv.Key) ([]byte, error) { +func (txn *tikvTxn) Get(ctx context.Context, k kv.Key) ([]byte, error) { tikvTxnCmdCountWithGet.Inc() start := time.Now() defer func() { tikvTxnCmdHistogramWithGet.Observe(time.Since(start).Seconds()) }() - ret, err := txn.us.Get(k) + ret, err := txn.us.Get(ctx, k) if kv.IsErrNotFound(err) { return nil, err } @@ -161,13 +162,19 @@ func (txn *tikvTxn) Get(k kv.Key) ([]byte, error) { } func (txn *tikvTxn) BatchGet(ctx context.Context, keys []kv.Key) (map[string][]byte, error) { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("tikvTxn.BatchGet", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + if txn.IsReadOnly() { return txn.snapshot.BatchGet(ctx, keys) } bufferValues := make([][]byte, len(keys)) shrinkKeys := make([]kv.Key, 0, len(keys)) for i, key := range keys { - val, err := txn.GetMemBuffer().Get(key) + val, err := txn.GetMemBuffer().Get(ctx, key) if kv.IsErrNotFound(err) { shrinkKeys = append(shrinkKeys, key) continue @@ -254,6 +261,12 @@ func (txn *tikvTxn) IsPessimistic() bool { } func (txn *tikvTxn) Commit(ctx context.Context) error { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("tikvTxn.Commit", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + if !txn.valid { return kv.ErrInvalidTxn } diff --git a/structure/hash.go b/structure/hash.go index ddad8d69d0344..8d16b626b7520 100644 --- a/structure/hash.go +++ b/structure/hash.go @@ -15,6 +15,7 @@ package structure import ( "bytes" + "context" "encoding/binary" "strconv" @@ -55,7 +56,7 @@ func (t *TxStructure) HSet(key []byte, field []byte, value []byte) error { // HGet gets the value of a hash field. func (t *TxStructure) HGet(key []byte, field []byte) ([]byte, error) { dataKey := t.encodeHashDataKey(key, field) - value, err := t.reader.Get(dataKey) + value, err := t.reader.Get(context.TODO(), dataKey) if kv.ErrNotExist.Equal(err) { err = nil } @@ -317,7 +318,7 @@ func (t *TxStructure) iterReverseHash(key []byte, fn func(k []byte, v []byte) (b } func (t *TxStructure) loadHashMeta(metaKey []byte) (hashMeta, error) { - v, err := t.reader.Get(metaKey) + v, err := t.reader.Get(context.TODO(), metaKey) if kv.ErrNotExist.Equal(err) { err = nil } @@ -339,7 +340,7 @@ func (t *TxStructure) loadHashMeta(metaKey []byte) (hashMeta, error) { } func (t *TxStructure) loadHashValue(dataKey []byte) ([]byte, error) { - v, err := t.reader.Get(dataKey) + v, err := t.reader.Get(context.TODO(), dataKey) if kv.ErrNotExist.Equal(err) { err = nil v = nil diff --git a/structure/list.go b/structure/list.go index 96e24345ae602..079a5d2765a23 100644 --- a/structure/list.go +++ b/structure/list.go @@ -14,6 +14,7 @@ package structure import ( + "context" "encoding/binary" "github.com/pingcap/errors" @@ -111,7 +112,7 @@ func (t *TxStructure) listPop(key []byte, left bool) ([]byte, error) { dataKey := t.encodeListDataKey(key, index) var data []byte - data, err = t.reader.Get(dataKey) + data, err = t.reader.Get(context.TODO(), dataKey) if err != nil { return nil, errors.Trace(err) } @@ -147,7 +148,7 @@ func (t *TxStructure) LGetAll(key []byte) ([][]byte, error) { length := int(meta.RIndex - meta.LIndex) elements := make([][]byte, 0, length) for index := meta.RIndex - 1; index >= meta.LIndex; index-- { - e, err := t.reader.Get(t.encodeListDataKey(key, index)) + e, err := t.reader.Get(context.TODO(), t.encodeListDataKey(key, index)) if err != nil { return nil, errors.Trace(err) } @@ -167,7 +168,7 @@ func (t *TxStructure) LIndex(key []byte, index int64) ([]byte, error) { index = adjustIndex(index, meta.LIndex, meta.RIndex) if index >= meta.LIndex && index < meta.RIndex { - return t.reader.Get(t.encodeListDataKey(key, index)) + return t.reader.Get(context.TODO(), t.encodeListDataKey(key, index)) } return nil, nil } @@ -213,7 +214,7 @@ func (t *TxStructure) LClear(key []byte) error { } func (t *TxStructure) loadListMeta(metaKey []byte) (listMeta, error) { - v, err := t.reader.Get(metaKey) + v, err := t.reader.Get(context.TODO(), metaKey) if kv.ErrNotExist.Equal(err) { err = nil } diff --git a/structure/string.go b/structure/string.go index 38482d15b9b49..b00082e74ea9f 100644 --- a/structure/string.go +++ b/structure/string.go @@ -14,6 +14,7 @@ package structure import ( + "context" "strconv" "github.com/pingcap/errors" @@ -32,7 +33,7 @@ func (t *TxStructure) Set(key []byte, value []byte) error { // Get gets the string value of a key. func (t *TxStructure) Get(key []byte) ([]byte, error) { ek := t.encodeStringDataKey(key) - value, err := t.reader.Get(ek) + value, err := t.reader.Get(context.TODO(), ek) if kv.ErrNotExist.Equal(err) { err = nil } diff --git a/table/index.go b/table/index.go index c05396fc3717d..3912d60003dc2 100644 --- a/table/index.go +++ b/table/index.go @@ -14,6 +14,8 @@ package table import ( + "context" + "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx" @@ -32,6 +34,7 @@ type CreateIdxOpt struct { SkipHandleCheck bool // If true, skip the handle constraint check. SkipCheck bool // If true, skip all the unique indices constraint check. kv.AssertionProto // If not nil, check assertion. + Ctx context.Context } // CreateIdxOptFunc is defined for the Create() method of Index interface. @@ -56,6 +59,14 @@ func WithAssertion(x kv.AssertionProto) CreateIdxOptFunc { } } +// WithCtx returns a CreateIdxFunc. +// This option is used to pass context.Context. +func WithCtx(ctx context.Context) CreateIdxOptFunc { + return func(opt *CreateIdxOpt) { + opt.Ctx = ctx + } +} + // Index is the interface for index data on KV store. type Index interface { // Meta returns IndexInfo. diff --git a/table/tables/index.go b/table/tables/index.go index bf9bd8d96db6a..570fb3c20095e 100644 --- a/table/tables/index.go +++ b/table/tables/index.go @@ -15,10 +15,12 @@ package tables import ( "bytes" + "context" "encoding/binary" "io" "unicode/utf8" + "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" @@ -196,15 +198,15 @@ func (c *index) GenIndexKey(sc *stmtctx.StatementContext, indexedValues []types. // Create creates a new entry in the kvIndex data. // If the index is unique and there is an existing entry with the same key, // Create will return the existing entry's handle as the first return value, ErrKeyExists as the second return value. -func (c *index) Create(ctx sessionctx.Context, rm kv.RetrieverMutator, indexedValues []types.Datum, h int64, opts ...table.CreateIdxOptFunc) (int64, error) { +func (c *index) Create(sctx sessionctx.Context, rm kv.RetrieverMutator, indexedValues []types.Datum, h int64, opts ...table.CreateIdxOptFunc) (int64, error) { var opt table.CreateIdxOpt for _, fn := range opts { fn(&opt) } ss := opt.AssertionProto - writeBufs := ctx.GetSessionVars().GetWriteStmtBufs() - skipCheck := ctx.GetSessionVars().LightningMode || ctx.GetSessionVars().StmtCtx.BatchCheck - key, distinct, err := c.GenIndexKey(ctx.GetSessionVars().StmtCtx, indexedValues, h, writeBufs.IndexKeyBuf) + writeBufs := sctx.GetSessionVars().GetWriteStmtBufs() + skipCheck := sctx.GetSessionVars().LightningMode || sctx.GetSessionVars().StmtCtx.BatchCheck + key, distinct, err := c.GenIndexKey(sctx.GetSessionVars().StmtCtx, indexedValues, h, writeBufs.IndexKeyBuf) if err != nil { return 0, err } @@ -227,8 +229,19 @@ func (c *index) Create(ctx sessionctx.Context, rm kv.RetrieverMutator, indexedVa return 0, err } + ctx := opt.Ctx + if ctx != nil { + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { + span1 := span.Tracer().StartSpan("index.Create", opentracing.ChildOf(span.Context())) + defer span1.Finish() + ctx = opentracing.ContextWithSpan(ctx, span1) + } + } else { + ctx = context.TODO() + } + var value []byte - value, err = rm.Get(key) + value, err = rm.Get(ctx, key) if kv.IsErrNotFound(err) { err = rm.Set(key, EncodeHandle(h)) if ss != nil { @@ -324,7 +337,7 @@ func (c *index) Exist(sc *stmtctx.StatementContext, rm kv.RetrieverMutator, inde return false, 0, err } - value, err := rm.Get(key) + value, err := rm.Get(context.TODO(), key) if kv.IsErrNotFound(err) { return false, 0, nil } diff --git a/table/tables/partition_test.go b/table/tables/partition_test.go index 8d3efd2f8a5d7..85aebd15c1ff9 100644 --- a/table/tables/partition_test.go +++ b/table/tables/partition_test.go @@ -83,10 +83,10 @@ PARTITION BY RANGE ( id ) ( // Check that add record writes to the partition, rather than the table. txn, err := ts.se.Txn(true) c.Assert(err, IsNil) - val, err := txn.Get(tables.PartitionRecordKey(p0.ID, rid)) + val, err := txn.Get(context.TODO(), tables.PartitionRecordKey(p0.ID, rid)) c.Assert(err, IsNil) c.Assert(len(val), Greater, 0) - _, err = txn.Get(tables.PartitionRecordKey(tbInfo.ID, rid)) + _, err = txn.Get(context.TODO(), tables.PartitionRecordKey(tbInfo.ID, rid)) c.Assert(kv.ErrNotExist.Equal(err), IsTrue) // Cover more code. @@ -174,10 +174,10 @@ func (ts *testSuite) TestHashPartitionAddRecord(c *C) { // Check that add record writes to the partition, rather than the table. txn, err := ts.se.Txn(true) c.Assert(err, IsNil) - val, err := txn.Get(tables.PartitionRecordKey(p0.ID, rid)) + val, err := txn.Get(context.TODO(), tables.PartitionRecordKey(p0.ID, rid)) c.Assert(err, IsNil) c.Assert(len(val), Greater, 0) - _, err = txn.Get(tables.PartitionRecordKey(tbInfo.ID, rid)) + _, err = txn.Get(context.TODO(), tables.PartitionRecordKey(tbInfo.ID, rid)) c.Assert(kv.ErrNotExist.Equal(err), IsTrue) // Cover more code. @@ -210,10 +210,10 @@ func (ts *testSuite) TestHashPartitionAddRecord(c *C) { c.Assert(err, IsNil) txn, err = ts.se.Txn(true) c.Assert(err, IsNil) - val, err = txn.Get(tables.PartitionRecordKey(tbInfo.Partition.Definitions[i].ID, rid)) + val, err = txn.Get(context.TODO(), tables.PartitionRecordKey(tbInfo.Partition.Definitions[i].ID, rid)) c.Assert(err, IsNil) c.Assert(len(val), Greater, 0) - _, err = txn.Get(tables.PartitionRecordKey(tbInfo.ID, rid)) + _, err = txn.Get(context.TODO(), tables.PartitionRecordKey(tbInfo.ID, rid)) c.Assert(kv.ErrNotExist.Equal(err), IsTrue) } _, err = ts.se.Execute(context.Background(), "drop table if exists t1, t2;") diff --git a/table/tables/tables.go b/table/tables/tables.go index 74bb680c2f8dc..89f33a7193876 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -18,6 +18,7 @@ package tables import ( + "context" "encoding/binary" "math" "strings" @@ -574,9 +575,9 @@ func (t *tableCommon) genIndexKeyStr(colVals []types.Datum) (string, error) { } // addIndices adds data into indices. If any key is duplicated, returns the original handle. -func (t *tableCommon) addIndices(ctx sessionctx.Context, recordID int64, r []types.Datum, rm kv.RetrieverMutator, +func (t *tableCommon) addIndices(sctx sessionctx.Context, recordID int64, r []types.Datum, rm kv.RetrieverMutator, opts []table.CreateIdxOptFunc) (int64, error) { - txn, err := ctx.Txn(true) + txn, err := sctx.Txn(true) if err != nil { return 0, err } @@ -586,14 +587,20 @@ func (t *tableCommon) addIndices(ctx sessionctx.Context, recordID int64, r []typ for _, fn := range opts { fn(&opt) } - skipCheck := ctx.GetSessionVars().LightningMode || ctx.GetSessionVars().StmtCtx.BatchCheck + var ctx context.Context + if opt.Ctx != nil { + ctx = opt.Ctx + } else { + ctx = context.Background() + } + skipCheck := sctx.GetSessionVars().LightningMode || sctx.GetSessionVars().StmtCtx.BatchCheck if t.meta.PKIsHandle && !skipCheck && !opt.SkipHandleCheck { - if err := CheckHandleExists(ctx, t, recordID, nil); err != nil { + if err := CheckHandleExists(ctx, sctx, t, recordID, nil); err != nil { return recordID, err } } - writeBufs := ctx.GetSessionVars().GetWriteStmtBufs() + writeBufs := sctx.GetSessionVars().GetWriteStmtBufs() indexVals := writeBufs.IndexValsBuf for _, v := range t.WritableIndices() { var err2 error @@ -610,7 +617,7 @@ func (t *tableCommon) addIndices(ctx sessionctx.Context, recordID int64, r []typ dupKeyErr = kv.ErrKeyExists.FastGen("Duplicate entry '%s' for key '%s'", entryKey, v.Meta().Name) txn.SetOption(kv.PresumeKeyNotExistsError, dupKeyErr) } - if dupHandle, err := v.Create(ctx, rm, indexVals, recordID, opts...); err != nil { + if dupHandle, err := v.Create(sctx, rm, indexVals, recordID, opts...); err != nil { if kv.ErrKeyExists.Equal(err) { return dupHandle, dupKeyErr } @@ -631,7 +638,7 @@ func (t *tableCommon) RowWithCols(ctx sessionctx.Context, h int64, cols []*table if err != nil { return nil, err } - value, err := txn.Get(key) + value, err := txn.Get(context.TODO(), key) if err != nil { return nil, err } @@ -1084,16 +1091,16 @@ func FindIndexByColName(t table.Table, name string) table.Index { // CheckHandleExists check whether recordID key exists. if not exists, return nil, // otherwise return kv.ErrKeyExists error. -func CheckHandleExists(ctx sessionctx.Context, t table.Table, recordID int64, data []types.Datum) error { +func CheckHandleExists(ctx context.Context, sctx sessionctx.Context, t table.Table, recordID int64, data []types.Datum) error { if pt, ok := t.(*partitionedTable); ok { info := t.Meta().GetPartitionInfo() - pid, err := pt.locatePartition(ctx, info, data) + pid, err := pt.locatePartition(sctx, info, data) if err != nil { return err } t = pt.GetPartition(pid) } - txn, err := ctx.Txn(true) + txn, err := sctx.Txn(true) if err != nil { return err } @@ -1102,7 +1109,7 @@ func CheckHandleExists(ctx sessionctx.Context, t table.Table, recordID int64, da e := kv.ErrKeyExists.FastGen("Duplicate entry '%d' for key 'PRIMARY'", recordID) txn.SetOption(kv.PresumeKeyNotExistsError, e) defer txn.DelOption(kv.PresumeKeyNotExistsError) - _, err = txn.Get(recordKey) + _, err = txn.Get(ctx, recordKey) if err == nil { return e } else if !kv.ErrNotExist.Equal(err) { diff --git a/util/admin/admin.go b/util/admin/admin.go index fce94e497bc93..2dbbe77fd7ade 100644 --- a/util/admin/admin.go +++ b/util/admin/admin.go @@ -626,7 +626,7 @@ func makeRowDecoder(t table.Table, decodeCol []*table.Column, genExpr map[model. // genExprs use to calculate generated column value. func rowWithCols(sessCtx sessionctx.Context, txn kv.Retriever, t table.Table, h int64, cols []*table.Column, rowDecoder *decoder.RowDecoder) ([]types.Datum, error) { key := t.RecordKey(h) - value, err := txn.Get(key) + value, err := txn.Get(context.TODO(), key) if err != nil { return nil, errors.Trace(err) } diff --git a/util/prefix_helper_test.go b/util/prefix_helper_test.go index d4b485cfa4c3c..2c0543406ac9e 100644 --- a/util/prefix_helper_test.go +++ b/util/prefix_helper_test.go @@ -140,7 +140,7 @@ func (s *testPrefixSuite) TestPrefix(c *C) { c.Assert(err, IsNil) err = util.DelKeyWithPrefix(txn, []byte("key")) c.Assert(err, IsNil) - _, err = txn.Get(k) + _, err = txn.Get(context.TODO(), k) c.Assert(terror.ErrorEqual(kv.ErrNotExist, err), IsTrue) err = txn.Commit(context.Background()) From 9d248f56cb91ce14bb2e0ebdd232b8b89ae303dc Mon Sep 17 00:00:00 2001 From: lysu Date: Fri, 9 Aug 2019 14:46:55 +0800 Subject: [PATCH 183/196] expression: remove `fmt.Println` in `Constant.EvalJSON` (#11694) --- expression/constant.go | 1 - 1 file changed, 1 deletion(-) diff --git a/expression/constant.go b/expression/constant.go index 360da451325bd..0533984c29235 100644 --- a/expression/constant.go +++ b/expression/constant.go @@ -284,7 +284,6 @@ func (c *Constant) EvalJSON(ctx sessionctx.Context, _ chunk.Row) (json.BinaryJSO if err != nil { return json.BinaryJSON{}, true, err } - fmt.Println("const eval json", val.GetMysqlJSON().String()) c.Value.SetMysqlJSON(val.GetMysqlJSON()) c.GetType().Tp = mysql.TypeJSON } else { From fc9def1e5fc306970886c23e524a53bde55bf7ac Mon Sep 17 00:00:00 2001 From: satoru Date: Fri, 9 Aug 2019 14:55:33 +0800 Subject: [PATCH 184/196] ddl,meta: Generate global IDs in bulk to reduce the number of lock acquisition (#11635) --- ddl/ddl.go | 14 +++------ meta/meta.go | 17 ++++++++++ meta/meta_test.go | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/ddl/ddl.go b/ddl/ddl.go index 59cade07c7e96..75709a5a41e96 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -512,10 +512,8 @@ func (d *ddl) GetInfoSchemaWithInterceptor(ctx sessionctx.Context) infoschema.In } func (d *ddl) genGlobalIDs(count int) ([]int64, error) { - ret := make([]int64, count) + var ret []int64 err := kv.RunInNewTxn(d.store, true, func(txn kv.Transaction) error { - var err error - failpoint.Inject("mockGenGlobalIDFail", func(val failpoint.Value) { if val.(bool) { failpoint.Return(errors.New("gofail genGlobalIDs error")) @@ -523,13 +521,9 @@ func (d *ddl) genGlobalIDs(count int) ([]int64, error) { }) m := meta.NewMeta(txn) - for i := 0; i < count; i++ { - ret[i], err = m.GenGlobalID() - if err != nil { - return err - } - } - return nil + var err error + ret, err = m.GenGlobalIDs(count) + return err }) return ret, err diff --git a/meta/meta.go b/meta/meta.go index 98e0c3ab7ab53..9de7188aeb6ed 100644 --- a/meta/meta.go +++ b/meta/meta.go @@ -117,6 +117,23 @@ func (m *Meta) GenGlobalID() (int64, error) { return m.txn.Inc(mNextGlobalIDKey, 1) } +// GenGlobalIDs generates the next n global IDs. +func (m *Meta) GenGlobalIDs(n int) ([]int64, error) { + globalIDMutex.Lock() + defer globalIDMutex.Unlock() + + newID, err := m.txn.Inc(mNextGlobalIDKey, int64(n)) + if err != nil { + return nil, err + } + origID := newID - int64(n) + ids := make([]int64, 0, n) + for i := origID + 1; i <= newID; i++ { + ids = append(ids, i) + } + return ids, nil +} + // GetGlobalID gets current global id. func (m *Meta) GetGlobalID() (int64, error) { return m.txn.GetInt64(mNextGlobalIDKey) diff --git a/meta/meta_test.go b/meta/meta_test.go index 2d0ef05f6605b..b08e1b79c85ec 100644 --- a/meta/meta_test.go +++ b/meta/meta_test.go @@ -17,6 +17,7 @@ import ( "context" "math" "strconv" + "sync" "testing" "time" @@ -57,6 +58,24 @@ func (s *testSuite) TestMeta(c *C) { c.Assert(err, IsNil) c.Assert(n, Equals, int64(1)) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ids, err := t.GenGlobalIDs(3) + c.Assert(err, IsNil) + anyMatch(c, ids, []int64{2, 3, 4}, []int64{6, 7, 8}) + }() + + wg.Add(1) + go func() { + defer wg.Done() + ids, err := t.GenGlobalIDs(4) + c.Assert(err, IsNil) + anyMatch(c, ids, []int64{5, 6, 7, 8}, []int64{2, 3, 4, 5}) + }() + wg.Wait() + n, err = t.GetSchemaVersion() c.Assert(err, IsNil) c.Assert(n, Equals, int64(0)) @@ -406,3 +425,64 @@ func (s *testSuite) TestDDL(c *C) { err = txn1.Commit(context.Background()) c.Assert(err, IsNil) } + +func (s *testSuite) BenchmarkGenGlobalIDs(c *C) { + defer testleak.AfterTest(c)() + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + defer store.Close() + + txn, err := store.Begin() + c.Assert(err, IsNil) + defer txn.Rollback() + + t := meta.NewMeta(txn) + + c.ResetTimer() + var ids []int64 + for i := 0; i < c.N; i++ { + ids, _ = t.GenGlobalIDs(10) + } + c.Assert(ids, HasLen, 10) + c.Assert(ids[9], Equals, int64(c.N)*10) +} + +func (s *testSuite) BenchmarkGenGlobalIDOneByOne(c *C) { + defer testleak.AfterTest(c)() + store, err := mockstore.NewMockTikvStore() + c.Assert(err, IsNil) + defer store.Close() + + txn, err := store.Begin() + c.Assert(err, IsNil) + defer txn.Rollback() + + t := meta.NewMeta(txn) + + c.ResetTimer() + var id int64 + for i := 0; i < c.N; i++ { + for j := 0; j < 10; j++ { + id, _ = t.GenGlobalID() + } + } + c.Assert(id, Equals, int64(c.N)*10) +} + +func anyMatch(c *C, ids []int64, candidates ...[]int64) { + var match bool +OUTER: + for _, cand := range candidates { + if len(ids) != len(cand) { + continue + } + for i, v := range cand { + if ids[i] != v { + continue OUTER + } + } + match = true + break + } + c.Assert(match, IsTrue) +} From d51a3e067e5552d473739c9bf759524bbb897855 Mon Sep 17 00:00:00 2001 From: Kenan Yao Date: Fri, 9 Aug 2019 17:00:33 +0800 Subject: [PATCH 185/196] planner: use HistColl containing all columns for row width estimation (#11689) --- planner/core/exhaust_physical_plans.go | 10 +++++----- planner/core/find_best_task.go | 12 ++++++------ planner/core/logical_plan_builder.go | 6 +++--- planner/core/logical_plans.go | 7 +++++-- planner/core/planbuilder.go | 6 +++--- planner/core/stats.go | 1 + planner/core/task.go | 16 ++++++++-------- statistics/table.go | 22 +++++++++++++++++++++- 8 files changed, 52 insertions(+), 28 deletions(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index f9d15c23a008a..22c388bbf081b 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -536,12 +536,12 @@ func (p *LogicalJoin) constructInnerTableScanTask(ds *DataSource, pk *expression for i := range ds.stats.Cardinality { ds.stats.Cardinality[i] = 1 } - rowSize := ds.tableStats.HistColl.GetAvgRowSize(ds.tableCols, false) + rowSize := ds.TblColHists.GetAvgRowSize(ds.TblCols, false) copTask := &copTask{ tablePlan: ts, indexPlanFinished: true, cst: scanFactor * rowSize * ts.stats.RowCount, - tableStats: ds.tableStats, + tblColHists: ds.TblColHists, } selStats := ts.stats.Scale(selectionFactor) ts.addPushedDownSelection(copTask, selStats) @@ -594,9 +594,9 @@ func (p *LogicalJoin) constructInnerIndexScanTask(ds *DataSource, idx *model.Ind } is.stats = ds.tableStats.ScaleByExpectCnt(rowCount) cop := &copTask{ - indexPlan: is, - tableStats: ds.tableStats, - tableCols: ds.tableCols, + indexPlan: is, + tblColHists: ds.TblColHists, + tblCols: ds.TblCols, } if !isCoveringIndex(ds.schema.Columns, is.Index.Columns, is.Table.PKIsHandle) { // On this way, it's double read case. diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 268d93f971d7b..8cc30412a8582 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -489,9 +489,9 @@ func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, candid } rowCount := path.countAfterAccess cop := &copTask{ - indexPlan: is, - tableStats: ds.tableStats, - tableCols: ds.tableCols, + indexPlan: is, + tblColHists: ds.TblColHists, + tblCols: ds.TblCols, } if !candidate.isSingleScan { // On this way, it's double read case. @@ -552,7 +552,7 @@ func (is *PhysicalIndexScan) indexScanRowSize(idx *model.IndexInfo, ds *DataSour } else { scanCols = is.schema.Columns } - return ds.tableStats.HistColl.GetAvgRowSize(scanCols, true) + return ds.TblColHists.GetAvgRowSize(scanCols, true) } // TODO: refactor this part, we should not call Clone in fact. @@ -818,7 +818,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid copTask := &copTask{ tablePlan: ts, indexPlanFinished: true, - tableStats: ds.tableStats, + tblColHists: ds.TblColHists, } task = copTask // Adjust number of rows we actually need to scan if prop.ExpectedCnt is smaller than the count we calculated. @@ -845,7 +845,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid // we still need to assume values are uniformly distributed. For simplicity, we use uniform-assumption // for all columns now, as we do in `deriveStatsByFilter`. ts.stats = ds.tableStats.ScaleByExpectCnt(rowCount) - rowSize := ds.tableStats.HistColl.GetAvgRowSize(ds.tableCols, false) + rowSize := ds.TblColHists.GetAvgRowSize(ds.TblCols, false) copTask.cst = rowCount * rowSize * scanFactor if candidate.isMatchProp { if prop.Items[0].Desc { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index b25ec243d38a1..9faee975b1c39 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2267,7 +2267,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L possibleAccessPaths: possiblePaths, Columns: make([]*model.ColumnInfo, 0, len(columns)), partitionNames: tn.PartitionNames, - tableCols: make([]*expression.Column, 0, len(columns)), + TblCols: make([]*expression.Column, 0, len(columns)), }.Init(b.ctx) var handleCol *expression.Column @@ -2288,7 +2288,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L handleCol = newCol } schema.Append(newCol) - ds.tableCols = append(ds.tableCols, newCol) + ds.TblCols = append(ds.TblCols, newCol) } // We append an extra handle column to the schema when "ds" is not a memory // table e.g. table in the "INFORMATION_SCHEMA" database, and the handle @@ -2298,7 +2298,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName) (L ds.Columns = append(ds.Columns, model.NewExtraHandleColInfo()) handleCol = ds.newExtraHandleSchemaCol() schema.Append(handleCol) - ds.tableCols = append(ds.tableCols, handleCol) + ds.TblCols = append(ds.TblCols, handleCol) } if handleCol != nil { schema.TblID2Handle[tableInfo.ID] = []*expression.Column{handleCol} diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 4612d4b5fc7d1..8f14372144784 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -358,9 +358,12 @@ type DataSource struct { // handleCol represents the handle column for the datasource, either the // int primary key column or extra handle column. handleCol *expression.Column - // tableCols contains the original columns of table before being pruned, and it + // TblCols contains the original columns of table before being pruned, and it // is used for estimating table scan cost. - tableCols []*expression.Column + TblCols []*expression.Column + // TblColHists contains the Histogram of all original table columns, + // it is converted from statisticTable, and used for IO/network cost estimating. + TblColHists *statistics.HistColl } // accessPath indicates the way we access a table: by using single index, or by using multiple indexes, diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 84baf9e28e162..099b330b59bb5 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -789,9 +789,9 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(ctx context.Context, dbName ts := PhysicalTableScan{Columns: tblReaderCols, Table: is.Table}.Init(b.ctx) ts.SetSchema(tblSchema) cop := &copTask{ - indexPlan: is, - tablePlan: ts, - tableStats: is.stats, + indexPlan: is, + tablePlan: ts, + tblColHists: is.stats.HistColl, } ts.HandleIdx = pkOffset is.initSchema(id, idx, true) diff --git a/planner/core/stats.go b/planner/core/stats.go index 4b977e5e168b7..25d30fbd1fc83 100644 --- a/planner/core/stats.go +++ b/planner/core/stats.go @@ -104,6 +104,7 @@ func (ds *DataSource) deriveStatsByFilter(conds expression.CNFExprs) { tableStats.Cardinality[i] = ds.getColumnNDV(col.ID) } ds.tableStats = tableStats + ds.TblColHists = ds.statisticTable.ID2UniqueID(ds.TblCols) selectivity, nodes, err := tableStats.HistColl.Selectivity(ds.ctx, conds) if err != nil { logutil.BgLogger().Debug("an error happened, use the default selectivity", zap.Error(err)) diff --git a/planner/core/task.go b/planner/core/task.go index 9d545955307ca..7f8a0784b9554 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -22,8 +22,8 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/aggregation" - "github.com/pingcap/tidb/planner/property" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/statistics" "github.com/pingcap/tidb/types" ) @@ -51,12 +51,12 @@ type copTask struct { // In double read case, it may output one more column for handle(row id). // We need to prune it, so we add a project do this. doubleReadNeedProj bool - // tableStats stores the original stats of DataSource, it is used to get + // tblColHists stores the original stats of DataSource, it is used to get // average row width when computing network cost. - tableStats *property.StatsInfo - // tableCols store the original columns of DataSource before being pruned, it + tblColHists *statistics.HistColl + // tblCols stores the original columns of DataSource before being pruned, it // is used to compute average row width when computing scan cost. - tableCols []*expression.Column + tblCols []*expression.Column } func (t *copTask) invalid() bool { @@ -119,13 +119,13 @@ func (t *copTask) finishIndexPlan() { cnt := t.count() t.indexPlanFinished = true // Network cost of transferring rows of index scan to TiDB. - t.cst += cnt * netWorkFactor * t.tableStats.HistColl.GetAvgRowSize(t.indexPlan.Schema().Columns, true) + t.cst += cnt * netWorkFactor * t.tblColHists.GetAvgRowSize(t.indexPlan.Schema().Columns, true) if t.tablePlan == nil { return } // Calculate the IO cost of table scan here because we cannot know its stats until we finish index plan. t.tablePlan.(*PhysicalTableScan).stats = t.indexPlan.statsInfo() - rowSize := t.tableStats.HistColl.GetAvgRowSize(t.tableCols, false) + rowSize := t.tblColHists.GetAvgRowSize(t.tblCols, false) t.cst += cnt * rowSize * scanFactor } @@ -355,7 +355,7 @@ func finishCopTask(ctx sessionctx.Context, task task) task { t.finishIndexPlan() // Network cost of transferring rows of table scan to TiDB. if t.tablePlan != nil { - t.cst += t.count() * netWorkFactor * t.tableStats.HistColl.GetAvgRowSize(t.tablePlan.Schema().Columns, false) + t.cst += t.count() * netWorkFactor * t.tblColHists.GetAvgRowSize(t.tablePlan.Schema().Columns, false) } t.cst /= copIterWorkers newTask := &rootTask{ diff --git a/statistics/table.go b/statistics/table.go index 5e3f4db8ee7eb..b2ba11a76241a 100644 --- a/statistics/table.go +++ b/statistics/table.go @@ -295,6 +295,26 @@ func GetOrdinalOfRangeCond(sc *stmtctx.StatementContext, ran *ranger.Range) int return len(ran.LowVal) } +// ID2UniqueID generates a new HistColl whose `Columns` is built from UniqueID of given columns. +func (coll *HistColl) ID2UniqueID(columns []*expression.Column) *HistColl { + cols := make(map[int64]*Column) + for _, col := range columns { + colHist, ok := coll.Columns[col.ID] + if ok { + cols[col.UniqueID] = colHist + } + } + newColl := &HistColl{ + PhysicalID: coll.PhysicalID, + HavePhysicalID: coll.HavePhysicalID, + Pseudo: coll.Pseudo, + Count: coll.Count, + ModifyCount: coll.ModifyCount, + Columns: cols, + } + return newColl +} + // GenerateHistCollFromColumnInfo generates a new HistColl whose ColID2IdxID and IdxID2ColIDs is built from the given parameter. func (coll *HistColl) GenerateHistCollFromColumnInfo(infos []*model.ColumnInfo, columns []*expression.Column) *HistColl { newColHistMap := make(map[int64]*Column) @@ -654,7 +674,7 @@ func (coll *HistColl) GetAvgRowSize(cols []*expression.Column, isEncodedKey bool colHist, ok := coll.Columns[col.UniqueID] // Normally this would not happen, it is for compatibility with old version stats which // does not include TotColSize. - if !ok || (colHist.TotColSize == 0 && (colHist.NullCount != coll.Count)) { + if !ok || (!colHist.IsHandle && colHist.TotColSize == 0 && (colHist.NullCount != coll.Count)) { size += pseudoColSize continue } From 31a8e2192e567a75e5dd56b63fdbbf5e517ce5e7 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Fri, 9 Aug 2019 17:13:32 +0800 Subject: [PATCH 186/196] store/tikv,util/logutil: add a function to convert proto.Message to hex format (#11550) --- store/tikv/split_region.go | 9 +++-- util/logutil/hex.go | 78 ++++++++++++++++++++++++++++++++++++++ util/logutil/hex_test.go | 64 +++++++++++++++++++++++++++++++ util/logutil/log_test.go | 2 + 4 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 util/logutil/hex.go create mode 100644 util/logutil/hex_test.go diff --git a/store/tikv/split_region.go b/store/tikv/split_region.go index 0c20f2058d3c2..701a2734f8f0a 100644 --- a/store/tikv/split_region.go +++ b/store/tikv/split_region.go @@ -16,6 +16,7 @@ package tikv import ( "bytes" "context" + "encoding/hex" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/kvrpcpb" @@ -45,7 +46,7 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key, scatter bool) (regionID uint64, } if bytes.Equal(splitKey, loc.StartKey) { logutil.BgLogger().Info("skip split region", - zap.Binary("at", splitKey)) + zap.String("at", hex.EncodeToString(splitKey))) return 0, nil } res, err := sender.SendReq(bo, req, loc.Region, readTimeoutShort) @@ -65,9 +66,9 @@ func (s *tikvStore) SplitRegion(splitKey kv.Key, scatter bool) (regionID uint64, } splitRegion := res.Resp.(*kvrpcpb.SplitRegionResponse) logutil.BgLogger().Info("split region complete", - zap.Binary("at", splitKey), - zap.Stringer("new region left", splitRegion.GetLeft()), - zap.Stringer("new region right", splitRegion.GetRight())) + zap.String("at", hex.EncodeToString(splitKey)), + zap.Stringer("new region left", logutil.Hex(splitRegion.GetLeft())), + zap.Stringer("new region right", logutil.Hex(splitRegion.GetRight()))) left := splitRegion.GetLeft() if left == nil { return 0, nil diff --git a/util/logutil/hex.go b/util/logutil/hex.go new file mode 100644 index 0000000000000..e8ca2faeccefe --- /dev/null +++ b/util/logutil/hex.go @@ -0,0 +1,78 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by aprettyPrintlicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "reflect" + "strings" + + "github.com/golang/protobuf/proto" +) + +// Hex defines a fmt.Stringer for proto.Message. +// We can't define the String() method on proto.Message, but we can wrap it. +func Hex(msg proto.Message) fmt.Stringer { + return hexStringer{msg} +} + +type hexStringer struct { + proto.Message +} + +func (h hexStringer) String() string { + val := reflect.ValueOf(h.Message) + var w bytes.Buffer + prettyPrint(&w, val) + return w.String() +} + +func prettyPrint(w io.Writer, val reflect.Value) { + tp := val.Type() + switch val.Kind() { + case reflect.Slice: + elemType := tp.Elem() + if elemType.Kind() == reflect.Uint8 { + fmt.Fprintf(w, "%s", hex.EncodeToString(val.Bytes())) + } else { + fmt.Fprintf(w, "%s", val.Interface()) + } + case reflect.Struct: + fmt.Fprintf(w, "{") + for i := 0; i < val.NumField(); i++ { + fv := val.Field(i) + ft := tp.Field(i) + if strings.HasPrefix(ft.Name, "XXX_") { + continue + } + if i != 0 { + fmt.Fprintf(w, " ") + } + fmt.Fprintf(w, "%s:", ft.Name) + prettyPrint(w, fv) + } + fmt.Fprintf(w, "}") + case reflect.Ptr: + if val.IsNil() { + fmt.Fprintf(w, "%v", val.Interface()) + } else { + prettyPrint(w, reflect.Indirect(val)) + } + default: + fmt.Fprintf(w, "%v", val.Interface()) + } +} diff --git a/util/logutil/hex_test.go b/util/logutil/hex_test.go new file mode 100644 index 0000000000000..44c989f9da4c8 --- /dev/null +++ b/util/logutil/hex_test.go @@ -0,0 +1,64 @@ +// Copyright 2017 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil_test + +import ( + "bytes" + "encoding/hex" + "reflect" + + . "github.com/pingcap/check" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/util/logutil" +) + +var _ = Suite(&testHexSuite{}) + +type testHexSuite struct{} + +func (s *testHexSuite) SetUpSuite(c *C) {} + +func (s *testHexSuite) SetUpTest(c *C) {} + +func (s *testHexSuite) TestHex(c *C) { + var region metapb.Region + region.Id = 6662 + region.StartKey = []byte{'t', 200, '\\', 000, 000, 000, '\\', 000, 000, 000, 37, '-', 000, 000, 000, 000, 000, 000, 000, 37} + region.EndKey = []byte("3asg3asd") + + c.Assert(logutil.Hex(®ion).String(), Equals, "{Id:6662 StartKey:74c85c0000005c000000252d0000000000000025 EndKey:3361736733617364 RegionEpoch: Peers:[]}") +} + +func (s *testHexSuite) TestPrettyPrint(c *C) { + var buf bytes.Buffer + + byteSlice := []byte("asd2fsdafs中文3af") + logutil.PrettyPrint(&buf, reflect.ValueOf(byteSlice)) + c.Assert(buf.String(), Equals, "61736432667364616673e4b8ade69687336166") + c.Assert(buf.String(), Equals, hex.EncodeToString(byteSlice)) + buf.Reset() + + // Go reflect can't distinguish uint8 from byte! + intSlice := []uint8{1, 2, 3, uint8('a'), uint8('b'), uint8('c'), uint8('\'')} + logutil.PrettyPrint(&buf, reflect.ValueOf(intSlice)) + c.Assert(buf.String(), Equals, "01020361626327") + buf.Reset() + + var ran kv.KeyRange + ran.StartKey = kv.Key("_txxey23_i263") + ran.EndKey = nil + logutil.PrettyPrint(&buf, reflect.ValueOf(ran)) + c.Assert(buf.String(), Equals, "{StartKey:5f747878657932335f69323633 EndKey:}") +} diff --git a/util/logutil/log_test.go b/util/logutil/log_test.go index 8a3920971eb47..b93ae639ba0d2 100644 --- a/util/logutil/log_test.go +++ b/util/logutil/log_test.go @@ -41,6 +41,8 @@ const ( zapLogWithKeyValPattern = `\[\d\d\d\d/\d\d/\d\d \d\d:\d\d:\d\d.\d\d\d\ (\+|-)\d\d:\d\d\] \[(FATAL|ERROR|WARN|INFO|DEBUG)\] \[([\w_%!$@.,+~-]+|\\.)+:\d+\] \[.*\] \[ctxKey=.*\] (\[.*=.*\]).*\n` ) +var PrettyPrint = prettyPrint + func Test(t *testing.T) { TestingT(t) } From df2075d74590d2e5338070d2d65c40c79dd66634 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Fri, 9 Aug 2019 20:35:37 +0800 Subject: [PATCH 187/196] *: remove the transaction kv count limit (#11141) --- config/config.go | 4 ---- config/config.toml.example | 6 ------ config/config_test.go | 3 --- executor/seqtest/seq_executor_test.go | 29 +++++++++++++++------------ kv/kv.go | 2 -- kv/mem_buffer_test.go | 9 --------- kv/memdb_buffer.go | 4 ---- store/tikv/2pc.go | 3 +-- tidb-server/main.go | 1 - 9 files changed, 17 insertions(+), 44 deletions(-) diff --git a/config/config.go b/config/config.go index 8db018422171c..242eb659be0f8 100644 --- a/config/config.go +++ b/config/config.go @@ -37,8 +37,6 @@ const ( MaxLogFileSize = 4096 // MB MinPessimisticTTL = time.Second * 15 MaxPessimisticTTL = time.Second * 120 - // DefTxnEntryCountLimit is the default value of TxnEntryCountLimit. - DefTxnEntryCountLimit = 300 * 1000 // DefTxnTotalSizeLimit is the default value of TxnTxnTotalSizeLimit. DefTxnTotalSizeLimit = 100 * 1024 * 1024 ) @@ -198,7 +196,6 @@ type Performance struct { PseudoEstimateRatio float64 `toml:"pseudo-estimate-ratio" json:"pseudo-estimate-ratio"` ForcePriority string `toml:"force-priority" json:"force-priority"` BindInfoLease string `toml:"bind-info-lease" json:"bind-info-lease"` - TxnEntryCountLimit uint64 `toml:"txn-entry-count-limit" json:"txn-entry-count-limit"` TxnTotalSizeLimit uint64 `toml:"txn-total-size-limit" json:"txn-total-size-limit"` } @@ -369,7 +366,6 @@ var defaultConf = Config{ PseudoEstimateRatio: 0.8, ForcePriority: "NO_PRIORITY", BindInfoLease: "3s", - TxnEntryCountLimit: DefTxnEntryCountLimit, TxnTotalSizeLimit: DefTxnTotalSizeLimit, }, ProxyProtocol: ProxyProtocol{ diff --git a/config/config.toml.example b/config/config.toml.example index 045d5a64a90a5..08e7e907a7e21 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -180,12 +180,6 @@ force-priority = "NO_PRIORITY" # Bind info lease duration, which influences the duration of loading bind info and handling invalid bind. bind-info-lease = "3s" -# The limitation of the number for the entries in one transaction. -# If using TiKV as the storage, the entry represents a key/value pair. -# WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster. -# Please adjust this configuration carefully. -txn-entry-count-limit = 300000 - # The limitation of the size in byte for the entries in one transaction. # If using TiKV as the storage, the entry represents a key/value pair. # WARNING: Do not set the value too large, otherwise it will make a very large impact on the TiKV cluster. diff --git a/config/config_test.go b/config/config_test.go index 5a0d1519e7d9f..e0c1a9e60b4f8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -38,7 +38,6 @@ func (s *testConfigSuite) TestConfig(c *C) { conf.Binlog.Enable = true conf.Binlog.IgnoreError = true conf.Binlog.Strategy = "hash" - conf.Performance.TxnEntryCountLimit = 1000 conf.Performance.TxnTotalSizeLimit = 1000 conf.TiKVClient.CommitTimeout = "10s" configFile := "config.toml" @@ -66,7 +65,6 @@ enable-table-lock = true delay-clean-table-lock = 5 split-region-max-num=10000 [performance] -txn-entry-count-limit=2000 txn-total-size-limit=2000 [tikv-client] commit-timeout="41s" @@ -83,7 +81,6 @@ max-batch-size=128 c.Assert(conf.Binlog.Strategy, Equals, "hash") // Test that the value will be overwritten by the config file. - c.Assert(conf.Performance.TxnEntryCountLimit, Equals, uint64(2000)) c.Assert(conf.Performance.TxnTotalSizeLimit, Equals, uint64(2000)) c.Assert(conf.TiKVClient.CommitTimeout, Equals, "41s") diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index 3936675404586..c10ac5d426f38 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -794,12 +794,12 @@ func (s *seqTestSuite) TestCartesianProduct(c *C) { } func (s *seqTestSuite) TestBatchInsertDelete(c *C) { - originLimit := atomic.LoadUint64(&kv.TxnEntryCountLimit) + originLimit := atomic.LoadUint64(&kv.TxnTotalSizeLimit) defer func() { - atomic.StoreUint64(&kv.TxnEntryCountLimit, originLimit) + atomic.StoreUint64(&kv.TxnTotalSizeLimit, originLimit) }() // Set the limitation to a small value, make it easier to reach the limitation. - atomic.StoreUint64(&kv.TxnEntryCountLimit, 100) + atomic.StoreUint64(&kv.TxnTotalSizeLimit, 5000) tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -827,19 +827,22 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { tk.MustExec("insert into batch_insert (c) select * from batch_insert;") r = tk.MustQuery("select count(*) from batch_insert;") r.Check(testkit.Rows("160")) + tk.MustExec("insert into batch_insert (c) select * from batch_insert;") + r = tk.MustQuery("select count(*) from batch_insert;") + r.Check(testkit.Rows("320")) // for on duplicate key - for i := 0; i < 160; i++ { + for i := 0; i < 320; i++ { tk.MustExec(fmt.Sprintf("insert into batch_insert_on_duplicate values(%d, %d);", i, i)) } r = tk.MustQuery("select count(*) from batch_insert_on_duplicate;") - r.Check(testkit.Rows("160")) + r.Check(testkit.Rows("320")) // This will meet txn too large error. _, err := tk.Exec("insert into batch_insert (c) select * from batch_insert;") c.Assert(err, NotNil) c.Assert(kv.ErrTxnTooLarge.Equal(err), IsTrue) r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("160")) + r.Check(testkit.Rows("320")) // for on duplicate key _, err = tk.Exec(`insert into batch_insert_on_duplicate select * from batch_insert_on_duplicate as tt @@ -847,23 +850,23 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { c.Assert(err, NotNil) c.Assert(kv.ErrTxnTooLarge.Equal(err), IsTrue, Commentf("%v", err)) r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("160")) + r.Check(testkit.Rows("320")) // Change to batch inset mode and batch size to 50. tk.MustExec("set @@session.tidb_batch_insert=1;") tk.MustExec("set @@session.tidb_dml_batch_size=50;") tk.MustExec("insert into batch_insert (c) select * from batch_insert;") r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("320")) + r.Check(testkit.Rows("640")) // Enlarge the batch size to 150 which is larger than the txn limitation (100). // So the insert will meet error. - tk.MustExec("set @@session.tidb_dml_batch_size=150;") + tk.MustExec("set @@session.tidb_dml_batch_size=600;") _, err = tk.Exec("insert into batch_insert (c) select * from batch_insert;") c.Assert(err, NotNil) c.Assert(kv.ErrTxnTooLarge.Equal(err), IsTrue) r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("320")) + r.Check(testkit.Rows("640")) // Set it back to 50. tk.MustExec("set @@session.tidb_dml_batch_size=50;") @@ -872,7 +875,7 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { on duplicate key update batch_insert_on_duplicate.id=batch_insert_on_duplicate.id+1000;`) c.Assert(err, IsNil) r = tk.MustQuery("select count(*) from batch_insert_on_duplicate;") - r.Check(testkit.Rows("160")) + r.Check(testkit.Rows("320")) // Disable BachInsert mode in transition. tk.MustExec("begin;") @@ -881,7 +884,7 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { c.Assert(kv.ErrTxnTooLarge.Equal(err), IsTrue) tk.MustExec("rollback;") r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("320")) + r.Check(testkit.Rows("640")) tk.MustExec("drop table if exists com_batch_insert") tk.MustExec("create table com_batch_insert (c int)") @@ -900,7 +903,7 @@ func (s *seqTestSuite) TestBatchInsertDelete(c *C) { c.Assert(err, NotNil) c.Assert(kv.ErrTxnTooLarge.Equal(err), IsTrue) r = tk.MustQuery("select count(*) from batch_insert;") - r.Check(testkit.Rows("320")) + r.Check(testkit.Rows("640")) // Enable batch delete and set batch size to 50. tk.MustExec("set @@session.tidb_batch_delete=on;") tk.MustExec("set @@session.tidb_dml_batch_size=50;") diff --git a/kv/kv.go b/kv/kv.go index a826a9756b63b..21cc6cc3f2317 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -73,8 +73,6 @@ const ( var ( // TxnEntrySizeLimit is limit of single entry size (len(key) + len(value)). TxnEntrySizeLimit = 6 * 1024 * 1024 - // TxnEntryCountLimit is limit of number of entries in the MemBuffer. - TxnEntryCountLimit uint64 = config.DefTxnEntryCountLimit // TxnTotalSizeLimit is limit of the sum of all entry size. TxnTotalSizeLimit uint64 = config.DefTxnTotalSizeLimit ) diff --git a/kv/mem_buffer_test.go b/kv/mem_buffer_test.go index 767fb064e01f7..297e5da7b55be 100644 --- a/kv/mem_buffer_test.go +++ b/kv/mem_buffer_test.go @@ -223,15 +223,6 @@ func (s *testKVSuite) TestBufferLimit(c *C) { c.Assert(err, IsNil) err = buffer.Set([]byte("yz"), make([]byte, 499)) c.Assert(err, NotNil) // buffer size limit - - buffer = NewMemDbBuffer(DefaultTxnMembufCap).(*memDbBuffer) - buffer.bufferLenLimit = 10 - for i := 0; i < 10; i++ { - err = buffer.Set([]byte{byte(i)}, []byte{byte(i)}) - c.Assert(err, IsNil) - } - err = buffer.Set([]byte("x"), []byte("y")) - c.Assert(err, NotNil) // buffer len limit } var opCnt = 100000 diff --git a/kv/memdb_buffer.go b/kv/memdb_buffer.go index f4fff437433a6..953b7933c9847 100644 --- a/kv/memdb_buffer.go +++ b/kv/memdb_buffer.go @@ -46,7 +46,6 @@ func NewMemDbBuffer(cap int) MemBuffer { return &memDbBuffer{ db: memdb.New(comparer.DefaultComparer, cap), entrySizeLimit: TxnEntrySizeLimit, - bufferLenLimit: atomic.LoadUint64(&TxnEntryCountLimit), bufferSizeLimit: atomic.LoadUint64(&TxnTotalSizeLimit), } } @@ -99,9 +98,6 @@ func (m *memDbBuffer) Set(k Key, v []byte) error { if m.Size() > int(m.bufferSizeLimit) { return ErrTxnTooLarge.GenWithStack("transaction too large, size:%d", m.Size()) } - if m.Len() > int(m.bufferLenLimit) { - return ErrTxnTooLarge.GenWithStack("transaction too large, len:%d", m.Len()) - } return errors.Trace(err) } diff --git a/store/tikv/2pc.go b/store/tikv/2pc.go index 70086747225fc..5f6ecd79537db 100644 --- a/store/tikv/2pc.go +++ b/store/tikv/2pc.go @@ -230,8 +230,7 @@ func (c *twoPhaseCommitter) initKeysAndMutations() error { mutation.asserted = true } - entrylimit := atomic.LoadUint64(&kv.TxnEntryCountLimit) - if len(keys) > int(entrylimit) || size > int(kv.TxnTotalSizeLimit) { + if size > int(kv.TxnTotalSizeLimit) { return kv.ErrTxnTooLarge } const logEntryCount = 10000 diff --git a/tidb-server/main.go b/tidb-server/main.go index 13ad3ffc71fde..36077d2718bf7 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -486,7 +486,6 @@ func setGlobalVars() { } plannercore.AllowCartesianProduct.Store(cfg.Performance.CrossJoin) privileges.SkipWithGrant = cfg.Security.SkipGrantTable - kv.TxnEntryCountLimit = cfg.Performance.TxnEntryCountLimit kv.TxnTotalSizeLimit = cfg.Performance.TxnTotalSizeLimit priority := mysql.Str2Priority(cfg.Performance.ForcePriority) From 4995fe741ea527b3870731a6873d34a92403d3de Mon Sep 17 00:00:00 2001 From: gaoxingliang Date: Sat, 10 Aug 2019 16:22:45 +0800 Subject: [PATCH 188/196] planner/core: check the window func arg first before checking window specs (#11613) --- cmd/explaintest/main.go | 3 +- planner/core/logical_plan_builder.go | 51 ++++++++++++++++++++++++++++ planner/core/logical_plan_test.go | 8 +++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/cmd/explaintest/main.go b/cmd/explaintest/main.go index b29b05c93c25e..4e4f6defd9e05 100644 --- a/cmd/explaintest/main.go +++ b/cmd/explaintest/main.go @@ -357,10 +357,9 @@ func (t *tester) execute(query query) error { gotBuf := t.buf.Bytes()[offset:] buf := make([]byte, t.buf.Len()-offset) - if _, err = t.resultFD.ReadAt(buf, int64(offset)); err != nil { + if _, err = t.resultFD.ReadAt(buf, int64(offset)); !(err == nil || err == io.EOF) { return errors.Trace(errors.Errorf("run \"%v\" at line %d err, we got \n%s\nbut read result err %s", st.Text(), query.Line, gotBuf, err)) } - if !bytes.Equal(gotBuf, buf) { return errors.Trace(errors.Errorf("run \"%v\" at line %d err, we need:\n%s\nbut got:\n%s\n", query.Query, query.Line, buf, gotBuf)) } diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 9faee975b1c39..b257b6287bca6 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -2099,6 +2099,11 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L var windowMapper map[*ast.WindowFuncExpr]int if hasWindowFuncField { windowFuncs := extractWindowFuncs(sel.Fields.Fields) + // we need to check the func args first before we check the window spec + err := b.checkWindowFuncArgs(ctx, p, windowFuncs, windowAggMap) + if err != nil { + return nil, err + } groupedFuncs, err := b.groupWindowFuncs(windowFuncs) if err != nil { return nil, err @@ -3056,6 +3061,35 @@ func (b *PlanBuilder) buildProjectionForWindow(ctx context.Context, p LogicalPla return proj, propertyItems[:lenPartition], propertyItems[lenPartition:], newArgList, nil } +func (b *PlanBuilder) buildArgs4WindowFunc(ctx context.Context, p LogicalPlan, args []ast.ExprNode, aggMap map[*ast.AggregateFuncExpr]int) ([]expression.Expression, error) { + b.optFlag |= flagEliminateProjection + + newArgList := make([]expression.Expression, 0, len(args)) + // use below index for created a new col definition + // it's okay here because we only want to return the args used in window function + newColIndex := 0 + for _, arg := range args { + newArg, np, err := b.rewrite(ctx, arg, p, aggMap, true) + if err != nil { + return nil, err + } + p = np + switch newArg.(type) { + case *expression.Column, *expression.Constant: + newArgList = append(newArgList, newArg) + continue + } + col := &expression.Column{ + ColName: model.NewCIStr(fmt.Sprintf("%d_proj_window_%d", p.ID(), newColIndex)), + UniqueID: b.ctx.GetSessionVars().AllocPlanColumnID(), + RetType: newArg.GetType(), + } + newColIndex += 1 + newArgList = append(newArgList, col) + } + return newArgList, nil +} + func (b *PlanBuilder) buildByItemsForWindow( ctx context.Context, p LogicalPlan, @@ -3220,6 +3254,23 @@ func (b *PlanBuilder) buildWindowFunctionFrame(ctx context.Context, spec *ast.Wi return frame, err } +func (b *PlanBuilder) checkWindowFuncArgs(ctx context.Context, p LogicalPlan, windowFuncExprs []*ast.WindowFuncExpr, windowAggMap map[*ast.AggregateFuncExpr]int) error { + for _, windowFuncExpr := range windowFuncExprs { + args, err := b.buildArgs4WindowFunc(ctx, p, windowFuncExpr.Args, windowAggMap) + if err != nil { + return err + } + desc, err := aggregation.NewWindowFuncDesc(b.ctx, windowFuncExpr.F, args) + if err != nil { + return err + } + if desc == nil { + return ErrWrongArguments.GenWithStackByArgs(strings.ToLower(windowFuncExpr.F)) + } + } + return nil +} + func getAllByItems(itemsBuf []*ast.ByItem, spec *ast.WindowSpec) []*ast.ByItem { itemsBuf = itemsBuf[:0] if spec.PartitionBy != nil { diff --git a/planner/core/logical_plan_test.go b/planner/core/logical_plan_test.go index c79b630efb2b1..a2843b98fc6f8 100644 --- a/planner/core/logical_plan_test.go +++ b/planner/core/logical_plan_test.go @@ -2469,6 +2469,14 @@ func (s *testPlanSuite) TestWindowFunction(c *C) { sql: "SELECT NTH_VALUE(a, 1) FROM LAST IGNORE NULLS over (partition by b order by b), a FROM t", result: "[planner:1235]This version of TiDB doesn't yet support 'IGNORE NULLS'", }, + { + sql: "SELECT NTH_VALUE(fieldA, ATAN(-1)) OVER (w1) AS 'ntile', fieldA, fieldB FROM ( SELECT a AS fieldA, b AS fieldB FROM t ) as te WINDOW w1 AS ( ORDER BY fieldB ASC, fieldA DESC )", + result: "[planner:1210]Incorrect arguments to nth_value", + }, + { + sql: "SELECT NTH_VALUE(fieldA, -1) OVER (w1 PARTITION BY fieldB ORDER BY fieldB , fieldA ) AS 'ntile', fieldA, fieldB FROM ( SELECT a AS fieldA, b AS fieldB FROM t ) as temp WINDOW w1 AS ( ORDER BY fieldB ASC, fieldA DESC )", + result: "[planner:1210]Incorrect arguments to nth_value", + }, } s.Parser.EnableWindowFunc(true) From ff71c0c746d0ad39ee46dd7bac2b150ee52ff6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BA=84=E5=A4=A9=E7=BF=BC?= Date: Sat, 10 Aug 2019 19:12:38 +0800 Subject: [PATCH 189/196] expression: fix last_day incompatible with mysql (#11704) --- expression/builtin_time.go | 9 ++++----- expression/builtin_time_test.go | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/expression/builtin_time.go b/expression/builtin_time.go index 39c45fdd7b131..d93eccbffc5f7 100644 --- a/expression/builtin_time.go +++ b/expression/builtin_time.go @@ -6416,14 +6416,13 @@ func (b *builtinLastDaySig) evalTime(row chunk.Row) (types.Time, bool, error) { return types.Time{}, true, handleInvalidTimeError(b.ctx, err) } tm := arg.Time - var day int - year, month := tm.Year(), tm.Month() - if month == 0 { + year, month, day := tm.Year(), tm.Month(), tm.Day() + if month == 0 || day == 0 { return types.Time{}, true, handleInvalidTimeError(b.ctx, types.ErrIncorrectDatetimeValue.GenWithStackByArgs(arg.String())) } - day = types.GetLastDay(year, month) + lastDay := types.GetLastDay(year, month) ret := types.Time{ - Time: types.FromDate(year, month, day, 0, 0, 0, 0), + Time: types.FromDate(year, month, lastDay, 0, 0, 0, 0), Type: mysql.TypeDate, Fsp: types.DefaultFsp, } diff --git a/expression/builtin_time_test.go b/expression/builtin_time_test.go index ed7e784cc231a..f37af43e18618 100644 --- a/expression/builtin_time_test.go +++ b/expression/builtin_time_test.go @@ -2676,6 +2676,7 @@ func (s *testEvaluatorSuite) TestLastDay(c *C) { "2007-10-07 23:59:61", "2005-00-00", "2005-00-01", + "2243-01 00:00:00", 123456789} for _, i := range testsNull { From 3076e632c89e4ff9dc0ecd72af5b90ef4d0ffa6c Mon Sep 17 00:00:00 2001 From: amyangfei Date: Sun, 11 Aug 2019 17:36:20 +0800 Subject: [PATCH 190/196] *: use bytes.Equal to check byte slice equivalent (#11706) --- executor/analyze.go | 2 +- executor/load_data.go | 2 +- store/tikv/region_cache.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index 9c95df3397d29..144be79ec8525 100755 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -687,7 +687,7 @@ func (e *AnalyzeFastExec) getNextSampleKey(bo *tikv.Backoffer, startKey kv.Key) if err != nil { return nil, err } - if bytes.Compare(loc.StartKey, e.sampTasks[prefixLen].Location.EndKey) == 0 { + if bytes.Equal(loc.StartKey, e.sampTasks[prefixLen].Location.EndKey) { startKey = loc.StartKey break } diff --git a/executor/load_data.go b/executor/load_data.go index 0f055b33bd396..36f090aea32d0 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -506,7 +506,7 @@ func (e *LoadDataInfo) getFieldsFromLine(line []byte) ([]field, error) { for { eol, f := reader.GetField() f = f.escape() - if bytes.Compare(f.str, null) == 0 && !f.enclosed { + if bytes.Equal(f.str, null) && !f.enclosed { f.str = []byte{'N'} f.maybeNull = true } diff --git a/store/tikv/region_cache.go b/store/tikv/region_cache.go index 36827778095e9..eda9a5260713d 100644 --- a/store/tikv/region_cache.go +++ b/store/tikv/region_cache.go @@ -650,7 +650,7 @@ func (c *RegionCache) loadRegion(bo *Backoffer, key []byte, isEndKey bool) (*Reg if len(meta.Peers) == 0 { return nil, errors.New("receive Region with no peer") } - if isEndKey && !searchPrev && bytes.Compare(meta.StartKey, key) == 0 && len(meta.StartKey) != 0 { + if isEndKey && !searchPrev && bytes.Equal(meta.StartKey, key) && len(meta.StartKey) != 0 { searchPrev = true continue } From bc4a32431ff628cee756e0de6b581475e653e651 Mon Sep 17 00:00:00 2001 From: Huang Zexin Date: Mon, 12 Aug 2019 11:13:10 +0800 Subject: [PATCH 191/196] =?UTF-8?q?expression:=20remove=20unnecessary=20co?= =?UTF-8?q?nvertIntToUint.=20fix=20error=20in=E2=80=A6=20(#11640)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- expression/builtin_cast.go | 29 +++++++++-------------------- expression/integration_test.go | 25 +++++++++++++++++++++++++ types/convert.go | 1 + types/etc.go | 5 +---- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 366c48def1b80..0b8d0abd4977e 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -437,6 +437,9 @@ func (b *builtinCastIntAsIntSig) Clone() builtinFunc { func (b *builtinCastIntAsIntSig) evalInt(row chunk.Row) (res int64, isNull bool, err error) { res, isNull, err = b.args[0].EvalInt(b.ctx, row) + if isNull || err != nil { + return + } if b.inUnion && mysql.HasUnsignedFlag(b.tp.Flag) && res < 0 { res = 0 } @@ -463,10 +466,8 @@ func (b *builtinCastIntAsRealSig) evalReal(row chunk.Row) (res float64, isNull b } else if b.inUnion && val < 0 { res = 0 } else { - var uVal uint64 - sc := b.ctx.GetSessionVars().StmtCtx - uVal, err = types.ConvertIntToUint(sc, val, types.IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) - res = float64(uVal) + // recall that, int to float is different from uint to float + res = float64(uint64(val)) } return res, false, err } @@ -491,13 +492,7 @@ func (b *builtinCastIntAsDecimalSig) evalDecimal(row chunk.Row) (res *types.MyDe } else if b.inUnion && val < 0 { res = &types.MyDecimal{} } else { - var uVal uint64 - sc := b.ctx.GetSessionVars().StmtCtx - uVal, err = types.ConvertIntToUint(sc, val, types.IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) - if err != nil { - return res, false, err - } - res = types.NewDecFromUint(uVal) + res = types.NewDecFromUint(uint64(val)) } res, err = types.ProduceDecWithSpecifiedTp(res, b.tp, b.ctx.GetSessionVars().StmtCtx) return res, isNull, err @@ -521,13 +516,7 @@ func (b *builtinCastIntAsStringSig) evalString(row chunk.Row) (res string, isNul if !mysql.HasUnsignedFlag(b.args[0].GetType().Flag) { res = strconv.FormatInt(val, 10) } else { - var uVal uint64 - sc := b.ctx.GetSessionVars().StmtCtx - uVal, err = types.ConvertIntToUint(sc, val, types.IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) - if err != nil { - return res, false, err - } - res = strconv.FormatUint(uVal, 10) + res = strconv.FormatUint(uint64(val), 10) } res, err = types.ProduceStrWithSpecifiedTp(res, b.tp, b.ctx.GetSessionVars().StmtCtx, false) if err != nil { @@ -748,13 +737,13 @@ func (b *builtinCastRealAsIntSig) evalInt(row chunk.Row) (res int64, isNull bool return res, isNull, err } if !mysql.HasUnsignedFlag(b.tp.Flag) { - res, err = types.ConvertFloatToInt(val, types.IntergerSignedLowerBound(mysql.TypeLonglong), types.IntergerSignedUpperBound(mysql.TypeLonglong), mysql.TypeDouble) + res, err = types.ConvertFloatToInt(val, types.IntergerSignedLowerBound(mysql.TypeLonglong), types.IntergerSignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) } else if b.inUnion && val < 0 { res = 0 } else { var uintVal uint64 sc := b.ctx.GetSessionVars().StmtCtx - uintVal, err = types.ConvertFloatToUint(sc, val, types.IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeDouble) + uintVal, err = types.ConvertFloatToUint(sc, val, types.IntergerUnsignedUpperBound(mysql.TypeLonglong), mysql.TypeLonglong) res = int64(uintVal) } return res, isNull, err diff --git a/expression/integration_test.go b/expression/integration_test.go index df87ece0ff292..59bdae56242d7 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -2200,6 +2200,31 @@ func (s *testIntegrationSuite) TestBuiltin(c *C) { msg := strings.Split(err.Error(), " ") last := msg[len(msg)-1] c.Assert(last, Equals, "bigint") + tk.MustExec(`drop table tb5;`) + + // test builtinCastIntAsDecimalSig + tk.MustExec(`create table tb5(a bigint(64) unsigned, b decimal(64, 10));`) + tk.MustExec(`insert into tb5 (a, b) values (9223372036854775808, 9223372036854775808);`) + tk.MustExec(`insert into tb5 (select * from tb5 where a = b);`) + result = tk.MustQuery(`select * from tb5;`) + result.Check(testkit.Rows("9223372036854775808 9223372036854775808.0000000000", "9223372036854775808 9223372036854775808.0000000000")) + tk.MustExec(`drop table tb5;`) + + // test builtinCastIntAsRealSig + tk.MustExec(`create table tb5(a bigint(64) unsigned, b double(64, 10));`) + tk.MustExec(`insert into tb5 (a, b) values (13835058000000000000, 13835058000000000000);`) + tk.MustExec(`insert into tb5 (select * from tb5 where a = b);`) + result = tk.MustQuery(`select * from tb5;`) + result.Check(testkit.Rows("13835058000000000000 13835058000000000000", "13835058000000000000 13835058000000000000")) + tk.MustExec(`drop table tb5;`) + + // test builtinCastIntAsStringSig + tk.MustExec(`create table tb5(a bigint(64) unsigned,b varchar(50));`) + tk.MustExec(`insert into tb5(a, b) values (9223372036854775808, '9223372036854775808');`) + tk.MustExec(`insert into tb5(select * from tb5 where a = b);`) + result = tk.MustQuery(`select * from tb5;`) + result.Check(testkit.Rows("9223372036854775808 9223372036854775808", "9223372036854775808 9223372036854775808")) + tk.MustExec(`drop table tb5;`) // Test corner cases of cast string as datetime result = tk.MustQuery(`select cast("170102034" as datetime);`) diff --git a/types/convert.go b/types/convert.go index 4a8368332b5f7..1f733d50354d1 100644 --- a/types/convert.go +++ b/types/convert.go @@ -98,6 +98,7 @@ func IntergerSignedLowerBound(intType byte) int64 { } // ConvertFloatToInt converts a float64 value to a int value. +// `tp` is used in err msg, if there is overflow, this func will report err according to `tp` func ConvertFloatToInt(fval float64, lowerBound, upperBound int64, tp byte) (int64, error) { val := RoundFloat(fval) if val < float64(lowerBound) { diff --git a/types/etc.go b/types/etc.go index b8b5af64f38d5..e29c91171e5a7 100644 --- a/types/etc.go +++ b/types/etc.go @@ -83,10 +83,7 @@ func IsTemporalWithDate(tp byte) bool { // IsBinaryStr returns a boolean indicating // whether the field type is a binary string type. func IsBinaryStr(ft *FieldType) bool { - if ft.Collate == charset.CollationBin && IsString(ft.Tp) { - return true - } - return false + return ft.Collate == charset.CollationBin && IsString(ft.Tp) } // IsNonBinaryStr returns a boolean indicating From a08918ab7415e0d996cddd557ec6716b12613598 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 12 Aug 2019 11:53:03 +0800 Subject: [PATCH 192/196] fixup (#11688) --- server/server.go | 2 +- util/processinfo.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/server.go b/server/server.go index 906ffdc48dec6..d43d8db804654 100644 --- a/server/server.go +++ b/server/server.go @@ -483,6 +483,7 @@ func (cc *clientConn) connectInfo() *variable.ConnectionInfo { // ShowProcessList implements the SessionManager interface. func (s *Server) ShowProcessList() map[uint64]*util.ProcessInfo { s.rwlock.RLock() + defer s.rwlock.RUnlock() rs := make(map[uint64]*util.ProcessInfo, len(s.clients)) for _, client := range s.clients { if atomic.LoadInt32(&client.status) == connStatusWaitShutdown { @@ -492,7 +493,6 @@ func (s *Server) ShowProcessList() map[uint64]*util.ProcessInfo { rs[pi.ID] = pi } } - s.rwlock.RUnlock() return rs } diff --git a/util/processinfo.go b/util/processinfo.go index cff4b20f631dd..6338375de55fb 100644 --- a/util/processinfo.go +++ b/util/processinfo.go @@ -80,7 +80,11 @@ func (pi *ProcessInfo) txnStartTs(tz *time.Location) (txnStart string) { // ToRow returns []interface{} for the row data of // "SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST". func (pi *ProcessInfo) ToRow(tz *time.Location) []interface{} { - return append(pi.ToRowForShow(true), pi.StmtCtx.MemTracker.BytesConsumed(), pi.txnStartTs(tz)) + bytesConsumed := int64(0) + if pi.StmtCtx.MemTracker != nil { + bytesConsumed = pi.StmtCtx.MemTracker.BytesConsumed() + } + return append(pi.ToRowForShow(true), bytesConsumed, pi.txnStartTs(tz)) } // SessionManager is an interface for session manage. Show processlist and From 5cbb17c86b07accd82ac9162a2a64473d2065a9c Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Mon, 12 Aug 2019 14:51:05 +0800 Subject: [PATCH 193/196] executor: add reserve methods to `Column` for var-length types (#11699) --- expression/vectorized.go | 21 ++++--- util/chunk/column.go | 121 ++++++++++++++++++++++++++------------ util/chunk/column_test.go | 48 +++++++++++---- 3 files changed, 134 insertions(+), 56 deletions(-) diff --git a/expression/vectorized.go b/expression/vectorized.go index ae9c5a23777ab..a83fea092dd45 100644 --- a/expression/vectorized.go +++ b/expression/vectorized.go @@ -25,12 +25,13 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C tp := expr.GetType() switch tp.EvalType() { case types.ETInt: - result.PreAllocInt64(n) + result.ResizeInt64(n) v, isNull, err := expr.EvalInt(ctx, chunk.Row{}) if err != nil { return err } - if isNull { // all slots are set to null by PreAlloc() + if isNull { + result.SetNulls(0, n, true) return nil } i64s := result.Int64s() @@ -39,12 +40,13 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C } result.SetNulls(0, n, false) case types.ETReal: - result.PreAllocFloat64(n) + result.ResizeFloat64(n) v, isNull, err := expr.EvalReal(ctx, chunk.Row{}) if err != nil { return err } - if isNull { // all slots are set to null by PreAlloc() + if isNull { + result.SetNulls(0, n, true) return nil } f64s := result.Float64s() @@ -53,12 +55,13 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C } result.SetNulls(0, n, false) case types.ETDecimal: - result.PreAllocDecimal(n) + result.ResizeDecimal(n) v, isNull, err := expr.EvalDecimal(ctx, chunk.Row{}) if err != nil { return err } - if isNull { // all slots are set to null by PreAlloc() + if isNull { + result.SetNulls(0, n, true) return nil } ds := result.Decimals() @@ -82,7 +85,7 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C } } case types.ETDuration: - result.Reset() + result.ResizeDuration(n) v, isNull, err := expr.EvalDuration(ctx, chunk.Row{}) if err != nil { return err @@ -97,7 +100,7 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C } } case types.ETJson: - result.Reset() + result.ReserveJSON(n) v, isNull, err := expr.EvalJSON(ctx, chunk.Row{}) if err != nil { return err @@ -112,7 +115,7 @@ func genVecFromConstExpr(ctx sessionctx.Context, expr Expression, input *chunk.C } } case types.ETString: - result.Reset() + result.ReserveString(n) v, isNull, err := expr.EvalString(ctx, chunk.Row{}) if err != nil { return err diff --git a/util/chunk/column.go b/util/chunk/column.go index 09a2fb7c9058d..9609e2e6eede1 100644 --- a/util/chunk/column.go +++ b/util/chunk/column.go @@ -242,33 +242,57 @@ const ( sizeGoDuration = int(unsafe.Sizeof(time.Duration(0))) ) -// preAlloc allocates space for a fixed-length-type slice and resets all slots to null. -func (c *Column) preAlloc(length, typeSize int) { - nData := length * typeSize - if len(c.data) >= nData { - c.data = c.data[:nData] +// resize resizes the column so that it contains n elements, only valid for fixed-length types. +func (c *Column) resize(n, typeSize int) { + sizeData := n * typeSize + if cap(c.data) >= sizeData { + (*reflect.SliceHeader)(unsafe.Pointer(&c.data)).Len = sizeData } else { - c.data = make([]byte, nData) + c.data = make([]byte, sizeData) } - nBitmap := (length + 7) >> 3 - if len(c.nullBitmap) >= nBitmap { - c.nullBitmap = c.nullBitmap[:nBitmap] - for i := range c.nullBitmap { - // resets all slots to null. - c.nullBitmap[i] = 0 - } + sizeNulls := (n + 7) >> 3 + if cap(c.nullBitmap) >= sizeNulls { + (*reflect.SliceHeader)(unsafe.Pointer(&c.nullBitmap)).Len = sizeNulls } else { - c.nullBitmap = make([]byte, nBitmap) + c.nullBitmap = make([]byte, sizeNulls) } - if c.elemBuf != nil && len(c.elemBuf) >= typeSize { - c.elemBuf = c.elemBuf[:typeSize] + if cap(c.elemBuf) >= typeSize { + (*reflect.SliceHeader)(unsafe.Pointer(&c.elemBuf)).Len = typeSize } else { c.elemBuf = make([]byte, typeSize) } - c.length = length + c.length = n +} + +// reserve makes the column capacity be at least enough to contain n elements. +// this method is only valid for var-length types and estElemSize is the estimated size of this type. +func (c *Column) reserve(n, estElemSize int) { + sizeData := n * estElemSize + if cap(c.data) >= sizeData { + c.data = c.data[:0] + } else { + c.data = make([]byte, 0, sizeData) + } + + sizeNulls := (n + 7) >> 3 + if cap(c.nullBitmap) >= sizeNulls { + c.nullBitmap = c.nullBitmap[:0] + } else { + c.nullBitmap = make([]byte, 0, sizeNulls) + } + + sizeOffs := n + 1 + if cap(c.offsets) >= sizeOffs { + c.offsets = c.offsets[:1] + } else { + c.offsets = make([]int64, 1, sizeOffs) + } + + c.elemBuf = nil + c.length = 0 } // SetNull sets the rowIdx to null. @@ -313,38 +337,63 @@ func (c *Column) nullCount() int { return cnt } -// PreAllocInt64 allocates space for an int64 slice and resets all slots to null. -func (c *Column) PreAllocInt64(length int) { - c.preAlloc(length, sizeInt64) +// ResizeInt64 resizes the column so that it contains n int64 elements. +func (c *Column) ResizeInt64(n int) { + c.resize(n, sizeInt64) +} + +// ResizeUint64 resizes the column so that it contains n uint64 elements. +func (c *Column) ResizeUint64(n int) { + c.resize(n, sizeUint64) +} + +// ResizeFloat32 resizes the column so that it contains n float32 elements. +func (c *Column) ResizeFloat32(n int) { + c.resize(n, sizeFloat32) +} + +// ResizeFloat64 resizes the column so that it contains n float64 elements. +func (c *Column) ResizeFloat64(n int) { + c.resize(n, sizeFloat64) +} + +// ResizeDecimal resizes the column so that it contains n decimal elements. +func (c *Column) ResizeDecimal(n int) { + c.resize(n, sizeMyDecimal) +} + +// ResizeDuration resizes the column so that it contains n duration elements. +func (c *Column) ResizeDuration(n int) { + c.resize(n, sizeGoDuration) } -// PreAllocUint64 allocates space for a uint64 slice and resets all slots to null. -func (c *Column) PreAllocUint64(length int) { - c.preAlloc(length, sizeUint64) +// ReserveString changes the column capacity to store n string elements and set the length to zero. +func (c *Column) ReserveString(n int) { + c.reserve(n, 8) } -// PreAllocFloat32 allocates space for a float32 slice and resets all slots to null. -func (c *Column) PreAllocFloat32(length int) { - c.preAlloc(length, sizeFloat32) +// ReserveBytes changes the column capacity to store n bytes elements and set the length to zero. +func (c *Column) ReserveBytes(n int) { + c.reserve(n, 8) } -// PreAllocFloat64 allocates space for a float64 slice and resets all slots to null. -func (c *Column) PreAllocFloat64(length int) { - c.preAlloc(length, sizeFloat64) +// ReserveJSON changes the column capacity to store n JSON elements and set the length to zero. +func (c *Column) ReserveJSON(n int) { + c.reserve(n, 8) } -// PreAllocDecimal allocates space for a decimal slice and resets all slots to null. -func (c *Column) PreAllocDecimal(length int) { - c.preAlloc(length, sizeMyDecimal) +// ReserveSet changes the column capacity to store n set elements and set the length to zero. +func (c *Column) ReserveSet(n int) { + c.reserve(n, 8) } -// PreAllocDuration allocates space for a duration slice and resets all slots to null. -func (c *Column) PreAllocDuration(length int) { - c.preAlloc(length, sizeGoDuration) +// ReserveEnum changes the column capacity to store n enum elements and set the length to zero. +func (c *Column) ReserveEnum(n int) { + c.reserve(n, 8) } func (c *Column) castSliceHeader(header *reflect.SliceHeader, typeSize int) { - header.Data = uintptr(unsafe.Pointer(&c.data[0])) + header.Data = (*reflect.SliceHeader)(unsafe.Pointer(&c.data)).Data header.Len = c.length header.Cap = cap(c.data) / typeSize } diff --git a/util/chunk/column_test.go b/util/chunk/column_test.go index 5cc7a0dc0766b..2397bfda72e9c 100644 --- a/util/chunk/column_test.go +++ b/util/chunk/column_test.go @@ -581,7 +581,7 @@ func (s *testChunkSuite) TestReconstructVarLen(c *check.C) { func (s *testChunkSuite) TestPreAllocInt64(c *check.C) { col := NewColumn(types.NewFieldType(mysql.TypeLonglong), 128) - col.PreAllocInt64(256) + col.ResizeInt64(256) i64s := col.Int64s() c.Assert(len(i64s), check.Equals, 256) for i := 0; i < 256; i++ { @@ -597,7 +597,7 @@ func (s *testChunkSuite) TestPreAllocUint64(c *check.C) { tll := types.NewFieldType(mysql.TypeLonglong) tll.Flag |= mysql.UnsignedFlag col := NewColumn(tll, 128) - col.PreAllocUint64(256) + col.ResizeUint64(256) u64s := col.Uint64s() c.Assert(len(u64s), check.Equals, 256) for i := 0; i < 256; i++ { @@ -611,7 +611,7 @@ func (s *testChunkSuite) TestPreAllocUint64(c *check.C) { func (s *testChunkSuite) TestPreAllocFloat32(c *check.C) { col := newFixedLenColumn(sizeFloat32, 128) - col.PreAllocFloat32(256) + col.ResizeFloat32(256) f32s := col.Float32s() c.Assert(len(f32s), check.Equals, 256) for i := 0; i < 256; i++ { @@ -625,7 +625,7 @@ func (s *testChunkSuite) TestPreAllocFloat32(c *check.C) { func (s *testChunkSuite) TestPreAllocFloat64(c *check.C) { col := newFixedLenColumn(sizeFloat64, 128) - col.PreAllocFloat64(256) + col.ResizeFloat64(256) f64s := col.Float64s() c.Assert(len(f64s), check.Equals, 256) for i := 0; i < 256; i++ { @@ -639,7 +639,7 @@ func (s *testChunkSuite) TestPreAllocFloat64(c *check.C) { func (s *testChunkSuite) TestPreAllocDecimal(c *check.C) { col := newFixedLenColumn(sizeMyDecimal, 128) - col.PreAllocDecimal(256) + col.ResizeDecimal(256) ds := col.Decimals() c.Assert(len(ds), check.Equals, 256) for i := 0; i < 256; i++ { @@ -652,7 +652,7 @@ func (s *testChunkSuite) TestPreAllocDecimal(c *check.C) { func (s *testChunkSuite) TestNull(c *check.C) { col := newFixedLenColumn(sizeFloat64, 32) - col.PreAllocFloat64(1024) + col.ResizeFloat64(1024) c.Assert(col.nullCount(), check.Equals, 1024) notNulls := make(map[int]struct{}) @@ -667,21 +667,24 @@ func (s *testChunkSuite) TestNull(c *check.C) { c.Assert(col.IsNull(idx), check.Equals, false) } - col.PreAllocFloat64(8) + col.ResizeFloat64(8) + col.SetNulls(0, 8, true) col.SetNull(7, false) c.Assert(col.nullCount(), check.Equals, 7) - col.PreAllocFloat64(8) + col.ResizeFloat64(8) + col.SetNulls(0, 8, true) c.Assert(col.nullCount(), check.Equals, 8) - col.PreAllocFloat64(9) + col.ResizeFloat64(9) + col.SetNulls(0, 9, true) col.SetNull(8, false) c.Assert(col.nullCount(), check.Equals, 8) } func (s *testChunkSuite) TestSetNulls(c *check.C) { col := newFixedLenColumn(sizeFloat64, 32) - col.PreAllocFloat64(1024) + col.ResizeFloat64(1024) c.Assert(col.nullCount(), check.Equals, 1024) col.SetNulls(0, 1024, false) @@ -707,6 +710,29 @@ func (s *testChunkSuite) TestSetNulls(c *check.C) { } } +func (s *testChunkSuite) TestResizeReserve(c *check.C) { + cI64s := newFixedLenColumn(sizeInt64, 0) + c.Assert(cI64s.length, check.Equals, 0) + for i := 0; i < 100; i++ { + t := rand.Intn(1024) + cI64s.ResizeInt64(t) + c.Assert(cI64s.length, check.Equals, t) + c.Assert(len(cI64s.Int64s()), check.Equals, t) + } + cI64s.ResizeInt64(0) + c.Assert(cI64s.length, check.Equals, 0) + c.Assert(len(cI64s.Int64s()), check.Equals, 0) + + cStrs := newVarLenColumn(0, nil) + for i := 0; i < 100; i++ { + t := rand.Intn(1024) + cStrs.ReserveString(t) + c.Assert(cStrs.length, check.Equals, 0) + } + cStrs.ReserveString(0) + c.Assert(cStrs.length, check.Equals, 0) +} + func BenchmarkDurationRow(b *testing.B) { chk1 := NewChunkWithCapacity([]*types.FieldType{types.NewFieldType(mysql.TypeDuration)}, 1024) col1 := chk1.Column(0) @@ -748,7 +774,7 @@ func BenchmarkDurationVec(b *testing.B) { b.ResetTimer() for k := 0; k < b.N; k++ { - result.PreAllocDuration(1024) + result.ResizeDuration(1024) for i := 0; i < 1024; i++ { d1 := types.Duration{Duration: ds1[i]} d2 := types.Duration{Duration: ds2[i]} From 9828c2ef65b2967d79bb255da4c0640a2777a71e Mon Sep 17 00:00:00 2001 From: Lingyu Song Date: Mon, 12 Aug 2019 15:51:08 +0800 Subject: [PATCH 194/196] fix field type error (#11713) --- executor/load_data.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/executor/load_data.go b/executor/load_data.go index 36f090aea32d0..4b020a5ff4a3a 100644 --- a/executor/load_data.go +++ b/executor/load_data.go @@ -353,15 +353,15 @@ type fieldWriter struct { pos int enclosedChar byte fieldTermChar byte - term *string + term string isEnclosed bool isLineStart bool isFieldStart bool - ReadBuf *[]byte + ReadBuf []byte OutputBuf []byte } -func (w *fieldWriter) Init(enclosedChar byte, fieldTermChar byte, readBuf *[]byte, term *string) { +func (w *fieldWriter) Init(enclosedChar byte, fieldTermChar byte, readBuf []byte, term string) { w.isEnclosed = false w.isLineStart = true w.isFieldStart = true @@ -376,8 +376,8 @@ func (w *fieldWriter) putback() { } func (w *fieldWriter) getChar() (bool, byte) { - if w.pos < len(*w.ReadBuf) { - ret := (*w.ReadBuf)[w.pos] + if w.pos < len(w.ReadBuf) { + ret := w.ReadBuf[w.pos] w.pos++ return true, ret } @@ -386,9 +386,9 @@ func (w *fieldWriter) getChar() (bool, byte) { func (w *fieldWriter) isTerminator() bool { chkpt, isterm := w.pos, true - for i := 1; i < len(*w.term); i++ { + for i := 1; i < len(w.term); i++ { flag, ch := w.getChar() - if !flag || ch != (*w.term)[i] { + if !flag || ch != w.term[i] { isterm = false break } @@ -502,7 +502,7 @@ func (e *LoadDataInfo) getFieldsFromLine(line []byte) ([]field, error) { return fields, nil } - reader.Init(e.FieldsInfo.Enclosed, e.FieldsInfo.Terminated[0], &line, &e.FieldsInfo.Terminated) + reader.Init(e.FieldsInfo.Enclosed, e.FieldsInfo.Terminated[0], line, e.FieldsInfo.Terminated) for { eol, f := reader.GetField() f = f.escape() From 140718d230f530c0db91dd22392b9b5d8328fcfc Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Mon, 12 Aug 2019 16:32:15 +0800 Subject: [PATCH 195/196] executor: fix auto retry when transaction has select for update (#11714) --- executor/executor.go | 12 ++++++++---- executor/point_get.go | 6 +----- executor/point_get_test.go | 15 +++++++++++++++ session/session.go | 2 +- session/tidb.go | 3 ++- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index 73890001eee50..f6050b219a4b4 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -760,7 +760,6 @@ func (e *SelectLockExec) Open(ctx context.Context) error { } txnCtx := e.ctx.GetSessionVars().TxnCtx - txnCtx.ForUpdate = true for id := range e.Schema().TblID2Handle { // This operation is only for schema validator check. txnCtx.UpdateDeltaForTable(id, 0, 0, map[int64]int64{}) @@ -790,13 +789,18 @@ func (e *SelectLockExec) Next(ctx context.Context, req *chunk.Chunk) error { } return nil } + return doLockKeys(ctx, e.ctx, e.keys...) +} + +func doLockKeys(ctx context.Context, se sessionctx.Context, keys ...kv.Key) error { + se.GetSessionVars().TxnCtx.ForUpdate = true // Lock keys only once when finished fetching all results. - txn, err := e.ctx.Txn(true) + txn, err := se.Txn(true) if err != nil { return err } - forUpdateTS := e.ctx.GetSessionVars().TxnCtx.GetForUpdateTS() - return txn.LockKeys(ctx, forUpdateTS, e.keys...) + forUpdateTS := se.GetSessionVars().TxnCtx.GetForUpdateTS() + return txn.LockKeys(ctx, forUpdateTS, keys...) } // LimitExec represents limit executor diff --git a/executor/point_get.go b/executor/point_get.go index e34c74f30c5d1..665742635a5e3 100644 --- a/executor/point_get.go +++ b/executor/point_get.go @@ -145,11 +145,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error { func (e *PointGetExecutor) lockKeyIfNeeded(ctx context.Context, key []byte) error { if e.lock { - txn, err := e.ctx.Txn(true) - if err != nil { - return err - } - return txn.LockKeys(ctx, e.ctx.GetSessionVars().TxnCtx.GetForUpdateTS(), kv.Key(key)) + return doLockKeys(ctx, e.ctx, key) } return nil } diff --git a/executor/point_get_test.go b/executor/point_get_test.go index cfbba81dd6a88..25fcee26c8ad7 100644 --- a/executor/point_get_test.go +++ b/executor/point_get_test.go @@ -549,3 +549,18 @@ func (s *testPointGetSuite) TestIssue10677(c *C) { tk.MustQuery("desc select * from t where pk = '1.0'").Check(testkit.Rows("Point_Get_1 1.00 root table:t, handle:1")) tk.MustQuery("select * from t where pk = '1.0'").Check(testkit.Rows("1")) } + +func (s *testPointGetSuite) TestForUpdateRetry(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.Exec("drop table if exists t") + tk.MustExec("create table t(pk int primary key, c int)") + tk.MustExec("insert into t values (1, 1), (2, 2)") + tk.MustExec("set @@tidb_disable_txn_auto_retry = 0") + tk.MustExec("begin") + tk.MustQuery("select * from t where pk = 1 for update") + tk2 := testkit.NewTestKitWithInit(c, s.store) + tk2.MustExec("update t set c = c + 1 where pk = 1") + tk.MustExec("update t set c = c + 1 where pk = 2") + _, err := tk.Exec("commit") + c.Assert(session.ErrForUpdateCantRetry.Equal(err), IsTrue) +} diff --git a/session/session.go b/session/session.go index a383398dffd45..f56e760599eef 100644 --- a/session/session.go +++ b/session/session.go @@ -631,7 +631,7 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) { connID := s.sessionVars.ConnectionID s.sessionVars.RetryInfo.Retrying = true if s.sessionVars.TxnCtx.ForUpdate { - err = errForUpdateCantRetry.GenWithStackByArgs(connID) + err = ErrForUpdateCantRetry.GenWithStackByArgs(connID) return err } diff --git a/session/tidb.go b/session/tidb.go index 1834367da4e32..1c0cbaca96c0f 100644 --- a/session/tidb.go +++ b/session/tidb.go @@ -328,8 +328,9 @@ func ResultSetToStringSlice(ctx context.Context, s Session, rs sqlexec.RecordSet return sRows, nil } +// Session errors. var ( - errForUpdateCantRetry = terror.ClassSession.New(codeForUpdateCantRetry, + ErrForUpdateCantRetry = terror.ClassSession.New(codeForUpdateCantRetry, mysql.MySQLErrName[mysql.ErrForUpdateCantRetry]) ) From 922f569136f0dd4357957db865fdc460068f8c4d Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Mon, 12 Aug 2019 16:39:41 +0800 Subject: [PATCH 196/196] util/rowcodec: add new row codec lib (#7597) --- util/rowcodec/common.go | 198 +++++++++++++++++++++ util/rowcodec/decoder.go | 213 ++++++++++++++++++++++ util/rowcodec/encoder.go | 316 +++++++++++++++++++++++++++++++++ util/rowcodec/rowcodec_test.go | 200 +++++++++++++++++++++ 4 files changed, 927 insertions(+) create mode 100644 util/rowcodec/common.go create mode 100644 util/rowcodec/decoder.go create mode 100644 util/rowcodec/encoder.go create mode 100644 util/rowcodec/rowcodec_test.go diff --git a/util/rowcodec/common.go b/util/rowcodec/common.go new file mode 100644 index 0000000000000..3ce6f9621755b --- /dev/null +++ b/util/rowcodec/common.go @@ -0,0 +1,198 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/pingcap/errors" +) + +// CodecVer is the constant number that represent the new row format. +const CodecVer = 128 + +var errInvalidCodecVer = errors.New("invalid codec version") + +// row is the struct type used to access a row. +// There are two types of row, small and large. +// A small row takes one byte colID and two bytes offset, optimized for most cases. +// If the max colID is larger than 255 or total value size is larger than 65535, the row type would be large. +// A large row takes four bytes colID and four bytes offset. +type row struct { + isLarge bool + numNotNullCols uint16 + numNullCols uint16 + data []byte + + // for small rows + colIDs []byte + offsets []uint16 + + // for large row + colIDs32 []uint32 + offsets32 []uint32 +} + +func (r *row) getData(i int) []byte { + var start, end uint32 + if r.isLarge { + if i > 0 { + start = r.offsets32[i-1] + } + end = r.offsets32[i] + } else { + if i > 0 { + start = uint32(r.offsets[i-1]) + } + end = uint32(r.offsets[i]) + } + return r.data[start:end] +} + +func (r *row) setRowData(rowData []byte) error { + if rowData[0] != CodecVer { + return errInvalidCodecVer + } + r.isLarge = rowData[1]&1 > 0 + r.numNotNullCols = binary.LittleEndian.Uint16(rowData[2:]) + r.numNullCols = binary.LittleEndian.Uint16(rowData[4:]) + cursor := 6 + if r.isLarge { + colIDsLen := int(r.numNotNullCols+r.numNullCols) * 4 + r.colIDs32 = bytesToU32Slice(rowData[cursor : cursor+colIDsLen]) + cursor += colIDsLen + offsetsLen := int(r.numNotNullCols) * 4 + r.offsets32 = bytesToU32Slice(rowData[cursor : cursor+offsetsLen]) + cursor += offsetsLen + } else { + colIDsLen := int(r.numNotNullCols + r.numNullCols) + r.colIDs = rowData[cursor : cursor+colIDsLen] + cursor += colIDsLen + offsetsLen := int(r.numNotNullCols) * 2 + r.offsets = bytes2U16Slice(rowData[cursor : cursor+offsetsLen]) + cursor += offsetsLen + } + r.data = rowData[cursor:] + return nil +} + +func bytesToU32Slice(b []byte) []uint32 { + if len(b) == 0 { + return nil + } + var u32s []uint32 + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&u32s)) + hdr.Len = len(b) / 4 + hdr.Cap = hdr.Len + hdr.Data = uintptr(unsafe.Pointer(&b[0])) + return u32s +} + +func bytes2U16Slice(b []byte) []uint16 { + if len(b) == 0 { + return nil + } + var u16s []uint16 + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&u16s)) + hdr.Len = len(b) / 2 + hdr.Cap = hdr.Len + hdr.Data = uintptr(unsafe.Pointer(&b[0])) + return u16s +} + +func u16SliceToBytes(u16s []uint16) []byte { + if len(u16s) == 0 { + return nil + } + var b []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + hdr.Len = len(u16s) * 2 + hdr.Cap = hdr.Len + hdr.Data = uintptr(unsafe.Pointer(&u16s[0])) + return b +} + +func u32SliceToBytes(u32s []uint32) []byte { + if len(u32s) == 0 { + return nil + } + var b []byte + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + hdr.Len = len(u32s) * 4 + hdr.Cap = hdr.Len + hdr.Data = uintptr(unsafe.Pointer(&u32s[0])) + return b +} + +func encodeInt(buf []byte, iVal int64) []byte { + var tmp [8]byte + if int64(int8(iVal)) == iVal { + buf = append(buf, byte(iVal)) + } else if int64(int16(iVal)) == iVal { + binary.LittleEndian.PutUint16(tmp[:], uint16(iVal)) + buf = append(buf, tmp[:2]...) + } else if int64(int32(iVal)) == iVal { + binary.LittleEndian.PutUint32(tmp[:], uint32(iVal)) + buf = append(buf, tmp[:4]...) + } else { + binary.LittleEndian.PutUint64(tmp[:], uint64(iVal)) + buf = append(buf, tmp[:8]...) + } + return buf +} + +func decodeInt(val []byte) int64 { + switch len(val) { + case 1: + return int64(int8(val[0])) + case 2: + return int64(int16(binary.LittleEndian.Uint16(val))) + case 4: + return int64(int32(binary.LittleEndian.Uint32(val))) + default: + return int64(binary.LittleEndian.Uint64(val)) + } +} + +func encodeUint(buf []byte, uVal uint64) []byte { + var tmp [8]byte + if uint64(uint8(uVal)) == uVal { + buf = append(buf, byte(uVal)) + } else if uint64(uint16(uVal)) == uVal { + binary.LittleEndian.PutUint16(tmp[:], uint16(uVal)) + buf = append(buf, tmp[:2]...) + } else if uint64(uint32(uVal)) == uVal { + binary.LittleEndian.PutUint32(tmp[:], uint32(uVal)) + buf = append(buf, tmp[:4]...) + } else { + binary.LittleEndian.PutUint64(tmp[:], uint64(uVal)) + buf = append(buf, tmp[:8]...) + } + return buf +} + +func decodeUint(val []byte) uint64 { + switch len(val) { + case 1: + return uint64(val[0]) + case 2: + return uint64(binary.LittleEndian.Uint16(val)) + case 4: + return uint64(binary.LittleEndian.Uint32(val)) + default: + return binary.LittleEndian.Uint64(val) + } +} diff --git a/util/rowcodec/decoder.go b/util/rowcodec/decoder.go new file mode 100644 index 0000000000000..f452b0deb2f63 --- /dev/null +++ b/util/rowcodec/decoder.go @@ -0,0 +1,213 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "math" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" +) + +// Decoder decodes the row to chunk.Chunk. +type Decoder struct { + row + requestColIDs []int64 + handleColID int64 + requestTypes []*types.FieldType + origDefaults [][]byte + loc *time.Location +} + +// NewDecoder creates a NewDecoder. +// requestColIDs is the columnIDs to decode. tps is the field types for request columns. +// origDefault is the original default value in old format, if the column ID is not found in the row, +// the origDefault will be used. +func NewDecoder(requestColIDs []int64, handleColID int64, tps []*types.FieldType, origDefaults [][]byte, + sc *stmtctx.StatementContext) (*Decoder, error) { + xOrigDefaultVals := make([][]byte, len(origDefaults)) + for i := 0; i < len(origDefaults); i++ { + if len(origDefaults[i]) == 0 { + continue + } + xDefaultVal, err := convertDefaultValue(origDefaults[i], sc) + if err != nil { + return nil, err + } + xOrigDefaultVals[i] = xDefaultVal + } + return &Decoder{ + requestColIDs: requestColIDs, + handleColID: handleColID, + requestTypes: tps, + origDefaults: xOrigDefaultVals, + loc: sc.TimeZone, + }, nil +} + +func convertDefaultValue(defaultVal []byte, sc *stmtctx.StatementContext) (colVal []byte, err error) { + var d types.Datum + _, d, err = codec.DecodeOne(defaultVal) + if err != nil { + return + } + return encodeDatum(nil, d, sc) +} + +// Decode decodes a row to chunk. +func (decoder *Decoder) Decode(rowData []byte, handle int64, chk *chunk.Chunk) error { + err := decoder.setRowData(rowData) + if err != nil { + return err + } + for colIdx, colID := range decoder.requestColIDs { + if colID == decoder.handleColID { + chk.AppendInt64(colIdx, handle) + continue + } + // Search the column in not-null columns array. + i, j := 0, int(decoder.numNotNullCols) + var found bool + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + var v int64 + if decoder.isLarge { + v = int64(decoder.colIDs32[h]) + } else { + v = int64(decoder.colIDs[h]) + } + if v < colID { + i = h + 1 + } else if v > colID { + j = h + } else { + found = true + colData := decoder.getData(h) + err := decoder.decodeColData(colIdx, colData, chk) + if err != nil { + return err + } + break + } + } + if found { + continue + } + // Search the column in null columns array. + i, j = int(decoder.numNotNullCols), int(decoder.numNotNullCols+decoder.numNullCols) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + var v int64 + if decoder.isLarge { + v = int64(decoder.colIDs32[h]) + } else { + v = int64(decoder.colIDs[h]) + } + if v < colID { + i = h + 1 + } else if v > colID { + j = h + } else { + found = true + break + } + } + if found || decoder.origDefaults[colIdx] == nil { + chk.AppendNull(colIdx) + } else { + err := decoder.decodeColData(colIdx, decoder.origDefaults[colIdx], chk) + if err != nil { + return err + } + } + } + return nil +} + +func (decoder *Decoder) decodeColData(colIdx int, colData []byte, chk *chunk.Chunk) error { + ft := decoder.requestTypes[colIdx] + switch ft.Tp { + case mysql.TypeLonglong, mysql.TypeLong, mysql.TypeInt24, mysql.TypeShort, mysql.TypeTiny, mysql.TypeYear: + if mysql.HasUnsignedFlag(ft.Flag) { + chk.AppendUint64(colIdx, decodeUint(colData)) + } else { + chk.AppendInt64(colIdx, decodeInt(colData)) + } + case mysql.TypeFloat: + chk.AppendFloat32(colIdx, float32(math.Float64frombits(decodeUint(colData)))) + case mysql.TypeDouble: + chk.AppendFloat64(colIdx, math.Float64frombits(decodeUint(colData))) + case mysql.TypeVarString, mysql.TypeVarchar, mysql.TypeString, + mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + chk.AppendBytes(colIdx, colData) + case mysql.TypeNewDecimal: + _, dec, _, _, err := codec.DecodeDecimal(colData) + if err != nil { + return err + } + chk.AppendMyDecimal(colIdx, dec) + case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: + var t types.Time + t.Type = ft.Tp + t.Fsp = ft.Decimal + err := t.FromPackedUint(decodeUint(colData)) + if err != nil { + return err + } + if ft.Tp == mysql.TypeTimestamp && !t.IsZero() { + err = t.ConvertTimeZone(time.UTC, decoder.loc) + if err != nil { + return err + } + } + chk.AppendTime(colIdx, t) + case mysql.TypeDuration: + var dur types.Duration + dur.Duration = time.Duration(decodeInt(colData)) + dur.Fsp = ft.Decimal + chk.AppendDuration(colIdx, dur) + case mysql.TypeEnum: + // ignore error deliberately, to read empty enum value. + enum, err := types.ParseEnumValue(ft.Elems, decodeUint(colData)) + if err != nil { + enum = types.Enum{} + } + chk.AppendEnum(colIdx, enum) + case mysql.TypeSet: + set, err := types.ParseSetValue(ft.Elems, decodeUint(colData)) + if err != nil { + return err + } + chk.AppendSet(colIdx, set) + case mysql.TypeBit: + byteSize := (ft.Flen + 7) >> 3 + chk.AppendBytes(colIdx, types.NewBinaryLiteralFromUint(decodeUint(colData), byteSize)) + case mysql.TypeJSON: + var j json.BinaryJSON + j.TypeCode = colData[0] + j.Value = colData[1:] + chk.AppendJSON(colIdx, j) + default: + return errors.Errorf("unknown type %d", ft.Tp) + } + return nil +} diff --git a/util/rowcodec/encoder.go b/util/rowcodec/encoder.go new file mode 100644 index 0000000000000..4b5cb2e46144e --- /dev/null +++ b/util/rowcodec/encoder.go @@ -0,0 +1,316 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "math" + "sort" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/parser/terror" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/codec" +) + +// Encoder is used to encode a row. +type Encoder struct { + row + tempColIDs []int64 + values []types.Datum + tempData []byte + sc *stmtctx.StatementContext +} + +// NewEncoder creates a new Encoder with column IDs. +func NewEncoder(colIDs []int64, sc *stmtctx.StatementContext) *Encoder { + return &Encoder{ + tempColIDs: colIDs, + sc: sc, + } +} + +func (encoder *Encoder) reset() { + encoder.isLarge = false + encoder.numNotNullCols = 0 + encoder.numNullCols = 0 + encoder.data = encoder.data[:0] + encoder.values = encoder.values[:0] +} + +// Encode encodes a row from a datums slice. +func (encoder *Encoder) Encode(values []types.Datum, buf []byte) ([]byte, error) { + encoder.reset() + encoder.values = append(encoder.values, values...) + for i, colID := range encoder.tempColIDs { + if colID > 255 { + encoder.isLarge = true + } + if values[i].IsNull() { + encoder.numNullCols++ + } else { + encoder.numNotNullCols++ + } + } + return encoder.build(buf) +} + +func (encoder *Encoder) build(buf []byte) ([]byte, error) { + r := &encoder.row + // Separate null and not-null column IDs. + numCols := len(encoder.tempColIDs) + nullIdx := numCols - int(r.numNullCols) + notNullIdx := 0 + if r.isLarge { + encoder.initColIDs32() + encoder.initOffsets32() + } else { + encoder.initColIDs() + encoder.initOffsets() + } + for i, colID := range encoder.tempColIDs { + if encoder.values[i].IsNull() { + if r.isLarge { + r.colIDs32[nullIdx] = uint32(colID) + } else { + r.colIDs[nullIdx] = byte(colID) + } + nullIdx++ + } else { + if r.isLarge { + r.colIDs32[notNullIdx] = uint32(colID) + } else { + r.colIDs[notNullIdx] = byte(colID) + } + encoder.values[notNullIdx] = encoder.values[i] + notNullIdx++ + } + } + if r.isLarge { + largeNotNullSorter := (*largeNotNullSorter)(encoder) + sort.Sort(largeNotNullSorter) + if r.numNullCols > 0 { + largeNullSorter := (*largeNullSorter)(encoder) + sort.Sort(largeNullSorter) + } + } else { + smallNotNullSorter := (*smallNotNullSorter)(encoder) + sort.Sort(smallNotNullSorter) + if r.numNullCols > 0 { + smallNullSorter := (*smallNullSorter)(encoder) + sort.Sort(smallNullSorter) + } + } + for i := 0; i < notNullIdx; i++ { + var err error + r.data, err = encodeDatum(r.data, encoder.values[i], encoder.sc) + if err != nil { + return nil, errors.Trace(err) + } + if len(r.data) > math.MaxUint16 && !r.isLarge { + // We need to convert the row to large row. + encoder.initColIDs32() + for j := 0; j < numCols; j++ { + r.colIDs32[j] = uint32(r.colIDs[j]) + } + encoder.initOffsets32() + for j := 0; j <= i; j++ { + r.offsets32[j] = uint32(r.offsets[j]) + } + r.isLarge = true + } + if r.isLarge { + r.offsets32[i] = uint32(len(r.data)) + } else { + r.offsets[i] = uint16(len(r.data)) + } + } + buf = append(buf, CodecVer) + flag := byte(0) + if r.isLarge { + flag = 1 + } + buf = append(buf, flag) + buf = append(buf, byte(r.numNotNullCols), byte(r.numNotNullCols>>8)) + buf = append(buf, byte(r.numNullCols), byte(r.numNullCols>>8)) + if r.isLarge { + buf = append(buf, u32SliceToBytes(r.colIDs32)...) + buf = append(buf, u32SliceToBytes(r.offsets32)...) + } else { + buf = append(buf, r.colIDs...) + buf = append(buf, u16SliceToBytes(r.offsets)...) + } + buf = append(buf, r.data...) + return buf, nil +} + +func encodeDatum(buf []byte, d types.Datum, sc *stmtctx.StatementContext) ([]byte, error) { + switch d.Kind() { + case types.KindInt64: + buf = encodeInt(buf, d.GetInt64()) + case types.KindUint64: + buf = encodeUint(buf, d.GetUint64()) + case types.KindString, types.KindBytes: + buf = append(buf, d.GetBytes()...) + case types.KindFloat32, types.KindFloat64: + buf = encodeUint(buf, uint64(math.Float64bits(d.GetFloat64()))) + case types.KindMysqlDecimal: + var err error + buf, err = codec.EncodeDecimal(buf, d.GetMysqlDecimal(), d.Length(), d.Frac()) + if terror.ErrorEqual(err, types.ErrTruncated) { + err = sc.HandleTruncate(err) + } else if terror.ErrorEqual(err, types.ErrOverflow) { + err = sc.HandleOverflow(err, err) + } + if err != nil { + return nil, errors.Trace(err) + } + case types.KindMysqlTime: + t := d.GetMysqlTime() + // Encoding timestamp need to consider timezone. + // If it's not in UTC, transform to UTC first. + if t.Type == mysql.TypeTimestamp && sc.TimeZone != time.UTC { + err := t.ConvertTimeZone(sc.TimeZone, time.UTC) + if err != nil { + return nil, errors.Trace(err) + } + } + v, err := t.ToPackedUint() + if err != nil { + return nil, errors.Trace(err) + } + buf = encodeUint(buf, v) + case types.KindMysqlDuration: + buf = encodeInt(buf, int64(d.GetMysqlDuration().Duration)) + case types.KindMysqlEnum: + buf = encodeUint(buf, uint64(d.GetMysqlEnum().ToNumber())) + case types.KindMysqlSet: + buf = encodeUint(buf, uint64(d.GetMysqlSet().ToNumber())) + case types.KindMysqlBit, types.KindBinaryLiteral: + val, err := types.BinaryLiteral(d.GetBytes()).ToInt(sc) + if err != nil { + terror.Log(errors.Trace(err)) + } + buf = encodeUint(buf, val) + case types.KindMysqlJSON: + j := d.GetMysqlJSON() + buf = append(buf, j.TypeCode) + buf = append(buf, j.Value...) + default: + return nil, errors.Errorf("unsupport encode type %d", d.Kind()) + } + return buf, nil +} + +func (encoder *Encoder) initColIDs() { + numCols := int(encoder.numNotNullCols + encoder.numNullCols) + if cap(encoder.colIDs) >= numCols { + encoder.colIDs = encoder.colIDs[:numCols] + } else { + encoder.colIDs = make([]byte, numCols) + } +} + +func (encoder *Encoder) initColIDs32() { + numCols := int(encoder.numNotNullCols + encoder.numNullCols) + if cap(encoder.colIDs32) >= numCols { + encoder.colIDs32 = encoder.colIDs32[:numCols] + } else { + encoder.colIDs32 = make([]uint32, numCols) + } +} + +func (encoder *Encoder) initOffsets() { + if cap(encoder.offsets) >= int(encoder.numNotNullCols) { + encoder.offsets = encoder.offsets[:encoder.numNotNullCols] + } else { + encoder.offsets = make([]uint16, encoder.numNotNullCols) + } +} + +func (encoder *Encoder) initOffsets32() { + if cap(encoder.offsets32) >= int(encoder.numNotNullCols) { + encoder.offsets32 = encoder.offsets32[:encoder.numNotNullCols] + } else { + encoder.offsets32 = make([]uint32, encoder.numNotNullCols) + } +} + +/* + We define several sorters to avoid switch cost in sort functions. +*/ + +type largeNotNullSorter Encoder + +func (s *largeNotNullSorter) Less(i, j int) bool { + return s.colIDs32[i] < s.colIDs32[j] +} + +func (s *largeNotNullSorter) Len() int { + return int(s.numNotNullCols) +} + +func (s *largeNotNullSorter) Swap(i, j int) { + s.colIDs32[i], s.colIDs32[j] = s.colIDs32[j], s.colIDs32[i] + s.values[i], s.values[j] = s.values[j], s.values[i] +} + +type smallNotNullSorter Encoder + +func (s *smallNotNullSorter) Less(i, j int) bool { + return s.colIDs[i] < s.colIDs[j] +} + +func (s *smallNotNullSorter) Len() int { + return int(s.numNotNullCols) +} + +func (s *smallNotNullSorter) Swap(i, j int) { + s.colIDs[i], s.colIDs[j] = s.colIDs[j], s.colIDs[i] + s.values[i], s.values[j] = s.values[j], s.values[i] +} + +type smallNullSorter Encoder + +func (s *smallNullSorter) Less(i, j int) bool { + nullCols := s.colIDs[s.numNotNullCols:] + return nullCols[i] < nullCols[j] +} + +func (s *smallNullSorter) Len() int { + return int(s.numNullCols) +} + +func (s *smallNullSorter) Swap(i, j int) { + nullCols := s.colIDs[s.numNotNullCols:] + nullCols[i], nullCols[j] = nullCols[j], nullCols[i] +} + +type largeNullSorter Encoder + +func (s *largeNullSorter) Less(i, j int) bool { + nullCols := s.colIDs32[s.numNotNullCols:] + return nullCols[i] < nullCols[j] +} + +func (s *largeNullSorter) Len() int { + return int(s.numNullCols) +} + +func (s *largeNullSorter) Swap(i, j int) { + nullCols := s.colIDs32[s.numNotNullCols:] + nullCols[i], nullCols[j] = nullCols[j], nullCols[i] +} diff --git a/util/rowcodec/rowcodec_test.go b/util/rowcodec/rowcodec_test.go new file mode 100644 index 0000000000000..4324404b3225a --- /dev/null +++ b/util/rowcodec/rowcodec_test.go @@ -0,0 +1,200 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package rowcodec + +import ( + "math" + "testing" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/types/json" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/codec" +) + +func TestT(t *testing.T) { + TestingT(t) +} + +var _ = Suite(&testSuite{}) + +type testSuite struct{} + +func (s *testSuite) TestRowCodec(c *C) { + colIDs := []int64{1, 2, 3, 4, 5, 10} + rb := NewEncoder(colIDs, new(stmtctx.StatementContext)) + dt, err := types.ParseDatetime(rb.sc, "2018-01-19 03:14:07.999999") + c.Assert(err, IsNil) + datums := types.MakeDatums( + dt, + "abc", + nil, + 1, + types.NewDecFromInt(1), + "abc", + ) + newRow, err := rb.Encode(datums, nil) + c.Check(err, IsNil) + s.checkDecode(c, rb.sc, newRow) + + // Test large column ID + rb.tempColIDs = []int64{1, 2, 3, 4, 5, 512} + newRow, err = rb.Encode(datums, nil) + c.Check(err, IsNil) + s.checkDecode(c, rb.sc, newRow) + + // Test large column value + rb.tempColIDs = []int64{1, 2, 3, 4, 5, 10} + datums[5] = types.NewBytesDatum(make([]byte, 65536)) + newRow, err = rb.Encode(datums, nil) + c.Check(err, IsNil) + s.checkDecode(c, rb.sc, newRow) +} + +func (s *testSuite) TestIntCodec(c *C) { + uints := []uint64{255, math.MaxUint16, math.MaxUint32, math.MaxUint32 + 1} + sizes := []int{1, 2, 4, 8} + for i, v := range uints { + data := encodeUint(nil, v) + c.Assert(len(data), Equals, sizes[i]) + c.Assert(decodeUint(data), Equals, v) + } + + ints := []int64{127, math.MaxInt16, math.MaxInt32, math.MaxInt32 + 1} + for i, v := range ints { + data := encodeInt(nil, v) + c.Assert(len(data), Equals, sizes[i]) + c.Assert(decodeInt(data), Equals, v) + } +} + +func (s *testSuite) TestMoreTypes(c *C) { + colIDs := []int64{1, 2, 3, 4, 5, 6, 7, 8} + sc := new(stmtctx.StatementContext) + sc.TimeZone = time.Local + rb := NewEncoder(colIDs, sc) + ts, err := types.ParseTimestampFromNum(rb.sc, 20181111090909) + c.Assert(err, IsNil) + datums := types.MakeDatums( + float32(1.0), + float64(1.0), + ts, + types.Duration{Duration: time.Minute}, + types.Enum{Name: "a", Value: 1}, + types.Set{Name: "a", Value: 1}, + json.CreateBinary("abc"), + types.BitLiteral([]byte{101}), + ) + newRow, err := rb.Encode(datums, nil) + c.Check(err, IsNil) + fieldTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeFloat), + types.NewFieldType(mysql.TypeDouble), + {Tp: mysql.TypeTimestamp, Decimal: 0}, + types.NewFieldType(mysql.TypeDuration), + {Tp: mysql.TypeEnum, Elems: []string{"a"}}, + {Tp: mysql.TypeSet, Elems: []string{"a"}}, + types.NewFieldType(mysql.TypeJSON), + {Tp: mysql.TypeBit, Flen: 8}, + } + rd, err := NewDecoder(colIDs, 0, fieldTypes, make([][]byte, 8), rb.sc) + c.Assert(err, IsNil) + chk := chunk.NewChunkWithCapacity(fieldTypes, 1) + err = rd.Decode(newRow, 0, chk) + c.Assert(err, IsNil) + row := chk.GetRow(0) + c.Assert(row.GetFloat32(0), Equals, float32(1.0)) + c.Assert(row.GetFloat64(1), Equals, float64(1.0)) + c.Assert(row.GetTime(2).String(), Equals, ts.String()) + c.Assert(row.GetDuration(3, 0), Equals, datums[3].GetMysqlDuration()) + c.Assert(row.GetEnum(4), Equals, datums[4].GetMysqlEnum()) + c.Assert(row.GetSet(5), Equals, datums[5].GetMysqlSet()) + c.Assert(row.GetJSON(6), DeepEquals, datums[6].GetMysqlJSON()) + c.Assert(row.GetBytes(7), DeepEquals, []byte(datums[7].GetMysqlBit())) +} + +func (s *testSuite) checkDecode(c *C, sc *stmtctx.StatementContext, newRow []byte) { + readRowTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeVarString), + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeNewDecimal), + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeLonglong), + } + readColIDS := []int64{2, 3, 4, 5, 6, 7} + defaultColVal, err := codec.EncodeValue(sc, nil, types.NewIntDatum(5)) + c.Assert(err, IsNil) + defaults := [][]byte{nil, defaultColVal, defaultColVal, nil, defaultColVal, nil} + + rd, err := NewDecoder(readColIDS, 7, readRowTypes, defaults, sc) + c.Assert(err, IsNil) + chk := chunk.NewChunkWithCapacity(readRowTypes, 1) + err = rd.Decode(newRow, 1000, chk) + c.Assert(err, IsNil) + row := chk.GetRow(0) + c.Assert(row.GetString(0), Equals, "abc") + c.Assert(row.IsNull(1), IsTrue) + c.Assert(row.GetInt64(2), Equals, int64(1)) + c.Assert(row.GetMyDecimal(3).String(), Equals, "1") + c.Assert(row.GetInt64(4), Equals, int64(5)) + c.Assert(row.GetInt64(5), Equals, int64(1000)) +} + +func BenchmarkEncode(b *testing.B) { + b.ReportAllocs() + oldRow := types.MakeDatums(1, "abc", 1.1) + xb := NewEncoder([]int64{1, 2, 3}, new(stmtctx.StatementContext)) + var buf []byte + var err error + for i := 0; i < b.N; i++ { + buf = buf[:0] + buf, err = xb.Encode(oldRow, buf) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecode(b *testing.B) { + b.ReportAllocs() + oldRow := types.MakeDatums(1, "abc", 1.1) + colIDs := []int64{-1, 2, 3} + tps := []*types.FieldType{ + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeString), + types.NewFieldType(mysql.TypeDouble), + } + xb := NewEncoder(colIDs, new(stmtctx.StatementContext)) + xRowData, err := xb.Encode(oldRow, nil) + if err != nil { + b.Fatal(err) + } + decoder, err := NewDecoder(colIDs, -1, tps, make([][]byte, 3), xb.sc) + if err != nil { + b.Fatal(err) + } + chk := chunk.NewChunkWithCapacity(tps, 1) + for i := 0; i < b.N; i++ { + chk.Reset() + err = decoder.Decode(xRowData, 1, chk) + if err != nil { + b.Fatal(err) + } + } +}