From f5053bc8f0b9594cdd5d45a7de8f110399eac49c Mon Sep 17 00:00:00 2001 From: 3pointer Date: Thu, 5 Aug 2021 13:42:25 +0800 Subject: [PATCH 01/16] Squashed 'br/' content from commit cf621d7c0 git-subtree-dir: br git-subtree-split: cf621d7c0c5eb5cc5d6a42816f84fd2aabb55bfc --- .codecov.yml | 22 + .dockerignore | 1 + .editorconfig | 10 + .gitattributes | 1 + .github/ISSUE_TEMPLATE/bug-report.md | 45 + .github/ISSUE_TEMPLATE/feature-request.md | 19 + .github/ISSUE_TEMPLATE/question.md | 24 + .github/challenge-bot.yml | 1 + .github/pull_request_template.md | 42 + .github/workflows/build.yml | 68 + .github/workflows/compatible_test.yml | 71 + .gitignore | 14 + .golangci.yml | 12 + COMPATIBILITY_TEST.md | 42 + CONTRIBUTING.md | 90 + LICENSE.md | 201 + Makefile | 267 + README.md | 137 + SECURITY.md | 33 + cmd/br/backup.go | 158 + cmd/br/cmd.go | 208 + cmd/br/debug.go | 389 + cmd/br/main.go | 61 + cmd/br/main_test.go | 44 + cmd/br/restore.go | 179 + cmd/tidb-lightning-ctl/main.go | 410 + cmd/tidb-lightning-ctl/main_test.go | 47 + cmd/tidb-lightning/main.go | 112 + cmd/tidb-lightning/main_test.go | 49 + compatibility/backup_cluster.yaml | 128 + .../application_default_credentials.json | 6 + compatibility/get_last_tags.sh | 24 + compatibility/prepare_backup.sh | 46 + compatibility/prepare_data/workload | 11 + docker-compose.yaml | 224 + docker/Dockerfile | 27 + docker/config/pd.toml | 18 + docker/config/tidb.toml | 9 + docker/config/tikv.toml | 25 + docker/gcs.env | 1 + docker/minio.env | 6 + ...2019-08-05-new-design-of-backup-restore.md | 151 + .../2019-09-09-BR-key-rewrite-disscussion.md | 89 + ...-design-of-reorganize-importSST-to-TiKV.md | 110 + .../arch-of-reorganized-importer.svg | 1 + docs/resources/download-sst-diagram.svg | 1 + docs/resources/solution3-of-key-rewrite.svg | 1 + errors.toml | 213 + go.mod1 | 61 + go.sum1 | 987 ++ images/arch.svg | 110 + metrics/alertmanager/lightning.rules.yml | 14 + metrics/grafana/br.json | 2969 +++++ metrics/grafana/lightning.json | 1604 +++ pkg/backup/check.go | 35 + pkg/backup/client.go | 949 ++ pkg/backup/client_test.go | 272 + pkg/backup/metrics.go | 31 + pkg/backup/push.go | 189 + pkg/backup/schema.go | 192 + pkg/backup/schema_test.go | 242 + pkg/cdclog/buffer.go | 217 + pkg/cdclog/decoder.go | 299 + pkg/cdclog/decoder_test.go | 168 + pkg/cdclog/puller.go | 184 + pkg/checksum/executor.go | 320 + pkg/checksum/executor_test.go | 132 + pkg/checksum/validate.go | 79 + pkg/conn/conn.go | 392 + pkg/conn/conn_test.go | 145 + pkg/errors/errors.go | 83 + pkg/glue/glue.go | 49 + pkg/gluetidb/glue.go | 172 + pkg/gluetikv/glue.go | 63 + pkg/gluetikv/glue_test.go | 20 + pkg/httputil/http.go | 22 + pkg/kv/checksum.go | 114 + pkg/kv/checksum_test.go | 91 + pkg/kv/kv.go | 502 + pkg/kv/kv_test.go | 91 + pkg/kv/session.go | 252 + pkg/kv/session_test.go | 35 + pkg/lightning/backend/backend.go | 481 + pkg/lightning/backend/backend_test.go | 410 + pkg/lightning/backend/importer/importer.go | 408 + .../backend/importer/importer_test.go | 293 + pkg/lightning/backend/kv/allocator.go | 63 + pkg/lightning/backend/kv/kv2sql.go | 62 + pkg/lightning/backend/kv/session.go | 338 + pkg/lightning/backend/kv/session_test.go | 35 + pkg/lightning/backend/kv/sql2kv.go | 533 + pkg/lightning/backend/kv/sql2kv_test.go | 602 + pkg/lightning/backend/kv/types.go | 54 + pkg/lightning/backend/local/duplicate.go | 633 + pkg/lightning/backend/local/iterator.go | 193 + pkg/lightning/backend/local/iterator_test.go | 254 + pkg/lightning/backend/local/key_adapter.go | 94 + .../backend/local/key_adapter_test.go | 154 + pkg/lightning/backend/local/local.go | 3183 +++++ pkg/lightning/backend/local/local_freebsd.go | 26 + pkg/lightning/backend/local/local_test.go | 763 ++ pkg/lightning/backend/local/local_unix.go | 91 + .../backend/local/local_unix_generic.go | 24 + pkg/lightning/backend/local/local_windows.go | 33 + pkg/lightning/backend/local/localhelper.go | 598 + .../backend/local/localhelper_test.go | 720 ++ pkg/lightning/backend/noop/noop.go | 184 + pkg/lightning/backend/tidb/tidb.go | 645 + pkg/lightning/backend/tidb/tidb_test.go | 387 + pkg/lightning/checkpoints/checkpoints.go | 1657 +++ .../checkpoints/checkpoints_file_test.go | 330 + .../checkpoints/checkpoints_sql_test.go | 531 + pkg/lightning/checkpoints/checkpoints_test.go | 307 + .../checkpointspb/file_checkpoints.pb.go | 2468 ++++ .../checkpointspb/file_checkpoints.proto | 72 + pkg/lightning/checkpoints/glue_checkpoint.go | 806 ++ pkg/lightning/checkpoints/tidb.go | 30 + pkg/lightning/common/conn.go | 111 + pkg/lightning/common/once_error.go | 46 + pkg/lightning/common/once_error_test.go | 59 + pkg/lightning/common/pause.go | 156 + pkg/lightning/common/pause_test.go | 190 + pkg/lightning/common/security.go | 162 + pkg/lightning/common/security_test.go | 99 + pkg/lightning/common/storage.go | 23 + pkg/lightning/common/storage_test.go | 34 + pkg/lightning/common/storage_unix.go | 74 + pkg/lightning/common/storage_windows.go | 50 + pkg/lightning/common/util.go | 396 + pkg/lightning/common/util_test.go | 218 + pkg/lightning/config/bytesize.go | 44 + pkg/lightning/config/bytesize_test.go | 129 + pkg/lightning/config/config.go | 898 ++ pkg/lightning/config/config_test.go | 694 ++ pkg/lightning/config/configlist.go | 153 + pkg/lightning/config/configlist_test.go | 133 + pkg/lightning/config/const.go | 32 + pkg/lightning/config/global.go | 285 + pkg/lightning/glue/glue.go | 191 + pkg/lightning/lightning.go | 693 ++ pkg/lightning/lightning_test.go | 564 + pkg/lightning/log/filter.go | 57 + pkg/lightning/log/filter_test.go | 96 + pkg/lightning/log/log.go | 220 + pkg/lightning/log/log_test.go | 51 + pkg/lightning/log/testlogger.go | 36 + pkg/lightning/manual/allocator.go | 20 + pkg/lightning/manual/manual.go | 65 + pkg/lightning/manual/manual_nocgo.go | 19 + pkg/lightning/metric/metric.go | 259 + pkg/lightning/metric/metric_test.go | 61 + pkg/lightning/mydump/bytes.go | 39 + pkg/lightning/mydump/csv/split_large_file.csv | 5 + pkg/lightning/mydump/csv_parser.go | 523 + pkg/lightning/mydump/csv_parser_test.go | 948 ++ pkg/lightning/mydump/examples/metadata | 2 + .../examples/mocker_test-schema-create.sql | 1 + .../mydump/examples/mocker_test.i-schema.sql | 6 + .../mydump/examples/mocker_test.i.sql | 1 + ...cker_test.report_case_high_risk-schema.sql | 9 + .../mocker_test.report_case_high_risk.sql | 1 + .../mocker_test.tbl_autoid-schema.sql | 8 + .../examples/mocker_test.tbl_autoid.sql | 10010 ++++++++++++++++ .../mocker_test.tbl_multi_index-schema.sql | 9 + .../examples/mocker_test.tbl_multi_index.sql | 10010 ++++++++++++++++ pkg/lightning/mydump/examples/test.parquet | Bin 0 -> 2686 bytes pkg/lightning/mydump/loader.go | 461 + pkg/lightning/mydump/loader_test.go | 596 + pkg/lightning/mydump/parquet_parser.go | 536 + pkg/lightning/mydump/parquet_parser_test.go | 267 + pkg/lightning/mydump/parser.go | 574 + pkg/lightning/mydump/parser.rl | 187 + pkg/lightning/mydump/parser_generated.go | 2515 ++++ pkg/lightning/mydump/parser_test.go | 884 ++ pkg/lightning/mydump/reader.go | 178 + pkg/lightning/mydump/reader_test.go | 182 + pkg/lightning/mydump/region.go | 404 + pkg/lightning/mydump/region_test.go | 334 + pkg/lightning/mydump/router.go | 338 + pkg/lightning/mydump/router_test.go | 256 + pkg/lightning/restore/check_info.go | 441 + pkg/lightning/restore/check_template.go | 130 + pkg/lightning/restore/checksum.go | 476 + pkg/lightning/restore/checksum_test.go | 410 + pkg/lightning/restore/meta_manager.go | 817 ++ pkg/lightning/restore/meta_manager_test.go | 265 + pkg/lightning/restore/restore.go | 2334 ++++ pkg/lightning/restore/restore_test.go | 2119 ++++ pkg/lightning/restore/table_restore.go | 887 ++ pkg/lightning/restore/tidb.go | 391 + pkg/lightning/restore/tidb_test.go | 498 + pkg/lightning/sigusr1_other.go | 20 + pkg/lightning/sigusr1_unix.go | 38 + pkg/lightning/tikv/tikv.go | 234 + pkg/lightning/tikv/tikv_test.go | 252 + pkg/lightning/verification/checksum.go | 107 + pkg/lightning/verification/checksum_test.go | 92 + pkg/lightning/web/progress.go | 186 + pkg/lightning/web/res.go | 22 + pkg/lightning/web/res_vfsdata.go | 202 + pkg/lightning/worker/worker.go | 65 + pkg/lightning/worker/worker_test.go | 56 + pkg/logutil/logging.go | 229 + pkg/logutil/logging_test.go | 210 + pkg/logutil/rate.go | 56 + pkg/membuf/buffer.go | 146 + pkg/membuf/buffer_test.go | 90 + pkg/metautil/metafile.go | 662 + pkg/metautil/metafile_test.go | 135 + pkg/mock/backend.go | 361 + pkg/mock/glue.go | 234 + pkg/mock/glue_checkpoint.go | 136 + pkg/mock/importer.go | 377 + pkg/mock/kv.go | 153 + pkg/mock/mock_cluster.go | 194 + pkg/mock/mock_cluster_test.go | 37 + pkg/mock/mockid/mockid.go | 36 + pkg/mock/s3iface.go | 4886 ++++++++ pkg/mock/storage/storage.go | 139 + pkg/pdutil/pd.go | 685 ++ pkg/pdutil/pd_test.go | 202 + pkg/pdutil/utils.go | 120 + pkg/redact/redact.go | 36 + pkg/redact/redact_test.go | 32 + pkg/restore/backoff.go | 116 + pkg/restore/backoff_test.go | 117 + pkg/restore/batcher.go | 375 + pkg/restore/batcher_test.go | 391 + pkg/restore/client.go | 1123 ++ pkg/restore/client_test.go | 249 + pkg/restore/db.go | 277 + pkg/restore/db_test.go | 222 + pkg/restore/import.go | 575 + pkg/restore/ingester.go | 605 + pkg/restore/log_client.go | 762 ++ pkg/restore/log_client_test.go | 127 + pkg/restore/merge.go | 155 + pkg/restore/merge_test.go | 297 + pkg/restore/pipeline_items.go | 275 + pkg/restore/range.go | 136 + pkg/restore/range_test.go | 85 + pkg/restore/split.go | 417 + pkg/restore/split_client.go | 551 + pkg/restore/split_test.go | 373 + pkg/restore/systable_restore.go | 217 + pkg/restore/util.go | 448 + pkg/restore/util_test.go | 270 + pkg/rtree/logging.go | 61 + pkg/rtree/logging_test.go | 47 + pkg/rtree/rtree.go | 220 + pkg/rtree/rtree_test.go | 186 + pkg/storage/compress.go | 131 + pkg/storage/compress_test.go | 42 + pkg/storage/flags.go | 22 + pkg/storage/gcs.go | 388 + pkg/storage/gcs_test.go | 271 + pkg/storage/local.go | 130 + pkg/storage/local_test.go | 68 + pkg/storage/local_unix.go | 19 + pkg/storage/local_windows.go | 13 + pkg/storage/noop.go | 71 + pkg/storage/parse.go | 172 + pkg/storage/parse_test.go | 176 + pkg/storage/s3.go | 745 ++ pkg/storage/s3_test.go | 1029 ++ pkg/storage/storage.go | 178 + pkg/storage/writer.go | 230 + pkg/storage/writer_test.go | 168 + pkg/summary/collector.go | 235 + pkg/summary/collector_test.go | 49 + pkg/summary/summary.go | 45 + pkg/task/backup.go | 532 + pkg/task/backup_raw.go | 243 + pkg/task/backup_test.go | 61 + pkg/task/common.go | 550 + pkg/task/common_test.go | 65 + pkg/task/restore.go | 575 + pkg/task/restore_log.go | 144 + pkg/task/restore_raw.go | 157 + pkg/task/restore_test.go | 23 + pkg/trace/tracing.go | 105 + pkg/trace/tracing_test.go | 60 + pkg/utils/dyn_pprof_other.go | 11 + pkg/utils/dyn_pprof_unix.go | 34 + pkg/utils/env.go | 33 + pkg/utils/env_test.go | 44 + pkg/utils/json.go | 188 + pkg/utils/json_test.go | 251 + pkg/utils/key.go | 92 + pkg/utils/key_test.go | 115 + pkg/utils/math.go | 63 + pkg/utils/math_test.go | 48 + pkg/utils/permission.go | 20 + pkg/utils/pprof.go | 72 + pkg/utils/progress.go | 155 + pkg/utils/progress_test.go | 77 + pkg/utils/retry.go | 71 + pkg/utils/safe_point.go | 139 + pkg/utils/safe_point_test.go | 139 + pkg/utils/schema.go | 117 + pkg/utils/schema_test.go | 301 + pkg/utils/utils_test.go | 13 + pkg/utils/worker.go | 99 + pkg/version/build/info.go | 59 + pkg/version/build/info_test.go | 33 + pkg/version/version.go | 241 + pkg/version/version_test.go | 297 + revive.toml | 52 + tests/README.md | 100 + tests/_utils/br_tikv_outage_util | 47 + tests/_utils/check_cluster_version | 26 + tests/_utils/check_contains | 24 + tests/_utils/check_not_contains | 24 + tests/_utils/generate_certs | 31 + tests/_utils/make_tiflash_config | 96 + tests/_utils/read_result | 21 + tests/_utils/run_br | 22 + tests/_utils/run_cdc | 22 + tests/_utils/run_curl | 32 + tests/_utils/run_lightning | 32 + tests/_utils/run_lightning_ctl | 30 + tests/_utils/run_pd_ctl | 22 + tests/_utils/run_services | 267 + tests/_utils/run_sql | 27 + tests/_utils/run_sql_in_container | 24 + tests/br_300_small_tables/run.sh | 105 + tests/br_backup_empty/run.sh | 53 + tests/br_backup_version/run.sh | 85 + tests/br_case_sensitive/run.sh | 46 + tests/br_clustered_index/run.sh | 196 + tests/br_db/run.sh | 74 + tests/br_db_online/run.sh | 54 + tests/br_db_online_newkv/run.sh | 78 + tests/br_db_skip/run.sh | 72 + tests/br_debug_meta/run.sh | 71 + tests/br_debug_meta/workload | 12 + tests/br_full/run.sh | 97 + tests/br_full/workload | 12 + tests/br_full_ddl/run.sh | 165 + tests/br_full_ddl/workload | 13 + tests/br_full_index/run.sh | 81 + tests/br_full_index/workload | 12 + tests/br_gcs/oauth.go | 25 + tests/br_gcs/run.sh | 146 + tests/br_gcs/workload | 12 + tests/br_history/run.sh | 67 + tests/br_history/workload | 12 + .../config/tidb-max-index-length.toml | 16 + tests/br_incompatible_tidb_config/run.sh | 125 + tests/br_incremental/run.sh | 58 + tests/br_incremental/workload | 12 + tests/br_incremental_ddl/run.sh | 74 + tests/br_incremental_index/run.sh | 74 + tests/br_incremental_only_ddl/run.sh | 70 + tests/br_incremental_same_table/run.sh | 86 + tests/br_insert_after_restore/run.sh | 80 + tests/br_key_locked/codec.go | 108 + tests/br_key_locked/locker.go | 349 + tests/br_key_locked/run.sh | 55 + tests/br_key_locked/workload | 12 + tests/br_log_restore/run.sh | 180 + tests/br_log_restore/workload | 12 + tests/br_log_test/run.sh | 43 + tests/br_log_test/workload | 12 + tests/br_move_backup/run.sh | 57 + tests/br_move_backup/workload | 12 + tests/br_other/run.sh | 190 + tests/br_range/run.sh | 39 + tests/br_rawkv/client.go | 373 + tests/br_rawkv/run.sh | 140 + tests/br_restore_TDE_enable/run.sh | 151 + tests/br_restore_TDE_enable/workload | 12 + tests/br_s3/run.sh | 159 + tests/br_s3/workload | 12 + tests/br_shuffle_leader/run.sh | 51 + tests/br_shuffle_leader/workload | 12 + tests/br_shuffle_region/run.sh | 52 + tests/br_shuffle_region/workload | 12 + tests/br_single_table/run.sh | 45 + tests/br_single_table/workload | 12 + tests/br_skip_checksum/run.sh | 86 + tests/br_skip_checksum/workload | 12 + tests/br_small_batch_size/run.sh | 78 + tests/br_small_batch_size/workload | 12 + tests/br_split_region_fail/run.sh | 84 + tests/br_split_region_fail/workload | 12 + tests/br_systables/run.sh | 102 + tests/br_systables/workload | 12 + tests/br_table_filter/run.sh | 120 + tests/br_table_partition/prepare.sh | 70 + tests/br_table_partition/run.sh | 61 + tests/br_tiflash/run.sh | 64 + tests/br_tikv_outage/run.sh | 36 + tests/br_tikv_outage/workload | 12 + tests/br_tikv_outage2/run.sh | 40 + tests/br_tikv_outage2/workload | 12 + tests/br_views_and_sequences/run.sh | 49 + tests/br_z_gc_safepoint/gc.go | 79 + tests/br_z_gc_safepoint/run.sh | 76 + tests/br_z_gc_safepoint/workload | 12 + tests/config/importer.toml | 4 + tests/config/ipsan.cnf | 11 + tests/config/pd.toml | 10 + tests/config/restore-tikv.toml | 25 + tests/config/root.cert | 9 + tests/config/root.key | 3 + tests/config/tidb.toml | 13 + tests/config/tikv.toml | 35 + tests/docker_compatible_gcs/prepare.sh | 25 + tests/docker_compatible_gcs/run.sh | 49 + tests/docker_compatible_s3/prepare.sh | 31 + tests/docker_compatible_s3/run.sh | 36 + tests/download_tools.sh | 87 + tests/lightning_alter_random/config.toml | 0 .../data/alter_random-schema-create.sql | 1 + .../data/alter_random.t-schema.sql | 4 + .../data/alter_random.t.sql | 5 + tests/lightning_alter_random/run.sh | 50 + .../lightning_auto_random_default/config.toml | 2 + .../data/auto_random-schema-create.sql | 1 + .../data/auto_random.t-schema.sql | 5 + .../data/auto_random.t.0.sql | 5 + .../data/auto_random.t.1.sql | 5 + tests/lightning_auto_random_default/run.sh | 61 + tests/lightning_black-white-list/config.toml | 2 + .../data/firstdb-schema-create.sql | 1 + .../data/firstdb.first-schema.sql | 1 + .../data/firstdb.first.1.sql | 1 + .../data/firstdb.first.2.sql | 1 + .../data/firstdb.second-schema.sql | 1 + .../data/firstdb.second.1.sql | 1 + .../data/mysql-schema-create.sql | 1 + .../data/mysql.testtable-schema.sql | 1 + .../data/seconddb-schema-create.sql | 1 + .../data/seconddb.fourth-schema.sql | 1 + .../data/seconddb.fourth.1.sql | 1 + .../data/seconddb.third-schema.sql | 1 + .../data/seconddb.third.1.sql | 1 + .../even-table-only.toml | 14 + .../firstdb-only.toml | 5 + tests/lightning_black-white-list/run.sh | 66 + tests/lightning_character_sets/auto.toml | 8 + tests/lightning_character_sets/binary.toml | 8 + tests/lightning_character_sets/gb18030.toml | 8 + .../gb18030/charsets-schema-create.sql | 1 + .../gb18030/charsets.gb18030-schema.sql | 1 + .../gb18030/charsets.gb18030.sql | 1 + .../mixed/charsets-schema-create.sql | 1 + .../mixed/charsets.mixed-schema.sql | 1 + .../mixed/charsets.mixed.sql | 1 + tests/lightning_character_sets/run.sh | 76 + tests/lightning_character_sets/utf8mb4.toml | 8 + .../utf8mb4/charsets-schema-create.sql | 1 + .../utf8mb4/charsets.utf8mb4-schema.sql | 1 + .../utf8mb4/charsets.utf8mb4.sql | 1 + tests/lightning_checkpoint/config.toml | 11 + tests/lightning_checkpoint/run.sh | 120 + tests/lightning_checkpoint_chunks/config.toml | 11 + tests/lightning_checkpoint_chunks/file.toml | 12 + tests/lightning_checkpoint_chunks/run.sh | 128 + .../lightning_checkpoint_columns/config.toml | 16 + tests/lightning_checkpoint_columns/run.sh | 49 + .../data/cpdt-schema-create.sql | 1 + .../data/cpdt.t-schema.sql | 1 + .../data/cpdt.t.sql | 1 + .../file.toml | 6 + .../mysql.toml | 6 + .../lightning_checkpoint_dirty_tableid/run.sh | 59 + .../lightning_checkpoint_engines/config.toml | 9 + .../data/cpeng-schema-create.sql | 1 + .../data/cpeng.a-schema.sql | 1 + .../data/cpeng.a.1.sql | 1 + .../data/cpeng.a.2.sql | 1 + .../data/cpeng.a.3.sql | 1 + .../data/cpeng.b-schema.sql | 1 + .../data/cpeng.b.1.sql | 4 + .../data/cpeng.b.2.sql | 1 + tests/lightning_checkpoint_engines/mysql.toml | 9 + tests/lightning_checkpoint_engines/run.sh | 103 + .../config.toml | 16 + .../disk_quota_checkpoint-schema-create.sql | 1 + .../data/disk_quota_checkpoint.t-schema.sql | 4 + .../data/disk_quota_checkpoint.t.0.sql | 4 + .../data/disk_quota_checkpoint.t.1.sql | 4 + .../data/disk_quota_checkpoint.t.2.sql | 4 + .../lightning_checkpoint_engines_order/run.sh | 52 + .../bad-data/cped-schema-create.sql | 1 + .../bad-data/cped.t-schema.sql | 1 + .../bad-data/cped.t.sql | 1 + .../file.toml | 10 + .../good-data/cped-schema-create.sql | 1 + .../good-data/cped.t-schema.sql | 1 + .../good-data/cped.t.sql | 1 + .../mysql.toml | 9 + .../lightning_checkpoint_error_destroy/run.sh | 52 + .../lightning_checkpoint_parquet/config.toml | 11 + tests/lightning_checkpoint_parquet/parquet.go | 65 + tests/lightning_checkpoint_parquet/run.sh | 61 + .../config.toml | 14 + .../data/cpts-schema-create.sql | 1 + .../data/cpts.cpts-schema.sql | 1 + .../data/cpts.cpts.1.sql | 4 + .../data/cpts.cpts.2.sql | 3 + .../lightning_checkpoint_timestamp/mysql.toml | 14 + tests/lightning_checkpoint_timestamp/run.sh | 46 + tests/lightning_cmdline_override/config.toml | 18 + .../data/cmdline_override-schema-create.sql | 1 + .../data/cmdline_override.t-schema.sql | 1 + .../data/cmdline_override.t.sql | 1 + tests/lightning_cmdline_override/run.sh | 18 + .../lightning_column_permutation/config.toml | 3 + .../data/perm-schema-create.sql | 1 + .../data/perm.test_perm-schema.sql | 22 + .../data/perm.test_perm.0.csv | 6 + tests/lightning_column_permutation/run.sh | 18 + tests/lightning_common_handle/config.toml | 2 + tests/lightning_common_handle/run.sh | 59 + .../lightning_concurrent-restore/config.toml | 6 + tests/lightning_concurrent-restore/run.sh | 48 + tests/lightning_csv/config.toml | 12 + .../lightning_csv/data/csv-schema-create.sql | 1 + .../data/csv.empty_strings-schema.sql | 5 + .../lightning_csv/data/csv.empty_strings.csv | 4 + .../lightning_csv/data/csv.escapes-schema.sql | 16 + tests/lightning_csv/data/csv.escapes.CSV | 6 + .../lightning_csv/data/csv.threads-schema.sql | 27 + tests/lightning_csv/data/csv.threads.csv | 43 + tests/lightning_csv/run.sh | 43 + tests/lightning_default-columns/config.toml | 2 + .../data/defcol-schema-create.sql | 1 + .../data/defcol.t-schema.sql | 6 + .../data/defcol.t.1.sql | 1 + .../data/defcol.t.2.sql | 1 + .../data/defcol.u-schema.sql | 4 + .../data/defcol.u.1.sql | 1 + tests/lightning_default-columns/run.sh | 35 + tests/lightning_disk_quota/config.toml | 8 + .../data/disk_quota-schema-create.sql | 1 + .../data/disk_quota.t-schema.sql | 12 + .../data/disk_quota.t.0.sql | 51 + .../data/disk_quota.t.1.sql | 51 + .../data/disk_quota.t.2.sql | 51 + .../data/disk_quota.t.3.sql | 51 + tests/lightning_disk_quota/run.sh | 75 + .../config1.toml | 30 + .../config2.toml | 30 + .../data/dup_detect-schema-create.sql | 1 + .../data/dup_detect.ta-schema.sql | 6 + .../data/dup_detect.ta.0.sql | 20 + .../data/dup_detect.ta.1.sql | 20 + .../data/dup_detect.tb-schema.sql | 6 + .../data/dup_detect.tb.0.sql | 20 + .../data/dup_detect.tb.1.sql | 20 + .../data/dup_detect.tc-schema.sql | 6 + .../data/dup_detect.tc.0.sql | 20 + .../data/dup_detect.tc.1.sql | 20 + .../data/dup_detect.td-schema.sql | 6 + .../data/dup_detect.td.0.sql | 20 + .../data/dup_detect.td.1.sql | 20 + .../data/dup_detect.te-schema.sql | 6 + .../data/dup_detect.te.0.sql | 20 + .../data/dup_detect.te.1.sql | 20 + .../data/dup_detect.tf-schema.sql | 6 + .../data/dup_detect.tf.0.sql | 20 + .../data/dup_detect.tf.1.sql | 20 + tests/lightning_duplicate_detection/run.sh | 45 + tests/lightning_error_summary/config.toml | 7 + .../data/error_summary-schema-create.sql | 1 + .../data/error_summary.a-schema.sql | 4 + .../data/error_summary.a.sql | 1 + .../data/error_summary.b-schema.sql | 4 + .../data/error_summary.b.sql | 1 + .../data/error_summary.c-schema.sql | 4 + .../data/error_summary.c.sql | 1 + tests/lightning_error_summary/run.sh | 67 + tests/lightning_examples/1.toml | 9 + tests/lightning_examples/131072.toml | 9 + tests/lightning_examples/512.toml | 9 + tests/lightning_examples/run.sh | 98 + tests/lightning_exotic_filenames/config.toml | 2 + .../data/xfn-schema-create.sql | 1 + .../data/xfn.etn-schema.sql | 1 + .../data/xfn.etn.sql | 7 + .../data/zwk-schema-create.sql | 1 + .../data/zwk.zwb-schema.sql | 1 + .../data/zwk.zwb.sql | 1 + tests/lightning_exotic_filenames/run.sh | 45 + tests/lightning_fail_fast/chunk.toml | 6 + .../data/fail_fast-schema-create.sql | 1 + .../data/fail_fast.tb-schema.sql | 1 + .../data/fail_fast.tb.0.csv | 5 + .../data/fail_fast.tb.1.csv | 5 + .../data/fail_fast.tb.2.csv | 5 + tests/lightning_fail_fast/engine.toml | 9 + tests/lightning_fail_fast/run.sh | 34 + tests/lightning_file_routing/config.toml | 43 + tests/lightning_file_routing/run.sh | 68 + tests/lightning_gcs/config.toml | 0 tests/lightning_gcs/run.sh | 102 + tests/lightning_generated_columns/config.toml | 0 .../data/gencol-schema-create.sql | 1 + .../data/gencol.nested-schema.sql | 7 + .../data/gencol.nested.0.sql | 1 + .../data/gencol.various_types-schema.sql | 19 + .../data/gencol.various_types.0.sql | 1 + tests/lightning_generated_columns/run.sh | 71 + tests/lightning_incremental/config.toml | 0 .../data/incr-schema-create.sql | 1 + .../data/incr.auto_random-schema.sql | 5 + .../data/incr.auto_random.sql | 5 + .../data/incr.pk_auto_inc-schema.sql | 4 + .../data/incr.pk_auto_inc.sql | 5 + .../data/incr.rowid_uk_inc-schema.sql | 4 + .../data/incr.rowid_uk_inc.sql | 5 + .../data/incr.uk_auto_inc-schema.sql | 4 + .../data/incr.uk_auto_inc.sql | 5 + .../data1/incr-schema-create.sql | 1 + .../data1/incr.auto_random-schema.sql | 5 + .../data1/incr.auto_random.sql | 5 + .../data1/incr.pk_auto_inc-schema.sql | 4 + .../data1/incr.pk_auto_inc.sql | 5 + .../data1/incr.rowid_uk_inc-schema.sql | 4 + .../data1/incr.rowid_uk_inc.sql | 5 + .../data1/incr.uk_auto_inc-schema.sql | 4 + .../data1/incr.uk_auto_inc.sql | 5 + tests/lightning_incremental/run.sh | 76 + tests/lightning_issue_282/config.toml | 0 .../data/issue282-schema-create.sql | 1 + .../data/issue282.t_access3-schema.sql | 4 + .../data/issue282.t_access3.sql | 5 + tests/lightning_issue_282/run.sh | 29 + tests/lightning_issue_410/config.toml | 0 .../data/issue410-schema-create.sql | 1 + .../data/issue410.row_flow_d-schema.sql | 28 + .../data/issue410.row_flow_d.0.csv | 2 + tests/lightning_issue_410/run.sh | 30 + tests/lightning_issue_519/config.toml | 8 + .../data/issue519-schema-create.sql | 1 + .../data/issue519.t-schema.sql | 1 + tests/lightning_issue_519/data/issue519.t.csv | 10 + tests/lightning_issue_519/run.sh | 25 + tests/lightning_local_backend/config.toml | 13 + .../data/cpeng-schema-create.sql | 1 + .../data/cpeng.a-schema.sql | 1 + .../data/cpeng.a.1.sql | 1 + .../data/cpeng.a.2.sql | 1 + .../data/cpeng.a.3.sql | 1 + .../data/cpeng.b-schema.sql | 1 + .../data/cpeng.b.1.sql | 4 + .../data/cpeng.b.2.sql | 1 + tests/lightning_local_backend/file.toml | 17 + tests/lightning_local_backend/mysql.toml | 17 + tests/lightning_local_backend/run.sh | 126 + tests/lightning_new_collation/config.toml | 0 tests/lightning_new_collation/run.sh | 59 + .../tidb-new-collation.toml | 11 + tests/lightning_no_schema/config.toml | 2 + tests/lightning_no_schema/data/noschema.t.sql | 15 + tests/lightning_no_schema/run.sh | 27 + tests/lightning_parquet/config.toml | 10 + .../data/export_info_ci.json | 1 + .../export_tables_info_ci_from_1_to_9.json | 1 + .../data/test/test.customer/_SUCCESS | 0 ...51c-49ba-bdf3-5864befff481-c000.gz.parquet | Bin 0 -> 15251 bytes .../data/test/test.district/_SUCCESS | 0 ...765-432a-8f18-cd17b4607f2a-c000.gz.parquet | Bin 0 -> 3626 bytes .../data/test/test.history/_SUCCESS | 0 ...169-4335-a93f-8805e02def97-c000.gz.parquet | Bin 0 -> 4133 bytes .../data/test/test.item/_SUCCESS | 0 ...c54-477f-9907-6e3eae50358b-c000.gz.parquet | Bin 0 -> 7078 bytes .../data/test/test.new_order/_SUCCESS | 0 ...629-4445-bd96-d34b6674b09d-c000.gz.parquet | Bin 0 -> 1140 bytes .../data/test/test.order_line/_SUCCESS | 0 ...f59-4ff6-b271-2e4b27ffbcf5-c000.gz.parquet | Bin 0 -> 5468 bytes .../data/test/test.orders/_SUCCESS | 0 ...2c0-4961-9bf7-32a0a04ffee5-c000.gz.parquet | Bin 0 -> 2941 bytes .../data/test/test.stock/_SUCCESS | 0 ...034-4d65-b375-ee55aa479215-c000.gz.parquet | Bin 0 -> 15085 bytes .../data/test/test.warehouse/_SUCCESS | 0 ...d2f-4c5c-8a2f-d162bde6c360-c000.gz.parquet | Bin 0 -> 2719 bytes tests/lightning_parquet/db.sql | 219 + tests/lightning_parquet/run.sh | 52 + tests/lightning_partitioned-table/config.toml | 2 + .../data/partitioned-schema-create.sql | 1 + .../data/partitioned.a-schema.sql | 1 + .../data/partitioned.a.sql | 10 + tests/lightning_partitioned-table/run.sh | 35 + tests/lightning_restore/config.toml | 5 + tests/lightning_restore/run.sh | 42 + tests/lightning_routes/config.toml | 10 + .../data/routes_a0-schema-create.sql | 1 + .../data/routes_a0.t0-schema.sql | 1 + .../lightning_routes/data/routes_a0.t0.1.sql | 1 + .../lightning_routes/data/routes_a0.t0.2.sql | 1 + .../data/routes_a0.t1-schema.sql | 1 + .../lightning_routes/data/routes_a0.t1.1.sql | 1 + .../data/routes_a1-schema-create.sql | 1 + .../data/routes_a1.s1-schema.sql | 1 + tests/lightning_routes/data/routes_a1.s1.sql | 1 + .../data/routes_a1.t2-schema.sql | 1 + tests/lightning_routes/data/routes_a1.t2.sql | 1 + tests/lightning_routes/run.sh | 25 + tests/lightning_row-format-v2/config.toml | 2 + .../data/rowformatv2-schema-create.sql | 1 + .../data/rowformatv2.t1-schema.sql | 258 + .../data/rowformatv2.t1.1.sql | 51 + tests/lightning_row-format-v2/run.sh | 28 + tests/lightning_s3/config.toml | 0 tests/lightning_s3/run.sh | 68 + tests/lightning_shard_rowid/config.toml | 0 .../data/shard_rowid-schema-create.sql | 1 + .../data/shard_rowid.shr-schema.sql | 7 + .../data/shard_rowid.shr.0.sql | 17 + tests/lightning_shard_rowid/run.sh | 50 + tests/lightning_source_linkfile/config.toml | 1 + tests/lightning_source_linkfile/run.sh | 60 + .../data/sqlmodedb-schema-create.sql | 1 + .../data/sqlmodedb.t-schema.sql | 7 + .../lightning_sqlmode/data/sqlmodedb.t.1.sql | 6 + tests/lightning_sqlmode/off.toml | 5 + tests/lightning_sqlmode/on.toml | 5 + tests/lightning_sqlmode/run.sh | 57 + .../data/dup-schema-create.sql | 1 + .../data/dup.dup-schema.sql | 1 + .../data/dup.dup.sql | 3 + .../lightning_tidb_duplicate_data/error.toml | 3 + .../lightning_tidb_duplicate_data/ignore.toml | 3 + .../replace.toml | 2 + tests/lightning_tidb_duplicate_data/run.sh | 64 + tests/lightning_tidb_rowid/config.toml | 0 .../data/rowid-schema-create.sql | 1 + .../data/rowid.explicit_tidb_rowid-schema.sql | 1 + .../data/rowid.explicit_tidb_rowid.sql | 11 + .../data/rowid.non_pk-schema.sql | 1 + .../data/rowid.non_pk.sql | 11 + .../data/rowid.non_pk_auto_inc-schema.sql | 9 + .../data/rowid.non_pk_auto_inc.sql | 26 + .../data/rowid.pre_rebase-schema.sql | 1 + .../data/rowid.pre_rebase.sql | 1 + .../data/rowid.specific_auto_inc-schema.sql | 1 + .../data/rowid.specific_auto_inc.sql | 7 + tests/lightning_tidb_rowid/run.sh | 75 + tests/lightning_tiflash/config.toml | 2 + tests/lightning_tiflash/run.sh | 71 + tests/lightning_too_many_columns/config.toml | 0 .../data/too_many_columns-schema-create.sql | 1 + .../data/too_many_columns.t-schema.sql | 259 + .../data/too_many_columns.t.0.csv | 2 + tests/lightning_too_many_columns/run.sh | 30 + tests/lightning_tool_135/config.toml | 2 + .../data/tool_135-schema-create.sql | 1 + .../data/tool_135.bar1-schema.sql | 1 + .../lightning_tool_135/data/tool_135.bar1.sql | 10 + .../data/tool_135.bar2-schema.sql | 1 + .../lightning_tool_135/data/tool_135.bar2.sql | 10 + .../data/tool_135.bar3-schema.sql | 1 + .../lightning_tool_135/data/tool_135.bar3.sql | 10 + .../data/tool_135.bar4-schema.sql | 1 + .../lightning_tool_135/data/tool_135.bar4.sql | 10 + .../data/tool_135.bar5-schema.sql | 1 + .../lightning_tool_135/data/tool_135.bar5.sql | 10 + tests/lightning_tool_135/run.sh | 94 + tests/lightning_tool_1420/config.toml | 2 + .../data/EE1420-schema-create.sql | 1 + .../data/EE1420.pt_role-schema.sql | 3 + .../data/EE1420.pt_role.sql | 1 + tests/lightning_tool_1420/run.sh | 24 + tests/lightning_tool_1472/config.toml | 2 + .../data/EE1472-schema-create.sql | 1 + .../data/EE1472.notpk-schema.sql | 5 + .../data/EE1472.notpk.1.sql | 8 + .../data/EE1472.notpk.2.sql | 8 + .../data/EE1472.pk-schema.sql | 3 + .../lightning_tool_1472/data/EE1472.pk.1.sql | 8 + .../lightning_tool_1472/data/EE1472.pk.2.sql | 8 + tests/lightning_tool_1472/run.sh | 31 + tests/lightning_tool_241/config.toml | 8 + .../data/qyjc-schema-create.sql | 1 + .../data/qyjc.q_alarm_group-schema.sql | 16 + .../data/qyjc.q_alarm_group.sql | 0 .../data/qyjc.q_alarm_message_log-schema.sql | 16 + .../data/qyjc.q_alarm_message_log.sql | 0 .../data/qyjc.q_alarm_receiver-schema.sql | 12 + .../data/qyjc.q_config-schema.sql | 9 + .../data/qyjc.q_fish_event-schema.sql | 15 + .../data/qyjc.q_fish_event.sql | 85 + .../qyjc.q_report_circular_data-schema.sql | 16 + .../data/qyjc.q_report_desc-schema.sql | 16 + .../data/qyjc.q_report_summary-schema.sql | 14 + .../data/qyjc.q_system_update-schema.sql | 7 + .../data/qyjc.q_system_update.sql | 0 .../data/qyjc.q_user_log-schema.sql | 12 + .../data/qyjc.q_user_log.sql | 0 tests/lightning_tool_241/run.sh | 45 + .../lightning_unused_config_keys/config.toml | 6 + .../data/unused_config_keys-schema-create.sql | 1 + tests/lightning_unused_config_keys/run.sh | 29 + tests/lightning_various_types/config.toml | 0 .../data/vt-schema-create.sql | 1 + .../data/vt.bigint-schema.sql | 3 + .../data/vt.bigint.0.sql | 8 + .../data/vt.binary-schema.sql | 4 + .../data/vt.binary.sql | 52 + .../data/vt.bit-schema.sql | 4 + tests/lightning_various_types/data/vt.bit.sql | 6 + .../data/vt.char-schema.sql | 4 + .../lightning_various_types/data/vt.char.sql | 51 + .../data/vt.datetime-schema.sql | 7 + .../data/vt.datetime.sql | 72 + .../data/vt.decimal-schema.sql | 4 + .../data/vt.decimal.sql | 51 + .../data/vt.double-schema.sql | 4 + .../data/vt.double.sql | 42 + .../data/vt.empty_strings-schema.sql | 4 + .../data/vt.empty_strings.sql | 1 + .../data/vt.enum-set-schema.sql | 26 + .../data/vt.enum-set.sql | 36 + .../data/vt.json-schema.sql | 4 + .../lightning_various_types/data/vt.json.sql | 106 + .../data/vt.precise_types-schema.sql | 6 + .../data/vt.precise_types.sql | 6 + tests/lightning_various_types/run.sh | 117 + tests/lightning_view/config.toml | 1 + .../lightning_view/data/db0-schema-create.sql | 1 + .../data/db0.v2-schema-view.sql | 13 + tests/lightning_view/data/db0.v2-schema.sql | 1 + .../lightning_view/data/db1-schema-create.sql | 1 + tests/lightning_view/data/db1.tbl-schema.sql | 1 + tests/lightning_view/data/db1.tbl.0.sql | 5 + .../data/db1.v1-schema-view.sql | 13 + tests/lightning_view/data/db1.v1-schema.sql | 1 + tests/lightning_view/run.sh | 36 + tests/run.sh | 64 + tests/run_compatible.sh | 46 + tests/up.sh | 188 + tidb-lightning.toml | 289 + tools/.gitignore | 1 + tools/Makefile | 34 + tools/check-errdoc.sh | 21 + tools/go.mod | 16 + tools/go.sum | 707 ++ tools/go_mod_guard.go | 35 + web/README.md | 93 + web/docs/InfoPage.png | Bin 0 -> 48339 bytes web/docs/ProgressPage.png | Bin 0 -> 41010 bytes web/docs/TableProgressPage.png | Bin 0 -> 47424 bytes web/docs/api.yaml | 521 + web/go.mod | 5 + web/go.sum | 1 + web/package-lock.json | 4867 ++++++++ web/package.json | 31 + web/public/index.html | 14 + web/src/ChunksProgressPanel.tsx | 114 + web/src/DottedProgress.tsx | 72 + web/src/EnginesProgressPanel.tsx | 72 + web/src/ErrorButton.tsx | 85 + web/src/InfoButton.tsx | 39 + web/src/InfoPage.tsx | 112 + web/src/MoveTaskButton.tsx | 123 + web/src/PauseButton.tsx | 33 + web/src/ProgressPage.tsx | 116 + web/src/RefreshButton.tsx | 124 + web/src/TableProgressCard.tsx | 113 + web/src/TableProgressPage.tsx | 73 + web/src/TaskButton.tsx | 139 + web/src/TitleBar.tsx | 95 + web/src/TitleLink.tsx | 33 + web/src/api.ts | 268 + web/src/index.tsx | 179 + web/src/json-bigint.d.ts | 17 + web/tsconfig.json | 19 + web/webpack.config.js | 38 + 872 files changed, 127054 insertions(+) create mode 100644 .codecov.yml create mode 120000 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/challenge-bot.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/compatible_test.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 COMPATIBILITY_TEST.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 cmd/br/backup.go create mode 100644 cmd/br/cmd.go create mode 100644 cmd/br/debug.go create mode 100644 cmd/br/main.go create mode 100644 cmd/br/main_test.go create mode 100644 cmd/br/restore.go create mode 100644 cmd/tidb-lightning-ctl/main.go create mode 100644 cmd/tidb-lightning-ctl/main_test.go create mode 100644 cmd/tidb-lightning/main.go create mode 100644 cmd/tidb-lightning/main_test.go create mode 100644 compatibility/backup_cluster.yaml create mode 100644 compatibility/credentials/application_default_credentials.json create mode 100644 compatibility/get_last_tags.sh create mode 100644 compatibility/prepare_backup.sh create mode 100644 compatibility/prepare_data/workload create mode 100644 docker-compose.yaml create mode 100644 docker/Dockerfile create mode 100644 docker/config/pd.toml create mode 100644 docker/config/tidb.toml create mode 100644 docker/config/tikv.toml create mode 100644 docker/gcs.env create mode 100644 docker/minio.env create mode 100644 docs/cn/2019-08-05-new-design-of-backup-restore.md create mode 100644 docs/cn/2019-09-09-BR-key-rewrite-disscussion.md create mode 100644 docs/cn/2019-09-17-design-of-reorganize-importSST-to-TiKV.md create mode 100644 docs/resources/arch-of-reorganized-importer.svg create mode 100644 docs/resources/download-sst-diagram.svg create mode 100644 docs/resources/solution3-of-key-rewrite.svg create mode 100644 errors.toml create mode 100644 go.mod1 create mode 100644 go.sum1 create mode 100644 images/arch.svg create mode 100644 metrics/alertmanager/lightning.rules.yml create mode 100644 metrics/grafana/br.json create mode 100644 metrics/grafana/lightning.json create mode 100644 pkg/backup/check.go create mode 100644 pkg/backup/client.go create mode 100644 pkg/backup/client_test.go create mode 100644 pkg/backup/metrics.go create mode 100644 pkg/backup/push.go create mode 100644 pkg/backup/schema.go create mode 100644 pkg/backup/schema_test.go create mode 100644 pkg/cdclog/buffer.go create mode 100644 pkg/cdclog/decoder.go create mode 100644 pkg/cdclog/decoder_test.go create mode 100644 pkg/cdclog/puller.go create mode 100644 pkg/checksum/executor.go create mode 100644 pkg/checksum/executor_test.go create mode 100644 pkg/checksum/validate.go create mode 100644 pkg/conn/conn.go create mode 100644 pkg/conn/conn_test.go create mode 100644 pkg/errors/errors.go create mode 100644 pkg/glue/glue.go create mode 100644 pkg/gluetidb/glue.go create mode 100644 pkg/gluetikv/glue.go create mode 100644 pkg/gluetikv/glue_test.go create mode 100644 pkg/httputil/http.go create mode 100644 pkg/kv/checksum.go create mode 100644 pkg/kv/checksum_test.go create mode 100644 pkg/kv/kv.go create mode 100644 pkg/kv/kv_test.go create mode 100644 pkg/kv/session.go create mode 100644 pkg/kv/session_test.go create mode 100644 pkg/lightning/backend/backend.go create mode 100644 pkg/lightning/backend/backend_test.go create mode 100644 pkg/lightning/backend/importer/importer.go create mode 100644 pkg/lightning/backend/importer/importer_test.go create mode 100644 pkg/lightning/backend/kv/allocator.go create mode 100644 pkg/lightning/backend/kv/kv2sql.go create mode 100644 pkg/lightning/backend/kv/session.go create mode 100644 pkg/lightning/backend/kv/session_test.go create mode 100644 pkg/lightning/backend/kv/sql2kv.go create mode 100644 pkg/lightning/backend/kv/sql2kv_test.go create mode 100644 pkg/lightning/backend/kv/types.go create mode 100644 pkg/lightning/backend/local/duplicate.go create mode 100644 pkg/lightning/backend/local/iterator.go create mode 100644 pkg/lightning/backend/local/iterator_test.go create mode 100644 pkg/lightning/backend/local/key_adapter.go create mode 100644 pkg/lightning/backend/local/key_adapter_test.go create mode 100644 pkg/lightning/backend/local/local.go create mode 100644 pkg/lightning/backend/local/local_freebsd.go create mode 100644 pkg/lightning/backend/local/local_test.go create mode 100644 pkg/lightning/backend/local/local_unix.go create mode 100644 pkg/lightning/backend/local/local_unix_generic.go create mode 100644 pkg/lightning/backend/local/local_windows.go create mode 100644 pkg/lightning/backend/local/localhelper.go create mode 100644 pkg/lightning/backend/local/localhelper_test.go create mode 100644 pkg/lightning/backend/noop/noop.go create mode 100644 pkg/lightning/backend/tidb/tidb.go create mode 100644 pkg/lightning/backend/tidb/tidb_test.go create mode 100644 pkg/lightning/checkpoints/checkpoints.go create mode 100644 pkg/lightning/checkpoints/checkpoints_file_test.go create mode 100644 pkg/lightning/checkpoints/checkpoints_sql_test.go create mode 100644 pkg/lightning/checkpoints/checkpoints_test.go create mode 100644 pkg/lightning/checkpoints/checkpointspb/file_checkpoints.pb.go create mode 100644 pkg/lightning/checkpoints/checkpointspb/file_checkpoints.proto create mode 100644 pkg/lightning/checkpoints/glue_checkpoint.go create mode 100644 pkg/lightning/checkpoints/tidb.go create mode 100644 pkg/lightning/common/conn.go create mode 100644 pkg/lightning/common/once_error.go create mode 100644 pkg/lightning/common/once_error_test.go create mode 100644 pkg/lightning/common/pause.go create mode 100644 pkg/lightning/common/pause_test.go create mode 100644 pkg/lightning/common/security.go create mode 100644 pkg/lightning/common/security_test.go create mode 100644 pkg/lightning/common/storage.go create mode 100644 pkg/lightning/common/storage_test.go create mode 100644 pkg/lightning/common/storage_unix.go create mode 100644 pkg/lightning/common/storage_windows.go create mode 100644 pkg/lightning/common/util.go create mode 100644 pkg/lightning/common/util_test.go create mode 100644 pkg/lightning/config/bytesize.go create mode 100644 pkg/lightning/config/bytesize_test.go create mode 100644 pkg/lightning/config/config.go create mode 100644 pkg/lightning/config/config_test.go create mode 100644 pkg/lightning/config/configlist.go create mode 100644 pkg/lightning/config/configlist_test.go create mode 100644 pkg/lightning/config/const.go create mode 100644 pkg/lightning/config/global.go create mode 100644 pkg/lightning/glue/glue.go create mode 100644 pkg/lightning/lightning.go create mode 100644 pkg/lightning/lightning_test.go create mode 100644 pkg/lightning/log/filter.go create mode 100644 pkg/lightning/log/filter_test.go create mode 100644 pkg/lightning/log/log.go create mode 100644 pkg/lightning/log/log_test.go create mode 100644 pkg/lightning/log/testlogger.go create mode 100644 pkg/lightning/manual/allocator.go create mode 100644 pkg/lightning/manual/manual.go create mode 100644 pkg/lightning/manual/manual_nocgo.go create mode 100644 pkg/lightning/metric/metric.go create mode 100644 pkg/lightning/metric/metric_test.go create mode 100644 pkg/lightning/mydump/bytes.go create mode 100644 pkg/lightning/mydump/csv/split_large_file.csv create mode 100644 pkg/lightning/mydump/csv_parser.go create mode 100644 pkg/lightning/mydump/csv_parser_test.go create mode 100644 pkg/lightning/mydump/examples/metadata create mode 100644 pkg/lightning/mydump/examples/mocker_test-schema-create.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.i-schema.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.i.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.report_case_high_risk-schema.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.report_case_high_risk.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.tbl_autoid-schema.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.tbl_autoid.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.tbl_multi_index-schema.sql create mode 100644 pkg/lightning/mydump/examples/mocker_test.tbl_multi_index.sql create mode 100644 pkg/lightning/mydump/examples/test.parquet create mode 100644 pkg/lightning/mydump/loader.go create mode 100644 pkg/lightning/mydump/loader_test.go create mode 100644 pkg/lightning/mydump/parquet_parser.go create mode 100644 pkg/lightning/mydump/parquet_parser_test.go create mode 100644 pkg/lightning/mydump/parser.go create mode 100644 pkg/lightning/mydump/parser.rl create mode 100644 pkg/lightning/mydump/parser_generated.go create mode 100644 pkg/lightning/mydump/parser_test.go create mode 100644 pkg/lightning/mydump/reader.go create mode 100644 pkg/lightning/mydump/reader_test.go create mode 100644 pkg/lightning/mydump/region.go create mode 100644 pkg/lightning/mydump/region_test.go create mode 100644 pkg/lightning/mydump/router.go create mode 100644 pkg/lightning/mydump/router_test.go create mode 100644 pkg/lightning/restore/check_info.go create mode 100644 pkg/lightning/restore/check_template.go create mode 100644 pkg/lightning/restore/checksum.go create mode 100644 pkg/lightning/restore/checksum_test.go create mode 100644 pkg/lightning/restore/meta_manager.go create mode 100644 pkg/lightning/restore/meta_manager_test.go create mode 100644 pkg/lightning/restore/restore.go create mode 100644 pkg/lightning/restore/restore_test.go create mode 100644 pkg/lightning/restore/table_restore.go create mode 100644 pkg/lightning/restore/tidb.go create mode 100644 pkg/lightning/restore/tidb_test.go create mode 100644 pkg/lightning/sigusr1_other.go create mode 100644 pkg/lightning/sigusr1_unix.go create mode 100644 pkg/lightning/tikv/tikv.go create mode 100644 pkg/lightning/tikv/tikv_test.go create mode 100644 pkg/lightning/verification/checksum.go create mode 100644 pkg/lightning/verification/checksum_test.go create mode 100644 pkg/lightning/web/progress.go create mode 100644 pkg/lightning/web/res.go create mode 100644 pkg/lightning/web/res_vfsdata.go create mode 100644 pkg/lightning/worker/worker.go create mode 100644 pkg/lightning/worker/worker_test.go create mode 100644 pkg/logutil/logging.go create mode 100644 pkg/logutil/logging_test.go create mode 100644 pkg/logutil/rate.go create mode 100644 pkg/membuf/buffer.go create mode 100644 pkg/membuf/buffer_test.go create mode 100644 pkg/metautil/metafile.go create mode 100644 pkg/metautil/metafile_test.go create mode 100644 pkg/mock/backend.go create mode 100644 pkg/mock/glue.go create mode 100644 pkg/mock/glue_checkpoint.go create mode 100644 pkg/mock/importer.go create mode 100644 pkg/mock/kv.go create mode 100644 pkg/mock/mock_cluster.go create mode 100644 pkg/mock/mock_cluster_test.go create mode 100644 pkg/mock/mockid/mockid.go create mode 100644 pkg/mock/s3iface.go create mode 100644 pkg/mock/storage/storage.go create mode 100644 pkg/pdutil/pd.go create mode 100644 pkg/pdutil/pd_test.go create mode 100644 pkg/pdutil/utils.go create mode 100644 pkg/redact/redact.go create mode 100644 pkg/redact/redact_test.go create mode 100644 pkg/restore/backoff.go create mode 100644 pkg/restore/backoff_test.go create mode 100644 pkg/restore/batcher.go create mode 100644 pkg/restore/batcher_test.go create mode 100644 pkg/restore/client.go create mode 100644 pkg/restore/client_test.go create mode 100644 pkg/restore/db.go create mode 100644 pkg/restore/db_test.go create mode 100644 pkg/restore/import.go create mode 100644 pkg/restore/ingester.go create mode 100644 pkg/restore/log_client.go create mode 100644 pkg/restore/log_client_test.go create mode 100644 pkg/restore/merge.go create mode 100644 pkg/restore/merge_test.go create mode 100644 pkg/restore/pipeline_items.go create mode 100644 pkg/restore/range.go create mode 100644 pkg/restore/range_test.go create mode 100644 pkg/restore/split.go create mode 100755 pkg/restore/split_client.go create mode 100644 pkg/restore/split_test.go create mode 100644 pkg/restore/systable_restore.go create mode 100644 pkg/restore/util.go create mode 100644 pkg/restore/util_test.go create mode 100644 pkg/rtree/logging.go create mode 100644 pkg/rtree/logging_test.go create mode 100644 pkg/rtree/rtree.go create mode 100644 pkg/rtree/rtree_test.go create mode 100644 pkg/storage/compress.go create mode 100644 pkg/storage/compress_test.go create mode 100644 pkg/storage/flags.go create mode 100644 pkg/storage/gcs.go create mode 100644 pkg/storage/gcs_test.go create mode 100644 pkg/storage/local.go create mode 100644 pkg/storage/local_test.go create mode 100644 pkg/storage/local_unix.go create mode 100644 pkg/storage/local_windows.go create mode 100644 pkg/storage/noop.go create mode 100644 pkg/storage/parse.go create mode 100644 pkg/storage/parse_test.go create mode 100644 pkg/storage/s3.go create mode 100644 pkg/storage/s3_test.go create mode 100644 pkg/storage/storage.go create mode 100644 pkg/storage/writer.go create mode 100644 pkg/storage/writer_test.go create mode 100644 pkg/summary/collector.go create mode 100644 pkg/summary/collector_test.go create mode 100644 pkg/summary/summary.go create mode 100644 pkg/task/backup.go create mode 100644 pkg/task/backup_raw.go create mode 100644 pkg/task/backup_test.go create mode 100644 pkg/task/common.go create mode 100644 pkg/task/common_test.go create mode 100644 pkg/task/restore.go create mode 100644 pkg/task/restore_log.go create mode 100644 pkg/task/restore_raw.go create mode 100644 pkg/task/restore_test.go create mode 100644 pkg/trace/tracing.go create mode 100644 pkg/trace/tracing_test.go create mode 100644 pkg/utils/dyn_pprof_other.go create mode 100644 pkg/utils/dyn_pprof_unix.go create mode 100644 pkg/utils/env.go create mode 100644 pkg/utils/env_test.go create mode 100644 pkg/utils/json.go create mode 100644 pkg/utils/json_test.go create mode 100644 pkg/utils/key.go create mode 100644 pkg/utils/key_test.go create mode 100644 pkg/utils/math.go create mode 100644 pkg/utils/math_test.go create mode 100644 pkg/utils/permission.go create mode 100644 pkg/utils/pprof.go create mode 100644 pkg/utils/progress.go create mode 100644 pkg/utils/progress_test.go create mode 100644 pkg/utils/retry.go create mode 100644 pkg/utils/safe_point.go create mode 100644 pkg/utils/safe_point_test.go create mode 100644 pkg/utils/schema.go create mode 100644 pkg/utils/schema_test.go create mode 100644 pkg/utils/utils_test.go create mode 100644 pkg/utils/worker.go create mode 100644 pkg/version/build/info.go create mode 100644 pkg/version/build/info_test.go create mode 100644 pkg/version/version.go create mode 100644 pkg/version/version_test.go create mode 100644 revive.toml create mode 100644 tests/README.md create mode 100644 tests/_utils/br_tikv_outage_util create mode 100755 tests/_utils/check_cluster_version create mode 100755 tests/_utils/check_contains create mode 100755 tests/_utils/check_not_contains create mode 100755 tests/_utils/generate_certs create mode 100755 tests/_utils/make_tiflash_config create mode 100755 tests/_utils/read_result create mode 100755 tests/_utils/run_br create mode 100755 tests/_utils/run_cdc create mode 100755 tests/_utils/run_curl create mode 100755 tests/_utils/run_lightning create mode 100755 tests/_utils/run_lightning_ctl create mode 100755 tests/_utils/run_pd_ctl create mode 100644 tests/_utils/run_services create mode 100755 tests/_utils/run_sql create mode 100755 tests/_utils/run_sql_in_container create mode 100644 tests/br_300_small_tables/run.sh create mode 100644 tests/br_backup_empty/run.sh create mode 100644 tests/br_backup_version/run.sh create mode 100644 tests/br_case_sensitive/run.sh create mode 100755 tests/br_clustered_index/run.sh create mode 100755 tests/br_db/run.sh create mode 100755 tests/br_db_online/run.sh create mode 100755 tests/br_db_online_newkv/run.sh create mode 100755 tests/br_db_skip/run.sh create mode 100644 tests/br_debug_meta/run.sh create mode 100644 tests/br_debug_meta/workload create mode 100755 tests/br_full/run.sh create mode 100644 tests/br_full/workload create mode 100755 tests/br_full_ddl/run.sh create mode 100644 tests/br_full_ddl/workload create mode 100755 tests/br_full_index/run.sh create mode 100644 tests/br_full_index/workload create mode 100644 tests/br_gcs/oauth.go create mode 100755 tests/br_gcs/run.sh create mode 100644 tests/br_gcs/workload create mode 100755 tests/br_history/run.sh create mode 100644 tests/br_history/workload create mode 100644 tests/br_incompatible_tidb_config/config/tidb-max-index-length.toml create mode 100755 tests/br_incompatible_tidb_config/run.sh create mode 100755 tests/br_incremental/run.sh create mode 100644 tests/br_incremental/workload create mode 100755 tests/br_incremental_ddl/run.sh create mode 100755 tests/br_incremental_index/run.sh create mode 100755 tests/br_incremental_only_ddl/run.sh create mode 100755 tests/br_incremental_same_table/run.sh create mode 100755 tests/br_insert_after_restore/run.sh create mode 100644 tests/br_key_locked/codec.go create mode 100644 tests/br_key_locked/locker.go create mode 100755 tests/br_key_locked/run.sh create mode 100644 tests/br_key_locked/workload create mode 100755 tests/br_log_restore/run.sh create mode 100644 tests/br_log_restore/workload create mode 100644 tests/br_log_test/run.sh create mode 100644 tests/br_log_test/workload create mode 100755 tests/br_move_backup/run.sh create mode 100644 tests/br_move_backup/workload create mode 100644 tests/br_other/run.sh create mode 100644 tests/br_range/run.sh create mode 100644 tests/br_rawkv/client.go create mode 100644 tests/br_rawkv/run.sh create mode 100755 tests/br_restore_TDE_enable/run.sh create mode 100644 tests/br_restore_TDE_enable/workload create mode 100755 tests/br_s3/run.sh create mode 100644 tests/br_s3/workload create mode 100755 tests/br_shuffle_leader/run.sh create mode 100644 tests/br_shuffle_leader/workload create mode 100755 tests/br_shuffle_region/run.sh create mode 100644 tests/br_shuffle_region/workload create mode 100755 tests/br_single_table/run.sh create mode 100644 tests/br_single_table/workload create mode 100755 tests/br_skip_checksum/run.sh create mode 100644 tests/br_skip_checksum/workload create mode 100755 tests/br_small_batch_size/run.sh create mode 100644 tests/br_small_batch_size/workload create mode 100644 tests/br_split_region_fail/run.sh create mode 100644 tests/br_split_region_fail/workload create mode 100644 tests/br_systables/run.sh create mode 100644 tests/br_systables/workload create mode 100755 tests/br_table_filter/run.sh create mode 100755 tests/br_table_partition/prepare.sh create mode 100755 tests/br_table_partition/run.sh create mode 100644 tests/br_tiflash/run.sh create mode 100644 tests/br_tikv_outage/run.sh create mode 100644 tests/br_tikv_outage/workload create mode 100644 tests/br_tikv_outage2/run.sh create mode 100644 tests/br_tikv_outage2/workload create mode 100755 tests/br_views_and_sequences/run.sh create mode 100644 tests/br_z_gc_safepoint/gc.go create mode 100755 tests/br_z_gc_safepoint/run.sh create mode 100644 tests/br_z_gc_safepoint/workload create mode 100644 tests/config/importer.toml create mode 100644 tests/config/ipsan.cnf create mode 100644 tests/config/pd.toml create mode 100644 tests/config/restore-tikv.toml create mode 100644 tests/config/root.cert create mode 100644 tests/config/root.key create mode 100644 tests/config/tidb.toml create mode 100644 tests/config/tikv.toml create mode 100755 tests/docker_compatible_gcs/prepare.sh create mode 100755 tests/docker_compatible_gcs/run.sh create mode 100755 tests/docker_compatible_s3/prepare.sh create mode 100755 tests/docker_compatible_s3/run.sh create mode 100755 tests/download_tools.sh create mode 100644 tests/lightning_alter_random/config.toml create mode 100644 tests/lightning_alter_random/data/alter_random-schema-create.sql create mode 100644 tests/lightning_alter_random/data/alter_random.t-schema.sql create mode 100644 tests/lightning_alter_random/data/alter_random.t.sql create mode 100644 tests/lightning_alter_random/run.sh create mode 100644 tests/lightning_auto_random_default/config.toml create mode 100644 tests/lightning_auto_random_default/data/auto_random-schema-create.sql create mode 100644 tests/lightning_auto_random_default/data/auto_random.t-schema.sql create mode 100644 tests/lightning_auto_random_default/data/auto_random.t.0.sql create mode 100644 tests/lightning_auto_random_default/data/auto_random.t.1.sql create mode 100644 tests/lightning_auto_random_default/run.sh create mode 100644 tests/lightning_black-white-list/config.toml create mode 100644 tests/lightning_black-white-list/data/firstdb-schema-create.sql create mode 100644 tests/lightning_black-white-list/data/firstdb.first-schema.sql create mode 100644 tests/lightning_black-white-list/data/firstdb.first.1.sql create mode 100644 tests/lightning_black-white-list/data/firstdb.first.2.sql create mode 100644 tests/lightning_black-white-list/data/firstdb.second-schema.sql create mode 100644 tests/lightning_black-white-list/data/firstdb.second.1.sql create mode 100644 tests/lightning_black-white-list/data/mysql-schema-create.sql create mode 100644 tests/lightning_black-white-list/data/mysql.testtable-schema.sql create mode 100644 tests/lightning_black-white-list/data/seconddb-schema-create.sql create mode 100644 tests/lightning_black-white-list/data/seconddb.fourth-schema.sql create mode 100644 tests/lightning_black-white-list/data/seconddb.fourth.1.sql create mode 100644 tests/lightning_black-white-list/data/seconddb.third-schema.sql create mode 100644 tests/lightning_black-white-list/data/seconddb.third.1.sql create mode 100644 tests/lightning_black-white-list/even-table-only.toml create mode 100644 tests/lightning_black-white-list/firstdb-only.toml create mode 100755 tests/lightning_black-white-list/run.sh create mode 100644 tests/lightning_character_sets/auto.toml create mode 100644 tests/lightning_character_sets/binary.toml create mode 100644 tests/lightning_character_sets/gb18030.toml create mode 100644 tests/lightning_character_sets/gb18030/charsets-schema-create.sql create mode 100644 tests/lightning_character_sets/gb18030/charsets.gb18030-schema.sql create mode 100644 tests/lightning_character_sets/gb18030/charsets.gb18030.sql create mode 100644 tests/lightning_character_sets/mixed/charsets-schema-create.sql create mode 100644 tests/lightning_character_sets/mixed/charsets.mixed-schema.sql create mode 100644 tests/lightning_character_sets/mixed/charsets.mixed.sql create mode 100755 tests/lightning_character_sets/run.sh create mode 100644 tests/lightning_character_sets/utf8mb4.toml create mode 100644 tests/lightning_character_sets/utf8mb4/charsets-schema-create.sql create mode 100644 tests/lightning_character_sets/utf8mb4/charsets.utf8mb4-schema.sql create mode 100644 tests/lightning_character_sets/utf8mb4/charsets.utf8mb4.sql create mode 100644 tests/lightning_checkpoint/config.toml create mode 100755 tests/lightning_checkpoint/run.sh create mode 100644 tests/lightning_checkpoint_chunks/config.toml create mode 100644 tests/lightning_checkpoint_chunks/file.toml create mode 100755 tests/lightning_checkpoint_chunks/run.sh create mode 100644 tests/lightning_checkpoint_columns/config.toml create mode 100755 tests/lightning_checkpoint_columns/run.sh create mode 100644 tests/lightning_checkpoint_dirty_tableid/data/cpdt-schema-create.sql create mode 100644 tests/lightning_checkpoint_dirty_tableid/data/cpdt.t-schema.sql create mode 100644 tests/lightning_checkpoint_dirty_tableid/data/cpdt.t.sql create mode 100644 tests/lightning_checkpoint_dirty_tableid/file.toml create mode 100644 tests/lightning_checkpoint_dirty_tableid/mysql.toml create mode 100755 tests/lightning_checkpoint_dirty_tableid/run.sh create mode 100644 tests/lightning_checkpoint_engines/config.toml create mode 100644 tests/lightning_checkpoint_engines/data/cpeng-schema-create.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.a-schema.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.a.1.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.a.2.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.a.3.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.b-schema.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.b.1.sql create mode 100644 tests/lightning_checkpoint_engines/data/cpeng.b.2.sql create mode 100644 tests/lightning_checkpoint_engines/mysql.toml create mode 100755 tests/lightning_checkpoint_engines/run.sh create mode 100644 tests/lightning_checkpoint_engines_order/config.toml create mode 100644 tests/lightning_checkpoint_engines_order/data/disk_quota_checkpoint-schema-create.sql create mode 100644 tests/lightning_checkpoint_engines_order/data/disk_quota_checkpoint.t-schema.sql create mode 100644 tests/lightning_checkpoint_engines_order/data/disk_quota_checkpoint.t.0.sql create mode 100644 tests/lightning_checkpoint_engines_order/data/disk_quota_checkpoint.t.1.sql create mode 100644 tests/lightning_checkpoint_engines_order/data/disk_quota_checkpoint.t.2.sql create mode 100644 tests/lightning_checkpoint_engines_order/run.sh create mode 100644 tests/lightning_checkpoint_error_destroy/bad-data/cped-schema-create.sql create mode 100644 tests/lightning_checkpoint_error_destroy/bad-data/cped.t-schema.sql create mode 100644 tests/lightning_checkpoint_error_destroy/bad-data/cped.t.sql create mode 100644 tests/lightning_checkpoint_error_destroy/file.toml create mode 100644 tests/lightning_checkpoint_error_destroy/good-data/cped-schema-create.sql create mode 100644 tests/lightning_checkpoint_error_destroy/good-data/cped.t-schema.sql create mode 100644 tests/lightning_checkpoint_error_destroy/good-data/cped.t.sql create mode 100644 tests/lightning_checkpoint_error_destroy/mysql.toml create mode 100755 tests/lightning_checkpoint_error_destroy/run.sh create mode 100644 tests/lightning_checkpoint_parquet/config.toml create mode 100644 tests/lightning_checkpoint_parquet/parquet.go create mode 100755 tests/lightning_checkpoint_parquet/run.sh create mode 100644 tests/lightning_checkpoint_timestamp/config.toml create mode 100644 tests/lightning_checkpoint_timestamp/data/cpts-schema-create.sql create mode 100644 tests/lightning_checkpoint_timestamp/data/cpts.cpts-schema.sql create mode 100644 tests/lightning_checkpoint_timestamp/data/cpts.cpts.1.sql create mode 100644 tests/lightning_checkpoint_timestamp/data/cpts.cpts.2.sql create mode 100644 tests/lightning_checkpoint_timestamp/mysql.toml create mode 100755 tests/lightning_checkpoint_timestamp/run.sh create mode 100644 tests/lightning_cmdline_override/config.toml create mode 100644 tests/lightning_cmdline_override/data/cmdline_override-schema-create.sql create mode 100644 tests/lightning_cmdline_override/data/cmdline_override.t-schema.sql create mode 100644 tests/lightning_cmdline_override/data/cmdline_override.t.sql create mode 100755 tests/lightning_cmdline_override/run.sh create mode 100644 tests/lightning_column_permutation/config.toml create mode 100644 tests/lightning_column_permutation/data/perm-schema-create.sql create mode 100644 tests/lightning_column_permutation/data/perm.test_perm-schema.sql create mode 100644 tests/lightning_column_permutation/data/perm.test_perm.0.csv create mode 100644 tests/lightning_column_permutation/run.sh create mode 100644 tests/lightning_common_handle/config.toml create mode 100644 tests/lightning_common_handle/run.sh create mode 100644 tests/lightning_concurrent-restore/config.toml create mode 100644 tests/lightning_concurrent-restore/run.sh create mode 100644 tests/lightning_csv/config.toml create mode 100644 tests/lightning_csv/data/csv-schema-create.sql create mode 100644 tests/lightning_csv/data/csv.empty_strings-schema.sql create mode 100644 tests/lightning_csv/data/csv.empty_strings.csv create mode 100644 tests/lightning_csv/data/csv.escapes-schema.sql create mode 100644 tests/lightning_csv/data/csv.escapes.CSV create mode 100644 tests/lightning_csv/data/csv.threads-schema.sql create mode 100644 tests/lightning_csv/data/csv.threads.csv create mode 100755 tests/lightning_csv/run.sh create mode 100644 tests/lightning_default-columns/config.toml create mode 100644 tests/lightning_default-columns/data/defcol-schema-create.sql create mode 100644 tests/lightning_default-columns/data/defcol.t-schema.sql create mode 100644 tests/lightning_default-columns/data/defcol.t.1.sql create mode 100644 tests/lightning_default-columns/data/defcol.t.2.sql create mode 100644 tests/lightning_default-columns/data/defcol.u-schema.sql create mode 100644 tests/lightning_default-columns/data/defcol.u.1.sql create mode 100755 tests/lightning_default-columns/run.sh create mode 100644 tests/lightning_disk_quota/config.toml create mode 100644 tests/lightning_disk_quota/data/disk_quota-schema-create.sql create mode 100644 tests/lightning_disk_quota/data/disk_quota.t-schema.sql create mode 100644 tests/lightning_disk_quota/data/disk_quota.t.0.sql create mode 100644 tests/lightning_disk_quota/data/disk_quota.t.1.sql create mode 100644 tests/lightning_disk_quota/data/disk_quota.t.2.sql create mode 100644 tests/lightning_disk_quota/data/disk_quota.t.3.sql create mode 100644 tests/lightning_disk_quota/run.sh create mode 100644 tests/lightning_duplicate_detection/config1.toml create mode 100644 tests/lightning_duplicate_detection/config2.toml create mode 100644 tests/lightning_duplicate_detection/data/dup_detect-schema-create.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.ta-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.ta.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.ta.1.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tb-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tb.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tb.1.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tc-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tc.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tc.1.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.td-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.td.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.td.1.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.te-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.te.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.te.1.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tf-schema.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tf.0.sql create mode 100644 tests/lightning_duplicate_detection/data/dup_detect.tf.1.sql create mode 100644 tests/lightning_duplicate_detection/run.sh create mode 100644 tests/lightning_error_summary/config.toml create mode 100644 tests/lightning_error_summary/data/error_summary-schema-create.sql create mode 100644 tests/lightning_error_summary/data/error_summary.a-schema.sql create mode 100644 tests/lightning_error_summary/data/error_summary.a.sql create mode 100644 tests/lightning_error_summary/data/error_summary.b-schema.sql create mode 100644 tests/lightning_error_summary/data/error_summary.b.sql create mode 100644 tests/lightning_error_summary/data/error_summary.c-schema.sql create mode 100644 tests/lightning_error_summary/data/error_summary.c.sql create mode 100755 tests/lightning_error_summary/run.sh create mode 100644 tests/lightning_examples/1.toml create mode 100644 tests/lightning_examples/131072.toml create mode 100644 tests/lightning_examples/512.toml create mode 100755 tests/lightning_examples/run.sh create mode 100644 tests/lightning_exotic_filenames/config.toml create mode 100644 tests/lightning_exotic_filenames/data/xfn-schema-create.sql create mode 100644 tests/lightning_exotic_filenames/data/xfn.etn-schema.sql create mode 100644 tests/lightning_exotic_filenames/data/xfn.etn.sql create mode 100644 tests/lightning_exotic_filenames/data/zwk-schema-create.sql create mode 100644 tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql create mode 100644 tests/lightning_exotic_filenames/data/zwk.zwb.sql create mode 100755 tests/lightning_exotic_filenames/run.sh create mode 100644 tests/lightning_fail_fast/chunk.toml create mode 100644 tests/lightning_fail_fast/data/fail_fast-schema-create.sql create mode 100644 tests/lightning_fail_fast/data/fail_fast.tb-schema.sql create mode 100644 tests/lightning_fail_fast/data/fail_fast.tb.0.csv create mode 100644 tests/lightning_fail_fast/data/fail_fast.tb.1.csv create mode 100644 tests/lightning_fail_fast/data/fail_fast.tb.2.csv create mode 100644 tests/lightning_fail_fast/engine.toml create mode 100755 tests/lightning_fail_fast/run.sh create mode 100644 tests/lightning_file_routing/config.toml create mode 100755 tests/lightning_file_routing/run.sh create mode 100644 tests/lightning_gcs/config.toml create mode 100644 tests/lightning_gcs/run.sh create mode 100644 tests/lightning_generated_columns/config.toml create mode 100644 tests/lightning_generated_columns/data/gencol-schema-create.sql create mode 100644 tests/lightning_generated_columns/data/gencol.nested-schema.sql create mode 100644 tests/lightning_generated_columns/data/gencol.nested.0.sql create mode 100644 tests/lightning_generated_columns/data/gencol.various_types-schema.sql create mode 100644 tests/lightning_generated_columns/data/gencol.various_types.0.sql create mode 100644 tests/lightning_generated_columns/run.sh create mode 100644 tests/lightning_incremental/config.toml create mode 100644 tests/lightning_incremental/data/incr-schema-create.sql create mode 100644 tests/lightning_incremental/data/incr.auto_random-schema.sql create mode 100644 tests/lightning_incremental/data/incr.auto_random.sql create mode 100644 tests/lightning_incremental/data/incr.pk_auto_inc-schema.sql create mode 100644 tests/lightning_incremental/data/incr.pk_auto_inc.sql create mode 100644 tests/lightning_incremental/data/incr.rowid_uk_inc-schema.sql create mode 100644 tests/lightning_incremental/data/incr.rowid_uk_inc.sql create mode 100644 tests/lightning_incremental/data/incr.uk_auto_inc-schema.sql create mode 100644 tests/lightning_incremental/data/incr.uk_auto_inc.sql create mode 100644 tests/lightning_incremental/data1/incr-schema-create.sql create mode 100644 tests/lightning_incremental/data1/incr.auto_random-schema.sql create mode 100644 tests/lightning_incremental/data1/incr.auto_random.sql create mode 100644 tests/lightning_incremental/data1/incr.pk_auto_inc-schema.sql create mode 100644 tests/lightning_incremental/data1/incr.pk_auto_inc.sql create mode 100644 tests/lightning_incremental/data1/incr.rowid_uk_inc-schema.sql create mode 100644 tests/lightning_incremental/data1/incr.rowid_uk_inc.sql create mode 100644 tests/lightning_incremental/data1/incr.uk_auto_inc-schema.sql create mode 100644 tests/lightning_incremental/data1/incr.uk_auto_inc.sql create mode 100644 tests/lightning_incremental/run.sh create mode 100644 tests/lightning_issue_282/config.toml create mode 100644 tests/lightning_issue_282/data/issue282-schema-create.sql create mode 100644 tests/lightning_issue_282/data/issue282.t_access3-schema.sql create mode 100644 tests/lightning_issue_282/data/issue282.t_access3.sql create mode 100644 tests/lightning_issue_282/run.sh create mode 100644 tests/lightning_issue_410/config.toml create mode 100644 tests/lightning_issue_410/data/issue410-schema-create.sql create mode 100644 tests/lightning_issue_410/data/issue410.row_flow_d-schema.sql create mode 100644 tests/lightning_issue_410/data/issue410.row_flow_d.0.csv create mode 100644 tests/lightning_issue_410/run.sh create mode 100644 tests/lightning_issue_519/config.toml create mode 100644 tests/lightning_issue_519/data/issue519-schema-create.sql create mode 100644 tests/lightning_issue_519/data/issue519.t-schema.sql create mode 100644 tests/lightning_issue_519/data/issue519.t.csv create mode 100755 tests/lightning_issue_519/run.sh create mode 100644 tests/lightning_local_backend/config.toml create mode 100644 tests/lightning_local_backend/data/cpeng-schema-create.sql create mode 100644 tests/lightning_local_backend/data/cpeng.a-schema.sql create mode 100644 tests/lightning_local_backend/data/cpeng.a.1.sql create mode 100644 tests/lightning_local_backend/data/cpeng.a.2.sql create mode 100644 tests/lightning_local_backend/data/cpeng.a.3.sql create mode 100644 tests/lightning_local_backend/data/cpeng.b-schema.sql create mode 100644 tests/lightning_local_backend/data/cpeng.b.1.sql create mode 100644 tests/lightning_local_backend/data/cpeng.b.2.sql create mode 100644 tests/lightning_local_backend/file.toml create mode 100644 tests/lightning_local_backend/mysql.toml create mode 100755 tests/lightning_local_backend/run.sh create mode 100644 tests/lightning_new_collation/config.toml create mode 100644 tests/lightning_new_collation/run.sh create mode 100644 tests/lightning_new_collation/tidb-new-collation.toml create mode 100644 tests/lightning_no_schema/config.toml create mode 100644 tests/lightning_no_schema/data/noschema.t.sql create mode 100644 tests/lightning_no_schema/run.sh create mode 100644 tests/lightning_parquet/config.toml create mode 100644 tests/lightning_parquet/data/export_info_ci.json create mode 100644 tests/lightning_parquet/data/export_tables_info_ci_from_1_to_9.json create mode 100644 tests/lightning_parquet/data/test/test.customer/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.customer/part-00000-c3744aeb-351c-49ba-bdf3-5864befff481-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.district/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.district/part-00000-f61f4bef-6765-432a-8f18-cd17b4607f2a-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.history/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.history/part-00000-8cf0e97a-1169-4335-a93f-8805e02def97-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.item/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.item/part-00000-8905ded8-4c54-477f-9907-6e3eae50358b-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.new_order/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.new_order/part-00000-d868200e-b629-4445-bd96-d34b6674b09d-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.order_line/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.order_line/part-00000-e36fecd0-3f59-4ff6-b271-2e4b27ffbcf5-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.orders/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.orders/part-00000-b45481f5-92c0-4961-9bf7-32a0a04ffee5-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.stock/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.stock/part-00000-eef45943-3034-4d65-b375-ee55aa479215-c000.gz.parquet create mode 100644 tests/lightning_parquet/data/test/test.warehouse/_SUCCESS create mode 100644 tests/lightning_parquet/data/test/test.warehouse/part-00000-c6c33252-4d2f-4c5c-8a2f-d162bde6c360-c000.gz.parquet create mode 100644 tests/lightning_parquet/db.sql create mode 100755 tests/lightning_parquet/run.sh create mode 100644 tests/lightning_partitioned-table/config.toml create mode 100644 tests/lightning_partitioned-table/data/partitioned-schema-create.sql create mode 100644 tests/lightning_partitioned-table/data/partitioned.a-schema.sql create mode 100644 tests/lightning_partitioned-table/data/partitioned.a.sql create mode 100755 tests/lightning_partitioned-table/run.sh create mode 100644 tests/lightning_restore/config.toml create mode 100755 tests/lightning_restore/run.sh create mode 100644 tests/lightning_routes/config.toml create mode 100644 tests/lightning_routes/data/routes_a0-schema-create.sql create mode 100644 tests/lightning_routes/data/routes_a0.t0-schema.sql create mode 100644 tests/lightning_routes/data/routes_a0.t0.1.sql create mode 100644 tests/lightning_routes/data/routes_a0.t0.2.sql create mode 100644 tests/lightning_routes/data/routes_a0.t1-schema.sql create mode 100644 tests/lightning_routes/data/routes_a0.t1.1.sql create mode 100644 tests/lightning_routes/data/routes_a1-schema-create.sql create mode 100644 tests/lightning_routes/data/routes_a1.s1-schema.sql create mode 100644 tests/lightning_routes/data/routes_a1.s1.sql create mode 100644 tests/lightning_routes/data/routes_a1.t2-schema.sql create mode 100644 tests/lightning_routes/data/routes_a1.t2.sql create mode 100755 tests/lightning_routes/run.sh create mode 100644 tests/lightning_row-format-v2/config.toml create mode 100644 tests/lightning_row-format-v2/data/rowformatv2-schema-create.sql create mode 100644 tests/lightning_row-format-v2/data/rowformatv2.t1-schema.sql create mode 100644 tests/lightning_row-format-v2/data/rowformatv2.t1.1.sql create mode 100644 tests/lightning_row-format-v2/run.sh create mode 100644 tests/lightning_s3/config.toml create mode 100755 tests/lightning_s3/run.sh create mode 100644 tests/lightning_shard_rowid/config.toml create mode 100644 tests/lightning_shard_rowid/data/shard_rowid-schema-create.sql create mode 100644 tests/lightning_shard_rowid/data/shard_rowid.shr-schema.sql create mode 100644 tests/lightning_shard_rowid/data/shard_rowid.shr.0.sql create mode 100644 tests/lightning_shard_rowid/run.sh create mode 100644 tests/lightning_source_linkfile/config.toml create mode 100644 tests/lightning_source_linkfile/run.sh create mode 100644 tests/lightning_sqlmode/data/sqlmodedb-schema-create.sql create mode 100644 tests/lightning_sqlmode/data/sqlmodedb.t-schema.sql create mode 100644 tests/lightning_sqlmode/data/sqlmodedb.t.1.sql create mode 100644 tests/lightning_sqlmode/off.toml create mode 100644 tests/lightning_sqlmode/on.toml create mode 100755 tests/lightning_sqlmode/run.sh create mode 100644 tests/lightning_tidb_duplicate_data/data/dup-schema-create.sql create mode 100644 tests/lightning_tidb_duplicate_data/data/dup.dup-schema.sql create mode 100644 tests/lightning_tidb_duplicate_data/data/dup.dup.sql create mode 100644 tests/lightning_tidb_duplicate_data/error.toml create mode 100644 tests/lightning_tidb_duplicate_data/ignore.toml create mode 100644 tests/lightning_tidb_duplicate_data/replace.toml create mode 100644 tests/lightning_tidb_duplicate_data/run.sh create mode 100644 tests/lightning_tidb_rowid/config.toml create mode 100644 tests/lightning_tidb_rowid/data/rowid-schema-create.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.explicit_tidb_rowid-schema.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.explicit_tidb_rowid.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.non_pk.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc-schema.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.pre_rebase-schema.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.pre_rebase.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.specific_auto_inc-schema.sql create mode 100644 tests/lightning_tidb_rowid/data/rowid.specific_auto_inc.sql create mode 100755 tests/lightning_tidb_rowid/run.sh create mode 100644 tests/lightning_tiflash/config.toml create mode 100644 tests/lightning_tiflash/run.sh create mode 100644 tests/lightning_too_many_columns/config.toml create mode 100644 tests/lightning_too_many_columns/data/too_many_columns-schema-create.sql create mode 100644 tests/lightning_too_many_columns/data/too_many_columns.t-schema.sql create mode 100644 tests/lightning_too_many_columns/data/too_many_columns.t.0.csv create mode 100644 tests/lightning_too_many_columns/run.sh create mode 100644 tests/lightning_tool_135/config.toml create mode 100644 tests/lightning_tool_135/data/tool_135-schema-create.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar1-schema.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar1.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar2-schema.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar2.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar3-schema.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar3.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar4-schema.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar4.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar5-schema.sql create mode 100644 tests/lightning_tool_135/data/tool_135.bar5.sql create mode 100755 tests/lightning_tool_135/run.sh create mode 100644 tests/lightning_tool_1420/config.toml create mode 100644 tests/lightning_tool_1420/data/EE1420-schema-create.sql create mode 100644 tests/lightning_tool_1420/data/EE1420.pt_role-schema.sql create mode 100644 tests/lightning_tool_1420/data/EE1420.pt_role.sql create mode 100755 tests/lightning_tool_1420/run.sh create mode 100644 tests/lightning_tool_1472/config.toml create mode 100644 tests/lightning_tool_1472/data/EE1472-schema-create.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.notpk-schema.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.notpk.1.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.notpk.2.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.pk-schema.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.pk.1.sql create mode 100644 tests/lightning_tool_1472/data/EE1472.pk.2.sql create mode 100755 tests/lightning_tool_1472/run.sh create mode 100644 tests/lightning_tool_241/config.toml create mode 100644 tests/lightning_tool_241/data/qyjc-schema-create.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_alarm_group-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_alarm_group.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_alarm_message_log-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_alarm_message_log.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_alarm_receiver-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_config-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_fish_event-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_fish_event.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_report_circular_data-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_report_desc-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_report_summary-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_system_update-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_system_update.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_user_log-schema.sql create mode 100644 tests/lightning_tool_241/data/qyjc.q_user_log.sql create mode 100755 tests/lightning_tool_241/run.sh create mode 100644 tests/lightning_unused_config_keys/config.toml create mode 100644 tests/lightning_unused_config_keys/data/unused_config_keys-schema-create.sql create mode 100755 tests/lightning_unused_config_keys/run.sh create mode 100644 tests/lightning_various_types/config.toml create mode 100644 tests/lightning_various_types/data/vt-schema-create.sql create mode 100644 tests/lightning_various_types/data/vt.bigint-schema.sql create mode 100644 tests/lightning_various_types/data/vt.bigint.0.sql create mode 100644 tests/lightning_various_types/data/vt.binary-schema.sql create mode 100644 tests/lightning_various_types/data/vt.binary.sql create mode 100644 tests/lightning_various_types/data/vt.bit-schema.sql create mode 100644 tests/lightning_various_types/data/vt.bit.sql create mode 100644 tests/lightning_various_types/data/vt.char-schema.sql create mode 100644 tests/lightning_various_types/data/vt.char.sql create mode 100644 tests/lightning_various_types/data/vt.datetime-schema.sql create mode 100644 tests/lightning_various_types/data/vt.datetime.sql create mode 100644 tests/lightning_various_types/data/vt.decimal-schema.sql create mode 100644 tests/lightning_various_types/data/vt.decimal.sql create mode 100644 tests/lightning_various_types/data/vt.double-schema.sql create mode 100644 tests/lightning_various_types/data/vt.double.sql create mode 100644 tests/lightning_various_types/data/vt.empty_strings-schema.sql create mode 100644 tests/lightning_various_types/data/vt.empty_strings.sql create mode 100644 tests/lightning_various_types/data/vt.enum-set-schema.sql create mode 100644 tests/lightning_various_types/data/vt.enum-set.sql create mode 100644 tests/lightning_various_types/data/vt.json-schema.sql create mode 100644 tests/lightning_various_types/data/vt.json.sql create mode 100644 tests/lightning_various_types/data/vt.precise_types-schema.sql create mode 100644 tests/lightning_various_types/data/vt.precise_types.sql create mode 100755 tests/lightning_various_types/run.sh create mode 100644 tests/lightning_view/config.toml create mode 100644 tests/lightning_view/data/db0-schema-create.sql create mode 100644 tests/lightning_view/data/db0.v2-schema-view.sql create mode 100644 tests/lightning_view/data/db0.v2-schema.sql create mode 100644 tests/lightning_view/data/db1-schema-create.sql create mode 100644 tests/lightning_view/data/db1.tbl-schema.sql create mode 100644 tests/lightning_view/data/db1.tbl.0.sql create mode 100644 tests/lightning_view/data/db1.v1-schema-view.sql create mode 100644 tests/lightning_view/data/db1.v1-schema.sql create mode 100755 tests/lightning_view/run.sh create mode 100755 tests/run.sh create mode 100755 tests/run_compatible.sh create mode 100755 tests/up.sh create mode 100644 tidb-lightning.toml create mode 100644 tools/.gitignore create mode 100644 tools/Makefile create mode 100755 tools/check-errdoc.sh create mode 100644 tools/go.mod create mode 100644 tools/go.sum create mode 100644 tools/go_mod_guard.go create mode 100644 web/README.md create mode 100644 web/docs/InfoPage.png create mode 100644 web/docs/ProgressPage.png create mode 100644 web/docs/TableProgressPage.png create mode 100644 web/docs/api.yaml create mode 100644 web/go.mod create mode 100644 web/go.sum create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/public/index.html create mode 100644 web/src/ChunksProgressPanel.tsx create mode 100644 web/src/DottedProgress.tsx create mode 100644 web/src/EnginesProgressPanel.tsx create mode 100644 web/src/ErrorButton.tsx create mode 100644 web/src/InfoButton.tsx create mode 100644 web/src/InfoPage.tsx create mode 100644 web/src/MoveTaskButton.tsx create mode 100644 web/src/PauseButton.tsx create mode 100644 web/src/ProgressPage.tsx create mode 100644 web/src/RefreshButton.tsx create mode 100644 web/src/TableProgressCard.tsx create mode 100644 web/src/TableProgressPage.tsx create mode 100644 web/src/TaskButton.tsx create mode 100644 web/src/TitleBar.tsx create mode 100644 web/src/TitleLink.tsx create mode 100644 web/src/api.ts create mode 100644 web/src/index.tsx create mode 100644 web/src/json-bigint.d.ts create mode 100644 web/tsconfig.json create mode 100644 web/webpack.config.js diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000000000..9fc8064faee5c --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,22 @@ +codecov: + require_ci_to_pass: yes + +comment: + layout: "reach, diff, flags, files" + behavior: default + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + require_head: yes # [yes :: must have a head report to post] + branches: # branch names that can post comment + - "master" + +coverage: + status: + project: + default: + # Allow the coverage to drop by 3% + target: 85% + threshold: 3% + branches: + - master + patch: off diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 0000000000000..3e4e48b0b5fe6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000..43c6a002cce45 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +# tab_size = 4 spaces +[*.go] +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000..ba35fa1000ef6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*_generated.go linguist-generated=true diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000000000..555efb841bd26 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,45 @@ +--- +name: "🐛 Bug Report" +about: Something isn't working as expected +title: '' +labels: 'type/bug ' +--- + +Please answer these questions before submitting your issue. Thanks! + +1. What did you do? +If possible, provide a recipe for reproducing the error. + + +2. What did you expect to see? + + + +3. What did you see instead? + + + +4. What version of BR and TiDB/TiKV/PD are you using? + + + +5. Operation logs + - Please upload `br.log` for BR if possible + - Please upload `tidb-lightning.log` for TiDB-Lightning if possible + - Please upload `tikv-importer.log` from TiKV-Importer if possible + - Other interesting logs + + +6. Configuration of the cluster and the task + - `tidb-lightning.toml` for TiDB-Lightning if possible + - `tikv-importer.toml` for TiKV-Importer if possible + - `topology.yml` if deployed by TiUP + + +7. Screenshot/exported-PDF of Grafana dashboard or metrics' graph in Prometheus if possible diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000000000..ed6b4c5b0bf7c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,19 @@ +--- +name: "🚀 Feature Request" +about: I have a suggestion +labels: 'type/feature-request' +--- + +## Feature Request + +### Describe your feature request related problem: + + +### Describe the feature you'd like: + + +### Describe alternatives you've considered: + + +### Teachability, Documentation, Adoption, Migration Strategy: + diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000000000..23a8118377288 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,24 @@ +--- +name: "\U0001F914 Question" +labels: "type/question" +about: Usage question that isn't answered in docs or discussion + +--- + +## Question + + + diff --git a/.github/challenge-bot.yml b/.github/challenge-bot.yml new file mode 100644 index 0000000000000..15d2f38ece965 --- /dev/null +++ b/.github/challenge-bot.yml @@ -0,0 +1 @@ +defaultSigLabel: sig/migrate diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000000..7f64aa17286f2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,42 @@ + + +### What problem does this PR solve? + + +### What is changed and how it works? + + +### Check List + +Tests + + - Unit test + - Integration test + - Manual test (add detailed scripts or steps below) + - No code + +Code changes + + - Has exported function/method change + - Has exported variable/fields change + - Has interface methods change + - Has persistent data change + +Side effects + + - Possible performance regression + - Increased code complexity + - Breaking backward compatibility + +Related changes + + - Need to cherry-pick to the release branch + - Need to update the documentation + +### Release note + + - + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000000..472def46ae2a0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,68 @@ +name: build +on: + push: + branches: + - master + - 'release-[0-9].[0-9]*' + paths-ignore: + - '**.html' + - '**.md' + - 'CNAME' + - 'LICENSE' + - 'docs/**' + - 'tests/**' + - 'docker/**' + - '.github/workflows/**.yml' + pull_request: + branches: + - master + - 'release-[0-9].[0-9]*' + paths-ignore: + - '**.html' + - '**.md' + - 'CNAME' + - 'LICENSE' + - 'docs/**' + - 'tests/**' + - 'docker/**' + - '.github/workflows/**.yml' + +jobs: + compile: + name: ${{ matrix.os }} / ${{ matrix.target}} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-latest + target: x86_64-apple-darwin + + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + + - os: windows-latest + target: x86_64-pc-windows-msvc + steps: + - uses: actions/checkout@v2.1.0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Run build + run: make build + + compile-freebsd: + name: Compile for FreeBSD job + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.1.0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Compile for FreeBSD + run: GOOS=freebsd make build diff --git a/.github/workflows/compatible_test.yml b/.github/workflows/compatible_test.yml new file mode 100644 index 0000000000000..9fbf0c52e83ac --- /dev/null +++ b/.github/workflows/compatible_test.yml @@ -0,0 +1,71 @@ +name: compatibility-test + +on: + push: + branches: + - master + - 'release-[0-9].[0-9]*' + paths-ignore: + - '**.html' + - '**.md' + - 'CNAME' + - 'LICENSE' + - 'docs/**' + - 'tests/**' + - 'docker/**' + - '.github/workflows/**.yml' + pull_request: + branches: + - master + - 'release-[0-9].[0-9]*' + paths-ignore: + - '**.html' + - '**.md' + - 'CNAME' + - 'LICENSE' + - 'docs/**' + - 'tests/**' + - 'docker/**' + - '.github/workflows/**.yml' + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + + - name: Free disk space + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + docker volume prune -f + docker image prune -f + + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Generate compatibility test backup data + timeout-minutes: 15 + run: sh compatibility/prepare_backup.sh + + - name: Start server + run: | + TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml rm -s -v + TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml build + TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml up --remove-orphans -d + TAG=nightly PORT_SUFFIX=1 docker-compose -f compatibility/backup_cluster.yaml exec -T control make compatibility_test + + - name: Collect component log + if: ${{ failure() }} + run: | + tar czvf ${{ github.workspace }}/logs.tar.gz /tmp/br/docker/backup_logs/* + + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: logs + path: ${{ github.workspace }}/logs.tar.gz diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..c29d04732ce40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +/br +/bin +/.idea +/docker/data/ +/docker/logs/ +*.swp +.DS_Store +/go.mod +/go.sum + +# for the web interface +web/node_modules/ +web/dist/ +.vscode/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000000..0cb2a9b651251 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,12 @@ +linters-settings: + gocyclo: + min-complexity: 40 + +issues: + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # TODO Remove it. + - path: split_client.go + text: "SA1019:" + linters: + - staticcheck diff --git a/COMPATIBILITY_TEST.md b/COMPATIBILITY_TEST.md new file mode 100644 index 0000000000000..b5580835baee8 --- /dev/null +++ b/COMPATIBILITY_TEST.md @@ -0,0 +1,42 @@ +# Compatibility test + +## Background + +We had some incompatibility issues in the past, which made BR cannot restore backed up data in some situations. +So we need a test workflow to check the compatiblity. + +## Goal + +- Ensure backward compatibility for restoring data from the previous 3 minor versions + +## Workflow + +### Data Preparation + +This workflow needs previous backup data. To get this data. we perform the following steps + +- Run a TiDB cluster with previous version. +- Run backup jobs with corresponding BR version, with different storages (s3, gcs). + +Given we test for the previous 3 versions, and there are 2 different storage systems, we will produce 6 backup archives for 6 separate compatibility tests. + +### Test Content + +- Start TiDB cluster with nightly version. +- Build BR binary with current directory. +- Use BR to restore different version backup data one by one. +- Make sure restore data is expected. + +### Running tests + +Start a cluster with docker-compose and Build br with latest version. + +```sh +docker-compose -f docker-compose.yaml rm -s -v && \ +docker-compose -f docker-compose.yaml build && \ +docker-compose -f docker-compose.yaml up --remove-orphans +``` + +```sh +docker-compose -f docker-compose.yaml control make compatibility_test +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000..1f2846471a7a1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# How to contribute + +This document outlines some of the conventions on development workflow, commit +message formatting, contact points and other resources to make it easier to get +your contribution accepted. + +## Getting started + +- Fork the repository on GitHub. +- Read the README.md for build instructions. +- Play with the project, submit bugs, submit patches! + +## Building BR + +Developing BR requires: + +* [Go 1.16+](http://golang.org/doc/code.html) +* An internet connection to download the dependencies + +Simply run `make` to build the program. + +```sh +make +``` + +### Running tests + +This project contains unit tests and integration tests with coverage collection. +See [tests/README.md](./tests/README.md) for how to execute and add tests. + +### Updating dependencies + +BR uses [Go 1.11 module](https://github.com/golang/go/wiki/Modules) to manage dependencies. +To add or update a dependency: use the `go mod edit` command to change the dependency. + +## Contribution flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work. This is usually `master`. +- Make commits of logical units and add test case if the change fixes a bug or adds new functionality. +- Run tests and make sure all the tests are passed. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Submit a pull request. +- Your PR must receive LGTMs from two maintainers. + +Thanks for your contributions! + +### Code style + +The coding style suggested by the Golang community is used in BR. +See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details. + +Please follow this style to make BR easy to review, maintain and develop. + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +restore: add comment for variable declaration + +Improve documentation. +``` + +The format can be described more formally as follows: + +``` +: + + + +