Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DBaaS ： tiflash upgrade to v5.2.0 failed #2781

Closed
apollodafoni opened this issue Aug 26, 2021 · 1 comment · Fixed by #3019
Closed

DBaaS ： tiflash upgrade to v5.2.0 failed #2781

apollodafoni opened this issue Aug 26, 2021 · 1 comment · Fixed by #3019
Assignees
@JinheLin
Labels
severity/critical type/bug The issue is confirmed as a bug.

Comments

@apollodafoni
Copy link

tiflash log :
Logging information to /data0/logs/server.log
Logging errors to /data0/logs/error.log
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:36] ["Welcome To RaftStore Proxy"]
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:38] ["Git Commit Hash: 7d4e73b43e8ddf47946378208b6db25df8d4e1e6"]
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:38] ["Git Commit Branch: HEAD"]
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:38] ["UTC Build Time: 2021-08-24 11:33:44"]
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:38] ["Rust Version: rustc 1.53.0-nightly (16bf626a3 2021-04-14)"]
[2021/08/25 08:17:14.059 +00:00] [INFO] [lib.rs:38] ["Storage Engine: tiflash"]
[2021/08/25 08:17:14.060 +00:00] [INFO] [lib.rs:38] ["Prometheus Prefix: tiflash_proxy_"]
[2021/08/25 08:17:14.060 +00:00] [INFO] [lib.rs:38] ["Profile: release"]
[2021/08/25 08:17:14.061 +00:00] [INFO] [mod.rs:70] ["memory limit in bytes: 68145410048, cpu cores quota: 8"]
[2021/08/25 08:17:14.061 +00:00] [WARN] [server.rs:1179] ["check: kernel"] [err="kernel parameters net.core.somaxconn got 128, expect 32768"]
[2021/08/25 08:17:14.061 +00:00] [WARN] [server.rs:1179] ["check: kernel"] [err="check_kernel_params failed No such file or directory (os error 2)"]
[2021/08/25 08:17:14.061 +00:00] [WARN] [server.rs:1179] ["check: kernel"] [err="kernel parameters vm.swappiness got 30, expect 0"]
[2021/08/25 08:17:14.063 +00:00] [INFO] [util.rs:536] ["connecting to PD endpoint"] [endpoints=db-pd.tidb1373933076657971398.svc:2379]
[2021/08/25 08:17:14.063 +00:00] [INFO] [] ["Disabling AF_INET6 sockets because ::1 is not available."]
[2021/08/25 08:17:14.068 +00:00] [INFO] [] ["TCP_USER_TIMEOUT is available. TCP_USER_TIMEOUT will be used thereafter"]
[2021/08/25 08:17:14.075 +00:00] [INFO] [] ["New connected subchannel at 0x7fb158627320 for subchannel 0x7fb1586331c0"]
[2021/08/25 08:17:14.077 +00:00] [INFO] [util.rs:536] ["connecting to PD endpoint"] [endpoints=https://db-pd-1.db-pd-peer.tidb1373933076657971398.svc:2379]
[2021/08/25 08:17:14.087 +00:00] [INFO] [] ["New connected subchannel at 0x7fb157e7afc0 for subchannel 0x7fb157e2c1c0"]
[2021/08/25 08:17:14.089 +00:00] [INFO] [util.rs:536] ["connecting to PD endpoint"] [endpoints=https://db-pd-2.db-pd-peer.tidb1373933076657971398.svc:2379]
[2021/08/25 08:17:14.098 +00:00] [INFO] [] ["New connected subchannel at 0x7fb15767afc0 for subchannel 0x7fb15762c1c0"]
[2021/08/25 08:17:14.099 +00:00] [INFO] [util.rs:650] ["connected to PD member"] [endpoints=https://db-pd-2.db-pd-peer.tidb1373933076657971398.svc:2379]
[2021/08/25 08:17:14.099 +00:00] [INFO] [util.rs:528] ["all PD endpoints are consistent"] [endpoints="["db-pd.tidb1373933076657971398.svc:2379"]"]
[2021/08/25 08:17:14.099 +00:00] [INFO] [server.rs:393] ["connect to PD cluster"] [cluster_id=7000205040155539449]
[2021/08/25 08:17:14.099 +00:00] [INFO] [config.rs:1943] ["readpool.storage.use-unified-pool is not set, set to true by default"]
[2021/08/25 08:17:14.099 +00:00] [INFO] [config.rs:1966] ["readpool.coprocessor.use-unified-pool is not set, set to true by default"]
[2021/08/25 08:17:14.127 +00:00] [INFO] [server.rs:1187] ["beginning system configuration check"]
[2021/08/25 08:17:14.127 +00:00] [INFO] [config.rs:898] ["data dir"] [mount_fs="FsInfo { tp: "ext4", opts: "rw,relatime,data=ordered", mnt_dir: "/data0", fsname: "/dev/nvme1n1" }"] [data_path=/data0/proxy]
[2021/08/25 08:17:14.127 +00:00] [INFO] [config.rs:898] ["data dir"] [mount_fs="FsInfo { tp: "ext4", opts: "rw,relatime,data=ordered", mnt_dir: "/data0", fsname: "/dev/nvme1n1" }"] [data_path=/data0/proxy/raft]
[2021/08/25 08:17:14.128 +00:00] [INFO] [server.rs:362] ["using config"] [config="{"log-level":"info","log-file":"","log-format":"text","slow-log-file":"","slow-log-threshold":"1s","log-rotation-timespan":"1d","log-rotation-size":"300MiB","panic-when-unexpected-key-or-data":false,"enable-io-snoop":true,"abort-on-panic":false,"memory-usage-limit":"6055MiB","memory-usage-high-water":0.9,"readpool":{"unified":{"min-thread-count":1,"max-thread-count":4,"stack-size":"10MiB","max-tasks-per-worker":2000},"storage":{"use-unified-pool":true,"high-concurrency":4,"normal-concurrency":4,"low-concurrency":4,"max-tasks-per-worker-high":2000,"max-tasks-per-worker-normal":2000,"max-tasks-per-worker-low":2000,"stack-size":"10MiB"},"coprocessor":{"use-unified-pool":true,"high-concurrency":6,"normal-concurrency":6,"low-concurrency":6,"max-tasks-per-worker-high":2000,"max-tasks-per-worker-normal":2000,"max-tasks-per-worker-low":2000,"stack-size":"10MiB"}},"server":{"addr":"0.0.0.0:20170","advertise-addr":"db-tiflash-0.db-tiflash-peer.tidb1373933076657971398.svc:20170","engine-addr":"db-tiflash-0.db-tiflash-peer.tidb1373933076657971398.svc:3930","engine-store-version":"v5.2.0","engine-store-git-hash":"ca0ee5ab6de5912be984135d36e1a3a0d561094b","status-addr":"0.0.0.0:20292","advertise-status-addr":"db-tiflash-0.db-tiflash-peer.tidb1373933076657971398.svc:20292","status-thread-pool-size":1,"max-grpc-send-msg-len":10485760,"raft-client-grpc-send-msg-buffer":524288,"raft-client-queue-size":8192,"raft-msg-max-batch-size":128,"grpc-compression-type":"none","grpc-concurrency":5,"grpc-concurrent-stream":1024,"grpc-raft-conn-num":1,"grpc-memory-pool-quota":9223372036854775807,"grpc-stream-initial-window-size":"2MiB","grpc-keepalive-time":"10s","grpc-keepalive-timeout":"3s","concurrent-send-snap-limit":32,"concurrent-recv-snap-limit":32,"end-point-recursion-limit":1000,"end-point-stream-channel-size":8,"end-point-batch-row-limit":64,"end-point-stream-batch-row-limit":128,"end-point-enable-batch-if-possible":true,"end-point-request-max-handle-duration":"1m","end-point-max-concurrency":8,"snap-max-write-bytes-per-sec":"100MiB","snap-max-total-size":"0KiB","stats-concurrency":1,"heavy-load-threshold":300,"heavy-load-wait-duration":"1ms","enable-request-batch":true,"background-thread-count":4,"end-point-slow-log-threshold":"1s","forward-max-connections-per-address":4,"labels":{"engine":"tiflash"}},"storage":{"data-dir":"/data0/proxy","gc-ratio-threshold":1.1,"max-key-size":4096,"scheduler-concurrency":524288,"scheduler-worker-pool-size":4,"scheduler-pending-write-threshold":"100MiB","reserve-space":"1GiB","enable-async-apply-prewrite":false,"enable-ttl":false,"ttl-check-poll-interval":"12h","block-cache":{"shared":true,"capacity":"3633MiB","num-shard-bits":6,"strict-capacity-limit":false,"high-pri-pool-ratio":0.8,"memory-allocator":"nodump"},"io-rate-limit":{"max-bytes-per-sec":"0KiB","mode":"write-only","strict":false,"foreground-read-priority":"high","foreground-write-priority":"high","flush-priority":"high","level-zero-compaction-priority":"medium","compaction-priority":"low","replication-priority":"high","load-balance-priority":"high","gc-priority":"high","import-priority":"medium","export-priority":"medium","other-priority":"high"}},"pd":{"endpoints":["db-pd.tidb1373933076657971398.svc:2379"],"retry-interval":"300ms","retry-max-count":9223372036854775807,"retry-log-every":10,"update-interval":"10m","enable-forwarding":false},"metric":{"job":"tikv"},"raftstore":{"prevote":true,"raftdb-path":"/data0/proxy/raft","capacity":"0KiB","raft-base-tick-interval":"1s","raft-heartbeat-ticks":2,"raft-election-timeout-ticks":10,"raft-min-election-timeout-ticks":10,"raft-max-election-timeout-ticks":20,"raft-max-size-per-msg":"1MiB","raft-max-inflight-msgs":256,"raft-entry-max-size":"8MiB","raft-log-gc-tick-interval":"10s","raft-log-gc-threshold":50,"raft-log-gc-count-limit":73728,"raft-log-gc-size-limit":"72MiB","raft-log-reserve-max-ticks":6,"raft-engine-purge-interval":"10s","raft-entry-cache-life-time":"30s","raft-reject-transfer-leader-duration":"3s","split-region-check-tick-interval":"10s","region-split-check-diff":"6MiB","region-compact-check-interval":"5m","region-compact-check-step":100,"region-compact-min-tombstones":10000,"region-compact-tombstones-percent":30,"pd-heartbeat-tick-interval":"1m","pd-store-heartbeat-tick-interval":"10s","snap-mgr-gc-tick-interval":"1m","snap-gc-timeout":"4h","lock-cf-compact-interval":"10m","lock-cf-compact-bytes-threshold":"256MiB","notify-capacity":40960,"messages-per-tick":4096,"max-peer-down-duration":"5m","max-leader-missing-duration":"2h","abnormal-leader-missing-duration":"10m","peer-stale-state-check-interval":"5m","leader-transfer-max-log-lag":128,"snap-apply-batch-size":"10MiB","snap-handle-pool-size":2,"region-worker-tick-interval":"500ms","consistency-check-interval":"0s","report-region-flow-interval":"1m","raft-store-max-leader-lease":"9s","right-derive-when-split":true,"allow-remove-leader":false,"merge-max-log-gap":10,"merge-check-tick-interval":"2s","use-delete-range":false,"cleanup-import-sst-interval":"10m","local-read-batch-size":1024,"apply-max-batch-size":256,"apply-pool-size":2,"apply-reschedule-duration":"5s","apply-low-priority-pool-size":1,"store-max-batch-size":256,"store-pool-size":2,"store-reschedule-duration":"5s","store-low-priority-pool-size":0,"future-poll-size":1,"hibernate-regions":true,"dev-assert":false,"apply-yield-duration":"500ms","perf-level":1,"store-batch-retry-recv-timeout":"4ms"},"coprocessor":{"split-region-on-table":false,"batch-split-limit":10,"region-max-size":"144MiB","region-split-size":"96MiB","region-max-keys":1440000,"region-split-keys":960000,"consistency-check-method":"mvcc","perf-level":2},"coprocessor-v2":{"coprocessor-plugin-directory":null},"rocksdb":{"info-log-level":"info","wal-recovery-mode":2,"wal-dir":"","wal-ttl-seconds":0,"wal-size-limit":"0KiB","max-total-wal-size":"4GiB","max-background-jobs":8,"max-background-flushes":3,"max-manifest-file-size":"128MiB","create-if-missing":true,"max-open-files":40960,"enable-statistics":true,"stats-dump-period":"10m","compaction-readahead-size":"0KiB","info-log-max-size":"1GiB","info-log-roll-time":"0s","info-log-keep-log-file-num":10,"info-log-dir":"","rate-bytes-per-sec":"10GiB","rate-limiter-refill-period":"100ms","rate-limiter-mode":2,"rate-limiter-auto-tuned":true,"bytes-per-sync":"1MiB","wal-bytes-per-sync":"512KiB","max-sub-compactions":3,"writable-file-max-buffer-size":"1MiB","use-direct-io-for-flush-and-compaction":false,"enable-pipelined-write":true,"enable-multi-batch-write":true,"enable-unordered-write":false,"defaultcf":{"block-size":"64KiB","block-cache-size":"128MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":true,"whole-key-filtering":true,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","lz4","lz4","lz4","zstd","zstd"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"512MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":4,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":3,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"256GiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":true,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"zstd","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"normal","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"writecf":{"block-size":"64KiB","block-cache-size":"128MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":false,"whole-key-filtering":false,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","lz4","lz4","lz4","zstd","zstd"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"512MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":4,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":3,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"256GiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":true,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"zstd","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"read-only","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"lockcf":{"block-size":"16KiB","block-cache-size":"128MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":false,"whole-key-filtering":true,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","no","no","no","no","no"],"write-buffer-size":"32MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"128MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":1,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":0,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"256GiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":false,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"disable","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"read-only","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"raftcf":{"block-size":"16KiB","block-cache-size":"128MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":true,"whole-key-filtering":true,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","no","no","no","no","no"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"128MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":1,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":0,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"256GiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":false,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"disable","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"read-only","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"titan":{"enabled":false,"dirname":"","disable-gc":false,"max-background-gc":4,"purge-obsolete-files-period":"10s"}},"raftdb":{"wal-recovery-mode":2,"wal-dir":"","wal-ttl-seconds":0,"wal-size-limit":"0KiB","max-total-wal-size":"4GiB","max-background-jobs":4,"max-background-flushes":1,"max-manifest-file-size":"20MiB","create-if-missing":true,"max-open-files":40960,"enable-statistics":true,"stats-dump-period":"10m","compaction-readahead-size":"0KiB","info-log-max-size":"1GiB","info-log-roll-time":"0s","info-log-keep-log-file-num":10,"info-log-dir":"","info-log-level":"info","max-sub-compactions":2,"writable-file-max-buffer-size":"1MiB","use-direct-io-for-flush-and-compaction":false,"enable-pipelined-write":true,"enable-unordered-write":false,"allow-concurrent-memtable-write":true,"bytes-per-sync":"1MiB","wal-bytes-per-sync":"512KiB","defaultcf":{"block-size":"64KiB","block-cache-size":"3249MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":false,"optimize-filters-for-hits":true,"whole-key-filtering":true,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","lz4","lz4","lz4","zstd","zstd"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"512MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":4,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":0,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"256GiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":false,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"disable","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"normal","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"titan":{"enabled":false,"dirname":"","disable-gc":false,"max-background-gc":4,"purge-obsolete-files-period":"10s"}},"raft-engine":{"enable":false,"dir":"/data0/proxy/raft-engine","recovery-mode":"tolerate-corrupted-tail-records","bytes-per-sync":"256KiB","target-file-size":"128MiB","purge-threshold":"10GiB","cache-limit":"1GiB","batch-compression-threshold":"8KiB"},"security":{"ca-path":"/var/lib/tiflash-tls/ca.crt","cert-path":"/var/lib/tiflash-tls/tls.crt","key-path":"/var/lib/tiflash-tls/tls.key","cert-allowed-cn":["81ac69e7"],"redact-info-log":null,"encryption":{"data-encryption-method":"aes256-ctr","data-key-rotation-period":"7d","enable-file-dictionary-log":true,"file-dictionary-rewrite-threshold":1000000,"master-key":{"type":"kms","key-id":"arn:aws:kms:us-west-2:385595570414:key/9e865424-6e66-4f95-ae3b-f7c2a5c6687f","region":"us-west-2","endpoint":"","vendor":""},"previous-master-key":{"type":"plaintext"}}},"import":{"num-threads":4,"stream-channel-window":128,"import-mode-timeout":"10m"},"backup":{"num-threads":6,"batch-size":8,"sst-max-size":"144MiB"},"pessimistic-txn":{"wait-for-lock-timeout":"1s","wake-up-delay-duration":"20ms","pipelined":true},"gc":{"ratio-threshold":1.1,"batch-keys":512,"max-write-bytes-per-sec":"0KiB","enable-compaction-filter":true,"compaction-filter-skip-version-check":false},"split":{"qps-threshold":3000,"split-balance-score":0.25,"split-contained-score":0.5,"detect-times":10,"sample-num":20,"sample-threshold":100,"byte-threshold":31457280},"cdc":{"min-ts-interval":"1s","hibernate-regions-compatible":true,"incremental-scan-threads":4,"incremental-scan-concurrency":6,"incremental-scan-speed-limit":"128MiB","sink-memory-quota":"512MiB","old-value-cache-memory-quota":"512MiB","old-value-cache-size":0},"resolved-ts":{"enable":true,"advance-ts-interval":"1s","scan-lock-pool-size":2},"resource-metering":{"enabled":false,"agent-address":"","report-agent-interval":"1m","max-resource-groups":2000,"precision":"1s"}}"]
[2021/08/25 08:17:14.132 +00:00] [INFO] [lib.rs:56] ["Encryption init cloud backend"] [vendor=] [key_id=arn:aws:kms:us-west-2:385595570414:key/9e865424-6e66-4f95-ae3b-f7c2a5c6687f] [endpoint=] [region=us-west-2]
[2021/08/25 08:17:14.191 +00:00] [INFO] [mod.rs:96] ["encryption: found both of key dictionary and file dictionary."]
[2021/08/25 08:17:14.192 +00:00] [INFO] [mod.rs:494] ["encryption is enabled. method = Aes256Ctr"]
[2021/08/25 08:17:14.192 +00:00] [INFO] [server.rs:213] ["set raft-store proxy helper"]
[2021/08/25 08:17:14.192 +00:00] [INFO] [server.rs:213] ["wait for engine-store server to start"]
[2021/08/25 08:17:14.392 +00:00] [INFO] [server.rs:213] ["engine-store server is not running, make proxy exit"]
[2021/08/25 08:17:14.392 +00:00] [INFO] [mod.rs:381] ["Key rotate worker has been cancelled."]

server log :
[2021/08/25 06:54:30.038 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:54:30.039 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:54:30.053 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:30.053 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:54:30.053 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:54:30.252 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:54:30.252 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:30.252 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:54:30.252 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 06:54:30.998 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 06:54:30.999 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 06:54:30.999 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 06:54:30.999 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 06:54:31.199 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:54:31.199 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:54:31.209 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:31.209 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:54:31.209 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:54:31.410 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:54:31.410 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:31.411 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:54:31.411 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 06:54:44.042 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 06:54:44.042 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 06:54:44.043 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 06:54:44.244 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:54:44.244 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:54:44.253 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:44.254 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:54:44.254 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:54:44.454 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:54:44.454 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:54:44.455 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:54:44.455 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 06:55:07.031 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 06:55:07.032 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 06:55:07.032 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 06:55:07.232 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:55:07.232 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:55:07.242 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:55:07.242 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:55:07.242 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:55:07.443 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:55:07.443 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:55:07.443 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:55:07.443 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 06:55:57.065 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 06:55:57.065 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 06:55:57.065 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 06:55:57.066 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 06:55:57.266 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:55:57.266 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:55:57.276 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:55:57.276 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:55:57.276 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:55:57.477 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:55:57.477 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:55:57.477 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:55:57.477 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 06:57:23.055 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 06:57:23.055 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 06:57:23.055 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 06:57:23.056 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 06:57:23.257 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 06:57:23.257 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 06:57:23.267 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:57:23.267 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 06:57:23.267 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 06:57:23.467 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 06:57:23.467 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 06:57:23.467 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 06:57:23.467 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:00:09.049 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:00:09.050 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:00:09.050 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:00:09.250 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:00:09.250 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:00:09.260 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:00:09.260 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:00:09.260 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:00:09.461 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:00:09.461 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:00:09.461 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:00:09.461 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:05:17.042 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:05:17.043 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:05:17.043 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:05:17.243 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:05:17.243 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:05:17.254 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:05:17.254 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:05:17.254 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:05:17.454 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:05:17.454 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:05:17.454 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:05:17.454 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:10:27.058 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:10:27.058 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:10:27.059 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:10:27.060 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:10:27.060 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:10:27.260 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:10:27.260 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:10:27.270 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:10:27.270 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:10:27.270 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:10:27.472 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:10:27.472 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:10:27.472 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:10:27.472 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:15:34.043 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:15:34.044 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:15:34.044 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:15:34.044 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:15:34.244 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:15:34.244 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:15:34.255 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:15:34.256 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:15:34.256 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:15:34.455 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:15:34.455 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:15:34.455 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:15:34.455 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:20:45.064 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:20:45.065 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:20:45.065 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:20:45.065 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:20:45.265 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:20:45.265 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:20:45.275 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:20:45.275 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:20:45.275 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:20:45.477 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:20:45.477 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:20:45.477 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:20:45.477 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:25:55.052 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:25:55.053 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:25:55.053 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:25:55.253 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:25:55.253 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:25:55.263 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:25:55.263 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:25:55.263 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:25:55.463 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:25:55.463 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:25:55.463 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:25:55.463 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: Got jemalloc config: opt.background_thread 0, opt.max_background_threads 4"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: Try to use background_thread of jemalloc to handle purging asynchronously"] [thread_id=1]
[2021/08/25 07:31:06.044 +00:00] [INFO] [] ["Application: Set jemalloc.max_background_threads 1"] [thread_id=1]
[2021/08/25 07:31:06.045 +00:00] [INFO] [] ["Application: Set jemalloc.background_thread 1"] [thread_id=1]
[2021/08/25 07:31:06.045 +00:00] [INFO] [] ["Application: start raft store proxy"] [thread_id=1]
[2021/08/25 07:31:06.045 +00:00] [INFO] [] ["Application: wait for tiflash proxy initializing"] [thread_id=1]
[2021/08/25 07:31:06.245 +00:00] [INFO] [] ["Application: tiflash proxy is initialized"] [thread_id=1]
[2021/08/25 07:31:06.245 +00:00] [INFO] [] ["Application: encryption is enabled, method is Aes256Ctr"] [thread_id=1]
[2021/08/25 07:31:06.255 +00:00] [ERROR] [] ["Application: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:31:06.255 +00:00] [INFO] [] ["Application: let tiflash proxy shutdown"] [thread_id=1]
[2021/08/25 07:31:06.255 +00:00] [INFO] [] ["Application: wait for tiflash proxy thread to join"] [thread_id=1]
[2021/08/25 07:31:06.456 +00:00] [INFO] [] ["Application: tiflash proxy thread is joined"] [thread_id=1]
[2021/08/25 07:31:06.456 +00:00] [ERROR] [] ["Application: DB::Exception: The configuration "storage.main.dir" is empty. Please check your configuration file."] [thread_id=1]
[2021/08/25 07:31:06.456 +00:00] [INFO] [] ["Application: shutting down"] [thread_id=1]
[2021/08/25 07:31:06.456 +00:00] [INFO] [] ["BaseDaemon: Stop SignalListener thread"] [thread_id=2]
[2021/08/25 07:36:08.060 +00:00] [INFO] [] ["Starting daemon with revision 54381"] [thread_id=1]
[2021/08/25 07:36:08.060 +00:00] [INFO] [] ["TiFlash build info: TiFlash\nRelease Version: v5.2.0\nEdition: Community\nGit Commit Hash: ca0ee5a\nGit Branch: heads/refs/tags/v5.2.0\nUTC Build Time: 2021-08-24 11:53:42\nProfile: RELWITHDEBINFO\n"] [thread_id=1]
[2021/08/25 07:36:08.060 +00:00] [INFO] [] ["Application: starting up"] [thread_id=1]
[2021/08/25 07:36:08.060 +00:00] [INFO] [] ["Application: Got jemalloc version: 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756"] [thread_id=1]
[2021/08/25 07:36:08.060 +00:00] [INFO] [] ["Application: Not found environment variable MALLOC_CONF"] [thread_id=1]
[2021/08/25 07:36:08.060 +00:00] [INFO] [
@apollodafoni apollodafoni added type/bug The issue is confirmed as a bug. severity/critical labels Aug 26, 2021
@apollodafoni
Copy link
Author

default_profile = "default"
delta_index_cache_size = 3221225472
display_name = "TiFlash"
https_port = 8123
interserver_http_port = 9009
listen_host = "0.0.0.0"
mark_cache_size = 2147483648
max_memory_usage_for_all_queries = 32212254720
minmax_index_cache_size = 2147483648
path = "/data0/db"
path_realtime_mode = false
tcp_port_secure = 9000
tmp_path = "/data0/tmp"

[application]
runAsDaemon = true

[flash]
compact_log_min_period = 200
overlap_threshold = 0.6
service_addr = "0.0.0.0:3930"
tidb_status_addr = "db-tidb.tidb1373933076657971408.svc:10080"
[flash.flash_cluster]
cluster_manager_path = "/tiflash/flash_cluster_manager"
log = "/data0/logs/flash_cluster_manager.log"
master_ttl = 60
refresh_interval = 20
update_rule_interval = 10
[flash.proxy]
addr = "0.0.0.0:20170"
advertise-addr = "db-tiflash-POD_NUM.db-tiflash-peer.tidb1373933076657971408.svc:20170"
config = "/data0/proxy.toml"
data-dir = "/data0/proxy"

[logger]
count = 10
errorlog = "/data0/logs/error.log"
level = "information"
log = "/data0/logs/server.log"
size = "100M"

[profiles]
[profiles.default]
load_balancing = "random"
max_memory_usage = 10000000000
use_uncompressed_cache = 0
[profiles.readonly]
readonly = 1

[quotas]
[quotas.default]
[quotas.default.interval]
duration = 3600
errors = 0
execution_time = 0
queries = 0
read_rows = 0
result_rows = 0

[raft]
kvstore_path = "/data0/kvstore"
pd_addr = "db-pd.tidb1373933076657971408.svc:2379"
storage_engine = "dt"

[security]
ca_path = "/var/lib/tiflash-tls/ca.crt"
cert_allowed_cn = ["82aa2ce3"]
cert_path = "/var/lib/tiflash-tls/tls.crt"
key_path = "/var/lib/tiflash-tls/tls.key"

[status]
metrics_port = 8234

  queries = 0
  read_rows = 0
  result_rows = 0

[raft]
kvstore_path = "/data0/kvstore"
pd_addr = "db-pd.tidb1373933076657971408.svc:2379"
storage_engine = "dt"

[security]
ca_path = "/var/lib/tiflash-tls/ca.crt"
cert_allowed_cn = ["82aa2ce3"]
cert_path = "/var/lib/tiflash-tls/tls.crt"
key_path = "/var/lib/tiflash-tls/tls.key"

[status]
metrics_port = 8234

[storage]
[storage.io_rate_limit]
max_bytes_per_sec = 655360000

[users]
[users.default]
password = ""
profile = "default"
quota = "default"
[users.default.networks]
ip = "::/0"
[users.readonly]
password = ""
profile = "readonly"
quota = "default"
[users.readonly.networks]
ip = "::/0"

@JinheLin JinheLin assigned JinheLin and unassigned flowbehappy Sep 6, 2021
@JinheLin JinheLin mentioned this issue Sep 6, 2021
Merged
@ti-chi-bot ti-chi-bot mentioned this issue Sep 8, 2021
Merged
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@JinheLin JinheLin

Labels
severity/critical type/bug The issue is confirmed as a bug.
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

Fix storage configuration compatibility.
3 participants
@flowbehappy @JinheLin @apollodafoni