From 15751fdc09e694a0667037a7d8051f865173cad1 Mon Sep 17 00:00:00 2001 From: yibin Date: Tue, 15 Mar 2022 20:43:53 +0800 Subject: [PATCH 1/9] Fix two issues introduced by adding license header (#4286) close pingcap/tiflash#4284 --- release-centos7-llvm/env/loader | 2 +- tests/run-test.py | 6 ++--- .../data0/metadata/test%2Ddb.sql | 26 +++++++++---------- .../data0/metadata/test%2Ddb/test%2Dtbl.sql | 26 +++++++++---------- .../flash-1136-v3.1.0/data0/metadata/test.sql | 26 +++++++++---------- .../data0/metadata/test/test%2Dtbl.sql | 26 +++++++++---------- .../flash-1136/data0/metadata/test%2Ddb.sql | 26 +++++++++---------- .../data0/metadata/test%2Ddb/test%2Dtbl.sql | 26 +++++++++---------- .../flash-1136/data0/metadata/test.sql | 26 +++++++++---------- .../data0/metadata/test/test%2Dtbl.sql | 26 +++++++++---------- tests/testdata/issue-1055/metadata/t_45.sql | 26 +++++++++---------- .../issue-941/data0/metadata/test.sql | 26 +++++++++---------- .../data0/metadata/test/%23hello%2Dworld.sql | 26 +++++++++---------- .../testdata/oncall-1651/db/metadata/test.sql | 26 +++++++++---------- .../oncall-1651/db/metadata/test/abc.sql | 26 +++++++++---------- .../oncall-1651/db/metadata/test/emp_bak.sql | 26 +++++++++---------- .../db/metadata/test/emp_bak_49.sql | 26 +++++++++---------- .../db/metadata/test/emp_bak_50.sql | 26 +++++++++---------- .../db/metadata/test/emp_bak_51.sql | 26 +++++++++---------- .../db/metadata/test/emp_bak_52.sql | 26 +++++++++---------- 20 files changed, 237 insertions(+), 239 deletions(-) diff --git a/release-centos7-llvm/env/loader b/release-centos7-llvm/env/loader index a8861e8ca20..de1ff1a69d3 100755 --- a/release-centos7-llvm/env/loader +++ b/release-centos7-llvm/env/loader @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding:utf-8 -*- # Copyright 2022 PingCAP, Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -*- coding:utf-8 -*- import os import os.path as path import platform diff --git a/tests/run-test.py b/tests/run-test.py index f2c2d25ec5b..cac1c7de69a 100644 --- a/tests/run-test.py +++ b/tests/run-test.py @@ -1,3 +1,5 @@ +# !/usr/bin/python2 +# -*- coding:utf-8 -*- # Copyright 2022 PingCAP, Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -11,10 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# -*- coding:utf-8 -*- -# !/usr/bin/python2 - import os import sys import time diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql index 669622b184f..c97c56faf67 100644 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql +++ b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE `test-db` ENGINE = Ordinary diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql index 7f5c13adebc..18be51be751 100644 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql +++ b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test%2Ddb/test%2Dtbl.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE `test-tbl` ( diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql index e68d5d8467a..897b444f60b 100644 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql +++ b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE `test` ENGINE = Ordinary diff --git a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql index 67f5a43ea9c..f6e5c321a91 100644 --- a/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql +++ b/tests/testdata/flash-1136-v3.1.0/data0/metadata/test/test%2Dtbl.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE `test-tbl` ( diff --git a/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql b/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql index 669622b184f..c97c56faf67 100644 --- a/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql +++ b/tests/testdata/flash-1136/data0/metadata/test%2Ddb.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE `test-db` ENGINE = Ordinary diff --git a/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql b/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql index 7f5c13adebc..18be51be751 100644 --- a/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql +++ b/tests/testdata/flash-1136/data0/metadata/test%2Ddb/test%2Dtbl.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE `test-tbl` ( diff --git a/tests/testdata/flash-1136/data0/metadata/test.sql b/tests/testdata/flash-1136/data0/metadata/test.sql index e68d5d8467a..897b444f60b 100644 --- a/tests/testdata/flash-1136/data0/metadata/test.sql +++ b/tests/testdata/flash-1136/data0/metadata/test.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE `test` ENGINE = Ordinary diff --git a/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql b/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql index 67f5a43ea9c..f6e5c321a91 100644 --- a/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql +++ b/tests/testdata/flash-1136/data0/metadata/test/test%2Dtbl.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE `test-tbl` ( diff --git a/tests/testdata/issue-1055/metadata/t_45.sql b/tests/testdata/issue-1055/metadata/t_45.sql index c3a6cc11db4..9d93c1028a0 100644 --- a/tests/testdata/issue-1055/metadata/t_45.sql +++ b/tests/testdata/issue-1055/metadata/t_45.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE t_45 ( diff --git a/tests/testdata/issue-941/data0/metadata/test.sql b/tests/testdata/issue-941/data0/metadata/test.sql index e68d5d8467a..897b444f60b 100644 --- a/tests/testdata/issue-941/data0/metadata/test.sql +++ b/tests/testdata/issue-941/data0/metadata/test.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE `test` ENGINE = Ordinary diff --git a/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql b/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql index e40bc2e8dc5..f93ab0a91b4 100644 --- a/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql +++ b/tests/testdata/issue-941/data0/metadata/test/%23hello%2Dworld.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE `#hello-world` ( diff --git a/tests/testdata/oncall-1651/db/metadata/test.sql b/tests/testdata/oncall-1651/db/metadata/test.sql index 412cbbb8310..a92a7db72c5 100644 --- a/tests/testdata/oncall-1651/db/metadata/test.sql +++ b/tests/testdata/oncall-1651/db/metadata/test.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH DATABASE test ENGINE = Ordinary diff --git a/tests/testdata/oncall-1651/db/metadata/test/abc.sql b/tests/testdata/oncall-1651/db/metadata/test/abc.sql index baaf1da94b9..9ea2b02b274 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/abc.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/abc.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE abc ( diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql index 6d9e969bc6c..4be86529f02 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/emp_bak.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE emp_bak ( diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql index f06d78261ce..315b7f24731 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_49.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE emp_bak_49 ( diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql index c98ae467263..055be9e4d91 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_50.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE emp_bak_50 ( diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql index 1d4c29aaabc..f96af626e07 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_51.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE emp_bak_51 ( diff --git a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql index e21aae7033a..6e1bb90b0fb 100644 --- a/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql +++ b/tests/testdata/oncall-1651/db/metadata/test/emp_bak_52.sql @@ -1,16 +1,16 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +-- Copyright 2022 PingCAP, Ltd. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. ATTACH TABLE emp_bak_52 ( From db28b92a896e2b0aae56616c4fc2cab3b4e99618 Mon Sep 17 00:00:00 2001 From: JaySon Date: Tue, 15 Mar 2022 21:13:53 +0800 Subject: [PATCH 2/9] Use CRC64 as checksum algo for LogFile (#4261) ref pingcap/tiflash#3594 --- dbms/src/Storages/Page/V3/LogFile/LogFormat.h | 6 ++--- .../Storages/Page/V3/LogFile/LogReader.cpp | 2 +- dbms/src/Storages/Page/V3/LogFile/LogWriter.h | 12 ++++----- .../Storages/Page/V3/tests/gtest_wal_log.cpp | 26 +++++++++---------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/dbms/src/Storages/Page/V3/LogFile/LogFormat.h b/dbms/src/Storages/Page/V3/LogFile/LogFormat.h index b27d76bb377..c1151d1181f 100644 --- a/dbms/src/Storages/Page/V3/LogFile/LogFormat.h +++ b/dbms/src/Storages/Page/V3/LogFile/LogFormat.h @@ -43,13 +43,13 @@ static constexpr UInt8 MaxRecordType = RecyclableLastType; static constexpr UInt32 BLOCK_SIZE = 32 * 1024; static_assert(BLOCK_SIZE < std::numeric_limits::max()); -using ChecksumClass = Digest::CRC32; // TODO: CRC64 +using ChecksumClass = Digest::CRC64; using ChecksumType = ChecksumClass::HashType; -static constexpr UInt32 CHECKSUM_FIELD_SIZE = sizeof(ChecksumType); +static constexpr UInt32 CHECKSUM_FIELD_SIZE = ChecksumClass::hash_size; -// If the size of payload is larger than `BLOCK_SIZE`, it will be splited into +// If the size of payload is larger than `BLOCK_SIZE`, it will be splitted into // fragments. So `PAYLOAD_FIELD_SIZE` must be fit in UInt16. static constexpr UInt32 PAYLOAD_FIELD_SIZE = sizeof(UInt16); diff --git a/dbms/src/Storages/Page/V3/LogFile/LogReader.cpp b/dbms/src/Storages/Page/V3/LogFile/LogReader.cpp index 1be16293ba9..f901d75f86e 100644 --- a/dbms/src/Storages/Page/V3/LogFile/LogReader.cpp +++ b/dbms/src/Storages/Page/V3/LogFile/LogReader.cpp @@ -397,7 +397,7 @@ UInt8 LogReader::readPhysicalRecord(std::string_view * result, size_t * drop_siz } else if (err != 0) return err; - // else parse header successe. + // else parse header success. if (verify_checksum) { diff --git a/dbms/src/Storages/Page/V3/LogFile/LogWriter.h b/dbms/src/Storages/Page/V3/LogFile/LogWriter.h index a931e28a229..f2bf3bcb8f6 100644 --- a/dbms/src/Storages/Page/V3/LogFile/LogWriter.h +++ b/dbms/src/Storages/Page/V3/LogFile/LogWriter.h @@ -39,23 +39,23 @@ namespace PS::V3 * +-----+-------------+--+----+----------+------+-- ... ----+ * File | r0 | r1 |P | r2 | r3 | r4 | | * +-----+-------------+--+----+----------+------+-- ... ----+ - * <--- kBlockSize ------>|<-- kBlockSize ------>| + * <---- BlockSize ------>|<--- BlockSize ------>| * rn = variable size records * P = Padding * - * Data is written out in kBlockSize chunks. If next record does not fit + * Data is written out in BlockSize chunks. If next record does not fit * into the space left, the leftover space will be padded with \0. * * Legacy record format: * * +--------------+-----------+-----------+--- ... ---+ - * |CheckSum (4B) | Size (2B) | Type (1B) | Payload | + * |CheckSum (8B) | Size (2B) | Type (1B) | Payload | * +--------------+-----------+-----------+--- ... ---+ * - * CheckSum = 32bit hash computed over the record type and payload using checksum algo + * CheckSum = 64bit hash computed over the record type and payload using checksum algo (CRC64) * Size = Length of the payload data * Type = Type of record - * (kZeroType, kFullType, kFirstType, kLastType, kMiddleType ) + * (ZeroType, FullType, FirstType, LastType, MiddleType) * The type is used to group a bunch of records together to represent * blocks that are larger than kBlockSize * Payload = Byte stream as long as specified by the payload size @@ -63,7 +63,7 @@ namespace PS::V3 * Recyclable record format: * * +--------------+-----------+-----------+----------------+--- ... ---+ - * |CheckSum (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload | + * |CheckSum (8B) | Size (2B) | Type (1B) | Log number (4B)| Payload | * +--------------+-----------+-----------+----------------+--- ... ---+ * * Same as above, with the addition of diff --git a/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp b/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp index 8ea8ba4e2cc..70b5bb5ce89 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp @@ -216,19 +216,17 @@ class LogFileRWTest : public ::testing::TestWithParam> PageUtil::ftruncateFile(wr_file, writtenBytes() - bytes); } - void fixChecksum(int header_offset, int len, bool recyclable) + void fixChecksum(int header_offset, int payload_len, bool recyclable) { // Compute crc of type/len/data int header_size = recyclable ? Format::RECYCLABLE_HEADER_SIZE : Format::HEADER_SIZE; - Digest::CRC32 digest; + Format::ChecksumClass digest; - size_t crc_buff_size = header_size - Format::CHECKSUM_START_OFFSET + len; + size_t crc_buff_size = header_size - Format::CHECKSUM_START_OFFSET + payload_len; char crc_buff[crc_buff_size]; PageUtil::readFile(wr_file, header_offset + Format::CHECKSUM_START_OFFSET, crc_buff, crc_buff_size, nullptr); - digest.update( - crc_buff, - crc_buff_size); + digest.update(crc_buff, crc_buff_size); auto checksum = digest.checksum(); PageUtil::writeFile(wr_file, header_offset, reinterpret_cast(&checksum), sizeof(checksum), nullptr); @@ -421,7 +419,7 @@ TEST_P(LogFileRWTest, BadRecordType) TEST_P(LogFileRWTest, TruncatedTrailingRecordIsIgnored) { write("foo"); - shrinkSize(4); // Drop all payload as well as a header byte + shrinkSize(3 + sizeof(Format::MaxRecordType)); // Drop all payload as well as a header byte resetReader(); ASSERT_EQ("EOF", read()); // Truncated last record is ignored, not treated as an error @@ -439,7 +437,7 @@ TEST_P(LogFileRWTest, TruncatedTrailingRecordIsNotIgnored) } write("foo"); - shrinkSize(4); // Drop all payload as well as a header byte + shrinkSize(3 + sizeof(Format::MaxRecordType)); // Drop all payload as well as a header byte resetReader(WALRecoveryMode::AbsoluteConsistency); ASSERT_EQ("EOF", read()); // Truncated last record is ignored, not treated as an error @@ -461,8 +459,8 @@ TEST_P(LogFileRWTest, BadLength) write(repeatedString("bar", payload_size)); write("foo"); resetReader(); - // Least significant size byte is stored in header[4]. - incrementByte(4, 1); + // Least significant size byte is stored in header[SizePos]. + incrementByte(Format::CHECKSUM_FIELD_SIZE, 1); if (!recyclable_log) { ASSERT_EQ("foo", read()); @@ -512,11 +510,11 @@ TEST_P(LogFileRWTest, BadLengthAtEndIsNotIgnored) TEST_P(LogFileRWTest, ChecksumMismatch) { write("foooooo"); - incrementByte(0, 14); + incrementByte(0, Format::HEADER_SIZE + 7); ASSERT_EQ("EOF", read()); if (!recyclable_log) { - ASSERT_EQ(14, droppedBytes()); + ASSERT_EQ(Format::HEADER_SIZE + 7, droppedBytes()); ASSERT_EQ("OK", matchError("checksum mismatch")); } else @@ -574,7 +572,7 @@ TEST_P(LogFileRWTest, MissingLastIsIgnored) { write(repeatedString("bar", PS::V3::Format::BLOCK_SIZE)); // Remove the LAST block, including header. - shrinkSize(14); + shrinkSize(2 * (recyclable_log ? Format::RECYCLABLE_HEADER_SIZE : Format::HEADER_SIZE)); ASSERT_EQ("EOF", read()); ASSERT_EQ("", reportMessage()); ASSERT_EQ(0, droppedBytes()); @@ -591,7 +589,7 @@ TEST_P(LogFileRWTest, MissingLastIsNotIgnored) resetReader(WALRecoveryMode::AbsoluteConsistency); write(repeatedString("bar", PS::V3::Format::BLOCK_SIZE)); // Remove the LAST block, including header. - shrinkSize(14); + shrinkSize(2 * (recyclable_log ? Format::RECYCLABLE_HEADER_SIZE : Format::HEADER_SIZE)); ASSERT_EQ("EOF", read()); ASSERT_GT(droppedBytes(), 0); ASSERT_EQ("OK", matchError("Corruption: error reading trailing data")); From ee458169db10c1460967a32a469d251459a4e613 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 15 Mar 2022 21:49:52 +0800 Subject: [PATCH 3/9] Clean deprecated directories and internal links (#4278) ref pingcap/tiflash#4141 --- .ci/build.groovy | 53 --- .ci/integration_test.groovy | 76 ---- .ci/unit_test.groovy | 30 -- .ci/util.groovy | 222 ---------- README.md | 2 - cluster_manage/.gitignore | 5 - cluster_manage/README.md | 26 -- cluster_manage/build.sh | 65 --- cluster_manage/check_lib.py | 28 -- cluster_manage/conf.py | 38 -- cluster_manage/define.py | 32 -- cluster_manage/flash_cluster_manager.py | 383 ------------------ cluster_manage/flash_http_client.py | 61 --- cluster_manage/flash_tools.py | 66 --- cluster_manage/pd_client.py | 203 ---------- cluster_manage/placement_rule.py | 50 --- cluster_manage/release.sh | 24 -- cluster_manage/tidb_tools.py | 64 --- cluster_manage/tikv_util/.clang-format | 5 - cluster_manage/tikv_util/.gitignore | 2 - cluster_manage/tikv_util/__init__.py | 15 - cluster_manage/tikv_util/build.sh | 24 -- cluster_manage/tikv_util/check_region_cnt.cc | 44 -- cluster_manage/tikv_util/check_region_cnt.h | 37 -- cluster_manage/tikv_util/codec.h | 194 --------- cluster_manage/tikv_util/exception_util.h | 28 -- cluster_manage/tikv_util/export.cc | 42 -- cluster_manage/tikv_util/format.sh | 22 - cluster_manage/tikv_util/setup.py | 125 ------ cluster_manage/tikv_util/test.py | 78 ---- cluster_manage/tikv_util/tikv_key.cc | 82 ---- cluster_manage/tikv_util/tikv_key.h | 64 --- cluster_manage/tikv_util/type.h | 23 -- cluster_manage/timer.py | 67 --- cluster_manage/util.py | 156 ------- deprecated/release-centos7/Dockerfile-builder | 34 -- .../release-centos7/Dockerfile-builder-arm64 | 33 -- .../release-centos7/Dockerfile-builder-ci | 6 - .../Dockerfile-tiflash-centos7 | 14 - .../release-centos7/Dockerfile-tiflash-ci | 5 - .../Dockerfile-tiflash-ci-base | 12 - deprecated/release-centos7/Makefile | 40 -- deprecated/release-centos7/README.md | 38 -- .../release-centos7/build/build-release.sh | 28 -- .../release-centos7/build/build-tiflash-ci.sh | 168 -------- .../build/build-tiflash-proxy.sh | 31 -- .../build/build-tiflash-release.sh | 65 --- .../build/build-tiflash-ut-coverage.sh | 143 ------- deprecated/release-centos7/build/env.sh | 17 - .../release-centos7/build/fetch-ci-build.sh | 52 --- .../build/fix_compile_commands.py | 61 --- .../release-centos7/build/run-clang-tidy.py | 363 ----------------- deprecated/release-centos7/build/run-ut.sh | 41 -- .../release-centos7/build/static-analysis.sh | 75 ---- .../release-centos7/build/upload-ci-build.sh | 40 -- .../build/upload-ut-coverage.sh | 30 -- .../grpc-arm-compile.patch | 13 - .../prepare-environments/install-cmake.sh | 35 -- .../prepare-environments/install-gcc.sh | 44 -- .../prepare-environments/install-grpc.sh | 56 --- .../prepare-environments/install-libcurl.sh | 26 -- .../prepare-environments/install-llvm.sh | 51 --- .../prepare-environments/install-mysql-dev.sh | 30 -- .../prepare-environments/install-openssl.sh | 25 -- .../prepare-environments/util.sh | 26 -- format-diff.py | 32 +- .../scripts/build-tiflash-ci.sh | 62 --- .../scripts/build-tiflash-ut-coverage.sh | 42 -- release-centos7-llvm/scripts/env.sh | 20 - .../scripts/fetch-ci-build.sh | 60 --- .../scripts/run-format-check.sh | 31 -- release-centos7-llvm/scripts/run-ut.sh | 41 -- .../scripts/static-analysis.sh | 47 --- .../scripts/tiflash-ci-build.sh | 90 ---- .../scripts/tiflash-ci-finish.sh | 53 --- .../scripts/tiflash-ci-prepare.sh | 102 ----- .../scripts/tiflash-ut-coverage-build.sh | 59 --- .../scripts/tiflash-ut-coverage-finish.sh | 39 -- .../scripts/tiflash-ut-coverage-prepare.sh | 82 ---- .../scripts/upload-ci-build.sh | 44 -- .../scripts/upload-ut-coverage.sh | 90 ---- 81 files changed, 20 insertions(+), 4912 deletions(-) delete mode 100644 .ci/build.groovy delete mode 100644 .ci/integration_test.groovy delete mode 100644 .ci/unit_test.groovy delete mode 100644 .ci/util.groovy delete mode 100644 cluster_manage/.gitignore delete mode 100644 cluster_manage/README.md delete mode 100755 cluster_manage/build.sh delete mode 100644 cluster_manage/check_lib.py delete mode 100644 cluster_manage/conf.py delete mode 100644 cluster_manage/define.py delete mode 100644 cluster_manage/flash_cluster_manager.py delete mode 100644 cluster_manage/flash_http_client.py delete mode 100644 cluster_manage/flash_tools.py delete mode 100644 cluster_manage/pd_client.py delete mode 100644 cluster_manage/placement_rule.py delete mode 100755 cluster_manage/release.sh delete mode 100644 cluster_manage/tidb_tools.py delete mode 100644 cluster_manage/tikv_util/.clang-format delete mode 100644 cluster_manage/tikv_util/.gitignore delete mode 100644 cluster_manage/tikv_util/__init__.py delete mode 100755 cluster_manage/tikv_util/build.sh delete mode 100644 cluster_manage/tikv_util/check_region_cnt.cc delete mode 100644 cluster_manage/tikv_util/check_region_cnt.h delete mode 100644 cluster_manage/tikv_util/codec.h delete mode 100644 cluster_manage/tikv_util/exception_util.h delete mode 100644 cluster_manage/tikv_util/export.cc delete mode 100755 cluster_manage/tikv_util/format.sh delete mode 100644 cluster_manage/tikv_util/setup.py delete mode 100644 cluster_manage/tikv_util/test.py delete mode 100644 cluster_manage/tikv_util/tikv_key.cc delete mode 100644 cluster_manage/tikv_util/tikv_key.h delete mode 100644 cluster_manage/tikv_util/type.h delete mode 100644 cluster_manage/timer.py delete mode 100644 cluster_manage/util.py delete mode 100644 deprecated/release-centos7/Dockerfile-builder delete mode 100644 deprecated/release-centos7/Dockerfile-builder-arm64 delete mode 100644 deprecated/release-centos7/Dockerfile-builder-ci delete mode 100644 deprecated/release-centos7/Dockerfile-tiflash-centos7 delete mode 100644 deprecated/release-centos7/Dockerfile-tiflash-ci delete mode 100644 deprecated/release-centos7/Dockerfile-tiflash-ci-base delete mode 100644 deprecated/release-centos7/Makefile delete mode 100644 deprecated/release-centos7/README.md delete mode 100755 deprecated/release-centos7/build/build-release.sh delete mode 100755 deprecated/release-centos7/build/build-tiflash-ci.sh delete mode 100755 deprecated/release-centos7/build/build-tiflash-proxy.sh delete mode 100755 deprecated/release-centos7/build/build-tiflash-release.sh delete mode 100755 deprecated/release-centos7/build/build-tiflash-ut-coverage.sh delete mode 100644 deprecated/release-centos7/build/env.sh delete mode 100755 deprecated/release-centos7/build/fetch-ci-build.sh delete mode 100755 deprecated/release-centos7/build/fix_compile_commands.py delete mode 100755 deprecated/release-centos7/build/run-clang-tidy.py delete mode 100755 deprecated/release-centos7/build/run-ut.sh delete mode 100755 deprecated/release-centos7/build/static-analysis.sh delete mode 100755 deprecated/release-centos7/build/upload-ci-build.sh delete mode 100755 deprecated/release-centos7/build/upload-ut-coverage.sh delete mode 100644 deprecated/release-centos7/prepare-environments/grpc-arm-compile.patch delete mode 100755 deprecated/release-centos7/prepare-environments/install-cmake.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-gcc.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-grpc.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-libcurl.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-llvm.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-mysql-dev.sh delete mode 100755 deprecated/release-centos7/prepare-environments/install-openssl.sh delete mode 100644 deprecated/release-centos7/prepare-environments/util.sh delete mode 100755 release-centos7-llvm/scripts/build-tiflash-ci.sh delete mode 100755 release-centos7-llvm/scripts/build-tiflash-ut-coverage.sh delete mode 100644 release-centos7-llvm/scripts/env.sh delete mode 100755 release-centos7-llvm/scripts/fetch-ci-build.sh delete mode 100755 release-centos7-llvm/scripts/run-format-check.sh delete mode 100755 release-centos7-llvm/scripts/run-ut.sh delete mode 100755 release-centos7-llvm/scripts/static-analysis.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ci-build.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ci-finish.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ci-prepare.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ut-coverage-build.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ut-coverage-finish.sh delete mode 100755 release-centos7-llvm/scripts/tiflash-ut-coverage-prepare.sh delete mode 100755 release-centos7-llvm/scripts/upload-ci-build.sh delete mode 100755 release-centos7-llvm/scripts/upload-ut-coverage.sh diff --git a/.ci/build.groovy b/.ci/build.groovy deleted file mode 100644 index aff116ea7ee..00000000000 --- a/.ci/build.groovy +++ /dev/null @@ -1,53 +0,0 @@ -catchError { - def util = load('util.groovy') - - def tiflashTag = ({ - def m = ghprbCommentBody =~ /tiflash\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m) { - return "${m.group(1)}" - } - return params.tiflashTag ?: ghprbTargetBranch ?: 'master' - }).call() - - def CURWS = pwd() - - def NPROC = 5 - - util.runCheckoutAndBuilderClosure("build-tics-v2", CURWS) { - dir("${CURWS}/tics") { - stage("Build & Upload") { - timeout(time: 70, unit: 'MINUTES') { - container("builder") { - sh "NPROC=${NPROC} BUILD_BRANCH=${ghprbTargetBranch} ENABLE_FORMAT_CHECK=true ${CURWS}/tics/release-centos7-llvm/scripts/build-tiflash-ci.sh" - sh "PULL_ID=${ghprbPullId} COMMIT_HASH=${ghprbActualCommit} ${CURWS}/tics/release-centos7-llvm/scripts/upload-ci-build.sh" - } - } - } - stage("Static Analysis") { - timeout(time: 360, unit: 'MINUTES') { - container("builder") { - sh "NPROC=${NPROC} /build/tics/release-centos7-llvm/scripts/static-analysis.sh" - } - } - } - } - } -} - - -stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Build Result: `${currentBuild.currentResult}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - - echo "${msg}" - - if (currentBuild.currentResult != "SUCCESS") { - echo "Send slack here ..." - slackSend channel: '#jenkins-ci', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${msg}" - } -} diff --git a/.ci/integration_test.groovy b/.ci/integration_test.groovy deleted file mode 100644 index 4fe80ae2bbf..00000000000 --- a/.ci/integration_test.groovy +++ /dev/null @@ -1,76 +0,0 @@ -catchError { - def util = load('util.groovy') - - def tidbBranch = ({ - def m = params.ghprbCommentBody =~ /tidb\s*=\s*([^\s\\]+)(\s|\\|$)/ - if (m) { - return "${m.group(1)}" - } - return params.ghprbTargetBranch ?: 'master' - }).call() - - stage("Wait for ci build") { - echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}" - } - - node("${GO_BUILD_SLAVE}") { - def curws = pwd() - dir("/home/jenkins/agent/code-archive") { - container("golang") { - if(fileExists("/nfs/cache/git/src-tics.tar.gz")){ - timeout(5) { - sh """ - cp -R /nfs/cache/git/src-tics.tar.gz* ./ - mkdir -p ${curws}/tics - tar -xzf src-tics.tar.gz -C ${curws}/tics --strip-components=1 - """ - } - } - } - dir("${curws}/tics") { - util.checkoutTiCS("${params.ghprbActualCommit}", "${params.ghprbPullId}") - } - timeout(time: 60, unit: 'MINUTES') { - container("golang") { - sh """ - rm -rf ${curws}/tics/contrib - rm -rf ${curws}/tics/.git - - COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} bash -e ${curws}/tics/release-centos7-llvm/scripts/fetch-ci-build.sh - """ - } - } - } - stash includes: "tics/**", name: "git-code-tics", useDefaultExcludes: false - } - - def pod_label = "tics-integration-test-v1" - - parallel ( - "tidb ci test": { - def name = "tidb-ci-test" - util.runTest(pod_label, name, "tics/tests/tidb-ci", tidbBranch) - }, - "delta merge test": { - def name = "delta-merge-test" - util.runTest(pod_label, name, "tics/tests/delta-merge-test", tidbBranch) - }, - "fullstack test": { - def name = "fullstack-test" - util.runTest(pod_label, name, "tics/tests/fullstack-test", tidbBranch) - }, - "fullstack test2": { - def name = "fullstack-test2" - util.runTest(pod_label, name, "tics/tests/fullstack-test2", tidbBranch) - }, - ) -} - -stage('Summary') { - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "Result: `${currentBuild.currentResult}`" + "\n" + - "Elapsed Time: `${duration} mins`" + "\n" + - "${env.RUN_DISPLAY_URL}" - - echo "${msg}" -} diff --git a/.ci/unit_test.groovy b/.ci/unit_test.groovy deleted file mode 100644 index 719baf6628f..00000000000 --- a/.ci/unit_test.groovy +++ /dev/null @@ -1,30 +0,0 @@ -catchError { - def util = load('util.groovy') - - def CURWS = pwd() - - def NPROC = 5 - - util.runUnitTests("ut-tics-v2", CURWS, NPROC) - -} - -stage('Summary') { - println("Coverage detail: ${CI_COVERAGE_BASE_URL}/${BUILD_NUMBER}/cobertura/") - println("Coverage detail url is limited office network access\n") - - def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP) - def msg = "[#${ghprbPullId}: ${ghprbPullTitle}]" + "\n" + - "${ghprbPullLink}" + "\n" + - "${ghprbPullDescription}" + "\n" + - "Build Result: `${currentBuild.currentResult}`" + "\n" + - "Elapsed Time: `${duration} mins` " + "\n" + - "${env.RUN_DISPLAY_URL}" - - echo "${msg}" - - if (currentBuild.currentResult != "SUCCESS") { - echo "Send slack here ..." - slackSend channel: '#jenkins-ci', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${msg}" - } -} diff --git a/.ci/util.groovy b/.ci/util.groovy deleted file mode 100644 index 65b0d34560c..00000000000 --- a/.ci/util.groovy +++ /dev/null @@ -1,222 +0,0 @@ -def doCheckout(commit, refspec) { - checkout(changelog: false, poll: false, scm: [ - $class : "GitSCM", - branches : [ - [name: "${commit}"], - ], - userRemoteConfigs: [ - [ - url : "git@github.com:pingcap/tics.git", - refspec : refspec, - credentialsId: "github-sre-bot-ssh", - ] - ], - extensions : [ - [$class: 'PruneStaleBranch'], - [$class: 'CleanBeforeCheckout'], - ], - ]) -} - -def checkoutTiCS(commit, pullId) { - def refspec = "+refs/heads/*:refs/remotes/origin/*" - if (pullId) { - refspec = " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*" - } - try { - doCheckout(commit, refspec) - } catch (info) { - retry(2) { - echo "checkout failed, retry.." - sleep 5 - if (sh(returnStatus: true, script: '[ -d .git ] && git rev-parse --git-dir > /dev/null 2>&1') != 0) { - echo ".git already exist or not a valid git dir. Delete dir..." - deleteDir() - } - doCheckout(commit, refspec) - } - } -} - -def checkoutTiCSFull(commit, pullId) { - def refspec = "+refs/heads/*:refs/remotes/origin/*" - if (pullId) { - refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*" - } - checkout(changelog: false, poll: false, scm: [ - $class : "GitSCM", - branches : [ - [name: "${commit}"], - ], - userRemoteConfigs : [ - [ - url : "git@github.com:pingcap/tics.git", - refspec : refspec, - credentialsId: "github-sre-bot-ssh", - ] - ], - extensions : [ - [$class : 'SubmoduleOption', - disableSubmodules : false, - parentCredentials : true, - recursiveSubmodules: true, - trackingSubmodules : false, - reference : ''], - [$class: 'PruneStaleBranch'], - [$class: 'CleanBeforeCheckout'], - ], - doGenerateSubmoduleConfigurations: false, - ]) -} - -def runBuilderClosure(label, Closure body) { - podTemplate(name: label, label: label, instanceCap: 15, containers: [ - containerTemplate(name: 'docker', image: 'hub.pingcap.net/jenkins/docker:build-essential-java', - alwaysPullImage: true, envVars: [ - envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'), - ], ttyEnabled: true, command: 'cat'), - containerTemplate(name: 'builder', image: 'hub.pingcap.net/tiflash/tiflash-llvm-base:amd64', - alwaysPullImage: true, ttyEnabled: true, command: 'cat', - resourceRequestCpu: '5000m', resourceRequestMemory: '10Gi', - resourceLimitCpu: '10000m', resourceLimitMemory: '30Gi'), - ], - volumes: [ - nfsVolume(mountPath: '/home/jenkins/agent/ci-cached-code-daily', serverAddress: '172.16.5.22', - serverPath: '/mnt/ci.pingcap.net-nfs/git', readOnly: false), - ] - ) { - node(label) { - body() - } - } -} - -def runClosure(label, Closure body) { - podTemplate(name: label, label: label, instanceCap: 15, containers: [ - containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true, - resourceRequestCpu: '5000m', resourceRequestMemory: '10Gi', - resourceLimitCpu: '16000m', resourceLimitMemory: '32Gi'), - containerTemplate(name: 'docker', image: 'hub.pingcap.net/jenkins/docker:build-essential-java', - alwaysPullImage: true, envVars: [ - envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375'), - ], ttyEnabled: true, command: 'cat'), - ] - ) { - node(label) { - body() - } - } -} - -def runCheckoutAndBuilderClosure(label, curws, Closure body) { - runBuilderClosure(label) { - dir("${curws}/tics") { - stage("Checkout") { - container("docker") { - def repoDailyCache = "/home/jenkins/agent/ci-cached-code-daily/src-tics.tar.gz" - if (fileExists(repoDailyCache)) { - println "get code from nfs to reduce clone time" - sh """ - cp -R ${repoDailyCache} ./ - tar -xzf ${repoDailyCache} --strip-components=1 - rm -f src-tics.tar.gz - """ - sh "chown -R 1000:1000 ./" - } else { - sh "exit -1" - } - } - checkoutTiCSFull("${ghprbActualCommit}", "${ghprbPullId}") - } - } - body() - } -} - -def runTest(label, name, testPath, tidbBranch) { - runClosure(label) { - stage("Unstash") { - unstash 'git-code-tics' - dir("tics") { - timeout(time: 5, unit: 'MINUTES') { - container("docker") { - sh """ - pwd - DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} bash -e release-centos7-llvm/scripts/fetch-ci-build.sh - """ - } - } - } - } - dir(testPath) { - stage("Test") { - timeout(time: 60, unit: 'MINUTES') { - container("docker") { - try { - sh "pwd" - sh "TAG=${params.ghprbActualCommit} BRANCH=${tidbBranch} bash -xe ./run.sh" - } catch (e) { - sh "mv log ${name}-log" - archiveArtifacts(artifacts: "${name}-log/**/*.log", allowEmptyArchive: true) - sh "find ${name}-log -name '*.log' | xargs tail -n 500" - sh "docker ps -a" - throw e - } - } - } - } - } - } -} - -def runUnitTests(label, CURWS, NPROC) { - def NPROC_UT = NPROC * 2 - runCheckoutAndBuilderClosure(label, CURWS) { - dir("${CURWS}/tics") { - stage("Build") { - timeout(time: 70, unit: 'MINUTES') { - container("builder") { - sh "NPROC=${NPROC} BUILD_BRANCH=${ghprbTargetBranch} UPDATE_CCACHE=false ${CURWS}/tics/release-centos7-llvm/scripts/build-tiflash-ut-coverage.sh" - } - } - } - stage("Tests") { - timeout(time: 50, unit: 'MINUTES') { - container("builder") { - sh "NPROC=${NPROC_UT} /build/tics/release-centos7-llvm/scripts/run-ut.sh" - } - } - } - stage("Show UT Coverage") { - timeout(time: 20, unit: 'MINUTES') { - container("builder") { - sh "NPROC=${NPROC} BUILD_NUMBER=${BUILD_NUMBER} BUILD_BRANCH=${ghprbTargetBranch} /build/tics/release-centos7-llvm/scripts/upload-ut-coverage.sh" - sh """ - cp /tiflash/profile/diff-coverage ./ - cp /tiflash/coverage-report.tar.gz ./ - chown -R 1000:1000 diff-coverage coverage-report.tar.gz - """ - ut_coverage_result = sh(script: "cat diff-coverage", returnStdout: true).trim() - sh """ - rm -f comment-pr - curl -O http://fileserver.pingcap.net/download/comment-pr - chmod +x comment-pr - set +x - ./comment-pr \ - --token=$TOKEN \ - --owner=pingcap \ - --repo=tics \ - --number=${ghprbPullId} \ - --comment='${ut_coverage_result}' - set -x - """ - } - } - - archiveArtifacts artifacts: 'coverage-report.tar.gz', fingerprint: true - } - } - } -} - -return this diff --git a/README.md b/README.md index 1e171d8c5cb..9d5cd32d3c7 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,6 @@ Or you can also use system-wise toolchain if you can install `clang/compiler-rt/ ###### TiFlash Env -> For faster access to precompiled package in internal network, you can use [this link](http://fileserver.pingcap.net/download/development/tiflash-env/v1.0.0/tiflash-env-x86_64.tar.xz). - TiFlash Env can be created with the following commands (`docker` and `tar xz` are needed): ```bash diff --git a/cluster_manage/.gitignore b/cluster_manage/.gitignore deleted file mode 100644 index 6b1b5d54dad..00000000000 --- a/cluster_manage/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/dist/ -/build/ -*.spec -version.py -/tikv_util/.eggs/ diff --git a/cluster_manage/README.md b/cluster_manage/README.md deleted file mode 100644 index f3823d997a4..00000000000 --- a/cluster_manage/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# TiFlash Cluster Manage - -- It works as `Cluster Manage` for `TiFlash` to interact with `TiDB` / `PD` / `TiKV`, run with each flash node. -A master will be elected by using `etcd` to manage the whole flash cluster. Periodically scan tables with column -replicas configuration in tidb and try to build flash engine peers or monitor their status. -- language: `Python` `C/C++` - -## Prepare - -* install `python3.7`, `pybind11`, `pyinstaller`, `clang-format`, `dnspython`, `uri`, `requests`, `urllib3` -, `toml`, `C++17` -, `setuptools` - -* use `release.sh` and get dir `dist/flash_cluster_manager/`. - -## Run - -* show version -``` -./dist/flash_cluster_manager/flash_cluster_manager -v -``` - -* run cmd: -``` -./dist/flash_cluster_manager/flash_cluster_manager --config [path-to-flash-config.toml] -``` diff --git a/cluster_manage/build.sh b/cluster_manage/build.sh deleted file mode 100755 index e62d22d9f63..00000000000 --- a/cluster_manage/build.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ue - -pushd tikv_util -./build.sh - -popd - -python3 check_lib.py - -if [ $? != 0 ]; then - echo "check lib fail" - exit -1 -else - echo "check lib success" -fi - -commit_ts=$(git log -1 --format="%ct") - -if [ "$(uname)" == "Darwin" ]; then - commit_time=$(date -r $commit_ts +"%Y-%m-%d %H:%M:%S") -else - commit_time=$(date -d @$commit_ts +"%Y-%m-%d %H:%M:%S") -fi - -git_hash=$(git log -1 --format="%H") -git_branch=$(git symbolic-ref -q --short HEAD || git describe --tags --exact-match || echo "HEAD") -version_file='version.py' -git_hash_info="git_hash = '$git_hash'" -overwrite="true" - -if [ -f ${version_file} ]; then - tmp_hash=$(head -n 1 version.py) - if [ "$tmp_hash" == "$git_hash_info" ]; then - overwrite="false" - fi -fi - -if [ $overwrite == "true" ]; then - echo "start to overwrite $version_file" - echo "$git_hash_info" >$version_file - echo "commit_time = '$commit_time'" >>$version_file - echo "git_branch = '$git_branch'" >>$version_file -fi - -echo "" -echo "Cluster Manager Version Info" -cat $version_file -echo "" - -pyinstaller flash_cluster_manager.py -y --hidden-import pkg_resources.py2_warn diff --git a/cluster_manage/check_lib.py b/cluster_manage/check_lib.py deleted file mode 100644 index c82b246bcf1..00000000000 --- a/cluster_manage/check_lib.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if __name__ == '__main__': - try: - import dns - import requests - import uri - import urllib3 - import toml - import pybind11 - import setuptools - import etcd3 - except Exception as e: - print(e) - exit(-1) - exit(0) diff --git a/cluster_manage/conf.py b/cluster_manage/conf.py deleted file mode 100644 index 75ae8c832d7..00000000000 --- a/cluster_manage/conf.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging - -import version - -version_info = '' -for d in dir(version): - if not d.startswith('__'): - version_info += '{}: {}\n'.format(d, getattr(version, d)) - -parser = argparse.ArgumentParser(description='TiFlash Cluster Manager', formatter_class=argparse.RawTextHelpFormatter) -parser.add_argument('--version', '-v', help='show version', action='version', version=version_info) -parser.add_argument('--config', help='path of config file *.toml', required=True) -parser.add_argument('--log_level', help='log level', default='INFO', choices=['INFO', 'DEBUG', 'WARN']) -parser.add_argument('--check_online_update', help='check can do online rolling update for TiFlash', action='store_true') -parser.add_argument('--clean_pd_rules', help='clean all placement rules about tiflash in pd', action='store_true') - -args = parser.parse_args() - -import flash_tools - -flash_conf = flash_tools.FlashConfig(args.config) -log_level = logging._nameToLevel.get(args.log_level) diff --git a/cluster_manage/define.py b/cluster_manage/define.py deleted file mode 100644 index 115e7877c63..00000000000 --- a/cluster_manage/define.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -TIFLASH = 'tiflash' -TIFLASH_LABEL = {'key': 'engine', 'value': TIFLASH} -REGION_COUNT = 'region_count' -TIFLASH_REGION_COUNT = 'flash_region_count' -LOCATION_LABELS = 'location_labels' -REPLICA_COUNT = 'replica_count' -LEARNER = 'learner' -AVAILABLE = 'available' -PRIORITY = 'high_priority' -TIFLASH_CLUSTER_MUTEX_KEY = '/{}/cluster/leader'.format(TIFLASH) -TIFLASH_CLUSTER_HTTP_PORT = '/{}/cluster/http_port/'.format(TIFLASH) -LABEL_CONSTRAINTS = 'label_constraints' -DDL_GLOBAL_SCHEMA_VERSION = '/tidb/ddl/global_schema_version' -TIFLASH_LAST_HANDLED_SCHEMA_VERSION = '/{}/cluster/last_handled_schema_version'.format(TIFLASH) -TIFLASH_LAST_HANDLED_SCHEMA_VERSION_TSO_SPLIT = b'_tso_' -TIFLASH_LAST_HANDLED_SCHEMA_TIME_OUT = 300 diff --git a/cluster_manage/flash_cluster_manager.py b/cluster_manage/flash_cluster_manager.py deleted file mode 100644 index ed20c115e4c..00000000000 --- a/cluster_manage/flash_cluster_manager.py +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import sys -import time -from logging.handlers import RotatingFileHandler - -import conf -import define -import flash_http_client -import placement_rule -import tidb_tools -import util -from pd_client import PDClient, EtcdClient - -terminal: bool = False - - -def handle_receive_signal(signal_number, _): - print('Received signal: ', signal_number) - global terminal - terminal = handle_receive_signal - - -def wrap_try_get_lock(func): - def wrap_func(manager, *args, **kwargs): - manager.try_get_lock() - role, ts = manager.state - if role == TiFlashClusterManager.ROLE_MASTER: - return func(manager, *args, **kwargs) - else: - pass - - return wrap_func - - -class Store: - def __eq__(self, other): - return self.inner == other - - def __str__(self): - return str(self.inner) - - def __init__(self, pd_store): - self.inner = pd_store - address = self.inner['address'] - host, port = address.split(':') - self.address = '{}:{}'.format(host, port) - _, status_port = self.inner['status_address'].split(':') - self.tiflash_status_address = '{}:{}'.format(host, status_port) - - @property - def id(self): - return self.inner['id'] - - -class Table: - def __init__(self, total_region, flash_region): - self.total_region = total_region - self.flash_region = flash_region - - -class TiFlashClusterManager: - ROLE_INIT = 0 - ROLE_SECONDARY = 1 - ROLE_MASTER = 2 - - @staticmethod - def compute_cur_store(stores): - for _, store in stores.items(): - if store.address == conf.flash_conf.service_addr: - return store - - raise Exception("Can not tell current store.\nservice_addr: {},\nall tiflash stores: {}".format( - conf.flash_conf.service_addr, [store.inner for store in stores.values()])) - - def _try_refresh(self): - ori_role = self.state[0] - res = self.pd_client.etcd_client.refresh_ttl(self.cur_store.address) - if res == EtcdClient.EtcdOK: - self.state = [TiFlashClusterManager.ROLE_MASTER, time.time()] - if ori_role == TiFlashClusterManager.ROLE_INIT: - self.logger.debug('Continue become master') - elif res == EtcdClient.EtcdKeyNotFound: - self.state = [TiFlashClusterManager.ROLE_INIT, 0] - self.try_get_lock() - elif res == EtcdClient.EtcdValueNotEqual: - self.state = [TiFlashClusterManager.ROLE_SECONDARY, time.time()] - self.logger.debug('Refresh ttl fail (key not equal), become secondary') - else: - assert False - - def try_get_lock(self): - role, ts = self.state - if role == TiFlashClusterManager.ROLE_INIT: - if self.pd_client.etcd_client.try_init_mutex(self.cur_store.address): - self.state = [TiFlashClusterManager.ROLE_MASTER, time.time()] - self.logger.info('After init, become master') - else: - self.state = [TiFlashClusterManager.ROLE_SECONDARY, time.time()] - self.logger.info('After init, become secondary') - elif role == TiFlashClusterManager.ROLE_SECONDARY: - cur = time.time() - if cur >= ts + conf.flash_conf.cluster_master_ttl: - self.state = [TiFlashClusterManager.ROLE_INIT, 0] - self.logger.info('Timeout, become init') - self.try_get_lock() - else: - cur = time.time() - if cur >= ts + conf.flash_conf.cluster_refresh_interval: - self._try_refresh() - - def _update_http_port(self): - key = '{}{}'.format(define.TIFLASH_CLUSTER_HTTP_PORT, self.cur_store.address) - val = conf.flash_conf.http_addr - self.pd_client.etcd_client.update(key, val, max(conf.flash_conf.cluster_master_ttl, 120)) - - def __init__(self, pd_client: PDClient, tidb_status_addr_list, logger): - self.logger = logger - self.tidb_status_addr_list = tidb_status_addr_list - self.pd_client = pd_client - - def run(self): - if conf.args.check_online_update: - self.check_online_update_available() - elif conf.args.clean_pd_rules: - self.clean_pd_rules() - else: - self.run_one_round() - - def run_one_round(self): - self.stores = {} - self.cur_store = None - self._update_cluster() - self.state = [TiFlashClusterManager.ROLE_INIT, 0] - self._try_refresh() - self.ddl_global_schema_version = None - self.ddl_global_schema_check_tso = None - self.table_update() - - def _update_cluster(self): - prev_stores = self.stores - self.stores = {store_id: Store(store) for store_id, store in - self.pd_client.get_store_by_labels(define.TIFLASH_LABEL).items()} - if self.stores != prev_stores and prev_stores: - self.logger.info('Update all tiflash stores: from {} to {}'.format([k.inner for k in prev_stores.values()], - [k.inner for k in self.stores.values()])) - self.cur_store = self.compute_cur_store(self.stores) - self._update_http_port() - - def deal_with_region(self, region): - for peer in region.peers: - if peer.store_id == self.cur_store.id: - assert peer.is_learner - - def _check_and_make_rule(self, table, start_key, end_key, all_rules: dict): - rule_id = 'table-{}-r'.format(table['id']) - - need_new_rule = True - if rule_id in all_rules: - rule = all_rules[rule_id] - if rule.override and rule.start_key == start_key and rule.end_key == end_key and rule.label_constraints == placement_rule.DEFAULT_LABEL_CONSTRAINTS and rule.location_labels == \ - table[define.LOCATION_LABELS] and rule.count == table[ - define.REPLICA_COUNT] and rule.role == define.LEARNER: - need_new_rule = False - - if need_new_rule: - rules_new = placement_rule.make_rule(rule_id, start_key, end_key, table[define.REPLICA_COUNT], - table[define.LOCATION_LABELS]) - self.set_rule(util.obj_2_dict(rules_new)) - - all_rules.pop(rule_id, None) - return need_new_rule - - @wrap_try_get_lock - def set_rule(self, rule): - if self.pd_client.set_rule(rule) == 200: - self.logger.info('Set placement rule {}'.format(rule)) - else: - raise Exception('Set placement rule {} fail'.format(rule)) - - def compute_sync_data_process(self, table_id, start_key, end_key, replica_count): - stats_region: dict = self.pd_client.get_stats_region_by_range_json(start_key, end_key) - region_count = max(stats_region.get('count', 1), 1) - flash_region_count, err = flash_http_client.get_region_count_by_table(self.stores.values(), table_id, - replica_count) - if err: - self.logger.error('fail to get table replica sync status {}'.format(err)) - return region_count, flash_region_count - - @wrap_try_get_lock - def report_to_tidb(self, table, region_count, flash_region_count): - table_id = table['id'] - self.logger.info( - 'report to tidb: id {}, region_count {}, flash_region_count {}'.format(table_id, region_count, - flash_region_count)) - - error_list = [] - for idx, address in enumerate(self.tidb_status_addr_list): - try: - r = util.post_http( - '{}/tiflash/replica'.format(address, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION), - {'id': table_id, define.REGION_COUNT: region_count, - define.TIFLASH_REGION_COUNT: flash_region_count}) - util.check_status_code(r) - return - except Exception as e: - error_list.append((address, e)) - continue - - self.logger.error('can not report replica sync status to tidb: {}'.format(error_list)) - - @wrap_try_get_lock - def remove_rule(self, rule_id): - self._remove_rule(rule_id) - - def _remove_rule(self, rule_id): - self.pd_client.remove_rule(placement_rule.TIFLASH_GROUP_ID, rule_id) - self.logger.info('Remove placement rule {}'.format(rule_id)) - - def escape_table_update(self): - ddl_global_schema_version, _ = self.pd_client.etcd_client.get(define.DDL_GLOBAL_SCHEMA_VERSION) - - last_handled_schema_version_tso, _ = self.pd_client.etcd_client.get(define.TIFLASH_LAST_HANDLED_SCHEMA_VERSION) - - if ddl_global_schema_version is None: - ddl_global_schema_version = b'0' - - if last_handled_schema_version_tso is None: - last_handled_schema_version_tso = b'%d%b%d' % (0, define.TIFLASH_LAST_HANDLED_SCHEMA_VERSION_TSO_SPLIT, 0) - - last_handled_schema_version, last_handled_schema_tso = last_handled_schema_version_tso.split( - define.TIFLASH_LAST_HANDLED_SCHEMA_VERSION_TSO_SPLIT) - - cur_tso = int(time.time()) - - if ddl_global_schema_version == last_handled_schema_version and int( - last_handled_schema_tso) + define.TIFLASH_LAST_HANDLED_SCHEMA_TIME_OUT > cur_tso: - return True - - self.ddl_global_schema_version = ddl_global_schema_version - self.ddl_global_schema_check_tso = cur_tso - - return False - - def check_online_update_available(self): - from tikv_util import common - - self.stores = {store_id: Store(store) for store_id, store in - self.pd_client.get_store_by_labels(define.TIFLASH_LABEL).items()} - table_list = tidb_tools.db_flash_replica(self.tidb_status_addr_list) - error_list = [] - for table in table_list: - table_id = table['id'] - replica_count = table[define.REPLICA_COUNT] - st, ed = common.make_table_begin(table_id), common.make_table_end(table_id) - start_key, end_key = st.to_bytes(), ed.to_bytes() - region_count, flash_region_count = self.compute_sync_data_process(table_id, start_key, end_key, - replica_count) - if region_count != flash_region_count: - error_list.append( - 'Table {}, replica count {}, got {} matched region peers in TiFlash stores, expect {}.'.format( - table_id, replica_count, flash_region_count, region_count)) - if error_list: - self.logger.info('False.') - for e in error_list: - self.logger.info(' %s' % e) - else: - self.logger.info('True.') - - def clean_pd_rules(self): - all_rules = self.pd_client.get_group_rules(placement_rule.TIFLASH_GROUP_ID) - self.logger.info( - 'There are {} rules in pd with group-id `{}`'.format(len(all_rules), placement_rule.TIFLASH_GROUP_ID)) - for rule_id in all_rules.keys(): - self._remove_rule(rule_id) - - @wrap_try_get_lock - def table_update(self): - if self.escape_table_update(): - return - - from tikv_util import common - - all_replica_available = True - table_list = tidb_tools.db_flash_replica(self.tidb_status_addr_list) - all_rules = self.pd_client.get_group_rules(placement_rule.TIFLASH_GROUP_ID) - for table in table_list: - table_id = table['id'] - st, ed = common.make_table_begin(table_id), common.make_table_end(table_id) - start_key, end_key = st.to_bytes(), ed.to_bytes() - start_key_hex, end_key_hex = st.to_pd_key(), ed.to_pd_key() - self._check_and_make_rule(table, start_key_hex, end_key_hex, all_rules) - - if not table[define.AVAILABLE]: - if table.get(define.PRIORITY, False): - self.pd_client.set_accelerate_schedule(start_key_hex, end_key_hex) - self.logger.info('try to accelerate pd schedule for table {}'.format(table_id)) - - region_count, flash_region_count = self.compute_sync_data_process(table_id, start_key, end_key, 1) - self.report_to_tidb(table, region_count, flash_region_count) - all_replica_available = False - - for rule in all_rules.values(): - self.remove_rule(rule.id) - - if all_replica_available: - v = b'%b%b%d' % (self.ddl_global_schema_version, define.TIFLASH_LAST_HANDLED_SCHEMA_VERSION_TSO_SPLIT, - self.ddl_global_schema_check_tso) - self.pd_client.etcd_client.put(define.TIFLASH_LAST_HANDLED_SCHEMA_VERSION, v) - self.logger.info( - 'all replicas are available at global schema version {}'.format(int(self.ddl_global_schema_version))) - - -def get_tz_offset(): - import datetime - now_stamp = time.time() - local_time = datetime.datetime.fromtimestamp(now_stamp) - utc_time = datetime.datetime.utcfromtimestamp(now_stamp) - offset = local_time - utc_time - total_seconds = offset.total_seconds() - flag = '+' - if total_seconds < 0: - flag = '-' - total_seconds = -total_seconds - mm, ss = divmod(total_seconds, 60) - hh, mm = divmod(mm, 60) - tz_offset = "%s%02d:%02d" % (flag, hh, mm) - return tz_offset - - -def main(): - flash_conf = conf.flash_conf - - tz_offset = get_tz_offset() - - if conf.args.check_online_update or conf.args.clean_pd_rules: - root = logging.getLogger() - root.setLevel(logging.INFO) - handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.INFO) - root.addHandler(handler) - logger = root - else: - parent_path = os.path.dirname(flash_conf.log_path) - if not os.path.exists(parent_path): - os.makedirs(parent_path) - - # keep at most 10G log files - logging.basicConfig( - handlers=[RotatingFileHandler(flash_conf.log_path, maxBytes=1024 * 1024 * 500, backupCount=5)], - level=conf.log_level, - format='[%(asctime)s.%(msecs)03d {}] [%(levelname)s] [%(name)s] [%(message)s]'.format(tz_offset), - datefmt='%Y/%m/%d %H:%M:%S', - ) - logging.getLogger("requests").setLevel(logging.WARNING) - logging.getLogger("urllib3").setLevel(logging.WARNING) - logging.debug('\nCluster Manager Version Info\n{}'.format(conf.version_info)) - logger = logging.getLogger('TiFlashManager') - - try: - pd_client = PDClient(flash_conf.pd_addrs) - TiFlashClusterManager(pd_client, conf.flash_conf.tidb_status_addr, logger).run() - except Exception as e: - logging.exception(e) - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/flash_http_client.py b/cluster_manage/flash_http_client.py deleted file mode 100644 index c5635cf9fd3..00000000000 --- a/cluster_manage/flash_http_client.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import util - - -def curl_flash(address, params): - if type(params) != dict: - params = {'query': params} - r = util.curl_http(address, params) - return r - - -def get_region_count_by_table(store_list, table_id, replica_count): - from tikv_util import common - - checker = common.CheckRegionCnt() - err = [] - for store in store_list: - try: - res = util.curl_http('{}/tiflash/sync-status/{}'.format(store.tiflash_status_address, table_id)) - util.check_status_code(res) - checker.add(res.content) - except Exception as e: - err.append(e) - return checker.compute(replica_count), err - - -def get_regions_by_range(address, start_key, end_key): - sql = "DBGInvoke find_region_by_range(\'{}\',\'{}\', 1)".format(start_key, end_key) - res = curl_flash(address, sql).text - res = res.split('\n')[:-1] - res[1] = res[1].split(' ')[1:-1] - return res - - -def get_region_count_by_range(address, start_key, end_key): - sql = "DBGInvoke find_region_by_range(\'{}\',\'{}\')".format(start_key, end_key) - res = curl_flash(address, sql).text.split('\n') - return int(res[0]) - - -def main(): - pass - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/flash_tools.py b/cluster_manage/flash_tools.py deleted file mode 100644 index cb4441849c7..00000000000 --- a/cluster_manage/flash_tools.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import toml - -import util - - -class FlashConfig: - - def __init__(self, file_path): - self.conf_file_path = file_path - self.conf_toml = toml.load(self.conf_file_path, _dict=dict) - self.pd_addrs = util.compute_addr_list(self.conf_toml['raft']['pd_addr']) - if 'http_port' in self.conf_toml: - self.http_port = self.conf_toml['http_port'] - else: - self.http_port = self.conf_toml['https_port'] - - p = self.conf_toml['flash'] - service_addr = p['service_addr'] - if p['proxy'].get('config') is not None: - proxy_toml = toml.load(p['proxy']['config'], _dict=dict) - service_addr = proxy_toml.get('server', {}).get('engine-addr', service_addr) - service_addr = p['proxy'].get('engine-addr', service_addr) - service_addr = service_addr.strip() - host, port = [e.strip() for e in service_addr.split(':')] - self.service_addr = '{}:{}'.format(host, port) - self.http_addr = '{}:{}'.format(host, self.http_port) - self.tidb_status_addr = util.compute_addr_list(p['tidb_status_addr']) - flash_cluster = p.get('flash_cluster', {}) - self.cluster_master_ttl = flash_cluster.get('master_ttl', 60) - self.cluster_refresh_interval = min( - int(flash_cluster.get('refresh_interval', 20)), self.cluster_master_ttl) - self.update_rule_interval = int(flash_cluster.get('update_rule_interval', 10)) - self.log_path = flash_cluster.get('log', '{}/flash_cluster_manager.log'.format(self.conf_toml.get('tmp_path', '/tmp'))) - self.max_time_out = self.cluster_master_ttl - - self.enable_tls = False - if 'security' in self.conf_toml: - security = self.conf_toml['security'] - if 'ca_path' in security: - self.ca_path = security['ca_path'] - self.key_path = security['key_path'] - self.cert_path = security['cert_path'] - self.enable_tls = True - - -def main(): - pass - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/pd_client.py b/cluster_manage/pd_client.py deleted file mode 100644 index 7f7840cb41f..00000000000 --- a/cluster_manage/pd_client.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Optional - -import define -import etcd3 -import uri -import conf -import util - - -class EtcdClient: - EtcdOK = 0 - EtcdKeyNotFound = 1 - EtcdValueNotEqual = 2 - - def try_init_mutex(self, cluster_mutex_value): - val, meta = self.client.get(define.TIFLASH_CLUSTER_MUTEX_KEY) - if val is None: - lease = self.client.lease(conf.flash_conf.cluster_master_ttl) - if self.client.put_if_not_exists(define.TIFLASH_CLUSTER_MUTEX_KEY, cluster_mutex_value, lease=lease): - self.logger.info('Try to init master success, ttl: %d, create new key: %s', lease.ttl, - define.TIFLASH_CLUSTER_MUTEX_KEY) - return True - self.logger.info('Try to init master fail, key exists') - return False - - def refresh_ttl(self, cluster_mutex_value): - val, meta = self.client.get(define.TIFLASH_CLUSTER_MUTEX_KEY) - if val is None: - return self.EtcdKeyNotFound - if cluster_mutex_value != str(val, encoding="utf8"): - return self.EtcdValueNotEqual - list(self.client.refresh_lease(meta.lease_id)) - return self.EtcdOK - - def get_by_prefix(self, prefix): - return self.client.get_prefix(prefix) - - def get(self, key): - return self.client.get(key) - - def put(self, key, val): - return self.client.put(key, val) - - def update(self, key, value, ttl=conf.flash_conf.cluster_master_ttl): - ori_val, meta = self.client.get(key) - if ori_val is None or value != str(ori_val, encoding="utf8"): - self.client.put(key, value, self.client.lease(ttl)) - else: - list(self.client.refresh_lease(meta.lease_id)) - - def __init__(self, host, port): - self.logger = logging.getLogger('etcd.client') - kwargs = {"timeout": conf.flash_conf.max_time_out} - if conf.flash_conf.enable_tls: - kwargs["ca_cert"] = conf.flash_conf.ca_path - kwargs["cert_key"] = conf.flash_conf.key_path - kwargs["cert_cert"] = conf.flash_conf.cert_path - self.client = etcd3.client(host=host, port=port, **kwargs) - - -class PDClient: - PD_API_PREFIX = 'pd/api' - PD_API_VERSION = 'v1' - - def get_all_regions_json(self): - r = util.curl_http('{}/{}/{}/regions'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION)) - return util.try_get_json(r) - - def get_regions_by_key_json(self, key: str, limit=16): - r = util.curl_http( - '{}/{}/{}/regions/key'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION), - {'key': key, 'limit': limit}) - return util.try_get_json(r) - - def get_region_by_id_json(self, region_id: int): - r = util.curl_http( - '{}/{}/{}/region/id/{}'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION, region_id)) - return util.try_get_json(r) - - def get_all_stores_json(self): - r = util.curl_http( - '{}/{}/{}/stores'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION)) - return util.try_get_json(r) - - def get_members_json(self, *args): - url = args[0] if args else self.leader - r = util.curl_http( - '{}/{}/{}/members'.format(url, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION)) - return util.try_get_json(r) - - def get_stats_region_by_range_json(self, start_key, end_key): - r = util.curl_http( - '{}/{}/{}/stats/region'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION), - {'start_key': start_key, 'end_key': end_key}, - ) - return util.try_get_json(r) - - def get_group_rules(self, group): - r = util.curl_http( - '{}/{}/{}/config/rules/group/{}'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION, - group)) - res = util.try_get_json(r) - res = res if res is not None else {} - for e in res: - if not isinstance(e, dict): - raise Exception('Got placement rules fail: {}'.format(r.text)) - from placement_rule import PlacementRule - return {e['id']: PlacementRule(**e) for e in res} - - def get_all_rules(self): - r = util.curl_http( - '{}/{}/{}/config/rules'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION)) - res = util.try_get_json(r) - return res if res is not None else {} - - def get_rule(self, group, rule_id): - r = util.curl_http( - '{}/{}/{}/config/rule/{}/{}'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION, group, - rule_id)) - return util.try_get_json(r) - - def set_rule(self, rule): - r = util.post_http( - '{}/{}/{}/config/rule'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION), rule) - return r.status_code - - def set_accelerate_schedule(self, start_key, end_key): - r = util.post_http( - '{}/{}/{}/regions/accelerate-schedule'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION), - {'start_key': start_key, 'end_key': end_key}, ) - if r.status_code != 200: - raise Exception( - "fail to accelerate schedule range [{},{}), error msg: {}".format(start_key, end_key, r.text)) - - def remove_rule(self, group, rule_id): - r = util.delete_http( - '{}/{}/{}/config/rule/{}/{}'.format(self.leader, PDClient.PD_API_PREFIX, PDClient.PD_API_VERSION, group, - rule_id)) - return r.status_code - - def _try_update_leader_etcd(self, url): - resp = self.get_members_json(url) - leader = resp.get('leader', {}) - client_urls = leader.get('client_urls', []) - if client_urls: - _client_urls = [] - for member in resp.get('members', {}): - _client_urls.extend(member.get('client_urls', [])) - self.urls = _client_urls - self.leader = uri.URI(client_urls[0]).authority - _etcd_leader_uri = uri.URI(resp.get('etcd_leader', {}).get('client_urls', [])[0]) - self.etcd_client = EtcdClient(_etcd_leader_uri.host, _etcd_leader_uri.port) - - def _update_leader_etcd(self): - errors = [] - for url in self.urls: - try: - return self._try_update_leader_etcd(url) - except Exception as e: - errors.append(e) - raise Exception("can not find pd leader: {}".format(errors)) - - def get_store_by_labels(self, flash_label): - res = {} - all_stores = self.get_all_stores_json() - for store in all_stores['stores']: - store = store['store'] - for label in store.get('labels', []): - if label == flash_label: - res[store['id']] = store - return res - - def __init__(self, urls): - self.logger = logging.getLogger('pd.client') - - self.urls = urls - self.leader = "" - self.etcd_client: Optional[EtcdClient] = None - self._update_leader_etcd() - - -def main(): - pass - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/placement_rule.py b/cluster_manage/placement_rule.py deleted file mode 100644 index 959b17c3a42..00000000000 --- a/cluster_manage/placement_rule.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import define - -TIFLASH_GROUP_ID = define.TIFLASH -DEFAULT_LABEL_CONSTRAINTS = [{"key": "engine", "op": "in", "values": [define.TIFLASH]}] - -base_rule = { - "group_id": TIFLASH_GROUP_ID, - 'id': '', - "index": 0, - "override": True, - "start_key": None, - "end_key": None, - "role": define.LEARNER, - "count": 2, - define.LABEL_CONSTRAINTS: DEFAULT_LABEL_CONSTRAINTS, - define.LOCATION_LABELS: None -} - - -class PlacementRule: - def __init__(self, **entries): - self.__dict__.update(entries) - if not hasattr(self, define.LOCATION_LABELS): - self.location_labels = [] - - -def make_rule(rid: str, start_key, end_key, count, location_labels): - rule = PlacementRule(**base_rule) - rule.id = rid - rule.start_key = start_key - rule.end_key = end_key - rule.count = count - rule.location_labels = location_labels - return rule diff --git a/cluster_manage/release.sh b/cluster_manage/release.sh deleted file mode 100755 index e7976ec0fe8..00000000000 --- a/cluster_manage/release.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ue - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" - -echo "cluster_manager is deprecated" -mkdir -p ${SCRIPTPATH}/dist/flash_cluster_manager diff --git a/cluster_manage/tidb_tools.py b/cluster_manage/tidb_tools.py deleted file mode 100644 index 538b8032e73..00000000000 --- a/cluster_manage/tidb_tools.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md - -import util - - -def curl_tidb(address, uri): - r = util.curl_http('{}{}'.format(address, uri)) - return util.try_get_json(r) - - -def status(address): - return curl_tidb(address, '/status') - - -def table_by_id(address, table_id): - return curl_tidb(address, '/schema?table_id={}'.format(table_id)) - - -def db_info(address, table_id): - return curl_tidb(address, '/db-table/{}'.format(table_id)) - - -def db_schema(address, db): - return curl_tidb(address, '/schema/{}'.format(db)) - - -def db_all_schema(address): - return curl_tidb(address, '/schema') - - -def db_flash_replica(tidb_status_addr_list): - error_list = [] - for idx, address in enumerate(tidb_status_addr_list): - try: - return curl_tidb(address, '/tiflash/replica') - except Exception as e: - error_list.append((address, e)) - continue - - raise Exception('can not get tiflash replica info from tidb: {}'.format(error_list)) - - -def main(): - pass - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/tikv_util/.clang-format b/cluster_manage/tikv_util/.clang-format deleted file mode 100644 index 81ea492c437..00000000000 --- a/cluster_manage/tikv_util/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -BasedOnStyle: Google -DerivePointerAlignment: false -PointerAlignment: Middle -... diff --git a/cluster_manage/tikv_util/.gitignore b/cluster_manage/tikv_util/.gitignore deleted file mode 100644 index 9d22eb46a9c..00000000000 --- a/cluster_manage/tikv_util/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.o -*.so diff --git a/cluster_manage/tikv_util/__init__.py b/cluster_manage/tikv_util/__init__.py deleted file mode 100644 index bc3524fc9d3..00000000000 --- a/cluster_manage/tikv_util/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - diff --git a/cluster_manage/tikv_util/build.sh b/cluster_manage/tikv_util/build.sh deleted file mode 100755 index c03bc133ae3..00000000000 --- a/cluster_manage/tikv_util/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -is_force="$1" - -if [[ "$is_force" == "true" ]]; then - is_force="--force" -else - is_force="" -fi - -python3 setup.py build_ext --inplace ${is_force} diff --git a/cluster_manage/tikv_util/check_region_cnt.cc b/cluster_manage/tikv_util/check_region_cnt.cc deleted file mode 100644 index c6379321302..00000000000 --- a/cluster_manage/tikv_util/check_region_cnt.cc +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "check_region_cnt.h" - -#include - -#include - -void CheckRegionCnt::Add(std::string_view s) { - char region_id[20]; - size_t id_len = 0; - - auto p1 = s.find('\n'); - auto p2 = s.find('\n', p1 + 1); - for (size_t i = p1 + 1; i < p2; ++i) { - if (s[i] == ' ') { - region_id[id_len] = 0; - data_[std::atoll(region_id)] += 1; - id_len = 0; - } else { - region_id[id_len++] = s[i]; - } - } -} - -Int64 CheckRegionCnt::Compute(UInt64 expect_cnt) { - if (expect_cnt == 1) return data_.size(); - - return std::accumulate( - data_.begin(), data_.end(), Int64(0), - [&](auto sum, const auto & e) { return sum + (e.second >= expect_cnt); }); -} diff --git a/cluster_manage/tikv_util/check_region_cnt.h b/cluster_manage/tikv_util/check_region_cnt.h deleted file mode 100644 index 70d79bf1236..00000000000 --- a/cluster_manage/tikv_util/check_region_cnt.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "type.h" - -namespace pybind11 { -class bytes; -} // namespace pybind11 - -namespace py = pybind11; - -class CheckRegionCnt { - public: - void Add(std::string_view); - // Count how many regions that the number of TiFlash peers is greater or equal - // to replica_count - Int64 Compute(UInt64); - - private: - std::unordered_map data_; -}; diff --git a/cluster_manage/tikv_util/codec.h b/cluster_manage/tikv_util/codec.h deleted file mode 100644 index f5ba5d4ab44..00000000000 --- a/cluster_manage/tikv_util/codec.h +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "exception_util.h" -#include "type.h" - -static const size_t kEncGroupSize = 8; -static const uint8_t kEncMarker = static_cast(0xff); -static const char kEncAscPadding[kEncGroupSize] = {0}; - -static const char kTablePrefix = 't'; -static const char * kRecordPrefixSep = "_r"; - -static const UInt64 kSignMark = (UInt64)1 << 63u; - -inline void EncodeBytes(const std::string & ori_str, std::stringstream & ss) { - const size_t len = ori_str.size(); - size_t index = 0; - while (index <= len) { - size_t remain = len - index; - size_t pad = 0; - if (remain >= kEncGroupSize) { - ss.write(ori_str.data() + index, kEncGroupSize); - } else { - pad = kEncGroupSize - remain; - ss.write(ori_str.data() + index, remain); - ss.write(kEncAscPadding, pad); - } - ss.put(static_cast(kEncMarker - (uint8_t)pad)); - index += kEncGroupSize; - } -} - -inline std::string Encode(const std::string & ori_str) { - std::stringstream ss; - EncodeBytes(ori_str, ss); - return ss.str(); -} - -inline UInt64 EndianReverse(UInt64 x) { - UInt64 step32, step16; - step32 = x << 32u | x >> 32u; - step16 = (step32 & 0x0000FFFF0000FFFFULL) << 16u | - (step32 & 0xFFFF0000FFFF0000ULL) >> 16u; - return (step16 & 0x00FF00FF00FF00FFULL) << 8u | - (step16 & 0xFF00FF00FF00FF00ULL) >> 8u; -} - -inline UInt64 ToBigEndian(const UInt64 x) { return EndianReverse(x); } - -inline UInt64 EncodeUInt64(const UInt64 x) { return ToBigEndian(x); } - -inline UInt64 EncodeInt64(const Int64 x) { - return EncodeUInt64(static_cast(x) ^ kSignMark); -} - -inline UInt64 EncodeUInt64Desc(const UInt64 x) { return EncodeUInt64(~x); } - -inline UInt64 DecodeUInt64(const UInt64 x) { return ToBigEndian(x); } - -inline UInt64 DecodeUInt64Desc(const UInt64 x) { return ~DecodeUInt64(x); } - -inline Int64 DecodeInt64(const UInt64 x) { - return static_cast(DecodeUInt64(x) ^ kSignMark); -} - -inline std::string GenKeyByTable(const TableId tableId) { - std::string key(1 + 8, 0); - memcpy(key.data(), &kTablePrefix, 1); - auto big_endian_table_id = EncodeInt64(tableId); - memcpy(key.data() + 1, reinterpret_cast(&big_endian_table_id), - 8); - return Encode(key); -} - -template -inline std::string GenKey(const TableId tableId) { - return GenKeyByTable(tableId + 1); -} - -template <> -inline std::string GenKey(const TableId tableId) { - std::string key(1 + 8 + 2, 0); - memcpy(key.data(), &kTablePrefix, 1); - auto big_endian_table_id = EncodeInt64(tableId); - memcpy(key.data() + 1, reinterpret_cast(&big_endian_table_id), - 8); - memcpy(key.data() + 1 + 8, kRecordPrefixSep, 2); - return Encode(key); -} - -inline std::string GenKey(const TableId tableId, const HandleId handle) { - std::string key(1 + 8 + 2 + 8, 0); - memcpy(key.data(), &kTablePrefix, 1); - auto big_endian_table_id = EncodeInt64(tableId); - memcpy(key.data() + 1, reinterpret_cast(&big_endian_table_id), - 8); - memcpy(key.data() + 1 + 8, kRecordPrefixSep, 2); - auto big_endian_handle_id = EncodeInt64(handle); - memcpy(key.data() + 1 + 8 + 2, - reinterpret_cast(&big_endian_handle_id), 8); - return Encode(key); -} - -inline std::string ToPdKey(const char * key, const size_t len) { - std::string res(len * 2, 0); - size_t i = 0; - for (size_t k = 0; k < len; ++k) { - uint8_t o = key[k]; - res[i++] = o / 16; - res[i++] = o % 16; - } - - for (char & re : res) { - if (re < 10) - re = re + '0'; - else - re = re - 10 + 'A'; - } - return res; -} - -inline std::string ToPdKey(const std::string & key) { - return ToPdKey(key.data(), key.size()); -} - -inline std::string FromPdKey(const char * key, const size_t len) { - std::string res(len / 2, 0); - for (size_t k = 0; k < len; k += 2) { - int s[2]; - - for (size_t i = 0; i < 2; ++i) { - char p = key[k + i]; - if (p >= 'A') - s[i] = p - 'A' + 10; - else - s[i] = p - '0'; - } - - res[k / 2] = s[0] * 16 + s[1]; - } - return res; -} - -inline bool CheckKeyPaddingValid(const char * ptr, const UInt8 pad_size) { - UInt64 p = (*reinterpret_cast(ptr)) >> - ((kEncGroupSize - pad_size) * 8); - return p == 0; -} - -inline std::tuple DecodeTikvKeyFull( - const std::string & key) { - const size_t chunk_len = kEncGroupSize + 1; - std::string res; - res.reserve(key.size() / chunk_len * kEncGroupSize); - for (const char * ptr = key.data();; ptr += chunk_len) { - if (ptr + chunk_len > key.size() + key.data()) - throw Exception("Unexpected eof"); - auto marker = (UInt8) * (ptr + kEncGroupSize); - UInt8 pad_size = (kEncMarker - marker); - if (pad_size == 0) { - res.append(ptr, kEncGroupSize); - continue; - } - if (pad_size > kEncGroupSize) throw Exception("Key padding"); - res.append(ptr, kEncGroupSize - pad_size); - - if (!CheckKeyPaddingValid(ptr, pad_size)) - throw Exception("Key padding, wrong end"); - - // raw string and the offset of remaining string such as timestamp - return std::make_tuple(std::move(res), ptr - key.data() + chunk_len); - } -} - -inline std::string DecodeTikvKey(const std::string & key) { - return std::get<0>(DecodeTikvKeyFull(key)); -} diff --git a/cluster_manage/tikv_util/exception_util.h b/cluster_manage/tikv_util/exception_util.h deleted file mode 100644 index 57fac9e0cdf..00000000000 --- a/cluster_manage/tikv_util/exception_util.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -class Exception : std::exception { - public: - const char * what() const noexcept override { return msg_.data(); } - - explicit Exception(std::string msg) : msg_(std::move(msg)) {} - - private: - std::string msg_; -}; \ No newline at end of file diff --git a/cluster_manage/tikv_util/export.cc b/cluster_manage/tikv_util/export.cc deleted file mode 100644 index 0dee0a1455e..00000000000 --- a/cluster_manage/tikv_util/export.cc +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "check_region_cnt.h" -#include "tikv_key.h" -namespace py = pybind11; - -PYBIND11_MODULE(common, m) { - py::class_(m, "TikvKey") - .def(py::init()) - .def("size", &TikvKey::Size) - .def("compare", &TikvKey::Compare) - .def("to_bytes", &TikvKey::ToBytes) - .def("to_pd_key", &TikvKey::ToPdKey); - - py::class_(m, "CheckRegionCnt") - .def(py::init()) - .def("add", &CheckRegionCnt::Add) - .def("compute", &CheckRegionCnt::Compute); - - m.def("make_table_begin", &MakeTableBegin); - m.def("make_table_end", &MakeTableEnd); - m.def("make_whole_table_begin", &MakeWholeTableBegin); - m.def("make_whole_table_end", &MakeWholeTableEnd); - m.def("make_table_handle", &MakeTableHandle); - m.def("decode_pd_key", &DecodePdKey); - m.def("get_table_id", &GetTableId); - m.def("get_handle", &GetHandle); -} diff --git a/cluster_manage/tikv_util/format.sh b/cluster_manage/tikv_util/format.sh deleted file mode 100755 index f44fedca029..00000000000 --- a/cluster_manage/tikv_util/format.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -clang_format=$(bash -c "compgen -c clang-format | grep 'clang-format' | sort --version-sort | head -n1") - -if [[ ! -z ${clang_format} ]]; then - find ./ -name "*.h" -or -name "*.cc" | xargs ${clang_format} -i -else - echo clang-format missing. try to install -fi diff --git a/cluster_manage/tikv_util/setup.py b/cluster_manage/tikv_util/setup.py deleted file mode 100644 index 9eabba088a8..00000000000 --- a/cluster_manage/tikv_util/setup.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext -import sys -import setuptools - -__version__ = '0.0.1' - - -class GetPybindInclude(object): - """Helper class to determine the pybind11 include path - - The purpose of this class is to postpone importing pybind11 - until it is actually installed, so that the ``get_include()`` - method can be invoked. """ - - def __init__(self, user=False): - self.user = user - - def __str__(self): - import pybind11 - return pybind11.get_include(self.user) - - -ext_modules = [ - Extension( - 'common', - sources=['tikv_key.cc', 'export.cc', 'check_region_cnt.cc'], - include_dirs=[ - GetPybindInclude(), - GetPybindInclude(user=True) - ], - language='c++' - ), -] - - -# As of Python 3.6, CCompiler has a `has_flag` method. -# cf http://bugs.python.org/issue26689 -def has_flag(compiler, flagname): - """Return a boolean indicating whether a flag name is supported on - the specified compiler. - """ - import tempfile - with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f: - f.write('int main (int argc, char **argv) { return 0; }') - try: - compiler.compile([f.name], extra_postargs=[flagname]) - except setuptools.distutils.errors.CompileError: - return False - return True - - -def cpp_flag(compiler): - """Return the -std=c++[11/14/17] compiler flag. - - The newer version is prefered over c++11 (when it is available). - """ - flags = ['-std=c++17', '-std=c++14', '-std=c++11'] - - for flag in flags: - if has_flag(compiler, flag): - return flag - - raise RuntimeError('Unsupported compiler -- at least C++11 support ' - 'is needed!') - - -class BuildExt(build_ext): - """A custom build extension for adding compiler-specific options.""" - c_opts = { - 'unix': [], - } - l_opts = { - 'unix': [], - } - - if sys.platform == 'darwin': - darwin_opts = ['-stdlib=libc++', '-mmacosx-version-min=10.14'] - c_opts['unix'] += darwin_opts - l_opts['unix'] += darwin_opts - os.environ["CC"] = "clang" - os.environ["CXX"] = "clang" - - def build_extensions(self): - ct = self.compiler.compiler_type - opts = self.c_opts.get(ct, []) - link_opts = self.l_opts.get(ct, []) - if ct == 'unix': - opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version()) - opts.append(cpp_flag(self.compiler)) - if has_flag(self.compiler, '-fvisibility=hidden'): - opts.append('-fvisibility=hidden') - for ext in self.extensions: - ext.extra_compile_args = opts - ext.extra_link_args = link_opts - build_ext.build_extensions(self) - - -setup( - name='tikv_util', - version=__version__, - author='Tong Zhigao', - author_email='tongzhigao@pingcap.com', - ext_modules=ext_modules, - install_requires=['pybind11>=2.3'], - setup_requires=['pybind11>=2.3'], - cmdclass={'build_ext': BuildExt}, - zip_safe=False, -) diff --git a/cluster_manage/tikv_util/test.py b/cluster_manage/tikv_util/test.py deleted file mode 100644 index 93eff168b45..00000000000 --- a/cluster_manage/tikv_util/test.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest - -from tikv_util import common - - -class TestTikvKey(unittest.TestCase): - - def test_pd_tikv_key(self): - pd_key = '7480000000000000FF2D5F728000000000FF3921010000000000FA' - ori_tikv_key = b't\x80\x00\x00\x00\x00\x00\x00\xff-_r\x80\x00\x00\x00\x00\xff9!\x01\x00\x00\x00\x00\x00\xfa' - - self.assertEqual(len(ori_tikv_key) * 2, len(pd_key)) - - tikv_key = common.TikvKey(ori_tikv_key) - self.assertEqual(len(ori_tikv_key), tikv_key.size()) - self.assertEqual(ori_tikv_key, tikv_key.to_bytes()) - self.assertEqual(pd_key, tikv_key.to_pd_key()) - - p = common.decode_pd_key(pd_key, len(pd_key)) - self.assertEqual(p, ori_tikv_key) - - def test_start_end_of_table(self): - b1, e1 = common.make_table_begin(123), common.make_table_end(123) - self.assertEqual(b1.to_pd_key(), '7480000000000000FF7B5F720000000000FA') - self.assertEqual(e1.to_pd_key(), '7480000000000000FF7C00000000000000F8') - self.assertTrue(b1.compare(e1) < 0) - b2, e2 = common.make_table_begin(124), common.make_table_end(124) - self.assertTrue(e1.compare(b2) < 0) - b3, e3 = common.make_whole_table_begin(124), common.make_whole_table_end(124) - self.assertTrue(b3.compare(e1) == 0) - self.assertTrue(e3.compare(common.make_whole_table_begin(125)) == 0) - - def test_table_handle(self): - b, e = common.make_table_begin(123), common.make_table_end(123) - mx = common.make_table_handle(123, 9223372036854775807) - mn = common.make_table_handle(123, -9223372036854775808) - self.assertTrue(b.compare(mn) < 0) - self.assertTrue(mn.compare(mx) < 0) - self.assertTrue(mx.compare(e) < 0) - self.assertTrue(mx.compare(mx) == 0) - self.assertEqual(common.get_table_id(mx), 123) - self.assertEqual(common.get_handle(mx), 9223372036854775807) - self.assertEqual(common.get_handle(mn), -9223372036854775808) - - def test_region_cnt(self): - s1 = b'3\n1000 1234 7 \n' - s2 = b'3\n1000 7 9 \n' - c = common.CheckRegionCnt() - c.add(s1) - c.add(s2) - self.assertEqual(4, c.compute(1)) - self.assertEqual(2, c.compute(2)) - self.assertEqual(0, c.compute(3)) - - def test_exception(self): - o = b'12345' - k = common.TikvKey(o) - try: - common.get_table_id(k) - self.assertTrue(False) - except RuntimeError: - pass diff --git a/cluster_manage/tikv_util/tikv_key.cc b/cluster_manage/tikv_util/tikv_key.cc deleted file mode 100644 index bccd274d180..00000000000 --- a/cluster_manage/tikv_util/tikv_key.cc +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "tikv_key.h" - -#include - -#include "codec.h" -namespace py = pybind11; - -TikvKey::TikvKey(String && s) : key_(std::move(s)) {} -TikvKey::TikvKey(std::string_view s) : key_(s) {} - -TikvKey::TikvKey(TikvKey && kv) : key_(kv.key_) {} - -int TikvKey::Compare(const TikvKey & k) { return key().compare(k.key()); } - -size_t TikvKey::Size() const { return key().size(); } - -TikvKey::~TikvKey() {} - -py::bytes TikvKey::ToBytes() const { return key(); } - -py::str TikvKey::ToPdKey() const { return ::ToPdKey(key_); } - -const TikvKey::String & TikvKey::key() const { return key_; } - -py::bytes DecodePdKey(const char * s, const size_t len) { - return FromPdKey(s, len); -} - -TikvKey MakeTableBegin(const TableId table_id) { - return TikvKey(GenKey(table_id)); -} - -TikvKey MakeTableEnd(const TableId table_id) { - return TikvKey(GenKey(table_id)); -} - -TikvKey MakeWholeTableBegin(const TableId table_id) { - return TikvKey(GenKeyByTable(table_id)); -} - -TikvKey MakeWholeTableEnd(const TableId table_id) { - return TikvKey(GenKeyByTable(table_id + 1)); -} - -TikvKey MakeTableHandle(const TableId table_id, const HandleId handle) { - return TikvKey(GenKey(table_id, handle)); -} - -template -inline T Read(const char * s) { - return *(reinterpret_cast(s)); -} - -TableId GetTableIdRaw(const std::string & key) { - return DecodeInt64(Read(key.data() + 1)); -} - -HandleId GetHandleRaw(const std::string & key) { - return DecodeInt64(Read(key.data() + 1 + 8 + 2)); -} - -TableId GetTableId(const TikvKey & key) { - return GetTableIdRaw(DecodeTikvKey(key.key())); -} - -HandleId GetHandle(const TikvKey & key) { - return GetHandleRaw(DecodeTikvKey(key.key())); -} diff --git a/cluster_manage/tikv_util/tikv_key.h b/cluster_manage/tikv_util/tikv_key.h deleted file mode 100644 index 1b56666c1e4..00000000000 --- a/cluster_manage/tikv_util/tikv_key.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include "type.h" - -namespace pybind11 { -class bytes; -class str; -} // namespace pybind11 - -namespace py = pybind11; - -class TikvKey { - public: - using String = std::string; - - TikvKey() = default; - explicit TikvKey(String && s); - explicit TikvKey(std::string_view); - TikvKey(const TikvKey &) = delete; - TikvKey(TikvKey && kv); - - int Compare(const TikvKey & k); - - size_t Size() const; - - ~TikvKey(); - - py::bytes ToBytes() const; - - py::str ToPdKey() const; - - const String & key() const; - - private: - const String key_; -}; - -TikvKey MakeTableBegin(const TableId table_id); -TikvKey MakeTableEnd(const TableId table_id); -TikvKey MakeTableHandle(const TableId table_id, const HandleId handle); -TikvKey MakeWholeTableBegin(const TableId table_id); -TikvKey MakeWholeTableEnd(const TableId table_id); - -py::bytes DecodePdKey(const char * s, const size_t len); - -TableId GetTableId(const TikvKey & key); - -HandleId GetHandle(const TikvKey & key); \ No newline at end of file diff --git a/cluster_manage/tikv_util/type.h b/cluster_manage/tikv_util/type.h deleted file mode 100644 index 42aaa4ecad8..00000000000 --- a/cluster_manage/tikv_util/type.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -using Int64 = int64_t; -using UInt64 = uint64_t; -using UInt8 = uint8_t; -using TableId = Int64; -using HandleId = Int64; diff --git a/cluster_manage/timer.py b/cluster_manage/timer.py deleted file mode 100644 index 0d4144ced18..00000000000 --- a/cluster_manage/timer.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import heapq -import time - - -class Task: - def __init__(self, func, run_tso): - self.func = func - self.run_tso = run_tso - - def __lt__(self, other): - return self.run_tso < other.run_tso - - def __ge__(self, other): - return self.run_tso >= other.run_tso - - def __le__(self, other): - return self.run_tso <= other.run_tso - - -class Timer: - def __init__(self): - self.queue = [] - - def add(self, func, interval): - heapq.heappush(self.queue, Task(func, time.time() + interval)) - - def run(self): - while True: - if not len(self.queue): - time.sleep(0.2) - continue - task = heapq.heappop(self.queue) - now = time.time() - if now < task.run_tso: - time.sleep(task.run_tso - now) - task.func() - - -def main(): - timer = Timer() - - def run(): - print('x') - timer.add(run, time.time() + 2) - - timer.add(run, time.time() + 2) - timer.run() - - -if __name__ == '__main__': - main() diff --git a/cluster_manage/util.py b/cluster_manage/util.py deleted file mode 100644 index c1397ed0417..00000000000 --- a/cluster_manage/util.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import errno -import fcntl -import logging -import os -import socket -import time - -import requests - - -def wrap_run_time(func): - def wrap_func(*args, **kwargs): - bg = time.time() - r = func(*args, **kwargs) - print('time cost {}'.format(time.time() - bg)) - return r - - return wrap_func - - -class FLOCK(object): - def __init__(self, name): - self.obj = open(name, 'w') - self.fd = self.obj.fileno() - - def lock(self): - try: - fcntl.lockf(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - return True - except OSError: - logging.error( - 'Cannot lock file {}. Another instance in same directory is already running'.format(self.obj.name)) - return False - - -def gen_http_kwargs(): - import conf - kwargs = {"timeout": conf.flash_conf.max_time_out} - http_name = 'http' - if conf.flash_conf.enable_tls: - kwargs["verify"] = conf.flash_conf.ca_path - kwargs['cert'] = (conf.flash_conf.cert_path, conf.flash_conf.key_path) - http_name = 'https' - return http_name, kwargs - - -def curl_http(uri, params=None): - if params is None: - params = {} - http_name, kwargs = gen_http_kwargs() - r = requests.get('{}://{}'.format(http_name, uri), params, **kwargs) - return r - - -def check_status_code(r): - if r.status_code != 200: - raise Exception('unexpected status code {} from {}'.format(r.status_code, r.url)) - - -def try_get_json(r): - check_status_code(r) - return r.json() - - -def post_http(uri, params): - http_name, kwargs = gen_http_kwargs() - r = requests.post('{}://{}'.format(http_name, uri), json=params, **kwargs) - return r - - -def delete_http(uri): - http_name, kwargs = gen_http_kwargs() - r = requests.delete('{}://{}'.format(http_name, uri), **kwargs) - return r - - -def obj_2_dict(obj): - pr = {} - for name in dir(obj): - value = getattr(obj, name) - if not name.startswith('_') and not callable(value): - pr[name] = value - return pr - - -def make_compare_pd_key(key): - return (1, key) if key else (0, '') - - -def net_is_used(ip, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = s.connect_ex((ip, port)) - s.close() - return result == 0 - - -def pid_exists(pid): - """Check whether pid exists in the current process table. - UNIX only. - """ - if pid < 0: - return False - if pid == 0: - # According to "man 2 kill" PID 0 refers to every process - # in the process group of the calling process. - # On certain systems 0 is a valid PID but we have no way - # to know that in a portable fashion. - raise ValueError('invalid PID 0') - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - # ESRCH == No such process - return False - elif err.errno == errno.EPERM: - # EPERM clearly means there's a process to deny access to - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) - raise - else: - return True - - -def pid_exists2(pid): - if pid == 0: - return True - return pid_exists(pid) - - -def compute_addr_list(addrs): - return [e.strip() for e in addrs.split(',') if e] - - -def main(): - pass - - -if __name__ == '__main__': - main() diff --git a/deprecated/release-centos7/Dockerfile-builder b/deprecated/release-centos7/Dockerfile-builder deleted file mode 100644 index 135be4fc3f5..00000000000 --- a/deprecated/release-centos7/Dockerfile-builder +++ /dev/null @@ -1,34 +0,0 @@ -FROM hub.pingcap.net/jenkins/centos7_golang-1.12 - -USER root -WORKDIR /root/ - -ENV HOME /root/ -ENV LIBRARY_PATH "" -ENV LD_LIBRARY_PATH "" - -COPY prepare-environments /prepare-environments - -RUN yum remove -y epel-release - -RUN yum makecache \ - && yum install -y \ - ccache \ - devscripts \ - fakeroot \ - debhelper \ - libtool \ - ncurses-static \ - libtool-ltdl-devel \ - python3-devel \ - bzip2 \ - chrpath \ - && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain nightly \ - && cd /prepare-environments \ - && ./install-cmake.sh \ - && ./install-gcc.sh \ - && ./install-openssl.sh \ - && ./install-libcurl.sh \ - && ./install-grpc.sh \ - && yum remove openssl-devel \ - && yum clean all diff --git a/deprecated/release-centos7/Dockerfile-builder-arm64 b/deprecated/release-centos7/Dockerfile-builder-arm64 deleted file mode 100644 index caaff70ac6b..00000000000 --- a/deprecated/release-centos7/Dockerfile-builder-arm64 +++ /dev/null @@ -1,33 +0,0 @@ -FROM hub.pingcap.net/jenkins/centos7_arm64:cached - -USER root -WORKDIR /root/ - -ENV HOME /root/ -ENV LIBRARY_PATH "" -ENV LD_LIBRARY_PATH "" - -COPY prepare-environments /prepare-environments - -RUN yum remove -y epel-release - -RUN yum makecache \ - && yum install -y \ - ccache \ - devscripts \ - fakeroot \ - debhelper \ - libtool \ - ncurses-static \ - libtool-ltdl-devel \ - python3-devel \ - bzip2 \ - chrpath \ - && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain nightly \ - && cd /prepare-environments \ - && ./install-gcc.sh \ - && ./install-openssl.sh \ - && yum remove openssl-devel \ - && ./install-libcurl.sh \ - && ./install-grpc.sh \ - && yum clean all diff --git a/deprecated/release-centos7/Dockerfile-builder-ci b/deprecated/release-centos7/Dockerfile-builder-ci deleted file mode 100644 index 1dbe8444860..00000000000 --- a/deprecated/release-centos7/Dockerfile-builder-ci +++ /dev/null @@ -1,6 +0,0 @@ -FROM hub.pingcap.net/tiflash/tiflash-builder:latest - -USER root -WORKDIR /root/ - -ENV HOME /root/ diff --git a/deprecated/release-centos7/Dockerfile-tiflash-centos7 b/deprecated/release-centos7/Dockerfile-tiflash-centos7 deleted file mode 100644 index 6977fe3e69e..00000000000 --- a/deprecated/release-centos7/Dockerfile-tiflash-centos7 +++ /dev/null @@ -1,14 +0,0 @@ -FROM hub.pingcap.net/tiflash/centos:centos7.6.1810 - -USER root -WORKDIR /root/ - -ENV HOME /root/ -ENV TZ Asia/Shanghai -ENV LD_LIBRARY_PATH /tiflash - -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -COPY tiflash /tiflash - -ENTRYPOINT ["/tiflash/tiflash", "server"] diff --git a/deprecated/release-centos7/Dockerfile-tiflash-ci b/deprecated/release-centos7/Dockerfile-tiflash-ci deleted file mode 100644 index 542274ce784..00000000000 --- a/deprecated/release-centos7/Dockerfile-tiflash-ci +++ /dev/null @@ -1,5 +0,0 @@ -FROM hub.pingcap.net/tiflash/tiflash-ci-base - -COPY tiflash /tiflash - -ENTRYPOINT ["/tiflash/tiflash", "server"] diff --git a/deprecated/release-centos7/Dockerfile-tiflash-ci-base b/deprecated/release-centos7/Dockerfile-tiflash-ci-base deleted file mode 100644 index 1f7dd56fff1..00000000000 --- a/deprecated/release-centos7/Dockerfile-tiflash-ci-base +++ /dev/null @@ -1,12 +0,0 @@ -FROM hub.pingcap.net/tiflash/centos:7.9.2009-amd64 - -USER root -WORKDIR /root/ - -ENV HOME /root/ -ENV TZ Asia/Shanghai -ENV LD_LIBRARY_PATH /tiflash - -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN yum install mysql -y && yum clean all diff --git a/deprecated/release-centos7/Makefile b/deprecated/release-centos7/Makefile deleted file mode 100644 index f0241d6f792..00000000000 --- a/deprecated/release-centos7/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -image_builder_release: - docker build -f Dockerfile-builder -t hub.pingcap.net/tiflash/tiflash-builder . - -image_builder_ci: - docker tag hub.pingcap.net/tiflash/tiflash-builder hub.pingcap.net/tiflash/tiflash-builder-ci - -build_tiflash_release: - docker run --rm -v $(realpath ..):/tics hub.pingcap.net/tiflash/tiflash-builder /tics/release-centos7/build/build-release.sh - -build_tiflash_ci: - docker run --rm -v $(realpath ..):/tics hub.pingcap.net/tiflash/tiflash-builder-ci /tics/release-centos7/build/build-tiflash-ci.sh - -image_tiflash_release: - docker build -f Dockerfile-tiflash-centos7 -t hub.pingcap.net/tiflash/tiflash-server-centos7 . - -image_tiflash_ci: - docker build -f Dockerfile-tiflash-ci -t hub.pingcap.net/tiflash/tiflash-ci-centos7 . - -image_tiflash_ci_base: - docker build -f Dockerfile-tiflash-ci-base -t hub.pingcap.net/tiflash/tiflash-ci-base . - -image_builder_release_arm64: - docker build -f Dockerfile-builder-arm64 -t hub.pingcap.net/tiflash/tiflash-builder:arm64 . - -build_tiflash_release_arm64: - docker run --rm -v $(realpath ..):/tics hub.pingcap.net/tiflash/tiflash-builder:arm64 /tics/release-centos7/build/build-release.sh diff --git a/deprecated/release-centos7/README.md b/deprecated/release-centos7/README.md deleted file mode 100644 index f57738a846a..00000000000 --- a/deprecated/release-centos7/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# How to build tiflash-release-builder-image - -```shell -# build tiflash-release-builder-image -make image_builder_release - -# push tiflash-release-builder-image to hub -docker push hub.pingcap.net/tiflash/tiflash-builder -``` - -# How to build tiflash-ci-builder-image -```shell -# build tiflash-ci-builder-image -make image_builder_ci - -# push tiflash-ci-builder-image to hub -docker push hub.pingcap.net/tiflash/tiflash-builder-ci -``` - -# How to build tiflash-release-binary -```shell -# build tilfash-release-binary -make build_tiflash_release -``` - -The executable files are located in `tiflash` dir. - -# Deploy Enviroument Requirements - -Following OS are tested OK - -* CentOS 7.6 -* Ubuntu 16.04 and 18.04 - -Your system needs to install - -* GLibC 2.27+ (musl LibC is not OK) -* Libgcc 4.8+ diff --git a/deprecated/release-centos7/build/build-release.sh b/deprecated/release-centos7/build/build-release.sh deleted file mode 100755 index 03fa914b1a6..00000000000 --- a/deprecated/release-centos7/build/build-release.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -CMAKE_PREFIX_PATH=$1 - -set -ueox pipefail - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -SRCPATH=$(cd ${SCRIPTPATH}/../..; pwd -P) - -install_dir="$SRCPATH/release-centos7/tiflash" -if [[ -d "$install_dir" ]]; then rm -rf "${install_dir:?}"/*; else mkdir -p "$install_dir"; fi - -${SCRIPTPATH}/build-tiflash-proxy.sh -${SCRIPTPATH}/build-tiflash-release.sh "" "${CMAKE_PREFIX_PATH}" diff --git a/deprecated/release-centos7/build/build-tiflash-ci.sh b/deprecated/release-centos7/build/build-tiflash-ci.sh deleted file mode 100755 index 8211bd81f87..00000000000 --- a/deprecated/release-centos7/build/build-tiflash-ci.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -command -v ccache >/dev/null 2>&1 -if [[ $? != 0 ]]; then - echo "try to install ccache" - wget http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/ccache.x86_64.rpm - rpm -Uvh ccache.x86_64.rpm -else - echo "ccache has been installed" -fi - -command -v clang-format >/dev/null 2>&1 -if [[ $? != 0 ]]; then - curl -o "/usr/local/bin/clang-format" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/clang-format-12 - chmod +x "/usr/local/bin/clang-format" -else - echo "clang-format has been installed" -fi -clang-format --version - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -BUILD_BRANCH=${BUILD_BRANCH:-master} -ENABLE_FORMAT_CHECK=${ENABLE_FORMAT_CHECK:-false} - -if [[ "${ENABLE_FORMAT_CHECK}" == "true" ]]; then - python3 ${SRCPATH}/format-diff.py --repo_path "${SRCPATH}" --check_formatted --diff_from $(git merge-base origin/${BUILD_BRANCH} HEAD) --dump_diff_files_to "/tmp/tiflash-diff-files.json" - export ENABLE_FORMAT_CHECK=false -fi - -CI_CCACHE_USED_SRCPATH="/build/tics" -export INSTALL_DIR=${INSTALL_DIR:-"$SRCPATH/release-centos7/tiflash"} - -if [[ ${CI_CCACHE_USED_SRCPATH} != ${SRCPATH} ]]; then - rm -rf "${CI_CCACHE_USED_SRCPATH}" - mkdir -p /build && cd /build - cp -r ${SRCPATH} ${CI_CCACHE_USED_SRCPATH} - sh ${CI_CCACHE_USED_SRCPATH}/release-centos7/build/build-tiflash-ci.sh - exit 0 -fi - -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -ENABLE_TESTS=${ENABLE_TESTS:-1} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} -BUILD_UPDATE_DEBUG_CI_CCACHE=${BUILD_UPDATE_DEBUG_CI_CCACHE:-false} -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') -if [[ "${CMAKE_BUILD_TYPE}" != "Debug" ]]; then - ENABLE_TESTS=0 -fi -# https://cd.pingcap.net/blue/organizations/jenkins/build_tiflash_multi_branch/activity/ -# Each time after a new commit merged into target branch, a task about nightly build will be triggered. -# BUILD_UPDATE_DEBUG_CI_CCACHE is set true in order to build and upload ccache. -if [[ "${BUILD_UPDATE_DEBUG_CI_CCACHE}" != "false" ]]; then - echo "====== begin to build & upload ccache for ci debug build ======" - UPDATE_CCACHE=true NPROC=${NPROC} BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=Debug BUILD_UPDATE_DEBUG_CI_CCACHE=false sh ${SRCPATH}/release-centos7/build/build-tiflash-ci.sh - echo "====== finish build & upload ccache for ci debug build ======" -fi - -rm -rf "${INSTALL_DIR}" -mkdir -p "${INSTALL_DIR}" - -USE_CCACHE=ON -rm -rf "${SRCPATH}/.ccache" -cache_file="${SRCPATH}/ccache.tar" -rm -rf "${cache_file}" -curl -o "${cache_file}" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR} -cache_size=$(ls -l "${cache_file}" | awk '{print $5}') -min_size=$((1024000)) -if [[ ${cache_size} -gt ${min_size} ]]; then - echo "try to use ccache to accelerate compile speed" - cd "${SRCPATH}" - tar -xf ccache.tar -fi -ccache -o cache_dir="${SRCPATH}/.ccache" -ccache -o max_size=2G -ccache -o limit_multiple=0.99 -ccache -o hash_dir=false -ccache -o compression=true -ccache -o compression_level=6 -if [[ ${UPDATE_CCACHE} == "false" ]]; then - ccache -o read_only=true -else - ccache -o read_only=false -fi -ccache -z - -rm -rf ${SRCPATH}/libs/libtiflash-proxy -mkdir -p ${SRCPATH}/libs/libtiflash-proxy - -cd ${SRCPATH}/contrib/tiflash-proxy -proxy_git_hash=$(git log -1 --format="%H") -curl -o "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" \ - http://fileserver.pingcap.net/download/builds/pingcap/tiflash-proxy/${proxy_git_hash}/libtiflash_proxy.so -proxy_size=$(ls -l "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" | awk '{print $5}') -min_size=$((102400)) -if [[ ${proxy_size} -lt ${min_size} ]]; then - echo "try to build libtiflash_proxy.so" - export PATH=$PATH:$HOME/.cargo/bin - make release - echo "try to upload libtiflash_proxy.so" - cd target/release - curl -F builds/pingcap/tiflash-proxy/${proxy_git_hash}/libtiflash_proxy.so=@libtiflash_proxy.so http://fileserver.pingcap.net/upload - curl -o "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" http://fileserver.pingcap.net/download/builds/pingcap/tiflash-proxy/${proxy_git_hash}/libtiflash_proxy.so -fi - -chmod 0731 "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" - -BUILD_DIR="$SRCPATH/release-centos7/build-release" -rm -rf ${BUILD_DIR} -mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR} -cmake "$SRCPATH" \ - -DENABLE_TESTS=${ENABLE_TESTS} \ - -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ - -DUSE_CCACHE=${USE_CCACHE} \ - -DDEBUG_WITHOUT_DEBUG_INFO=ON - -make -j ${NPROC} tiflash - -if [[ "${CMAKE_BUILD_TYPE}" = "Debug" && ${ENABLE_TESTS} -ne 0 ]]; then - make -j ${NPROC} page_ctl - make -j ${NPROC} page_stress_testing - # build gtest in `release-centos7/build/build-tiflash-ut-coverage.sh` -fi - -ccache -s - -if [[ ${UPDATE_CCACHE} == "true" ]]; then - cd ${SRCPATH} - rm -rf ccache.tar - tar -cf ccache.tar .ccache - curl -F builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR}=@ccache.tar http://fileserver.pingcap.net/upload -fi - -# Reduce binary size by compressing. -objcopy --compress-debug-sections=zlib-gnu "${BUILD_DIR}/dbms/src/Server/tiflash" -cp -f "${BUILD_DIR}/dbms/src/Server/tiflash" "${INSTALL_DIR}/tiflash" -cp -f "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" "${INSTALL_DIR}/libtiflash_proxy.so" -ldd "${INSTALL_DIR}/tiflash" - -cd "${INSTALL_DIR}" -chrpath -d libtiflash_proxy.so "${INSTALL_DIR}/tiflash" -ldd "${INSTALL_DIR}/tiflash" -ls -lh "${INSTALL_DIR}" diff --git a/deprecated/release-centos7/build/build-tiflash-proxy.sh b/deprecated/release-centos7/build/build-tiflash-proxy.sh deleted file mode 100755 index e7e69865141..00000000000 --- a/deprecated/release-centos7/build/build-tiflash-proxy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=$( - cd ${SCRIPTPATH}/../.. - pwd -P -) -PATH=$PATH:/root/.cargo/bin - -cd ${SRCPATH}/contrib/tiflash-proxy - -make release diff --git a/deprecated/release-centos7/build/build-tiflash-release.sh b/deprecated/release-centos7/build/build-tiflash-release.sh deleted file mode 100755 index 187d3c62f5e..00000000000 --- a/deprecated/release-centos7/build/build-tiflash-release.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -CMAKE_BUILD_TYPE=$1 -CMAKE_PREFIX_PATH=$2 - -if [[ -z ${CMAKE_BUILD_TYPE} ]]; then - CMAKE_BUILD_TYPE="RELWITHDEBINFO" -fi - -DEFINE_CMAKE_PREFIX_PATH="" -if [[ ${CMAKE_PREFIX_PATH} ]]; then - DEFINE_CMAKE_PREFIX_PATH="-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" - echo "CMAKE_PREFIX_PATH is ${CMAKE_PREFIX_PATH}" -fi - -set -ueox pipefail - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -SRCPATH=$(cd ${SCRIPTPATH}/../..; pwd -P) -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} - -INSTALL_DIR="${SRCPATH}/release-centos7/tiflash" - -rm -rf ${SRCPATH}/libs/libtiflash-proxy -mkdir -p ${SRCPATH}/libs/libtiflash-proxy -ln -s ${SRCPATH}/contrib/tiflash-proxy/target/release/libtiflash_proxy.so ${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so - -BUILD_DIR="${SRCPATH}/release-centos7/build-release" -rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR} - -cmake "${SRCPATH}" ${DEFINE_CMAKE_PREFIX_PATH} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DENABLE_TESTING=OFF \ - -DENABLE_TESTS=OFF \ - -Wno-dev \ - -DUSE_CCACHE=OFF - -make -j ${NPROC} tiflash - -# Reduce binary size by compressing. -objcopy --compress-debug-sections=zlib-gnu "${BUILD_DIR}/dbms/src/Server/tiflash" - -cp -f "${BUILD_DIR}/dbms/src/Server/tiflash" "${INSTALL_DIR}/tiflash" -cp -f "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" "${INSTALL_DIR}/libtiflash_proxy.so" - -ldd "${INSTALL_DIR}/tiflash" - -cd "${INSTALL_DIR}" -chrpath -d libtiflash_proxy.so "${INSTALL_DIR}/tiflash" -ldd "${INSTALL_DIR}/tiflash" -ls -lh "${INSTALL_DIR}" diff --git a/deprecated/release-centos7/build/build-tiflash-ut-coverage.sh b/deprecated/release-centos7/build/build-tiflash-ut-coverage.sh deleted file mode 100755 index 90e24373575..00000000000 --- a/deprecated/release-centos7/build/build-tiflash-ut-coverage.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -mkdir -p /build - -command -v ccache >/dev/null 2>&1 -if [[ $? != 0 ]]; then - echo "try to install ccache" - wget http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/ccache.x86_64.rpm - rpm -Uvh ccache.x86_64.rpm -else - echo "ccache has been installed" -fi - -command -v gcovr >/dev/null 2>&1 -if [[ $? != 0 ]]; then - echo "try to install gcovr" - pushd /tmp - curl -o /tmp/gcovr.tar http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/gcovr.tar - rm -rf gcovr && tar xf gcovr.tar && rm -rf gcovr.tar - rm -rf /usr/bin/gcovr && ln -s /tmp/gcovr/gcovr /usr/bin/gcovr - popd -else - echo "gcovr has been installed" -fi - -set -ueox pipefail -BASE_NAME=$(basename $0) -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=Debug -BUILD_BRANCH=${BUILD_BRANCH:-master} - -CI_CCACHE_USED_SRCPATH="/build/tics" -export INSTALL_DIR=${INSTALL_DIR:-"/build/release-centos7/tiflash"} - -if [[ ${CI_CCACHE_USED_SRCPATH} != ${SRCPATH} ]]; then - rm -rf "${CI_CCACHE_USED_SRCPATH}" - cd /build - cp -r ${SRCPATH} ${CI_CCACHE_USED_SRCPATH} - sh ${CI_CCACHE_USED_SRCPATH}/release-centos7/build/${BASE_NAME} - exit 0 -fi - -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}-gcov.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') - -rm -rf "${INSTALL_DIR}" -mkdir -p "${INSTALL_DIR}" - -USE_CCACHE=ON -rm -rf "${SRCPATH}/.ccache" -cache_file="${SRCPATH}/ccache.tar" -rm -rf "${cache_file}" -curl -o "${cache_file}" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR} -cache_size=$(ls -l "${cache_file}" | awk '{print $5}') -min_size=$((1024000)) -if [[ ${cache_size} -gt ${min_size} ]]; then - echo "try to use ccache to accelerate compile speed" - cd "${SRCPATH}" - tar -xf ccache.tar -fi -ccache -o cache_dir="${SRCPATH}/.ccache" -ccache -o max_size=2G -ccache -o limit_multiple=0.99 -ccache -o hash_dir=false -ccache -o compression=true -ccache -o compression_level=6 -if [[ ${UPDATE_CCACHE} == "false" ]]; then - ccache -o read_only=true -else - ccache -o read_only=false -fi -ccache -z - -rm -rf ${SRCPATH}/libs/libtiflash-proxy -mkdir -p ${SRCPATH}/libs/libtiflash-proxy - -cd ${SRCPATH}/contrib/tiflash-proxy -proxy_git_hash=$(git log -1 --format="%H") - -while [[ true ]]; do - curl -o "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" \ - http://fileserver.pingcap.net/download/builds/pingcap/tiflash-proxy/${proxy_git_hash}/libtiflash_proxy.so - proxy_size=$(ls -l "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" | awk '{print $5}') - if [[ ${proxy_size} -lt $((102400)) ]]; then - echo "fail to get ci build tiflash proxy, sleep 60s" - sleep 60 - else - chmod 0731 "${SRCPATH}/libs/libtiflash-proxy/libtiflash_proxy.so" - break - fi -done - -BUILD_DIR="/build/release-centos7/build-release" -rm -rf ${BUILD_DIR} -mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR} -cmake "${SRCPATH}" \ - -DENABLE_TESTS=ON \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DUSE_CCACHE=${USE_CCACHE} \ - -DTEST_COVERAGE=ON \ - -DTEST_COVERAGE_XML=ON \ - -DDEBUG_WITHOUT_DEBUG_INFO=ON - -make -j ${NPROC} gtests_dbms gtests_libcommon gtests_libdaemon -mv "${BUILD_DIR}/dbms/gtests_dbms" "${INSTALL_DIR}/" -mv "${BUILD_DIR}/libs/libcommon/src/tests/gtests_libcommon" "${INSTALL_DIR}/" -mv "${BUILD_DIR}/libs/libdaemon/src/tests/gtests_libdaemon" "${INSTALL_DIR}/" - -ccache -s - -ls -lh "${INSTALL_DIR}" - -if [[ ${UPDATE_CCACHE} == "true" ]]; then - cd ${SRCPATH} - rm -rf ccache.tar - tar -cf ccache.tar .ccache - curl -F builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR}=@ccache.tar http://fileserver.pingcap.net/upload - exit 0 -fi diff --git a/deprecated/release-centos7/build/env.sh b/deprecated/release-centos7/build/env.sh deleted file mode 100644 index 3ed8c7f01a3..00000000000 --- a/deprecated/release-centos7/build/env.sh +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -TIFLASH_CI_BUILD_URI_PREFIX="builds/pingcap/tiflash/ci-cache/tmp/pr-build" -TIFLASH_CI_BUILD_PRE_FIX="http://fileserver.pingcap.net/download/${TIFLASH_CI_BUILD_URI_PREFIX}" -CI_BUILD_INFO_PRE_FIX="commit hash value" diff --git a/deprecated/release-centos7/build/fetch-ci-build.sh b/deprecated/release-centos7/build/fetch-ci-build.sh deleted file mode 100755 index e5033f641d3..00000000000 --- a/deprecated/release-centos7/build/fetch-ci-build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -source ${SCRIPTPATH}/env.sh - -DOWNLOAD_TAR=${DOWNLOAD_TAR:-false} - -rm -rf "${TAR_PATH}" -mkdir -p "${TAR_PATH}" -cd "${TAR_PATH}" -TAR_BIN_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/tiflash.tar.gz" -COMMIT_HASH_FILE_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/commit-hash" -while [[ true ]]; do - curl -o ./commit-hash ${COMMIT_HASH_FILE_URL} - if [[ $(grep -c "${CI_BUILD_INFO_PRE_FIX}" ./commit-hash) -ne '0' ]]; then - COMMIT_HASH_VAL=$(tail -n -1 ./commit-hash) - if [[ ${COMMIT_HASH_VAL} == ${COMMIT_HASH} ]]; then - if [[ ${DOWNLOAD_TAR} == "true" ]]; then - curl -o ./tiflash.tar.gz ${TAR_BIN_URL} - MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') - EXPECT_MD5_SUM=$(tail -n +2 ./commit-hash | head -n 1) - if [[ ${MD5_SUM} != ${EXPECT_MD5_SUM} ]]; then - echo "wrong md5sum got ${MD5_SUM} but expect ${EXPECT_MD5_SUM}" - exit -1 - fi - tar -zxf tiflash.tar.gz - fi - break - fi - fi - echo "fail to get ci build tiflash binary, sleep 60s" - sleep 60 -done diff --git a/deprecated/release-centos7/build/fix_compile_commands.py b/deprecated/release-centos7/build/fix_compile_commands.py deleted file mode 100755 index c4973afddf7..00000000000 --- a/deprecated/release-centos7/build/fix_compile_commands.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import os - - -def check_files(data, load_diff_files_from): - data_map = {e['file']: e for e in data} - remain = [] - try: - d = json.load(open(load_diff_files_from, 'r')) - except Exception: - return data - files = d['files'] - repo = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../../') - for e in files: - e = '{}{}'.format(repo, e) - e = data_map.get(e) - if e: - remain.append(e) - print('original {} files, remain {} files for static analysis'.format(len(data), len(remain))) - return remain - - -def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--file_path', help='path of `compile_commands.json`', required=True) - parser.add_argument('--includes', help='external include directories, split by `,`', required=True) - parser.add_argument('--load_diff_files_from', help='load diff file names from specific path', default=None) - args = parser.parse_args() - - includes = args.includes.split(',') - print('start to fix `{}` by adding external include directories {}'.format(args.file_path, includes)) - includes = ' '.join(['-I{}'.format(a) for a in includes]) - data = json.load(open(args.file_path, 'r')) - - if args.load_diff_files_from: - data = check_files(data, args.load_diff_files_from) - - for line in data: - line['command'] += ' ' + includes - json.dump(data, open(args.file_path, 'w'), indent=4) - print('process {} elements'.format(len(data))) - - -if __name__ == '__main__': - main() diff --git a/deprecated/release-centos7/build/run-clang-tidy.py b/deprecated/release-centos7/build/run-clang-tidy.py deleted file mode 100755 index e49fb2fa6ea..00000000000 --- a/deprecated/release-centos7/build/run-clang-tidy.py +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env python -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# ===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===# -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -# ===-----------------------------------------------------------------------===# - -# copy from https://clang.llvm.org/extra/doxygen/run-clang-tidy_8py_source.html - -# FIXME: Integrate with clang-tidy-diff.py - - -""" -Parallel clang-tidy runner -========================== - -Runs clang-tidy over all files in a compilation database. Requires clang-tidy -and clang-apply-replacements in $PATH. - -Example invocations. -- Run clang-tidy on all files in the current working directory with a default - set of checks and show warnings in the cpp files and all project headers. - run-clang-tidy.py $PWD - -- Fix all header guards. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard - -- Fix all header guards included from clang-tidy and header guards - for clang-tidy headers. - run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ - -header-filter=extra/clang-tidy - -Compilation database setup: -http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html -""" - -from __future__ import print_function - -import argparse -import glob -import json -import multiprocessing -import os -import re -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback - -try: - import yaml -except ImportError: - yaml = None - -is_py2 = sys.version[0] == '2' - -if is_py2: - import Queue as queue -else: - import queue as queue - - -def find_compilation_database(path): - """Adjusts the directory until a compilation database is found.""" - result = './' - while not os.path.isfile(os.path.join(result, path)): - if os.path.realpath(result) == '/': - print('Error: could not find compilation database.') - sys.exit(1) - result += '../' - return os.path.realpath(result) - - -def make_absolute(f, directory): - if os.path.isabs(f): - return f - return os.path.normpath(os.path.join(directory, f)) - - -def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path, - header_filter, allow_enabling_alpha_checkers, - extra_arg, extra_arg_before, quiet, config, - line_filter): - """Gets a command line for clang-tidy.""" - start = [clang_tidy_binary] - if allow_enabling_alpha_checkers: - start.append('-allow-enabling-analyzer-alpha-checkers') - if header_filter is not None: - start.append('-header-filter=' + header_filter) - if line_filter is not None: - start.append('-line-filter=' + line_filter) - if checks: - start.append('-checks=' + checks) - if tmpdir is not None: - start.append('-export-fixes') - # Get a temporary file. We immediately close the handle so clang-tidy can - # overwrite it. - (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) - os.close(handle) - start.append(name) - for arg in extra_arg: - start.append('-extra-arg=%s' % arg) - for arg in extra_arg_before: - start.append('-extra-arg-before=%s' % arg) - start.append('-p=' + build_path) - if quiet: - start.append('-quiet') - if config: - start.append('-config=' + config) - start.append(f) - return start - - -def merge_replacement_files(tmpdir, mergefile): - """Merge all replacement files in a directory into a single file""" - # The fixes suggested by clang-tidy >= 4.0.0 are given under - # the top level key 'Diagnostics' in the output yaml files - mergekey = "Diagnostics" - merged = [] - for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): - content = yaml.safe_load(open(replacefile, 'r')) - if not content: - continue # Skip empty files. - merged.extend(content.get(mergekey, [])) - - if merged: - # MainSourceFile: The key is required by the definition inside - # include/clang/Tooling/ReplacementsYaml.h, but the value - # is actually never used inside clang-apply-replacements, - # so we set it to '' here. - output = {'MainSourceFile': '', mergekey: merged} - with open(mergefile, 'w') as out: - yaml.safe_dump(output, out) - else: - # Empty the file: - open(mergefile, 'w').close() - - -def check_clang_apply_replacements_binary(args): - """Checks if invoking supplied clang-apply-replacements binary works.""" - try: - subprocess.check_call([args.clang_apply_replacements_binary, '--version']) - except: - print('Unable to run clang-apply-replacements. Is clang-apply-replacements ' - 'binary correctly specified?', file=sys.stderr) - traceback.print_exc() - sys.exit(1) - - -def apply_fixes(args, tmpdir): - """Calls clang-apply-fixes on a given directory.""" - invocation = [args.clang_apply_replacements_binary] - if args.format: - invocation.append('-format') - if args.style: - invocation.append('-style=' + args.style) - invocation.append(tmpdir) - subprocess.call(invocation) - - -def run_tidy(args, tmpdir, build_path, queue, lock, failed_files): - """Takes filenames out of queue and runs clang-tidy on them.""" - while True: - name = queue.get() - invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks, - tmpdir, build_path, args.header_filter, - args.allow_enabling_alpha_checkers, - args.extra_arg, args.extra_arg_before, - args.quiet, args.config, args.line_filter) - - proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, err = proc.communicate() - if proc.returncode != 0: - if proc.returncode < 0: - msg = "%s: terminated by signal %d\n" % (name, -proc.returncode) - err += msg.encode('utf-8') - failed_files.append(name) - with lock: - sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8')) - if len(err) > 0: - sys.stdout.flush() - sys.stderr.write(err.decode('utf-8')) - queue.task_done() - - -def main(): - parser = argparse.ArgumentParser(description='Runs clang-tidy over all files ' - 'in a compilation database. Requires ' - 'clang-tidy and clang-apply-replacements in ' - '$PATH.') - parser.add_argument('-allow-enabling-alpha-checkers', - action='store_true', help='allow alpha checkers from ' - 'clang-analyzer.') - parser.add_argument('-clang-tidy-binary', metavar='PATH', - default='clang-tidy', - help='path to clang-tidy binary') - parser.add_argument('-clang-apply-replacements-binary', metavar='PATH', - default='clang-apply-replacements', - help='path to clang-apply-replacements binary') - parser.add_argument('-checks', default=None, - help='checks filter, when not specified, use clang-tidy ' - 'default') - parser.add_argument('-config', default=None, - help='Specifies a configuration in YAML/JSON format: ' - ' -config="{Checks: \'*\', ' - ' CheckOptions: [{key: x, ' - ' value: y}]}" ' - 'When the value is empty, clang-tidy will ' - 'attempt to find a file named .clang-tidy for ' - 'each source file in its parent directories.') - parser.add_argument('-header-filter', default=None, - help='regular expression matching the names of the ' - 'headers to output diagnostics from. Diagnostics from ' - 'the main file of each translation unit are always ' - 'displayed.') - parser.add_argument('-line-filter', default=None, - help='List of files with line ranges to filter the' - 'warnings.') - if yaml: - parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes', - help='Create a yaml file to store suggested fixes in, ' - 'which can be applied with clang-apply-replacements.') - parser.add_argument('-j', type=int, default=0, - help='number of tidy instances to be run in parallel.') - parser.add_argument('--files', nargs='*', default=['.*'], - help='files to be processed (regex on path)') - parser.add_argument('-fix', action='store_true', help='apply fix-its') - parser.add_argument('-format', action='store_true', help='Reformat code ' - 'after applying fixes') - parser.add_argument('-style', default='file', help='The style of reformat ' - 'code after applying fixes') - parser.add_argument('-p', dest='build_path', - help='Path used to read a compile command database.') - parser.add_argument('-extra-arg', dest='extra_arg', - action='append', default=[], - help='Additional argument to append to the compiler ' - 'command line.') - parser.add_argument('-extra-arg-before', dest='extra_arg_before', - action='append', default=[], - help='Additional argument to prepend to the compiler ' - 'command line.') - parser.add_argument('-quiet', action='store_true', - help='Run clang-tidy in quiet mode') - args = parser.parse_args() - - db_path = 'compile_commands.json' - - if args.build_path is not None: - build_path = args.build_path - else: - # Find our database - build_path = find_compilation_database(db_path) - - try: - invocation = [args.clang_tidy_binary, '-list-checks'] - if args.allow_enabling_alpha_checkers: - invocation.append('-allow-enabling-analyzer-alpha-checkers') - invocation.append('-p=' + build_path) - if args.checks: - invocation.append('-checks=' + args.checks) - invocation.append('-') - if args.quiet: - # Even with -quiet we still want to check if we can call clang-tidy. - with open(os.devnull, 'w') as dev_null: - subprocess.check_call(invocation, stdout=dev_null) - else: - subprocess.check_call(invocation) - except: - print("Unable to run clang-tidy.", file=sys.stderr) - sys.exit(1) - - # Load the database and extract all files. - database = json.load(open(os.path.join(build_path, db_path))) - files = [make_absolute(entry['file'], entry['directory']) - for entry in database] - - max_task = args.j - if max_task == 0: - max_task = multiprocessing.cpu_count() - - tmpdir = None - if args.fix or (yaml and args.export_fixes): - check_clang_apply_replacements_binary(args) - tmpdir = tempfile.mkdtemp() - - # Build up a big regexy filter from all command line arguments. - file_name_re = re.compile('|'.join(args.files)) - - return_code = 0 - try: - # Spin up a bunch of tidy-launching threads. - task_queue = queue.Queue(max_task) - # List of files with a non-zero return code. - failed_files = [] - lock = threading.Lock() - for _ in range(max_task): - t = threading.Thread(target=run_tidy, - args=(args, tmpdir, build_path, task_queue, lock, failed_files)) - t.daemon = True - t.start() - - # Fill the queue with files. - for name in files: - if file_name_re.search(name): - task_queue.put(name) - - # Wait for all threads to be done. - task_queue.join() - if len(failed_files): - return_code = 1 - - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print('\nCtrl-C detected, goodbye.') - if tmpdir: - shutil.rmtree(tmpdir) - os.kill(0, 9) - - if yaml and args.export_fixes: - print('Writing fixes to ' + args.export_fixes + ' ...') - try: - merge_replacement_files(tmpdir, args.export_fixes) - except: - print('Error exporting fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if args.fix: - print('Applying fixes ...') - try: - apply_fixes(args, tmpdir) - except: - print('Error applying fixes.\n', file=sys.stderr) - traceback.print_exc() - return_code = 1 - - if tmpdir: - shutil.rmtree(tmpdir) - sys.exit(return_code) - - -if __name__ == '__main__': - main() diff --git a/deprecated/release-centos7/build/run-ut.sh b/deprecated/release-centos7/build/run-ut.sh deleted file mode 100755 index 21c95b2c6b1..00000000000 --- a/deprecated/release-centos7/build/run-ut.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" - -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -set -x - -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -OUTPUT_XML=${OUTPUT_XML:-false} - -INSTALL_DIR=${INSTALL_DIR:-"/build/release-centos7/tiflash"} - -rm -rf /tests && ln -s ${SRCPATH}/tests /tests -rm -rf /tiflash && ln -s ${INSTALL_DIR} /tiflash - -source /tests/docker/util.sh - -show_env - -ENV_VARS_PATH=/tests/docker/_env.sh OUTPUT_XML=${OUTPUT_XML} NPROC=${NPROC} /tests/run-gtest.sh diff --git a/deprecated/release-centos7/build/static-analysis.sh b/deprecated/release-centos7/build/static-analysis.sh deleted file mode 100755 index c4b8f350ce7..00000000000 --- a/deprecated/release-centos7/build/static-analysis.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -function check_src_path() { - SRC_PATH=$1 - if [[ ! -d "${SRC_PATH}" ]]; then - echo "no header file in ${SRC_PATH}" - exit 1 - fi -} - -ENABLE_CLANG_TIDY_CHECK=${ENABLE_CLANG_TIDY_CHECK:-true} - -if [[ "${ENABLE_CLANG_TIDY_CHECK}" == "true" ]]; then - command -v clang-tidy >/dev/null 2>&1 - if [[ $? != 0 ]]; then - curl -o "/usr/local/bin/clang-tidy" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/clang-tidy-12 - chmod +x "/usr/local/bin/clang-tidy" - # http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/clang-libs-12.0.0-1.module_el8.5.0+840+21214faf.x86_64.rpm - curl -o "/tmp/lib64-clang-12-include.tar.gz" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/lib64-clang-12-include.tar.gz - pushd /tmp - tar zxf lib64-clang-12-include.tar.gz - popd - else - echo "clang-tidy has been installed" - fi - clang-tidy --version -fi - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} - -# check with clang-tidy after build to generate kvproto & tipb & re2_st. -if [[ "${ENABLE_CLANG_TIDY_CHECK}" == "true" ]]; then - BUILD_DIR="${SRCPATH}/release-centos7/build-release" - # DO NOT remove build cache for `re2_st`. - #rm -rf ${BUILD_DIR} - - check_src_path "${SRCPATH}/contrib/kvproto/cpp/kvproto" - check_src_path "${SRCPATH}/contrib/tipb/cpp/tipb" - check_src_path "${BUILD_DIR}/contrib/re2_st/re2_st" - - cd ${BUILD_DIR} - cmake "${SRCPATH}" \ - -DENABLE_TESTS=0 \ - -DCMAKE_BUILD_TYPE=RELWITHDEBINFO \ - -DUSE_CCACHE=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - python3 ${SRCPATH}/release-centos7/build/fix_compile_commands.py --file_path=${BUILD_DIR}/compile_commands.json --includes=/tmp/usr/lib64/clang/12.0.0/include --load_diff_files_from "/tmp/tiflash-diff-files.json" - python3 ${SRCPATH}/release-centos7/build/run-clang-tidy.py -p ${BUILD_DIR} -j ${NPROC} --files ".*/tics/dbms/*" - ### - export ENABLE_CLANG_TIDY_CHECK=false -fi diff --git a/deprecated/release-centos7/build/upload-ci-build.sh b/deprecated/release-centos7/build/upload-ci-build.sh deleted file mode 100755 index e36799f489d..00000000000 --- a/deprecated/release-centos7/build/upload-ci-build.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} -source ${SCRIPTPATH}/env.sh -TAR_BIN_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/tiflash.tar.gz" -COMMIT_HASH_FILE_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/commit-hash" - -build_dir="$SRCPATH/release-centos7" -cd ${build_dir} -tar -zcf tiflash.tar.gz ./tiflash -MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') -echo "${CI_BUILD_INFO_PRE_FIX}" >./commit-hash -echo "${MD5_SUM}" >>./commit-hash -echo "${COMMIT_HASH}" >>./commit-hash - -curl -F ${TAR_BIN_URI}=@tiflash.tar.gz http://fileserver.pingcap.net/upload -curl -F ${COMMIT_HASH_FILE_URI}=@commit-hash http://fileserver.pingcap.net/upload diff --git a/deprecated/release-centos7/build/upload-ut-coverage.sh b/deprecated/release-centos7/build/upload-ut-coverage.sh deleted file mode 100755 index d22dcd72514..00000000000 --- a/deprecated/release-centos7/build/upload-ut-coverage.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SRCPATH=/build/tics -BUILD_DIR=/build/release-centos7/build-release -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} - -gcovr --xml -r ${SRCPATH} \ - -e "/usr/include/*" -e "/usr/local/*" -e "/usr/lib/*" \ - -e "${SRCPATH}/contrib/*" \ - -e "${SRCPATH}/dbms/src/Debug/*" \ - -e "${SRCPATH}/dbms/src/Client/*" \ - --object-directory=${BUILD_DIR} -o /tmp/tiflash_gcovr_coverage.xml -j ${NPROC} -s >/tmp/tiflash_gcovr_coverage.res - -cat /tmp/tiflash_gcovr_coverage.res diff --git a/deprecated/release-centos7/prepare-environments/grpc-arm-compile.patch b/deprecated/release-centos7/prepare-environments/grpc-arm-compile.patch deleted file mode 100644 index 1da2b1bd072..00000000000 --- a/deprecated/release-centos7/prepare-environments/grpc-arm-compile.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index b7f468f..a76ed75 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -117,7 +117,7 @@ endif() - if(CMAKE_COMPILER_IS_GNUCXX OR CLANG) - # Note clang-cl is odd and sets both CLANG and MSVC. We base our configuration - # primarily on our normal Clang one. -- set(C_CXX_FLAGS "-Werror -Wformat=2 -Wsign-compare -Wmissing-field-initializers -Wwrite-strings -Wvla") -+ set(C_CXX_FLAGS "-Wformat=2 -Wsign-compare -Wmissing-field-initializers -Wwrite-strings -Wvla") - if(MSVC) - # clang-cl sets different default warnings than clang. It also treats -Wall - # as -Weverything, to match MSVC. Instead -W3 is the alias for -Wall. diff --git a/deprecated/release-centos7/prepare-environments/install-cmake.sh b/deprecated/release-centos7/prepare-environments/install-cmake.sh deleted file mode 100755 index 3b115c38435..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-cmake.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail - -VERSION="3.10" -COMPLETE_VERSION="3.10.2" - -cd ~ -wget "https://download.pingcap.org/cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz" -tar zxvf cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz - -rm cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz -yum remove cmake -y - -ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/ccmake /usr/bin/ccmake -ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake -ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cmake-gui /usr/bin/cmake-gui -ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cpack /usr/bin/cpack -ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/ctest /usr/bin/ctest - -cmake --version diff --git a/deprecated/release-centos7/prepare-environments/install-gcc.sh b/deprecated/release-centos7/prepare-environments/install-gcc.sh deleted file mode 100755 index 7bf2010f9e1..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-gcc.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -e - -VERSION="gcc-7.4.0" -THREADS=$(nproc || grep -c ^processor /proc/cpuinfo) - -cd ~ -mkdir gcc -cd gcc -wget https://mirrors.ustc.edu.cn/gnu/gcc/${VERSION}/${VERSION}.tar.gz -tar xf "${VERSION}.tar.gz" -rm "${VERSION}.tar.gz" - -cd ${VERSION} -./contrib/download_prerequisites -mkdir gccbuild -cd gccbuild -../configure --enable-languages=c,c++ --disable-multilib -make -j $THREADS -make install -ln -s /usr/local/bin/gcc /usr/local/bin/cc - -yum remove -y gcc - -echo "/usr/local/lib64" | tee /etc/ld.so.conf.d/10_local-lib64.conf -ldconfig -gcc --version - -rm -rf ~/gcc diff --git a/deprecated/release-centos7/prepare-environments/install-grpc.sh b/deprecated/release-centos7/prepare-environments/install-grpc.sh deleted file mode 100755 index 9b48b4a6e6b..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-grpc.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -e - -VERSION="v1.26.0" -THREADS=$(nproc || grep -c ^processor /proc/cpuinfo) - -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -source ${SCRIPTPATH}/util.sh -is_arm=$(check_arm_arch) -CMAKE_FLAGS_ADD="" - -cd ~ -git clone https://github.com/grpc/grpc.git -b ${VERSION} -cd grpc -git submodule update --init - -if [[ ${is_arm} == 1 ]]; then - CMAKE_FLAGS_ADD="-DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 -DCOMPILE_HAVE_GNU_POSIX_REGEX=0" - cd third_party/boringssl - git apply ${SCRIPTPATH}/grpc-arm-compile.patch -fi - -cd ~/grpc -mkdir .build -cd .build -cmake .. -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS_ADD} -make -j ${THREADS} -make install - - -cd ~/grpc -rm -rf .build -mkdir .build -cd .build -cmake .. -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release ${CMAKE_FLAGS_ADD} -make -j ${THREADS} -make install - -protoc --version - -rm -rf ~/grpc diff --git a/deprecated/release-centos7/prepare-environments/install-libcurl.sh b/deprecated/release-centos7/prepare-environments/install-libcurl.sh deleted file mode 100755 index c9b144f4a0a..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-libcurl.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail - -NPROC=$(nproc || grep -c ^processor /proc/cpuinfo) - -wget https://curl.haxx.se/download/curl-7.74.0.tar.gz -tar zxf curl-7.74.0.tar.gz -cd curl-7.74.0 -./configure --enable-shared=false --enable-static=true -make -j ${NPROC} && make install - diff --git a/deprecated/release-centos7/prepare-environments/install-llvm.sh b/deprecated/release-centos7/prepare-environments/install-llvm.sh deleted file mode 100755 index b21ac5ad311..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-llvm.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -e - -BRANCH="tags/RELEASE_500/final" -THREADS=$(nproc || grep -c ^processor /proc/cpuinfo) - -yum install -y subversion - -cd ~ -mkdir llvm -cd llvm -svn co "http://llvm.org/svn/llvm-project/llvm/${BRANCH}" llvm - -cd llvm/tools -svn co "http://llvm.org/svn/llvm-project/cfe/${BRANCH}" clang -svn co "http://llvm.org/svn/llvm-project/lld/${BRANCH}" lld -svn co "http://llvm.org/svn/llvm-project/polly/${BRANCH}" polly - -cd clang/tools -svn co "http://llvm.org/svn/llvm-project/clang-tools-extra/${BRANCH}" extra - -cd ../../../.. -cd llvm/projects/ -svn co "http://llvm.org/svn/llvm-project/compiler-rt/${BRANCH}" compiler-rt -svn co "http://llvm.org/svn/llvm-project/libcxx/${BRANCH}" libcxx -svn co "http://llvm.org/svn/llvm-project/libcxxabi/${BRANCH}" libcxxabi - -cd ../.. -mkdir build -cd build/ -cmake -D CMAKE_BUILD_TYPE:STRING=Release ../llvm -make -j $THREADS -make install - -yum remove -y subversion -rm -rf ~/llvm diff --git a/deprecated/release-centos7/prepare-environments/install-mysql-dev.sh b/deprecated/release-centos7/prepare-environments/install-mysql-dev.sh deleted file mode 100755 index 14c2f66445c..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-mysql-dev.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail - -cd ~ -mkdir mysql -cd mysql - -wget http://dev.mysql.com/get/mysql57-community-release-el7-9.noarch.rpm -yum -y --nogpgcheck install mysql57-community-release-el7-9.noarch.rpm - -yum -y install mysql-community-devel -ln -s /usr/lib64/mysql/libmysqlclient.a /usr/lib64/libmysqlclient.a - -yum clean all -rm -rf ~/mysql diff --git a/deprecated/release-centos7/prepare-environments/install-openssl.sh b/deprecated/release-centos7/prepare-environments/install-openssl.sh deleted file mode 100755 index 173b49e83ce..00000000000 --- a/deprecated/release-centos7/prepare-environments/install-openssl.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail - -wget https://www.openssl.org/source/old/1.1.0/openssl-1.1.0f.tar.gz -tar xvf openssl-1.1.0f.tar.gz -cd openssl-1.1.0f -./config -fPIC no-shared no-afalgeng --prefix=/usr/local/opt/openssl --openssldir=/usr/local/opt/openssl -static -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -make -j ${NPROC} -make install diff --git a/deprecated/release-centos7/prepare-environments/util.sh b/deprecated/release-centos7/prepare-environments/util.sh deleted file mode 100644 index 1a8390f5244..00000000000 --- a/deprecated/release-centos7/prepare-environments/util.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -function check_arm_arch() { - local ARCH=$(uname -i) - if [[ "$ARCH" =~ ^(aarch64.*|AARCH64.*) || "$ARCH" == arm* ]]; then - echo 1 - else - echo 0 - fi -} - -export -f check_arm_arch diff --git a/format-diff.py b/format-diff.py index 3560384d04f..8aac65ec5a7 100755 --- a/format-diff.py +++ b/format-diff.py @@ -36,14 +36,19 @@ def main(): default=os.path.dirname(os.path.abspath(__file__))) parser.add_argument('--suffix', help='suffix of files to format, split by space', default=' '.join(default_suffix)) - parser.add_argument('--ignore_suffix', help='ignore files with suffix, split by space') - parser.add_argument('--diff_from', help='commit hash/branch to check git diff', default='HEAD') - parser.add_argument('--check_formatted', help='exit -1 if NOT formatted', action='store_true') - parser.add_argument('--dump_diff_files_to', help='dump diff file names to specific path', default=None) + parser.add_argument('--ignore_suffix', + help='ignore files with suffix, split by space') + parser.add_argument( + '--diff_from', help='commit hash/branch to check git diff', default='HEAD') + parser.add_argument('--check_formatted', + help='exit -1 if NOT formatted', action='store_true') + parser.add_argument('--dump_diff_files_to', + help='dump diff file names to specific path', default=None) args = parser.parse_args() default_suffix = args.suffix.strip().split(' ') if args.suffix else [] - ignore_suffix = args.ignore_suffix.strip().split(' ') if args.ignore_suffix else [] + ignore_suffix = args.ignore_suffix.strip().split( + ' ') if args.ignore_suffix else [] tics_repo_path = args.repo_path if not os.path.isabs(tics_repo_path): raise Exception("path of repo should be absolute") @@ -52,7 +57,8 @@ def main(): os.chdir(tics_repo_path) files_to_check = run_cmd('git diff HEAD --name-only') if args.diff_from == 'HEAD' else run_cmd( 'git diff {} --name-only'.format(args.diff_from)) - files_to_check = [os.path.join(tics_repo_path, s.strip()) for s in files_to_check] + files_to_check = [os.path.join(tics_repo_path, s.strip()) + for s in files_to_check] files_to_format = [] for f in files_to_check: if not any([f.endswith(e) for e in default_suffix]): @@ -69,21 +75,23 @@ def main(): if args.dump_diff_files_to: da = [e[len(tics_repo_path):] for e in files_to_format] - json.dump({'files': da, 'repo': tics_repo_path}, open(args.dump_diff_files_to, 'w')) - print('dump {} modified files info to {}'.format(len(da), args.dump_diff_files_to)) + json.dump({'files': da, 'repo': tics_repo_path}, + open(args.dump_diff_files_to, 'w')) + print('dump {} modified files info to {}'.format( + len(da), args.dump_diff_files_to)) if files_to_format: print('Files to format:\n {}'.format('\n '.join(files_to_format))) - - if args.check_formatted: - cmd = 'clang-format -i {}'.format(' '.join(files_to_format)) + for file in files_to_format: + cmd = 'clang-format -i {}'.format(file) if subprocess.Popen(cmd, shell=True, cwd=tics_repo_path).wait(): exit(-1) + + if args.check_formatted: diff_res = run_cmd('git diff --name-only') if diff_res: print('Error: found files NOT formatted') print(''.join(diff_res)) - print(''.join(run_cmd('git diff'))) exit(-1) else: print("Format check passed") diff --git a/release-centos7-llvm/scripts/build-tiflash-ci.sh b/release-centos7-llvm/scripts/build-tiflash-ci.sh deleted file mode 100755 index 4a59750c81c..00000000000 --- a/release-centos7-llvm/scripts/build-tiflash-ci.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -INSTALL_DIR="${SRCPATH}/${INSTALL_DIR_SUFFIX}" # use original path - -BUILD_UPDATE_DEBUG_CI_CCACHE=${BUILD_UPDATE_DEBUG_CI_CCACHE:-false} -CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -BUILD_BRANCH=${BUILD_BRANCH:-master} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} - -ENABLE_FORMAT_CHECK=${ENABLE_FORMAT_CHECK:-false} -if [[ "${ENABLE_FORMAT_CHECK}" == "true" ]]; then - BUILD_BRANCH=${BUILD_BRANCH} sh ${SRCPATH}/release-centos7-llvm/scripts/run-format-check.sh -fi - -if [[ ${CI_CCACHE_USED_SRCPATH} != ${SRCPATH} ]]; then - rm -rf "${CI_CCACHE_USED_SRCPATH}" - mkdir -p /build && cd /build - cp -r ${SRCPATH} ${CI_CCACHE_USED_SRCPATH} -fi - -# https://cd.pingcap.net/blue/organizations/jenkins/build_tiflash_multi_branch/activity/ -# Each time after a new commit merged into target branch, a task about nightly build will be triggered. -# BUILD_UPDATE_DEBUG_CI_CCACHE is set true in order to build and upload ccache. -if [[ "${BUILD_UPDATE_DEBUG_CI_CCACHE}" != "false" ]]; then - echo "====== begin to build & upload ccache for ci debug build ======" - UPDATE_CCACHE=true BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=Debug sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-prepare.sh - BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=Debug NPROC=${NPROC} INSTALL_DIR=${INSTALL_DIR} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-build.sh - UPDATE_CCACHE=true BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=Debug INSTALL_DIR=${INSTALL_DIR} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-finish.sh - echo "====== finish build & upload ccache for ci debug build ======" -fi - -UPDATE_CCACHE=${UPDATE_CCACHE} BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-prepare.sh -BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} NPROC=${NPROC} INSTALL_DIR=${INSTALL_DIR} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-build.sh -UPDATE_CCACHE=${UPDATE_CCACHE} BUILD_BRANCH=${BUILD_BRANCH} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} INSTALL_DIR=${INSTALL_DIR} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ci-finish.sh diff --git a/release-centos7-llvm/scripts/build-tiflash-ut-coverage.sh b/release-centos7-llvm/scripts/build-tiflash-ut-coverage.sh deleted file mode 100755 index f100f1e775b..00000000000 --- a/release-centos7-llvm/scripts/build-tiflash-ut-coverage.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -BUILD_BRANCH=${BUILD_BRANCH:-master} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -if [[ ${CI_CCACHE_USED_SRCPATH} != ${SRCPATH} ]]; then - rm -rf "${CI_CCACHE_USED_SRCPATH}" - mkdir -p /build && cd /build - cp -r ${SRCPATH} ${CI_CCACHE_USED_SRCPATH} -fi - -UPDATE_CCACHE=${UPDATE_CCACHE} BUILD_BRANCH=${BUILD_BRANCH} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ut-coverage-prepare.sh -NPROC=${NPROC} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ut-coverage-build.sh -UPDATE_CCACHE=${UPDATE_CCACHE} BUILD_BRANCH=${BUILD_BRANCH} sh ${CI_CCACHE_USED_SRCPATH}/release-centos7-llvm/scripts/tiflash-ut-coverage-finish.sh diff --git a/release-centos7-llvm/scripts/env.sh b/release-centos7-llvm/scripts/env.sh deleted file mode 100644 index 0eb235bf478..00000000000 --- a/release-centos7-llvm/scripts/env.sh +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CI_BUILD_INFO_PRE_FIX="commit hash value" -CI_CCACHE_USED_SRCPATH="/build/tics" -BUILD_DIR_SUFFIX="release-centos7-llvm/build-release" -BUILD_UT_DIR="/build/release-centos7-llvm/build-release" -INSTALL_DIR_SUFFIX="release-centos7-llvm/tiflash" -INSTALL_UT_DIR="/build/${INSTALL_DIR_SUFFIX}" diff --git a/release-centos7-llvm/scripts/fetch-ci-build.sh b/release-centos7-llvm/scripts/fetch-ci-build.sh deleted file mode 100755 index cd2bb9ee7b8..00000000000 --- a/release-centos7-llvm/scripts/fetch-ci-build.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -DOWNLOAD_TAR=${DOWNLOAD_TAR:-false} -TAR_PATH=${SRCPATH}/tests/.build -TIFLASH_CI_BUILD_URI_PREFIX="builds/pingcap/tiflash/ci-cache/tmp/pr-build" -TIFLASH_CI_BUILD_PRE_FIX="http://fileserver.pingcap.net/download/${TIFLASH_CI_BUILD_URI_PREFIX}" - -rm -rf "${TAR_PATH}" -mkdir -p "${TAR_PATH}" -cd "${TAR_PATH}" -TAR_BIN_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/tiflash.tar.gz" -COMMIT_HASH_FILE_URL="${TIFLASH_CI_BUILD_PRE_FIX}/${PULL_ID}/commit-hash" -while [[ true ]]; do - curl -o ./commit-hash ${COMMIT_HASH_FILE_URL} - if [[ $(grep -c "${CI_BUILD_INFO_PRE_FIX}" ./commit-hash) -ne '0' ]]; then - COMMIT_HASH_VAL=$(tail -n -1 ./commit-hash) - if [[ ${COMMIT_HASH_VAL} == ${COMMIT_HASH} ]]; then - if [[ ${DOWNLOAD_TAR} == "true" ]]; then - curl -o ./tiflash.tar.gz ${TAR_BIN_URL} - MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') - EXPECT_MD5_SUM=$(tail -n +2 ./commit-hash | head -n 1) - if [[ ${MD5_SUM} != ${EXPECT_MD5_SUM} ]]; then - echo "wrong md5sum got ${MD5_SUM} but expect ${EXPECT_MD5_SUM}" - exit -1 - fi - tar -zxf tiflash.tar.gz - fi - break - fi - fi - echo "fail to get ci build tiflash binary, sleep 60s" - sleep 60 -done diff --git a/release-centos7-llvm/scripts/run-format-check.sh b/release-centos7-llvm/scripts/run-format-check.sh deleted file mode 100755 index d2d069f2e9b..00000000000 --- a/release-centos7-llvm/scripts/run-format-check.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ex - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" - -SRCPATH=${1:-$( - cd ${SCRIPTPATH}/../.. - pwd -P -)} - -BUILD_BRANCH=${BUILD_BRANCH:-master} - -python3 ${SRCPATH}/format-diff.py --repo_path "${SRCPATH}" --check_formatted --diff_from $(git merge-base origin/${BUILD_BRANCH} HEAD) --dump_diff_files_to "/tmp/tiflash-diff-files.json" diff --git a/release-centos7-llvm/scripts/run-ut.sh b/release-centos7-llvm/scripts/run-ut.sh deleted file mode 100755 index bc88a678a25..00000000000 --- a/release-centos7-llvm/scripts/run-ut.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" - -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -set -x - -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -OUTPUT_XML=${OUTPUT_XML:-false} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh -INSTALL_DIR=${INSTALL_UT_DIR} - -rm -rf /tests && ln -s ${SRCPATH}/tests /tests -rm -rf /tiflash && ln -s ${INSTALL_DIR} /tiflash - -source /tests/docker/util.sh -export LLVM_PROFILE_FILE="/tiflash/profile/unit-test-%${NPROC}m.profraw" -show_env -ENV_VARS_PATH=/tests/docker/_env.sh OUTPUT_XML=${OUTPUT_XML} NPROC=${NPROC} /tests/run-gtest.sh diff --git a/release-centos7-llvm/scripts/static-analysis.sh b/release-centos7-llvm/scripts/static-analysis.sh deleted file mode 100755 index a2a997287d4..00000000000 --- a/release-centos7-llvm/scripts/static-analysis.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -ENABLE_CLANG_TIDY_CHECK=${ENABLE_CLANG_TIDY_CHECK:-true} - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -if [[ "${ENABLE_CLANG_TIDY_CHECK}" == "true" ]]; then - BUILD_DIR="${SRCPATH}/${BUILD_DIR_SUFFIX}" - - cd ${BUILD_DIR} - cmake "${SRCPATH}" \ - -DENABLE_TESTS=0 \ - -DCMAKE_BUILD_TYPE=RELWITHDEBINFO \ - -DUSE_CCACHE=OFF \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DRUN_HAVE_STD_REGEX=0 \ - -GNinja - python3 ${SRCPATH}/release-centos7-llvm/scripts/fix_compile_commands.py \ - --file_path=${BUILD_DIR}/compile_commands.json \ - --load_diff_files_from "/tmp/tiflash-diff-files.json" - python3 ${SRCPATH}/release-centos7-llvm/scripts/run-clang-tidy.py -p ${BUILD_DIR} -j ${NPROC} --files ".*/tics/dbms/*" -fi diff --git a/release-centos7-llvm/scripts/tiflash-ci-build.sh b/release-centos7-llvm/scripts/tiflash-ci-build.sh deleted file mode 100755 index 51371617b41..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ci-build.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -echo "INSTALL_DIR=${INSTALL_DIR}" -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -TIFLASH_PROXY_SRC=${SRCPATH}/contrib/tiflash-proxy -TIFLASH_PROXY_TAR_DIR=${TIFLASH_PROXY_SRC}/target/release -TIFLASH_PROXY_BIN_PATH=${TIFLASH_PROXY_TAR_DIR}/libtiflash_proxy.so - -rm -rf ${TIFLASH_PROXY_TAR_DIR} -mkdir -p ${TIFLASH_PROXY_TAR_DIR} - -if [[ -f /tmp/libtiflash_proxy.so ]]; then - BUILD_TIFLASH_PROXY=false - CMAKE_PREBUILT_LIBS_ROOT_ARG=-DPREBUILT_LIBS_ROOT="${TIFLASH_PROXY_SRC}" - cp /tmp/libtiflash_proxy.so ${TIFLASH_PROXY_BIN_PATH} -else - BUILD_TIFLASH_PROXY=true - CMAKE_PREBUILT_LIBS_ROOT_ARG="" - echo "need to build libtiflash_proxy.so" - export PATH=$PATH:$HOME/.cargo/bin -fi - -CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -BUILD_BRANCH=${BUILD_BRANCH:-master} -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -ENABLE_TESTS=${ENABLE_TESTS:-1} -USE_CCACHE=${USE_CCACHE:-ON} - -if [[ "${CMAKE_BUILD_TYPE}" != "Debug" ]]; then - ENABLE_TESTS=0 -fi - -rm -rf "${INSTALL_DIR}" -mkdir -p "${INSTALL_DIR}" - -BUILD_DIR="${SRCPATH}/${BUILD_DIR_SUFFIX}" -rm -rf ${BUILD_DIR} -mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR} -cmake "$SRCPATH" ${CMAKE_PREBUILT_LIBS_ROOT_ARG} \ - -DENABLE_TESTS=${ENABLE_TESTS} \ - -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ - -DUSE_CCACHE=${USE_CCACHE} \ - -DDEBUG_WITHOUT_DEBUG_INFO=ON \ - -DCMAKE_PREFIX_PATH="/usr/local" \ - -DRUN_HAVE_STD_REGEX=0 \ - -DUSE_INTERNAL_TIFLASH_PROXY=${BUILD_TIFLASH_PROXY} \ - -GNinja - -cmake --build . --target tiflash --parallel -cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" - -if [[ "${CMAKE_BUILD_TYPE}" == "Debug" && ${ENABLE_TESTS} -ne 0 ]]; then - ninja page_ctl - ninja page_stress_testing - # build gtest in `release-centos7/build/build-tiflash-ut-coverage.sh` -fi - -ccache -s - -# unset LD_LIBRARY_PATH before test -unset LD_LIBRARY_PATH -readelf -d "${INSTALL_DIR}/tiflash" -ldd "${INSTALL_DIR}/tiflash" -ls -lha "${INSTALL_DIR}" diff --git a/release-centos7-llvm/scripts/tiflash-ci-finish.sh b/release-centos7-llvm/scripts/tiflash-ci-finish.sh deleted file mode 100755 index 9a02a34dfed..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ci-finish.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -echo "INSTALL_DIR=${INSTALL_DIR}" - -CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -BUILD_BRANCH=${BUILD_BRANCH:-master} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} - -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}-llvm.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') - -# Download prebuilt tiflash proxy to accelerate CI build. -TIFLASH_PROXY_SRC=${SRCPATH}/contrib/tiflash-proxy -TIFLASH_PROXY_TAR_DIR=${TIFLASH_PROXY_SRC}/target/release - -if [[ -f /tmp/build_tiflash_proxy_flag ]]; then - proxy_git_hash=$(tail -n -1 /tmp/build_tiflash_proxy_flag) - cd ${INSTALL_DIR} - curl -F builds/pingcap/tiflash-proxy/${proxy_git_hash}-llvm/libtiflash_proxy.so=@libtiflash_proxy.so http://fileserver.pingcap.net/upload - rm -rf /tmp/build_tiflash_proxy_flag -fi - -if [[ ${UPDATE_CCACHE} != "false" ]]; then - cd ${SRCPATH} - rm -rf ccache.tar - tar -cf ccache.tar .ccache - curl -F builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR}=@ccache.tar http://fileserver.pingcap.net/upload -fi diff --git a/release-centos7-llvm/scripts/tiflash-ci-prepare.sh b/release-centos7-llvm/scripts/tiflash-ci-prepare.sh deleted file mode 100755 index fb297ba8bb4..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ci-prepare.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -command -v ccache >/dev/null 2>&1 -ccache_major=$(ccache --version | head - -n 1 | tr '.' ' ' | awk -e '{ print $3 }') -if [[ $? != 0 || $ccache_major -lt 4 ]]; then - echo "try to install ccache" - curl -o /usr/local/bin/ccache http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/ccache-4.5.1 - chmod +x /usr/local/bin/ccache -else - echo "ccache has been installed" -fi - -command -v clang-format >/dev/null 2>&1 -if [[ $? != 0 ]]; then - curl -o "/usr/local/bin/clang-format" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/clang-format-12 - chmod +x "/usr/local/bin/clang-format" -else - echo "clang-format has been installed" -fi -clang-format --version - -command -v clang-tidy >/dev/null 2>&1 -if [[ $? != 0 ]]; then - curl -o "/usr/local/bin/clang-tidy" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/clang-tidy-12 - chmod +x "/usr/local/bin/clang-tidy" -else - echo "clang-tidy has been installed" -fi -clang-tidy --version - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} -BUILD_BRANCH=${BUILD_BRANCH:-master} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}-llvm.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') - -rm -rf "${SRCPATH}/.ccache" -cache_file="${SRCPATH}/ccache.tar" -rm -rf "${cache_file}" -curl -o "${cache_file}" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR} -cache_size=$(ls -l "${cache_file}" | awk '{print $5}') -min_size=$((1024000)) -if [[ ${cache_size} -gt ${min_size} ]]; then - echo "try to use ccache to accelerate compile speed" - cd "${SRCPATH}" - tar -xf ccache.tar -fi -ccache -o cache_dir="${SRCPATH}/.ccache" -ccache -o max_size=2G -ccache -o limit_multiple=0.99 -ccache -o hash_dir=false -ccache -o compression=true -ccache -o compression_level=6 -if [[ ${UPDATE_CCACHE} == "false" ]]; then - ccache -o read_only=true -else - ccache -o read_only=false -fi -ccache -z - -TIFLASH_PROXY_BIN_PATH=/tmp/libtiflash_proxy.so -rm -rf ${TIFLASH_PROXY_BIN_PATH} - -cd ${SRCPATH}/contrib/tiflash-proxy -proxy_git_hash=$(git log -1 --format="%H") -curl -o ${TIFLASH_PROXY_BIN_PATH} \ - http://fileserver.pingcap.net/download/builds/pingcap/tiflash-proxy/${proxy_git_hash}-llvm/libtiflash_proxy.so -proxy_size=$(ls -l "${TIFLASH_PROXY_BIN_PATH}" | awk '{print $5}') -min_size=$((102400)) - -if [[ ${proxy_size} -lt ${min_size} ]]; then - rm -rf ${TIFLASH_PROXY_BIN_PATH} - echo "${proxy_git_hash}" >/tmp/build_tiflash_proxy_flag # use it as flag to upload binary -else - rm -rf /tmp/build_tiflash_proxy_flag - chmod 0731 "${TIFLASH_PROXY_BIN_PATH}" -fi diff --git a/release-centos7-llvm/scripts/tiflash-ut-coverage-build.sh b/release-centos7-llvm/scripts/tiflash-ut-coverage-build.sh deleted file mode 100755 index 06c7a09b1dd..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ut-coverage-build.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail -BASE_NAME=$(basename $0) -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=Debug -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -BUILD_DIR=${BUILD_UT_DIR} -INSTALL_DIR=${INSTALL_UT_DIR} - -rm -rf ${INSTALL_DIR} -mkdir -p ${INSTALL_DIR} - -rm -rf ${BUILD_DIR} -mkdir -p ${BUILD_DIR} && cd ${BUILD_DIR} - -cmake "${SRCPATH}" \ - -DENABLE_TESTS=ON \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DUSE_CCACHE=ON \ - -DTEST_LLVM_COVERAGE=ON \ - -DDEBUG_WITHOUT_DEBUG_INFO=ON \ - -DRUN_HAVE_STD_REGEX=0 \ - -DCMAKE_PREFIX_PATH="/usr/local" \ - -GNinja - -ninja gtests_dbms gtests_libcommon gtests_libdaemon -mv "${BUILD_DIR}/dbms/gtests_dbms" "${INSTALL_DIR}/" -mv "${BUILD_DIR}/libs/libcommon/src/tests/gtests_libcommon" "${INSTALL_DIR}/" -mv "${BUILD_DIR}/libs/libdaemon/src/tests/gtests_libdaemon" "${INSTALL_DIR}/" - -ccache -s - -ls -lh "${INSTALL_DIR}" diff --git a/release-centos7-llvm/scripts/tiflash-ut-coverage-finish.sh b/release-centos7-llvm/scripts/tiflash-ut-coverage-finish.sh deleted file mode 100755 index 1cfe8841f43..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ut-coverage-finish.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ueox pipefail - -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=Debug -BUILD_BRANCH=${BUILD_BRANCH:-master} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}-utcov-llvm.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') - -if [[ ${UPDATE_CCACHE} != "false" ]]; then - cd ${SRCPATH} - rm -rf ccache.tar - tar -cf ccache.tar .ccache - curl -F builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR}=@ccache.tar http://fileserver.pingcap.net/upload - exit 0 -fi diff --git a/release-centos7-llvm/scripts/tiflash-ut-coverage-prepare.sh b/release-centos7-llvm/scripts/tiflash-ut-coverage-prepare.sh deleted file mode 100755 index e5fe1a6aed9..00000000000 --- a/release-centos7-llvm/scripts/tiflash-ut-coverage-prepare.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -mkdir -p /build - -command -v ccache >/dev/null 2>&1 -ccache_major=$(ccache --version | head - -n 1 | tr '.' ' ' | awk -e '{ print $3 }') -if [[ $? != 0 || $ccache_major -lt 4 ]]; then - echo "try to install ccache" - curl -o /usr/local/bin/ccache http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/ccache-4.5.1 - chmod +x /usr/local/bin/ccache -else - echo "ccache has been installed" -fi - -command -v lcov >/dev/null 2>&1 -if [[ $? != 0 ]]; then - echo "try to install lcov" - pushd /tmp - wget http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/lcov-1.15-1.noarch.rpm - rpm -i lcov-1.15-1.noarch.rpm - rm -rf lcov-1.15-1.noarch.rpm - popd -else - echo "lcov has been installed" -fi - -set -ueox pipefail -BASE_NAME=$(basename $0) -scriptpath="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $scriptpath/../.. - pwd -P -)} - -CMAKE_BUILD_TYPE=Debug -BUILD_BRANCH=${BUILD_BRANCH:-master} -UPDATE_CCACHE=${UPDATE_CCACHE:-false} - -CCACHE_REMOTE_TAR="${BUILD_BRANCH}-${CMAKE_BUILD_TYPE}-utcov-llvm.tar" -CCACHE_REMOTE_TAR=$(echo "${CCACHE_REMOTE_TAR}" | tr 'A-Z' 'a-z') - -rm -rf "${SRCPATH}/.ccache" -cache_file="${SRCPATH}/ccache.tar" -rm -rf "${cache_file}" -curl -o "${cache_file}" http://fileserver.pingcap.net/download/builds/pingcap/tiflash/ci-cache/${CCACHE_REMOTE_TAR} -cache_size=$(ls -l "${cache_file}" | awk '{print $5}') -min_size=$((1024000)) -if [[ ${cache_size} -gt ${min_size} ]]; then - echo "try to use ccache to accelerate compile speed" - cd "${SRCPATH}" - tar -xf ccache.tar -fi -ccache -o cache_dir="${SRCPATH}/.ccache" -ccache -o max_size=2G -ccache -o limit_multiple=0.99 -ccache -o hash_dir=false -ccache -o compression=true -ccache -o compression_level=6 -if [[ ${UPDATE_CCACHE} == "false" ]]; then - ccache -o read_only=true -else - ccache -o read_only=false -fi - -ccache -z diff --git a/release-centos7-llvm/scripts/upload-ci-build.sh b/release-centos7-llvm/scripts/upload-ci-build.sh deleted file mode 100755 index 530660bcde2..00000000000 --- a/release-centos7-llvm/scripts/upload-ci-build.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail - -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd $SCRIPTPATH/../.. - pwd -P -)} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -TIFLASH_CI_BUILD_URI_PREFIX="builds/pingcap/tiflash/ci-cache/tmp/pr-build" -TIFLASH_CI_BUILD_PRE_FIX="http://fileserver.pingcap.net/download/${TIFLASH_CI_BUILD_URI_PREFIX}" - -TAR_BIN_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/tiflash.tar.gz" -COMMIT_HASH_FILE_URI="${TIFLASH_CI_BUILD_URI_PREFIX}/${PULL_ID}/commit-hash" - -cd "${SRCPATH}/${INSTALL_DIR_SUFFIX}/.." -tar -zcf tiflash.tar.gz ./tiflash -MD5_SUM=$(md5sum ./tiflash.tar.gz | awk '{ print $1 }') -echo "${CI_BUILD_INFO_PRE_FIX}" >./commit-hash -echo "${MD5_SUM}" >>./commit-hash -echo "${COMMIT_HASH}" >>./commit-hash - -curl -F ${TAR_BIN_URI}=@tiflash.tar.gz http://fileserver.pingcap.net/upload -curl -F ${COMMIT_HASH_FILE_URI}=@commit-hash http://fileserver.pingcap.net/upload diff --git a/release-centos7-llvm/scripts/upload-ut-coverage.sh b/release-centos7-llvm/scripts/upload-ut-coverage.sh deleted file mode 100755 index 9f0225956f3..00000000000 --- a/release-centos7-llvm/scripts/upload-ut-coverage.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# Copyright 2022 PingCAP, Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -ueox pipefail -SCRIPTPATH="$( - cd "$(dirname "$0")" - pwd -P -)" -SRCPATH=${1:-$( - cd ${SCRIPTPATH}/../.. - pwd -P -)} - -source ${SRCPATH}/release-centos7-llvm/scripts/env.sh - -SRCPATH=${CI_CCACHE_USED_SRCPATH} -BUILD_DIR={BUILD_UT_DIR} - -NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} - -llvm-profdata merge -sparse /tiflash/profile/*.profraw -o /tiflash/profile/merged.profdata - -llvm-cov export \ - /tiflash/gtests_dbms /tiflash/gtests_libcommon /tiflash/gtests_libdaemon \ - --format=lcov \ - --instr-profile /tiflash/profile/merged.profdata \ - --ignore-filename-regex "/usr/include/.*" \ - --ignore-filename-regex "/usr/local/.*" \ - --ignore-filename-regex "/usr/lib/.*" \ - --ignore-filename-regex ".*/contrib/.*" \ - --ignore-filename-regex ".*/dbms/src/Debug/.*" \ - --ignore-filename-regex ".*/dbms/src/Client/.*" \ - > /tiflash/profile/lcov.info - -mkdir -p /tiflash/report -genhtml /tiflash/profile/lcov.info -o /tiflash/report/ -cd /tiflash -tar -czf coverage-report.tar.gz report - -cd $SRCPATH -COMMIT_HASH_BASE=$(git merge-base origin/${BUILD_BRANCH} HEAD) -SOURCE_DELTA=$(git diff --name-only ${COMMIT_HASH_BASE} | { grep -E '.*\.(cpp|h|hpp|cc|c)$' || true; }) -echo '### Coverage for changed files' > /tiflash/profile/diff-coverage -echo '```' >> /tiflash/profile/diff-coverage - -if [[ -z ${SOURCE_DELTA} ]]; then - echo 'no c/c++ source change detected' >> /tiflash/profile/diff-coverage -else - llvm-cov report /tiflash/gtests_dbms /tiflash/gtests_libcommon /tiflash/gtests_libdaemon -instr-profile /tiflash/profile/merged.profdata $SOURCE_DELTA > "/tiflash/profile/diff-for-delta" - if [[ $(wc -l "/tiflash/profile/diff-for-delta" | awk -e '{printf $1;}') -gt 32 ]]; then - echo 'too many lines from llvm-cov, please refer to full report instead' >> /tiflash/profile/diff-coverage - else - cat /tiflash/profile/diff-for-delta >> /tiflash/profile/diff-coverage - fi -fi - -echo '```' >> /tiflash/profile/diff-coverage -echo '' >> /tiflash/profile/diff-coverage -echo '### Coverage summary' >> /tiflash/profile/diff-coverage -echo '```' >> /tiflash/profile/diff-coverage -llvm-cov report \ - --summary-only \ - --show-region-summary=false \ - --show-branch-summary=false \ - --ignore-filename-regex "/usr/include/.*" \ - --ignore-filename-regex "/usr/local/.*" \ - --ignore-filename-regex "/usr/lib/.*" \ - --ignore-filename-regex ".*/contrib/.*" \ - --ignore-filename-regex ".*/dbms/src/Debug/.*" \ - --ignore-filename-regex ".*/dbms/src/Client/.*" \ - /tiflash/gtests_dbms /tiflash/gtests_libcommon /tiflash/gtests_libdaemon -instr-profile /tiflash/profile/merged.profdata | \ - grep -E "^(TOTAL|Filename)" | \ - cut -d" " -f2- | sed -e 's/^[[:space:]]*//' | sed -e 's/Missed\ /Missed/g' | column -t >> /tiflash/profile/diff-coverage -echo '```' >> /tiflash/profile/diff-coverage -echo '' >> /tiflash/profile/diff-coverage -echo "[full coverage report](https://ci-internal.pingcap.net/job/tics_ghpr_unit_test/${BUILD_NUMBER}/artifact/coverage-report.tar.gz) (for internal network access only)" >> /tiflash/profile/diff-coverage -cat /tiflash/profile/diff-coverage From 752793bd7fef03c68bc680f11b03c64192515303 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 15 Mar 2022 22:17:52 +0800 Subject: [PATCH 4/9] fix bug that ExchangeReceiver is not cancelled if exception happens in union/agg block input stream (#4285) ref pingcap/tiflash#4229 --- .../ParallelAggregatingBlockInputStream.cpp | 3 +- dbms/src/DataStreams/UnionBlockInputStream.h | 33 ++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp index 4865662553d..447dfdfbed5 100644 --- a/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp +++ b/dbms/src/DataStreams/ParallelAggregatingBlockInputStream.cpp @@ -197,7 +197,8 @@ void ParallelAggregatingBlockInputStream::Handler::onException(std::exception_pt parent.exceptions[thread_num] = exception; /// can not cancel parent inputStream or the exception might be lost if (!parent.executed) - parent.processor.cancel(false); + /// kill the processor so ExchangeReceiver will be closed + parent.processor.cancel(true); } diff --git a/dbms/src/DataStreams/UnionBlockInputStream.h b/dbms/src/DataStreams/UnionBlockInputStream.h index 3bc71f21569..f43034c7a47 100644 --- a/dbms/src/DataStreams/UnionBlockInputStream.h +++ b/dbms/src/DataStreams/UnionBlockInputStream.h @@ -40,11 +40,11 @@ struct OutputData Block block; std::exception_ptr exception; - OutputData() {} + OutputData() = default; explicit OutputData(Block & block_) : block(block_) {} - explicit OutputData(std::exception_ptr & exception_) + explicit OutputData(const std::exception_ptr & exception_) : exception(exception_) {} }; @@ -57,12 +57,12 @@ struct OutputData BlockExtraInfo extra_info; std::exception_ptr exception; - OutputData() {} + OutputData() = default; OutputData(Block & block_, BlockExtraInfo & extra_info_) : block(block_) , extra_info(extra_info_) {} - explicit OutputData(std::exception_ptr & exception_) + explicit OutputData(const std::exception_ptr & exception_) : exception(exception_) {} }; @@ -279,6 +279,23 @@ class UnionBlockInputStream final : public IProfilingBlockInputStream * otherwise ParallelInputsProcessor can be blocked during insertion into the queue. */ OutputQueue output_queue; + std::mutex mu; + bool meet_exception = false; + + void handleException(const std::exception_ptr & exception) + { + std::unique_lock lock(mu); + if (meet_exception) + return; + meet_exception = true; + /// The order of the rows matters. If it is changed, then the situation is possible, + /// when before exception, an empty block (end of data) will be put into the queue, + /// and the exception is lost. + output_queue.emplace(exception); + /// can not cancel itself or the exception might be lost + /// kill the processor so ExchangeReceiver will be closed + processor.cancel(true); + } struct Handler { @@ -309,13 +326,7 @@ class UnionBlockInputStream final : public IProfilingBlockInputStream void onException(std::exception_ptr & exception, size_t /*thread_num*/) { - /// The order of the rows matters. If it is changed, then the situation is possible, - /// when before exception, an empty block (end of data) will be put into the queue, - /// and the exception is lost. - - parent.output_queue.emplace(exception); - /// can not cancel parent inputStream or the exception might be lost - parent.processor.cancel(false); /// Does not throw exceptions. + parent.handleException(exception); } String getName() const From 921dd4dd3e3c28df70f4f6cce6e290d0cbf9c324 Mon Sep 17 00:00:00 2001 From: Liqi Geng Date: Wed, 16 Mar 2022 12:41:52 +0800 Subject: [PATCH 5/9] Support dayofweek and dayofyear pushdown to tiflash (#4222) close pingcap/tiflash#4150 --- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 4 +- dbms/src/Functions/FunctionsDateTime.cpp | 1 + dbms/src/Functions/FunctionsDateTime.h | 27 ++- .../Functions/tests/gtest_dayofweekyear.cpp | 196 ++++++++++++++++++ .../fullstack-test/expr/day_of_weekyear.test | 42 ++++ 5 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 dbms/src/Functions/tests/gtest_dayofweekyear.cpp create mode 100644 tests/fullstack-test/expr/day_of_weekyear.test diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index b36454e6e97..fa58b131e0f 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -504,8 +504,8 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::DayName, "toDayName"}, {tipb::ScalarFuncSig::DayOfMonth, "toDayOfMonth"}, - //{tipb::ScalarFuncSig::DayOfWeek, "cast"}, - //{tipb::ScalarFuncSig::DayOfYear, "cast"}, + {tipb::ScalarFuncSig::DayOfWeek, "toDayOfWeek"}, + {tipb::ScalarFuncSig::DayOfYear, "toDayOfYear"}, //{tipb::ScalarFuncSig::WeekWithMode, "cast"}, //{tipb::ScalarFuncSig::WeekWithoutMode, "cast"}, diff --git a/dbms/src/Functions/FunctionsDateTime.cpp b/dbms/src/Functions/FunctionsDateTime.cpp index 056697d8e30..930bc9b6bd9 100644 --- a/dbms/src/Functions/FunctionsDateTime.cpp +++ b/dbms/src/Functions/FunctionsDateTime.cpp @@ -78,6 +78,7 @@ void registerFunctionsDateTime(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/dbms/src/Functions/FunctionsDateTime.h b/dbms/src/Functions/FunctionsDateTime.h index d95006c45b1..04621fbd63a 100644 --- a/dbms/src/Functions/FunctionsDateTime.h +++ b/dbms/src/Functions/FunctionsDateTime.h @@ -485,14 +485,36 @@ struct ToDayOfWeekImpl { return time_zone.toDayOfWeek(DayNum(d)); } - static inline UInt8 execute(UInt64, const DateLUTImpl &) + static inline UInt8 execute(UInt64 d, const DateLUTImpl &) { - throw Exception("Illegal type MyTime of argument for function toDayOfWeek", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + MyDateTime my_time(d); + return UInt8(my_time.weekDay() + 1); } using FactorTransform = ToMondayImpl; }; +struct ToDayOfYearImpl +{ + static constexpr auto name = "toDayOfYear"; + + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfYear(t); + } + static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfYear(DayNum(d)); + } + static inline UInt16 execute(UInt64 d, const DateLUTImpl &) + { + MyDateTime my_time(d); + return UInt16(my_time.yearDay()); + } + + using FactorTransform = ToStartOfYearImpl; +}; + struct ToHourImpl { static constexpr auto name = "toHour"; @@ -3373,6 +3395,7 @@ using FunctionToQuarter = FunctionDateOrDateTimeToSomething; using FunctionToDayOfMonth = FunctionDateOrDateTimeToSomething; using FunctionToDayOfWeek = FunctionDateOrDateTimeToSomething; +using FunctionToDayOfYear = FunctionDateOrDateTimeToSomething; using FunctionToHour = FunctionDateOrDateTimeToSomething; using FunctionToMinute = FunctionDateOrDateTimeToSomething; using FunctionToSecond = FunctionDateOrDateTimeToSomething; diff --git a/dbms/src/Functions/tests/gtest_dayofweekyear.cpp b/dbms/src/Functions/tests/gtest_dayofweekyear.cpp new file mode 100644 index 00000000000..243bdd0c403 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_dayofweekyear.cpp @@ -0,0 +1,196 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include +#include + +namespace DB::tests +{ +class TestDayOfWeekYear : public DB::tests::FunctionTest +{ +}; + +TEST_F(TestDayOfWeekYear, TestDayOfWeek) +try +{ + /// ColumnVector(nullable) + const String func_name = "toDayOfWeek"; + static auto const nullable_datetime_type_ptr = makeNullable(std::make_shared(6)); + static auto const datetime_type_ptr = std::make_shared(6); + static auto const date_type_ptr = std::make_shared(); + auto data_col_ptr = createColumn>( + { + {}, // Null + // FIXME: https://github.com/pingcap/tiflash/issues/4186 + // MyDateTime(2022, 12, 0, 1, 1, 1, 1).toPackedUInt(), + // MyDateTime(2022, 13, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(), + MyDateTime(2022, 3, 14, 9, 8, 7, 6).toPackedUInt(), + MyDateTime(2022, 3, 15, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 16, 1, 2, 3, 4).toPackedUInt(), + MyDateTime(2022, 3, 17, 4, 3, 2, 1).toPackedUInt(), + MyDateTime(2022, 3, 18, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt(), + }) + .column; + auto input_col = ColumnWithTypeAndName(data_col_ptr, nullable_datetime_type_ptr, "input"); + auto output_col = createColumn>({{}, 5, 1, 2, 3, 4, 5, 6, 7}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnVector(non-null) + data_col_ptr = createColumn( + { + MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(), + MyDateTime(2022, 3, 14, 9, 8, 7, 6).toPackedUInt(), + MyDateTime(2022, 3, 15, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 16, 1, 2, 3, 4).toPackedUInt(), + MyDateTime(2022, 3, 17, 4, 3, 2, 1).toPackedUInt(), + MyDateTime(2022, 3, 18, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt(), + }) + .column; + input_col = ColumnWithTypeAndName(data_col_ptr, datetime_type_ptr, "input"); + output_col = createColumn({5, 1, 2, 3, 4, 5, 6, 7}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(non-null) + input_col = ColumnWithTypeAndName(createConstColumn(1, MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt()).column, datetime_type_ptr, "input"); + output_col = createConstColumn(1, {7}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(nullable) + input_col = ColumnWithTypeAndName(createConstColumn>(1, MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt()).column, nullable_datetime_type_ptr, "input"); + output_col = createConstColumn>(1, {7}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(nullable(null)) + input_col = ColumnWithTypeAndName(createConstColumn>(1, {}).column, nullable_datetime_type_ptr, "input"); + output_col = createConstColumn>(1, {}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// MyDate ColumnVector(non-null) + data_col_ptr = createColumn( + { + MyDate(1969, 1, 2).toPackedUInt(), + MyDate(2022, 3, 13).toPackedUInt(), + MyDate(2022, 3, 14).toPackedUInt(), + MyDate(2022, 3, 15).toPackedUInt(), + MyDate(2022, 3, 16).toPackedUInt(), + MyDate(2022, 3, 17).toPackedUInt(), + MyDate(2022, 3, 18).toPackedUInt(), + MyDate(2022, 3, 19).toPackedUInt(), + }) + .column; + input_col = ColumnWithTypeAndName(data_col_ptr, date_type_ptr, "input"); + output_col = createColumn({5, 1, 2, 3, 4, 5, 6, 7}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); +} +CATCH + +TEST_F(TestDayOfWeekYear, TestDayOfYear) +try +{ + /// ColumnVector(nullable) + const String func_name = "toDayOfYear"; + static auto const nullable_datetime_type_ptr = makeNullable(std::make_shared(6)); + static auto const datetime_type_ptr = std::make_shared(6); + static auto const date_type_ptr = std::make_shared(); + auto data_col_ptr = createColumn>( + { + {}, // Null + // FIXME: https://github.com/pingcap/tiflash/issues/4186 + // MyDateTime(2022, 12, 0, 1, 1, 1, 1).toPackedUInt(), + // MyDateTime(2022, 13, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(), + MyDateTime(2022, 3, 14, 9, 8, 7, 6).toPackedUInt(), + MyDateTime(2022, 3, 15, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 16, 1, 2, 3, 4).toPackedUInt(), + MyDateTime(2022, 3, 17, 4, 3, 2, 1).toPackedUInt(), + MyDateTime(2022, 3, 18, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(1900, 12, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2020, 12, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 12, 31, 1, 1, 1, 1).toPackedUInt(), + }) + .column; + auto input_col = ColumnWithTypeAndName(data_col_ptr, nullable_datetime_type_ptr, "input"); + auto output_col = createColumn>({{}, 2, 72, 73, 74, 75, 76, 77, 78, 365, 366, 365}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnVector(non-null) + data_col_ptr = createColumn( + { + MyDateTime(1969, 1, 2, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 3, 13, 6, 7, 8, 9).toPackedUInt(), + MyDateTime(2022, 3, 14, 9, 8, 7, 6).toPackedUInt(), + MyDateTime(2022, 3, 15, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 16, 1, 2, 3, 4).toPackedUInt(), + MyDateTime(2022, 3, 17, 4, 3, 2, 1).toPackedUInt(), + MyDateTime(2022, 3, 18, 0, 0, 0, 0).toPackedUInt(), + MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(1900, 12, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2020, 12, 31, 1, 1, 1, 1).toPackedUInt(), + MyDateTime(2022, 12, 31, 1, 1, 1, 1).toPackedUInt(), + }) + .column; + input_col = ColumnWithTypeAndName(data_col_ptr, datetime_type_ptr, "input"); + output_col = createColumn({2, 72, 73, 74, 75, 76, 77, 78, 365, 366, 365}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(non-null) + input_col = ColumnWithTypeAndName(createConstColumn(1, MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt()).column, datetime_type_ptr, "input"); + output_col = createConstColumn(1, {78}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(nullable) + input_col = ColumnWithTypeAndName(createConstColumn>(1, MyDateTime(2022, 3, 19, 1, 1, 1, 1).toPackedUInt()).column, nullable_datetime_type_ptr, "input"); + output_col = createConstColumn>(1, {78}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// ColumnConst(nullable(null)) + input_col = ColumnWithTypeAndName(createConstColumn>(1, {}).column, nullable_datetime_type_ptr, "input"); + output_col = createConstColumn>(1, {}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); + + /// MyDate ColumnVector(non-null) + data_col_ptr = createColumn( + { + MyDate(1969, 1, 2).toPackedUInt(), + MyDate(2022, 3, 13).toPackedUInt(), + MyDate(2022, 3, 14).toPackedUInt(), + MyDate(2022, 3, 15).toPackedUInt(), + MyDate(2022, 3, 16).toPackedUInt(), + MyDate(2022, 3, 17).toPackedUInt(), + MyDate(2022, 3, 18).toPackedUInt(), + MyDate(2022, 3, 19).toPackedUInt(), + MyDate(1900, 12, 31).toPackedUInt(), + MyDate(2020, 12, 31).toPackedUInt(), + MyDate(2022, 12, 31).toPackedUInt(), + }) + .column; + input_col = ColumnWithTypeAndName(data_col_ptr, date_type_ptr, "input"); + output_col = createColumn({2, 72, 73, 74, 75, 76, 77, 78, 365, 366, 365}); + ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); +} +CATCH + +} // namespace DB::tests diff --git a/tests/fullstack-test/expr/day_of_weekyear.test b/tests/fullstack-test/expr/day_of_weekyear.test new file mode 100644 index 00000000000..ae63599a731 --- /dev/null +++ b/tests/fullstack-test/expr/day_of_weekyear.test @@ -0,0 +1,42 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql> drop table if exists test.t; +mysql> create table test.t(a char(20), b datetime, c date); +mysql> insert into test.t values('', '1970-1-1 12:12:12', '1970-1-1'); +mysql> insert into test.t values('123', '1989-6-6 12:12:12', '1989-6-6'); +mysql> insert into test.t values('2022-3-10', '2000-3-4 12:12:12', '2000-3-4'); +mysql> alter table test.t set tiflash replica 1; + +func> wait_table test t + +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select dayofweek(a), dayofweek(b), dayofweek(c) from test.t; ++--------------+--------------+--------------+ +| dayofweek(a) | dayofweek(b) | dayofweek(c) | ++--------------+--------------+--------------+ +| NULL | 5 | 5 | +| NULL | 3 | 3 | +| 5 | 7 | 7 | ++--------------+--------------+--------------+ + +mysql> set @@tidb_isolation_read_engines='tiflash'; set @@tidb_enforce_mpp = 1; select dayofyear(a), dayofyear(b), dayofyear(c) from test.t; ++--------------+--------------+--------------+ +| dayofyear(a) | dayofyear(b) | dayofyear(c) | ++--------------+--------------+--------------+ +| NULL | 1 | 1 | +| NULL | 157 | 157 | +| 69 | 64 | 64 | ++--------------+--------------+--------------+ + +mysql> drop table if exists test.t; From dfed057ce5d60d8282dacbc2793174dcb9f8e4f5 Mon Sep 17 00:00:00 2001 From: bestwoody <89765764+bestwoody@users.noreply.github.com> Date: Wed, 16 Mar 2022 14:52:56 +0800 Subject: [PATCH 6/9] refine tiflash shutdown logic (#4291) * 1.add metrics of calldata&mpptunnel 2.refine shutdown logic Signed-off-by: bestwoody * update * Apply suggestions from code review Co-authored-by: Fu Zhe * Update dbms/src/Flash/EstablishCall.cpp Co-authored-by: Fu Zhe * add harm limit to wait Signed-off-by: bestwoody * fix Signed-off-by: bestwoody Co-authored-by: Fu Zhe --- dbms/src/Common/TiFlashMetrics.h | 3 +++ dbms/src/Flash/EstablishCall.cpp | 25 ++++++++++++++++++++++--- dbms/src/Flash/EstablishCall.h | 4 ++++ dbms/src/Flash/Mpp/MPPTunnel.cpp | 7 +++++++ dbms/src/Flash/Mpp/MPPTunnel.h | 7 +++++++ dbms/src/Server/Server.cpp | 15 +++++++++------ 6 files changed, 52 insertions(+), 9 deletions(-) diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h index 682d9c93470..9a57e2b1c5c 100644 --- a/dbms/src/Common/TiFlashMetrics.h +++ b/dbms/src/Common/TiFlashMetrics.h @@ -169,6 +169,9 @@ namespace DB F(type_decode, {{"type", "decode"}}, ExpBuckets{0.0005, 2, 20}), F(type_write, {{"type", "write"}}, ExpBuckets{0.0005, 2, 20})) \ M(tiflash_server_info, "Indicate the tiflash server info, and the value is the start timestamp (s).", Gauge, \ F(start_time, {"version", TiFlashBuildInfo::getReleaseVersion()}, {"hash", TiFlashBuildInfo::getGitHash()})) \ + M(tiflash_object_count, "Number of objects", Gauge, \ + F(type_count_of_establish_calldata, {"type", "count_of_establish_calldata"}), \ + F(type_count_of_mpptunnel, {"type", "count_of_mpptunnel"})) \ M(tiflash_thread_count, "Number of threads", Gauge, \ F(type_max_threads_of_thdpool, {"type", "thread_pool_total_max"}), \ F(type_active_threads_of_thdpool, {"type", "thread_pool_active"}), \ diff --git a/dbms/src/Flash/EstablishCall.cpp b/dbms/src/Flash/EstablishCall.cpp index 04080530d58..cf9d2e2ed6f 100644 --- a/dbms/src/Flash/EstablishCall.cpp +++ b/dbms/src/Flash/EstablishCall.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -26,12 +27,18 @@ EstablishCallData::EstablishCallData(AsyncFlashService * service, grpc::ServerCo , responder(&ctx) , state(NEW_REQUEST) { + GET_METRIC(tiflash_object_count, type_count_of_establish_calldata).Increment(); // As part of the initial CREATE state, we *request* that the system // start processing requests. In this request, "this" acts are // the tag uniquely identifying the request. service->RequestEstablishMPPConnection(&ctx, &request, &responder, cq, notify_cq, this); } +EstablishCallData::~EstablishCallData() +{ + GET_METRIC(tiflash_object_count, type_count_of_establish_calldata).Decrement(); +} + EstablishCallData * EstablishCallData::spawn(AsyncFlashService * service, grpc::ServerCompletionQueue * cq, grpc::ServerCompletionQueue * notify_cq, const std::shared_ptr> & is_shutdown) { return new EstablishCallData(service, cq, notify_cq, is_shutdown); @@ -53,7 +60,9 @@ void EstablishCallData::tryFlushOne() void EstablishCallData::responderFinish(const grpc::Status & status) { - if (!(*is_shutdown)) + if (*is_shutdown) + finishTunnelAndResponder(); + else responder.Finish(status, this); } @@ -79,7 +88,10 @@ void EstablishCallData::initRpc() bool EstablishCallData::write(const mpp::MPPDataPacket & packet) { if (*is_shutdown) - return false; + { + finishTunnelAndResponder(); + return true; + } responder.Write(packet, this); return true; } @@ -116,11 +128,18 @@ void EstablishCallData::cancel() delete this; return; } + finishTunnelAndResponder(); +} + +void EstablishCallData::finishTunnelAndResponder() +{ state = FINISH; if (mpp_tunnel) + { mpp_tunnel->consumerFinish("grpc writes failed.", true); //trigger mpp tunnel finish work + } grpc::Status status(static_cast(GRPC_STATUS_UNKNOWN), "Consumer exits unexpected, grpc writes failed."); - responderFinish(status); + responder.Finish(status, this); } void EstablishCallData::proceed() diff --git a/dbms/src/Flash/EstablishCall.h b/dbms/src/Flash/EstablishCall.h index 8aeacf84c39..3b81b9da6c1 100644 --- a/dbms/src/Flash/EstablishCall.h +++ b/dbms/src/Flash/EstablishCall.h @@ -51,6 +51,8 @@ class EstablishCallData : public PacketWriter grpc::ServerCompletionQueue * notify_cq, const std::shared_ptr> & is_shutdown); + ~EstablishCallData(); + bool write(const mpp::MPPDataPacket & packet) override; void tryFlushOne() override; @@ -79,6 +81,8 @@ class EstablishCallData : public PacketWriter void initRpc(); + void finishTunnelAndResponder(); + void responderFinish(const grpc::Status & status); std::mutex mu; diff --git a/dbms/src/Flash/Mpp/MPPTunnel.cpp b/dbms/src/Flash/Mpp/MPPTunnel.cpp index ac8cb33e0fe..99c7b296355 100644 --- a/dbms/src/Flash/Mpp/MPPTunnel.cpp +++ b/dbms/src/Flash/Mpp/MPPTunnel.cpp @@ -49,11 +49,15 @@ MPPTunnelBase::MPPTunnelBase( , log(getMPPTaskLog(log_, tunnel_id)) { assert(!(is_local && is_async)); + GET_METRIC(tiflash_object_count, type_count_of_mpptunnel).Increment(); } template MPPTunnelBase::~MPPTunnelBase() { + SCOPE_EXIT({ + GET_METRIC(tiflash_object_count, type_count_of_mpptunnel).Decrement(); + }); try { { @@ -308,6 +312,9 @@ void MPPTunnelBase::consumerFinish(const String & err_msg, bool need_loc send_queue.finish(); auto rest_work = [this, &err_msg] { + // it's safe to call it multiple times + if (finished && consumer_state.errHasSet()) + return; finished = true; // must call setError in the critical area to keep consistent with `finished` from outside. consumer_state.setError(err_msg); diff --git a/dbms/src/Flash/Mpp/MPPTunnel.h b/dbms/src/Flash/Mpp/MPPTunnel.h index 0b568cdb385..cdf4b382e09 100644 --- a/dbms/src/Flash/Mpp/MPPTunnel.h +++ b/dbms/src/Flash/Mpp/MPPTunnel.h @@ -173,11 +173,18 @@ class MPPTunnelBase : private boost::noncopyable void setError(const String & err_msg) { promise.set_value(err_msg); + err_has_set = true; + } + + bool errHasSet() const + { + return err_has_set.load(); } private: std::promise promise; std::shared_future future; + std::atomic err_has_set{false}; }; ConsumerState consumer_state; diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 044bf701c2a..35ec453a70b 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -642,14 +642,17 @@ class Server::FlashGrpcServerHolder ~FlashGrpcServerHolder() { - *is_shutdown = true; - const int wait_calldata_after_shutdown_interval_ms = 500; - std::this_thread::sleep_for(std::chrono::milliseconds(wait_calldata_after_shutdown_interval_ms)); // sleep 500ms to let operations of calldata called by MPPTunnel done. /// Shut down grpc server. - // wait 5 seconds for pending rpcs to gracefully stop - gpr_timespec deadline{5, 0, GPR_TIMESPAN}; LOG_FMT_INFO(log, "Begin to shut down flash grpc server"); - flash_grpc_server->Shutdown(deadline); + flash_grpc_server->Shutdown(); + *is_shutdown = true; + // Wait all existed MPPTunnels done to prevent crash. + // If all existed MPPTunnels are done, almost in all cases it means all existed MPPTasks and ExchangeReceivers are also done. + const int max_wait_cnt = 300; + int wait_cnt = 0; + while (GET_METRIC(tiflash_object_count, type_count_of_mpptunnel).Value() >= 1 && (wait_cnt++ < max_wait_cnt)) + std::this_thread::sleep_for(std::chrono::seconds(1)); + for (auto & cq : cqs) cq->Shutdown(); for (auto & cq : notify_cqs) From fa1e9ebf16ae4c76c70e3afdb38644e0258e2ca3 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Wed, 16 Mar 2022 16:37:52 +0800 Subject: [PATCH 7/9] WALStore support multi path (#4085) ref pingcap/tiflash#3594 --- dbms/src/Storages/Page/V3/WAL/WALReader.cpp | 12 +++-- dbms/src/Storages/Page/V3/WALStore.cpp | 29 ++++++++--- dbms/src/Storages/Page/V3/WALStore.h | 9 ++-- .../Page/V3/tests/gtest_wal_store.cpp | 50 ++++++++++++++++--- .../Storages/tests/TiFlashStorageTestBasic.h | 6 +-- 5 files changed, 80 insertions(+), 26 deletions(-) diff --git a/dbms/src/Storages/Page/V3/WAL/WALReader.cpp b/dbms/src/Storages/Page/V3/WAL/WALReader.cpp index abc78d72562..bfb9b7617dc 100644 --- a/dbms/src/Storages/Page/V3/WAL/WALReader.cpp +++ b/dbms/src/Storages/Page/V3/WAL/WALReader.cpp @@ -36,17 +36,21 @@ LogFilenameSet WALStoreReader::listAllFiles( // [, , ...] std::vector> all_filenames; Strings filenames; - for (const auto & p : delegator->listPaths()) + for (const auto & parent_path : delegator->listPaths()) { - Poco::File directory(p); + String wal_parent_path = parent_path + WALStore::wal_folder_prefix; + Poco::File directory(wal_parent_path); if (!directory.exists()) + { directory.createDirectories(); + continue; + } + filenames.clear(); directory.list(filenames); - all_filenames.emplace_back(std::make_pair(p, std::move(filenames))); + all_filenames.emplace_back(std::make_pair(wal_parent_path, std::move(filenames))); filenames.clear(); } - assert(all_filenames.size() == 1); // TODO: multi-path LogFilenameSet log_files; for (const auto & [parent_path, filenames] : all_filenames) diff --git a/dbms/src/Storages/Page/V3/WALStore.cpp b/dbms/src/Storages/Page/V3/WALStore.cpp index 5d29845330c..215bd03b84a 100644 --- a/dbms/src/Storages/Page/V3/WALStore.cpp +++ b/dbms/src/Storages/Page/V3/WALStore.cpp @@ -65,6 +65,7 @@ WALStore::WALStore( : delegator(delegator_) , provider(provider_) , last_log_num(last_log_num_) + , wal_paths_index(0) , logger(&Poco::Logger::get("WALStore")) { } @@ -90,7 +91,7 @@ void WALStore::apply(const PageEntriesEdit & edit, const WriteLimiterPtr & write if (log_file == nullptr || log_file->writtenBytes() > PAGE_META_ROLL_SIZE) { auto log_num = last_log_num++; - auto [new_log_file, filename] = createLogWriter(delegator, provider, {log_num, 0}, logger, false); + auto [new_log_file, filename] = createLogWriter({log_num, 0}, false); (void)filename; log_file.swap(new_log_file); } @@ -100,13 +101,29 @@ void WALStore::apply(const PageEntriesEdit & edit, const WriteLimiterPtr & write } std::tuple, LogFilename> WALStore::createLogWriter( - PSDiskDelegatorPtr delegator, - const FileProviderPtr & provider, const std::pair & new_log_lvl, - Poco::Logger * logger, bool manual_flush) { - const auto path = delegator->defaultPath(); // TODO: multi-path + String path; + + if (delegator->numPaths() == 1) + { + path = delegator->defaultPath(); + } + else + { + const auto & paths = delegator->listPaths(); + + if (wal_paths_index >= paths.size()) + { + wal_paths_index = 0; + } + path = paths[wal_paths_index]; + wal_paths_index++; + } + + path += wal_folder_prefix; + LogFilename log_filename = LogFilename{ (manual_flush ? LogFileStage::Temporary : LogFileStage::Normal), new_log_lvl.first, @@ -172,7 +189,7 @@ bool WALStore::saveSnapshot(FilesSnapshot && files_snap, PageEntriesEdit && dire // Use {largest_log_num + 1, 1} to save the `edit` const auto log_num = files_snap.persisted_log_files.rbegin()->log_num; // Create a temporary file for saving directory snapshot - auto [compact_log, log_filename] = createLogWriter(delegator, provider, {log_num, 1}, logger, /*manual_flush*/ true); + auto [compact_log, log_filename] = createLogWriter({log_num, 1}, /*manual_flush*/ true); { const String serialized = ser::serializeTo(directory_snap); ReadBufferFromString payload(serialized); diff --git a/dbms/src/Storages/Page/V3/WALStore.h b/dbms/src/Storages/Page/V3/WALStore.h index 742e3c462dc..5828a6eb638 100644 --- a/dbms/src/Storages/Page/V3/WALStore.h +++ b/dbms/src/Storages/Page/V3/WALStore.h @@ -82,7 +82,7 @@ using WALStoreReaderPtr = std::shared_ptr; class WALStore { public: - using ChecksumClass = Digest::CRC64; + constexpr static const char * wal_folder_prefix = "/wal"; static std::pair create( @@ -117,18 +117,17 @@ class WALStore const FileProviderPtr & provider_, Format::LogNumberType last_log_num_); - static std::tuple, LogFilename> + std::tuple, LogFilename> createLogWriter( - PSDiskDelegatorPtr delegator, - const FileProviderPtr & provider, const std::pair & new_log_lvl, - Poco::Logger * logger, bool manual_flush); PSDiskDelegatorPtr delegator; FileProviderPtr provider; mutable std::mutex log_file_mutex; Format::LogNumberType last_log_num; + // select next path for creating new logfile + UInt32 wal_paths_index; std::unique_ptr log_file; Poco::Logger * logger; diff --git a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp index 3fa46e56b55..21ca60ea089 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp @@ -211,22 +211,45 @@ TEST(WALLognameSetTest, ordering) } -class WALStoreTest : public DB::base::TiFlashStorageTestBasic +class WALStoreTest + : public DB::base::TiFlashStorageTestBasic + , public testing::WithParamInterface { +public: + WALStoreTest() + : multi_paths(GetParam()) + { + } + void SetUp() override { auto path = getTemporaryPath(); dropDataOnDisk(path); - // TODO: multi-path - delegator = std::make_shared(getTemporaryPath()); + if (!multi_paths) + { + delegator = std::make_shared(getTemporaryPath()); + } + else + { + // mock 8 dirs for multi-paths + Strings paths; + for (size_t i = 0; i < 8; ++i) + { + paths.emplace_back(fmt::format("{}/path_{}", path, i)); + } + delegator = std::make_shared(paths); + } } +private: + const bool multi_paths; + protected: PSDiskDelegatorPtr delegator; }; -TEST_F(WALStoreTest, FindCheckpointFile) +TEST_P(WALStoreTest, FindCheckpointFile) { Poco::Logger * log = &Poco::Logger::get("WALStoreTest"); auto path = getTemporaryPath(); @@ -276,7 +299,7 @@ TEST_F(WALStoreTest, FindCheckpointFile) } } -TEST_F(WALStoreTest, Empty) +TEST_P(WALStoreTest, Empty) { auto ctx = DB::tests::TiFlashTestEnv::getContext(); auto provider = ctx.getFileProvider(); @@ -299,7 +322,7 @@ TEST_F(WALStoreTest, Empty) ASSERT_EQ(num_callback_called, 0); } -TEST_F(WALStoreTest, ReadWriteRestore) +TEST_P(WALStoreTest, ReadWriteRestore) try { auto ctx = DB::tests::TiFlashTestEnv::getContext(); @@ -419,7 +442,7 @@ try } CATCH -TEST_F(WALStoreTest, ReadWriteRestore2) +TEST_P(WALStoreTest, ReadWriteRestore2) try { auto ctx = DB::tests::TiFlashTestEnv::getContext(); @@ -509,7 +532,7 @@ try } CATCH -TEST_F(WALStoreTest, ManyEdits) +TEST_P(WALStoreTest, ManyEdits) try { auto ctx = DB::tests::TiFlashTestEnv::getContext(); @@ -589,4 +612,15 @@ try } CATCH +INSTANTIATE_TEST_CASE_P( + Disks, + WALStoreTest, + ::testing::Bool(), + [](const ::testing::TestParamInfo & info) -> String { + const auto multi_path = info.param; + if (multi_path) + return "multi_disks"; + return "single_disk"; + }); + } // namespace DB::PS::V3::tests diff --git a/dbms/src/Storages/tests/TiFlashStorageTestBasic.h b/dbms/src/Storages/tests/TiFlashStorageTestBasic.h index e81dab139fe..c2e07ed87eb 100644 --- a/dbms/src/Storages/tests/TiFlashStorageTestBasic.h +++ b/dbms/src/Storages/tests/TiFlashStorageTestBasic.h @@ -67,7 +67,7 @@ class TiFlashStorageTestBasic : public ::testing::Test } protected: - void dropDataOnDisk(const String & path) + static void dropDataOnDisk(const String & path) { if (Poco::File file(path); file.exists()) { @@ -75,7 +75,7 @@ class TiFlashStorageTestBasic : public ::testing::Test } } - void createIfNotExist(const String & path) + static void createIfNotExist(const String & path) { if (Poco::File file(path); !file.exists()) file.createDirectories(); @@ -98,4 +98,4 @@ class TiFlashStorageTestBasic : public ::testing::Test std::unique_ptr db_context; }; } // namespace base -} // namespace DB \ No newline at end of file +} // namespace DB From b18b1d04a4b5b5a28d1e92a654033783372add32 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 16 Mar 2022 17:19:53 +0800 Subject: [PATCH 8/9] Limit build release parallelism (#4301) ref pingcap/tiflash#4141 --- release-centos7-llvm/scripts/build-tiflash-release.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-centos7-llvm/scripts/build-tiflash-release.sh b/release-centos7-llvm/scripts/build-tiflash-release.sh index c424d465742..d3eda1dff4e 100755 --- a/release-centos7-llvm/scripts/build-tiflash-release.sh +++ b/release-centos7-llvm/scripts/build-tiflash-release.sh @@ -35,6 +35,7 @@ set -ueox pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" SRCPATH=$(cd ${SCRIPTPATH}/../..; pwd -P) +NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} INSTALL_DIR="${SRCPATH}/release-centos7-llvm/tiflash" rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} @@ -51,7 +52,7 @@ cmake "${SRCPATH}" ${DEFINE_CMAKE_PREFIX_PATH} \ -DRUN_HAVE_STD_REGEX=0 \ -GNinja -cmake --build . --target tiflash --parallel +cmake --build . --target tiflash --parallel ${NPROC} cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" # unset LD_LIBRARY_PATH before test From 0f23a022bda61ae65553ca7c54098ba29d4f9087 Mon Sep 17 00:00:00 2001 From: SeaRise Date: Wed, 16 Mar 2022 18:11:53 +0800 Subject: [PATCH 9/9] add `HashJoinProbeBlockInputStream` (#4246) close pingcap/tiflash#4248 --- .../HashJoinProbeBlockInputStream.cpp | 69 +++++++++++++++++++ .../HashJoinProbeBlockInputStream.h | 55 +++++++++++++++ .../Coprocessor/DAGQueryBlockInterpreter.cpp | 4 +- 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp create mode 100644 dbms/src/DataStreams/HashJoinProbeBlockInputStream.h diff --git a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp new file mode 100644 index 00000000000..3657f099b74 --- /dev/null +++ b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.cpp @@ -0,0 +1,69 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +namespace DB +{ +HashJoinProbeBlockInputStream::HashJoinProbeBlockInputStream( + const BlockInputStreamPtr & input, + const ExpressionActionsPtr & join_probe_actions_, + const LogWithPrefixPtr & log_) + : log(getMPPTaskLog(log_, name)) + , join_probe_actions(join_probe_actions_) +{ + children.push_back(input); + + if (!join_probe_actions || join_probe_actions->getActions().size() != 1 + || join_probe_actions->getActions().back().type != ExpressionAction::Type::JOIN) + { + throw Exception("isn't valid join probe actions", ErrorCodes::LOGICAL_ERROR); + } +} + +Block HashJoinProbeBlockInputStream::getTotals() +{ + if (IProfilingBlockInputStream * child = dynamic_cast(&*children.back())) + { + totals = child->getTotals(); + join_probe_actions->executeOnTotals(totals); + } + + return totals; +} + +Block HashJoinProbeBlockInputStream::getHeader() const +{ + Block res = children.back()->getHeader(); + join_probe_actions->execute(res); + return res; +} + +Block HashJoinProbeBlockInputStream::readImpl() +{ + Block res = children.back()->read(); + if (!res) + return res; + + join_probe_actions->execute(res); + + // TODO split block if block.size() > settings.max_block_size + // https://github.com/pingcap/tiflash/issues/3436 + + return res; +} + +} // namespace DB diff --git a/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h new file mode 100644 index 00000000000..9c24ecd9190 --- /dev/null +++ b/dbms/src/DataStreams/HashJoinProbeBlockInputStream.h @@ -0,0 +1,55 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace DB +{ +class ExpressionActions; + +/** Executes a certain expression over the block. + * Basically the same as ExpressionBlockInputStream, + * but requires that there must be a join probe action in the Expression. + * + * The join probe action is different from the general expression + * and needs to be executed after join hash map building. + * We should separate it from the ExpressionBlockInputStream. + */ +class HashJoinProbeBlockInputStream : public IProfilingBlockInputStream +{ +private: + using ExpressionActionsPtr = std::shared_ptr; + static constexpr auto name = "HashJoinProbe"; + +public: + HashJoinProbeBlockInputStream( + const BlockInputStreamPtr & input, + const ExpressionActionsPtr & join_probe_actions_, + const LogWithPrefixPtr & log_); + + String getName() const override { return name; } + Block getTotals() override; + Block getHeader() const override; + +protected: + Block readImpl() override; + +private: + const LogWithPrefixPtr log; + ExpressionActionsPtr join_probe_actions; +}; + +} // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index e24d57e138c..6f5dfd24ffd 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -724,7 +724,7 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & } } for (auto & stream : pipeline.streams) - stream = std::make_shared(stream, chain.getLastActions(), taskLogger()); + stream = std::make_shared(stream, chain.getLastActions(), taskLogger()); /// add a project to remove all the useless column NamesWithAliases project_cols;