From 973dd87d3f0e339959662d6df08c721cef5760d7 Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 06:10:15 +0000 Subject: [PATCH 1/8] Fix changing default database by USE in session mode --- programs/local/LocalServer.cpp | 37 +++++++++++++++++++++ src/Interpreters/InterpreterUseQuery.cpp | 23 +++++++++++++ tests/test_usedb.py | 42 ++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 tests/test_usedb.py diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index baaf455b7bf..e8ba7a8ee65 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include "config.h" @@ -801,6 +802,42 @@ void LocalServer::processConfig() global_context->getUserDefinedSQLObjectsLoader().loadObjects(); LOG_DEBUG(log, "Loaded metadata."); + + /** Set default database if it is specified in default_database file. + * NOTE: We do it after loading metadata to let the '_local' and 'system' database initialization + * to be done correctly. + * I have tried to set default database during parsing '--database' option, but it leads to + * "Code: 82. DB::Exception: Database db_xxx already exists.: while loading database `db_xxx` + * from path .state_tmp_auxten_usedb_/metadata/db_xxx. (DATABASE_ALREADY_EXISTS)" + * This will also happen if we call: + * `clickhouse local --database=db_xxx --path=.state_tmp_auxten_usedb_ --query="select * FROM log_table_xxx"` + * with existing `db_xxx` database in the '.state_tmp_auxten_usedb_' directory. + */ + auto default_database_path = fs::path(path) / "default_database"; + if (std::filesystem::exists(default_database_path)) + { + std::ifstream ifs(default_database_path); + std::string user_default_database; + if (ifs.is_open()) + { + ifs >> user_default_database; + // strip default_database + user_default_database.erase( + std::remove_if( + user_default_database.begin(), user_default_database.end(), [](unsigned char x) { return std::isspace(x); }), + user_default_database.end()); + if (!user_default_database.empty()) + { + global_context->setCurrentDatabase(user_default_database); + LOG_DEBUG(log, "Set default database to {} recorded in {}", user_default_database, default_database_path); + } + ifs.close(); + } + else + { + LOG_ERROR(log, "Cannot read default database from {}", default_database_path); + } + } } else if (!config().has("no-system-tables")) { diff --git a/src/Interpreters/InterpreterUseQuery.cpp b/src/Interpreters/InterpreterUseQuery.cpp index b71f3a9cc1c..f8f413332fa 100644 --- a/src/Interpreters/InterpreterUseQuery.cpp +++ b/src/Interpreters/InterpreterUseQuery.cpp @@ -2,17 +2,40 @@ #include #include #include +#include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_OPEN_FILE; +} + BlockIO InterpreterUseQuery::execute() { const String & new_database = query_ptr->as().getDatabase(); getContext()->checkAccess(AccessType::SHOW_DATABASES, new_database); getContext()->getSessionContext()->setCurrentDatabase(new_database); + + // Save the current using database in default_database stored in getPath() + // for the case when the database is changed in chDB session. + // The default_database content is used in the LocalServer::processConfig() method. + auto default_database_path = fs::path(getContext()->getPath()) / "default_database"; + std::ofstream tmp_path_fs(default_database_path, std::ofstream::out | std::ofstream::trunc); + if (tmp_path_fs && tmp_path_fs.is_open()) + { + tmp_path_fs << new_database; + tmp_path_fs.close(); + } + else + { + throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Cannot open file {} for writing", default_database_path.string()); + } + return {}; } diff --git a/tests/test_usedb.py b/tests/test_usedb.py new file mode 100644 index 00000000000..0eed3e09589 --- /dev/null +++ b/tests/test_usedb.py @@ -0,0 +1,42 @@ +#!python3 + +import unittest +import shutil +import psutil +from chdb import session + + +test_state_dir = ".state_tmp_auxten_usedb_" +current_process = psutil.Process() +check_thread_count = False + + +class TestStateful(unittest.TestCase): + def setUp(self) -> None: + shutil.rmtree(test_state_dir, ignore_errors=True) + return super().setUp() + + def tearDown(self) -> None: + shutil.rmtree(test_state_dir, ignore_errors=True) + return super().tearDown() + + def test_path(self): + sess = session.Session(test_state_dir) + + sess.query("CREATE DATABASE IF NOT EXISTS db_xxx ENGINE = Atomic", "CSV") + ret = sess.query("SHOW DATABASES", "CSV") + self.assertIn("db_xxx", str(ret)) + + sess.query("CREATE TABLE IF NOT EXISTS db_xxx.log_table_xxx (x UInt8) ENGINE = Log;") + sess.query("INSERT INTO db_xxx.log_table_xxx VALUES (1), (2), (3), (4);") + + ret = sess.query("USE db_xxx; SELECT * FROM log_table_xxx", "Debug") + self.assertEqual(str(ret), "1\n2\n3\n4\n") + + sess.query("USE db_xxx") + ret = sess.query("SELECT * FROM log_table_xxx", "Debug") + self.assertEqual(str(ret), "1\n2\n3\n4\n") + + +if __name__ == '__main__': + unittest.main() From 393ea1bccb7a2c1694119675087342c548edff7f Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 09:33:44 +0000 Subject: [PATCH 2/8] Use py38 in macOS 11 --- .github/workflows/build_wheels.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index fa9075847a8..4221d7d9c60 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -273,12 +273,12 @@ jobs: matrix: os: [ macos-11 ] # python-version: [ "3.8", "3.9", "3.10"] - python-version: [ "3.11" ] + python-version: [ "3.8" ] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} steps: - - name: Install Offical Python 3.11 + - name: Install Offical Python 3.8 uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -287,7 +287,7 @@ jobs: python3 -VV python3-config --includes - name: Install clang++ for macOS - if: matrix.os == 'macos-11' || matrix.os == 'macos-12' + if: matrix.os == 'macos-11' run: | pwd uname -a From 27b1cf9e6d73e27b12c97e42eaee90bc21e862be Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 10:58:37 +0000 Subject: [PATCH 3/8] Debug macos 11 --- .github/workflows/build_wheels.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 4221d7d9c60..fb3f9617dd3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -273,13 +273,13 @@ jobs: matrix: os: [ macos-11 ] # python-version: [ "3.8", "3.9", "3.10"] - python-version: [ "3.8" ] + python-version: [ "3.8", "3.9", "3.10" ] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} steps: - name: Install Offical Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Check Python version @@ -336,20 +336,20 @@ jobs: df -h env: CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" - # - name: Remove /usr/local/bin/python3 - # run: | - # sudo rm -f /usr/local/bin/python3 + - name: Remove /usr/local/bin/python3 + run: | + sudo rm -f /usr/local/bin/python3 - name: Install dependencies for building wheels run: | python3 -m pip install -U pip tox pybind11 twine setuptools wheel - python3 -m pip install cibuildwheel==2.12.1 + python3 -m pip install cibuildwheel==2.16.2 - name: Build wheels run: python3 -m cibuildwheel --output-dir wheelhouse timeout-minutes: 300 env: CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" CIBW_DEBUG: 1 - CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh" + CIBW_BEFORE_BUILD: "which python3 && pip install -U pip tox pybind11 && python3 -m pybind11 --includes && bash -x gen_manifest.sh && bash chdb/build.sh" CIBW_BUILD_VERBOSITY: 3 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" CIBW_TEST_REQUIRES: "pyarrow pandas psutil" From ef8516326c6d9afc5c2bf334b8bfb1272fdb5170 Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 11:07:54 +0000 Subject: [PATCH 4/8] Build one version in each runner --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index fb3f9617dd3..3f05bb8e689 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -278,7 +278,7 @@ jobs: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} steps: - - name: Install Offical Python 3.8 + - name: Install Offical Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -351,7 +351,7 @@ jobs: CIBW_DEBUG: 1 CIBW_BEFORE_BUILD: "which python3 && pip install -U pip tox pybind11 && python3 -m pybind11 --includes && bash -x gen_manifest.sh && bash chdb/build.sh" CIBW_BUILD_VERBOSITY: 3 - CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + # CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" CIBW_TEST_REQUIRES: "pyarrow pandas psutil" CIBW_TEST_COMMAND: "cd {project} && make test" - name: Keep killall ccache and wait for ccache to finish From a459fe43176e7a6fcea710659267e68b34d84dfc Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 11:09:40 +0000 Subject: [PATCH 5/8] Fix builder name --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 3f05bb8e689..74b1784d597 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -266,7 +266,7 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} build_wheels_macos: - name: cbw on ${{ matrix.os }} + name: ${{ matrix.os }} py${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false From a0dba5dfba807b926f1370434d85caf84cd0074d Mon Sep 17 00:00:00 2001 From: auxten Date: Wed, 15 Nov 2023 20:43:34 +0800 Subject: [PATCH 6/8] Drop cibuildwheel --- .github/workflows/build_wheels.yml | 53 +++++++++++++++++------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 74b1784d597..cb7b2ee49f4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -322,12 +322,17 @@ jobs: key: ${{ matrix.os }} max-size: 5G append-timestamp: true - - name: Prepare chdb/build.sh + - name: Run chdb/build.sh + timeout-minutes: 300 run: | python3 -m pip install pybind11 export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@16)/bin/clang - export CXX=$(brew --prefix llvm@16)/bin/clang++ + export CXX=$(brew --prefix llvm@16)/bin/clang++ + bash gen_manifest.sh + bash ./chdb/build.sh + python3 -m pip install pandas pyarrow + bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Check ccache statistics run: | @@ -336,24 +341,6 @@ jobs: df -h env: CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install dependencies for building wheels - run: | - python3 -m pip install -U pip tox pybind11 twine setuptools wheel - python3 -m pip install cibuildwheel==2.16.2 - - name: Build wheels - run: python3 -m cibuildwheel --output-dir wheelhouse - timeout-minutes: 300 - env: - CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" - CIBW_DEBUG: 1 - CIBW_BEFORE_BUILD: "which python3 && pip install -U pip tox pybind11 && python3 -m pybind11 --includes && bash -x gen_manifest.sh && bash chdb/build.sh" - CIBW_BUILD_VERBOSITY: 3 - # CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" - CIBW_TEST_REQUIRES: "pyarrow pandas psutil" - CIBW_TEST_COMMAND: "cd {project} && make test" - name: Keep killall ccache and wait for ccache to finish if: always() run: | @@ -362,17 +349,37 @@ jobs: killall ccache; \ sleep 10; \ done + - name: Install dependencies for building wheels + run: | + python3 -m pip install -U pip tox pybind11 twine setuptools wheel + - name: Build wheels + timeout-minutes: 300 + run: | + export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@16)/bin/clang + export CXX=$(brew --prefix llvm@16)/bin/clang++ + make wheel + - name: Fix wheel platform tag + run: | + python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Run tests + run: | + python3 -m pip install dist/*.whl + python3 -m pip install pandas pyarrow psutil + python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + make test + continue-on-error: false - name: Show files - run: ls -lh wheelhouse + run: ls -lh dist shell: bash - uses: actions/upload-artifact@v3 with: - path: ./wheelhouse/*.whl + path: ./dist/*.whl - name: Upload pypi if: startsWith(github.ref, 'refs/tags/v') run: | python3 -m pip install twine - python3 -m twine upload wheelhouse/*.whl + python3 -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} From 0c6680221a820420a80cc0ac01a00532d1bc8638 Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 16 Nov 2023 02:30:22 +0000 Subject: [PATCH 7/8] Add Python 3.12 --- .github/workflows/build_arm_wheels.yml | 2 +- .github/workflows/build_wheels.yml | 4 ++-- chdb/build_mac_arm64.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_arm_wheels.yml b/.github/workflows/build_arm_wheels.yml index 7bd8073dc5e..5c14b0aaa5f 100644 --- a/.github/workflows/build_arm_wheels.yml +++ b/.github/workflows/build_arm_wheels.yml @@ -25,7 +25,7 @@ jobs: os: [ ubuntu-20.04 ] env: RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSIONS: "3.8 3.9 3.10 3.11" + PYTHON_VERSIONS: "3.8 3.9 3.10 3.11 3.12" steps: - name: Install clang++ for Ubuntu if: matrix.os == 'ubuntu-20.04' diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index cb7b2ee49f4..e46d03aba0a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-20.04 ] - python-version: [ "3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] # python-version: [ "3.7" ] env: RUNNER_OS: ${{ matrix.os }} @@ -149,7 +149,7 @@ jobs: matrix: os: [ macos-12 ] # python-version: [ "3.8", "3.9", "3.10", "3.11"] - python-version: [ "3.11" ] + python-version: [ "3.11", "3.12" ] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/chdb/build_mac_arm64.sh b/chdb/build_mac_arm64.sh index 9d00a100fd5..d48aa3765ef 100755 --- a/chdb/build_mac_arm64.sh +++ b/chdb/build_mac_arm64.sh @@ -51,7 +51,7 @@ if [ "${dest}" != "darwin-x86_64" ]; then fi -for PY_VER in 3.9.13 3.10.11 3.11.3; do +for PY_VER in 3.9.13 3.10.11 3.11.3 3.12.0; do if [ ! -f ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg ]; then wget https://www.python.org/ftp/python/${PY_VER}/python-${PY_VER}-macos11.pkg -O ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg fi From d17020f003e35dedf8499bf69db02aa2862e824a Mon Sep 17 00:00:00 2001 From: auxten Date: Thu, 16 Nov 2023 04:48:46 +0000 Subject: [PATCH 8/8] Install setuptools explicitly for Python 3.12.0 --- .github/workflows/build_wheels.yml | 4 ++-- chdb/build_linux_arm64.sh | 2 +- chdb/build_mac_arm64.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e46d03aba0a..25a021bcd89 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -87,7 +87,7 @@ jobs: clang++ --version - name: Run chdb/build.sh run: | - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export CC=/usr/bin/clang export CXX=/usr/bin/clang++ bash ./chdb/build.sh @@ -207,7 +207,7 @@ jobs: - name: Run chdb/build.sh timeout-minutes: 300 run: | - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@16)/bin/clang export CXX=$(brew --prefix llvm@16)/bin/clang++ diff --git a/chdb/build_linux_arm64.sh b/chdb/build_linux_arm64.sh index 410249ceb81..f6c1ece4bf2 100755 --- a/chdb/build_linux_arm64.sh +++ b/chdb/build_linux_arm64.sh @@ -10,7 +10,7 @@ for PY_VER in ${PYTHON_VERSIONS}; do cd ${PROJ_DIR} pyenv local "${PY_VER}" python3 --version - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export CC=/usr/bin/clang export CXX=/usr/bin/clang++ # Install universal2 pkg diff --git a/chdb/build_mac_arm64.sh b/chdb/build_mac_arm64.sh index d48aa3765ef..3188d1c428c 100755 --- a/chdb/build_mac_arm64.sh +++ b/chdb/build_mac_arm64.sh @@ -73,7 +73,7 @@ for PY_VER in 3.9.13 3.10.11 3.11.3 3.12.0; do exit 1 fi - python3 -m pip install -U pybind11 wheel build tox psutil + python3 -m pip install -U pybind11 wheel build tox psutil setuptools rm -rf ${PROJ_DIR}/buildlib ${PROJ_DIR}/chdb/build.sh