Fix changing default database by USE in session mode

.github/workflows/build_arm_wheels.yml

@@ -25,7 +25,7 @@ jobs:
         os: [ ubuntu-20.04 ]
     env:
       RUNNER_OS: ${{ matrix.os }}
-      PYTHON_VERSIONS: "3.8 3.9 3.10 3.11"
+      PYTHON_VERSIONS: "3.8 3.9 3.10 3.11 3.12"
     steps:
       - name: Install clang++ for Ubuntu
         if: matrix.os == 'ubuntu-20.04'

.github/workflows/build_wheels.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-20.04 ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11"]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         # python-version: [ "3.7" ]
     env:
       RUNNER_OS: ${{ matrix.os }}
@@ -87,7 +87,7 @@ jobs:
           clang++ --version
       - name: Run chdb/build.sh
         run: |
-          python3 -m pip install pybind11
+          python3 -m pip install pybind11 setuptools
           export CC=/usr/bin/clang
           export CXX=/usr/bin/clang++
           bash ./chdb/build.sh
@@ -149,7 +149,7 @@ jobs:
       matrix:
         os: [ macos-12 ]
         # python-version: [ "3.8", "3.9", "3.10", "3.11"]
-        python-version: [ "3.11" ]
+        python-version: [ "3.11", "3.12" ]
     env:
       RUNNER_OS: ${{ matrix.os }}
       PYTHON_VERSION: ${{ matrix.python-version }}
@@ -207,7 +207,7 @@ jobs:
       - name: Run chdb/build.sh
         timeout-minutes: 300
         run: |
-          python3 -m pip install pybind11
+          python3 -m pip install pybind11 setuptools
           export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@16)/bin/clang
           export CXX=$(brew --prefix llvm@16)/bin/clang++
@@ -266,28 +266,28 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
 
   build_wheels_macos:
-    name:  cbw on ${{ matrix.os }}
+    name:  ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ macos-11 ]
         # python-version: [ "3.8", "3.9", "3.10"]
-        python-version: [ "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10" ]
     env:
       RUNNER_OS: ${{ matrix.os }}
       PYTHON_VERSION: ${{ matrix.python-version }}
     steps:
-      - name: Install Offical Python 3.11
-        uses: actions/setup-python@v2
+      - name: Install Offical Python
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Check Python version
         run: |
           python3 -VV
           python3-config --includes
       - name: Install clang++ for macOS
-        if: matrix.os == 'macos-11' || matrix.os == 'macos-12'
+        if: matrix.os == 'macos-11'
         run: |
           pwd
           uname -a
@@ -322,12 +322,17 @@ jobs:
           key: ${{ matrix.os }}
           max-size: 5G
           append-timestamp: true
-      - name: Prepare chdb/build.sh
+      - name: Run chdb/build.sh
+        timeout-minutes: 300
         run: |
           python3 -m pip install pybind11
           export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@16)/bin/clang
-          export CXX=$(brew --prefix llvm@16)/bin/clang++ 
+          export CXX=$(brew --prefix llvm@16)/bin/clang++
+          bash gen_manifest.sh
+          bash ./chdb/build.sh
+          python3 -m pip install pandas pyarrow
+          bash -x ./chdb/test_smoke.sh
         continue-on-error: false
       - name: Check ccache statistics
         run: |
@@ -336,24 +341,6 @@ jobs:
           df -h
         env:
           CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++"
-      # - name: Remove /usr/local/bin/python3
-      #   run: |
-      #     sudo rm -f /usr/local/bin/python3
-      - name: Install dependencies for building wheels
-        run: |
-          python3 -m pip install -U pip tox pybind11 twine setuptools wheel
-          python3 -m pip install cibuildwheel==2.12.1
-      - name: Build wheels
-        run: python3 -m cibuildwheel --output-dir wheelhouse
-        timeout-minutes: 300
-        env:
-          CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++"
-          CIBW_DEBUG: 1
-          CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh"
-          CIBW_BUILD_VERBOSITY: 3
-          CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64"
-          CIBW_TEST_REQUIRES: "pyarrow pandas psutil"
-          CIBW_TEST_COMMAND: "cd {project} && make test"
       - name: Keep killall ccache and wait for ccache to finish
         if: always()
         run: |
@@ -362,17 +349,37 @@ jobs:
             killall ccache; \
             sleep 10; \
           done
+      - name: Install dependencies for building wheels
+        run: |
+          python3 -m pip install -U pip tox pybind11 twine setuptools wheel
+      - name: Build wheels
+        timeout-minutes: 300
+        run: |
+          export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export CC=$(brew --prefix llvm@16)/bin/clang
+          export CXX=$(brew --prefix llvm@16)/bin/clang++
+          make wheel
+      - name: Fix wheel platform tag
+        run: |
+          python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl
+      - name: Run tests
+        run: |
+          python3 -m pip install dist/*.whl
+          python3 -m pip install pandas pyarrow psutil
+          python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)"
+          make test
+        continue-on-error: false
       - name: Show files
-        run: ls -lh wheelhouse
+        run: ls -lh dist
         shell: bash
       - uses: actions/upload-artifact@v3
         with:
-          path: ./wheelhouse/*.whl
+          path: ./dist/*.whl
       - name: Upload pypi
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
           python3 -m pip install twine
-          python3 -m twine upload wheelhouse/*.whl
+          python3 -m twine upload dist/*.whl
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

chdb/build_linux_arm64.sh

@@ -10,7 +10,7 @@ for PY_VER in ${PYTHON_VERSIONS}; do
     cd ${PROJ_DIR}
     pyenv local "${PY_VER}"
     python3 --version
-    python3 -m pip install pybind11
+    python3 -m pip install pybind11 setuptools
     export CC=/usr/bin/clang
     export CXX=/usr/bin/clang++
     # Install universal2 pkg

chdb/build_mac_arm64.sh

@@ -51,7 +51,7 @@ if [ "${dest}" != "darwin-x86_64" ]; then
 fi
 
 
-for PY_VER in 3.9.13 3.10.11 3.11.3; do
+for PY_VER in 3.9.13 3.10.11 3.11.3 3.12.0; do
     if [ ! -f ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg ]; then
         wget https://www.python.org/ftp/python/${PY_VER}/python-${PY_VER}-macos11.pkg -O ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg
     fi
@@ -73,7 +73,7 @@ for PY_VER in 3.9.13 3.10.11 3.11.3; do
         exit 1
     fi
 
-    python3 -m pip install -U pybind11 wheel build tox psutil
+    python3 -m pip install -U pybind11 wheel build tox psutil setuptools
     rm -rf ${PROJ_DIR}/buildlib
 
     ${PROJ_DIR}/chdb/build.sh

programs/local/LocalServer.cpp

@@ -58,6 +58,7 @@
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
 #include <filesystem>
+#include <fstream>
 
 #include "config.h"
 
@@ -801,6 +802,42 @@ void LocalServer::processConfig()
         global_context->getUserDefinedSQLObjectsLoader().loadObjects();
 
         LOG_DEBUG(log, "Loaded metadata.");
+
+        /** Set default database if it is specified in default_database file.
+          * NOTE: We do it after loading metadata to let the '_local' and 'system' database initialization
+          * to be done correctly.
+          * I have tried to set default database during parsing '--database' option, but it leads to
+          *   "Code: 82. DB::Exception: Database db_xxx already exists.: while loading database `db_xxx` 
+          *      from path .state_tmp_auxten_usedb_/metadata/db_xxx. (DATABASE_ALREADY_EXISTS)"
+          * This will also happen if we call:
+          *   `clickhouse local --database=db_xxx --path=.state_tmp_auxten_usedb_ --query="select * FROM log_table_xxx"`
+          * with existing `db_xxx` database in the '.state_tmp_auxten_usedb_' directory.
+          */
+        auto default_database_path = fs::path(path) / "default_database";
+        if (std::filesystem::exists(default_database_path))
+        {
+            std::ifstream ifs(default_database_path);
+            std::string user_default_database;
+            if (ifs.is_open())
+            {
+                ifs >> user_default_database;
+                // strip default_database
+                user_default_database.erase(
+                    std::remove_if(
+                        user_default_database.begin(), user_default_database.end(), [](unsigned char x) { return std::isspace(x); }),
+                    user_default_database.end());
+                if (!user_default_database.empty())
+                {
+                    global_context->setCurrentDatabase(user_default_database);
+                    LOG_DEBUG(log, "Set default database to {} recorded in {}", user_default_database, default_database_path);
+                }
+                ifs.close();
+            }
+            else
+            {
+                LOG_ERROR(log, "Cannot read default database from {}", default_database_path);
+            }
+        }
     }
     else if (!config().has("no-system-tables"))
     {

src/Interpreters/InterpreterUseQuery.cpp

@@ -2,17 +2,40 @@
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterUseQuery.h>
 #include <Access/Common/AccessFlags.h>
+#include <Common/Exception.h>
 #include <Common/typeid_cast.h>
 
+#include <fstream>
 
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int CANNOT_OPEN_FILE;
+}
+
 BlockIO InterpreterUseQuery::execute()
 {
     const String & new_database = query_ptr->as<ASTUseQuery &>().getDatabase();
     getContext()->checkAccess(AccessType::SHOW_DATABASES, new_database);
     getContext()->getSessionContext()->setCurrentDatabase(new_database);
+
+    // Save the current using database in default_database stored in getPath()
+    // for the case when the database is changed in chDB session.
+    // The default_database content is used in the LocalServer::processConfig() method.
+    auto default_database_path = fs::path(getContext()->getPath()) / "default_database";
+    std::ofstream tmp_path_fs(default_database_path, std::ofstream::out | std::ofstream::trunc);
+    if (tmp_path_fs && tmp_path_fs.is_open())
+    {
+        tmp_path_fs << new_database;
+        tmp_path_fs.close();
+    }
+    else
+    {
+        throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Cannot open file {} for writing", default_database_path.string());
+    }
+
     return {};
 }
 

tests/test_usedb.py

@@ -0,0 +1,42 @@
+#!python3
+
+import unittest
+import shutil
+import psutil
+from chdb import session
+
+
+test_state_dir = ".state_tmp_auxten_usedb_"
+current_process = psutil.Process()
+check_thread_count = False
+
+
+class TestStateful(unittest.TestCase):
+    def setUp(self) -> None:
+        shutil.rmtree(test_state_dir, ignore_errors=True)
+        return super().setUp()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(test_state_dir, ignore_errors=True)
+        return super().tearDown()
+
+    def test_path(self):
+        sess = session.Session(test_state_dir)
+
+        sess.query("CREATE DATABASE IF NOT EXISTS db_xxx ENGINE = Atomic", "CSV")
+        ret = sess.query("SHOW DATABASES", "CSV")
+        self.assertIn("db_xxx", str(ret))
+
+        sess.query("CREATE TABLE IF NOT EXISTS db_xxx.log_table_xxx (x UInt8) ENGINE = Log;")
+        sess.query("INSERT INTO db_xxx.log_table_xxx VALUES (1), (2), (3), (4);")
+
+        ret = sess.query("USE db_xxx; SELECT * FROM log_table_xxx", "Debug")
+        self.assertEqual(str(ret), "1\n2\n3\n4\n")
+
+        sess.query("USE db_xxx")
+        ret = sess.query("SELECT * FROM log_table_xxx", "Debug")
+        self.assertEqual(str(ret), "1\n2\n3\n4\n")
+
+
+if __name__ == '__main__':
+    unittest.main()