Merge branch 'master' into autoloading_settings

.github/workflows/ci-integration-db-live.yaml

@@ -18,6 +18,7 @@ jobs:
 
     env:
       PLOOMBER_VERSION_CHECK_DISABLED: true
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
       - name: Checkout
@@ -43,18 +44,20 @@ jobs:
 
       - name: Install dependencies
         if: ${{ steps.secret-check.outputs.available == 'true' }}
-        run:
-          pip install ".[integration]"
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade nox
+          nox --session test_integration_snowflake --install-only
+
       - name: Integration Test
         if: ${{ steps.secret-check.outputs.available == 'true'}}
         env:
           SF_USERNAME: ${{ secrets.SF_USERNAME }}
           SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
           SF_DATABASE: ${{ secrets.SF_DATABASE }}
         run: |
-          # Run the integration test with live option
-          python -c "import sqlalchemy; print('Sqlalchemy version ', sqlalchemy.__version__)"
-          pytest src/tests/integration --verbose --live
+
+          nox --session test_integration_snowflake --no-install --reuse-existing-virtualenvs
 
       - name: Upload failed images artifacts
         uses: actions/upload-artifact@v3

.github/workflows/ci-integration-db.yaml

@@ -15,6 +15,7 @@ jobs:
 
     env:
       PLOOMBER_VERSION_CHECK_DISABLED: true
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
 
@@ -37,13 +38,13 @@ jobs:
           echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc
           source ~/.bashrc
 
-          pip install ".[integration]"
-          pip install "sqlalchemy>=2"
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade nox
+          nox --session test_integration --install-only
+
       - name: Integration Test
         run: |
-          # Run the integration test by pytest marker
-          python -c "import sqlalchemy; print('Sqlalchemy version ', sqlalchemy.__version__)"
-          pytest src/tests/integration --verbose
+          nox --session test_integration --no-install --reuse-existing-virtualenvs
 
       - name: Upload failed images artifacts
         uses: actions/upload-artifact@v3

.github/workflows/ci.yaml

@@ -77,6 +77,7 @@ jobs:
     if: needs.preliminary.outputs.check_doc_modified == 'failure'
     uses: ./.github/workflows/ci-integration-db-live.yaml
     secrets: inherit
+
   integration-test-non-live:
     needs: [preliminary]
     if: needs.preliminary.outputs.check_doc_modified == 'failure'
@@ -94,6 +95,7 @@ jobs:
 
     env:
       PLOOMBER_VERSION_CHECK_DISABLED: true
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
 
@@ -108,25 +110,19 @@ jobs:
       - name: Lint
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade pkgmt codespell
+          python -m pip install --upgrade pkgmt codespell nox
           pkgmt lint
           codespell
 
       - name: Install dependencies
         run: |
 
-          pip install .
-          # check package is importable
-          python -c "import sql"
-          pip install ".[dev]"
-          pip install sqlalchemy -U
+          nox --session test_unit --install-only
 
       - name: Test with pytest
         run: |
-          # ensure we're running sqlalchemy 2
-          python -c "import sqlalchemy; assert int(sqlalchemy.__version__.split('.')[0]) == 2"
-          # run unit tests
-          pytest --durations-min=5 --ignore=src/tests/integration
+
+          nox --session test_unit --no-install --reuse-existing-virtualenvs
 
       - name: Upload failed images artifacts
         uses: actions/upload-artifact@v3
@@ -147,6 +143,7 @@ jobs:
 
     env:
       PLOOMBER_VERSION_CHECK_DISABLED: true
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
 
@@ -160,20 +157,17 @@ jobs:
 
       - name: Lint
         run: |
-          python -m pip install --upgrade pip pkgmt
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade pkgmt nox
           pkgmt lint
 
       - name: Install dependencies
         run: |
-          pip install "sqlalchemy<2"
-          pip install ".[dev]"
+          nox --session test_unit_sqlalchemy_one --install-only
 
       - name: Test with pytest
         run: |
-          # ensure we're running sqlalchemy 1
-          python -c "import sqlalchemy; assert int(sqlalchemy.__version__.split('.')[0]) == 1"
-          # run tests
-          pytest --durations-min=5 --ignore=src/tests/integration
+          nox --session test_unit_sqlalchemy_one --no-install --reuse-existing-virtualenvs
 
       - name: Upload failed images artifacts sqlalchemyv1
         uses: actions/upload-artifact@v3

CHANGELOG.md

@@ -1,9 +1,11 @@
 # CHANGELOG
 
-## 0.8.1dev
+
+## 0.9.0dev
 * [Feature] Allow loading configuration value from a `pyproject.toml` file upon magic initialization (#689)
 * [Fix] Fix error that was incorrectly converted into a print message
-
+* [Fix] Modified histogram query to ensure histogram binning is done correctly (#751)
+* [Fix] Fix bug that caused the `COMMIT` not to work when the SQLAlchemy driver did not support `set_isolation_level`
 * [Fix] Fixed vertical color breaks in histograms (#702)
 
 ## 0.8.0 (2023-07-18)

doc/api/magic-plot.md

@@ -5,7 +5,7 @@ jupytext:
     extension: .md
     format_name: myst
     format_version: 0.13
-    jupytext_version: 1.14.6
+    jupytext_version: 1.14.7
 kernelspec:
   display_name: Python 3 (ipykernel)
   language: python
@@ -139,6 +139,12 @@ generate histograms without explicitly removing NULL entries.
 %sqlplot histogram --table penguins.csv --column body_mass_g 
 ```
 
+When plotting a histogram, it divides a range with the number of bins - 1 to calculate a bin size. Then, it applies round half down relative to the bin size and categorizes continuous values into bins to replicate right closed intervals from the ggplot histogram in R.
+
+![body_mass_g](../static/body_mass_g_R.png)
+
++++
+
 ### Number of bins
 
 ```{code-cell} ipython3

doc/community/developer-guide.md

@@ -106,11 +106,11 @@ Currently, these common errors are handled by providing more meaningful error me
 
 ## Managing Connections
 
-In our codebase, we manage connections to databases with a `Connection` object, this is required for the `%%sql magic` to work. Internally, this connection object has a sqlalchemy connection.
+In our codebase, we manage connections to databases with a `SQLAlchemy` and `DBAPIConnection` objects, this is required for the `%%sql magic` to work.
 
 ### Working with connections
 
-`Connection` should be exclusively used to manage database connections on the user's behalf and to obtain the current SQLAlchemy connection. We can access the current SQLAlchemy connection using `current.session`.
+`ConnectionManager` should be exclusively used to manage database connections on the user's behalf and to obtain the current connection. We can access the current connection using `current`.
 
 ```{code-cell} ipython3
 :tags: [remove-output]
@@ -120,13 +120,13 @@ In our codebase, we manage connections to databases with a `Connection` object,
 ```
 
 ```{code-cell} ipython3
-from sql.connection import Connection
+from sql.connection import ConnectionManager
 
-conn = Connection.current.session
+conn = ConnectionManager.current
 conn
 ```
 
-Functions that expect a `conn` (sometimes named `con`) input variable should only use SQLAlchemy connections.
+Functions that expect a `conn` (sometimes named `con`) input variable should only use connections.
 
 ```python
 def histogram(payload, table, column, bins, with_=None, conn=None):
@@ -139,9 +139,9 @@ When creating data for tests, we should use `sqlalchemy.create_engine` and avoid
 
 ```{code-cell} ipython3
 from sqlalchemy import create_engine
-from sql.connection import Connection
+from sql.connection import SQLAlchemyConnection
 
-conn = Connection(engine=create_engine("sqlite://"))
+conn = SQLAlchemyConnection(engine=create_engine("sqlite://"))
 
 conn.execute("CREATE TABLE some_table (name, age)")
 ```
@@ -375,17 +375,17 @@ We can use [SQLGlot](https://sqlglot.com/sqlglot.html) to build the general sql
 
 Then transpile to the sql which is supported by current connected dialect.
 
-Our `sql.connection.Connection._transpile_query` will automatically detect the dialect and transpile the SQL clause.
+Our `sql.SQLAlchemyConnection._transpile_query` will automatically detect the dialect and transpile the SQL clause.
 
 #### Example
 
 ```{code-cell} ipython3
 # Prepare connection
 from sqlglot import select, condition
-from sql import connection
+from sql.connection import SQLAlchemyConnection
 from sqlalchemy import create_engine
 
-conn = connection.Connection(engine=create_engine(url="sqlite://"))
+conn = SQLAlchemyConnection(engine=create_engine(url="sqlite://"))
 ```
 
 ```{code-cell} ipython3
@@ -416,11 +416,11 @@ We may provide `sqlglot.parse_one({source_sql_clause}, read={source_database_dia
 ##### Prepare connection
 
 ```{code-cell} ipython3
-from sql import connection
+from sql.connection import SQLAlchemyConnection
 from sqlalchemy import create_engine
 import sqlglot
 
-conn = connection.Connection(engine=create_engine(url="duckdb://"))
+conn = SQLAlchemyConnection(engine=create_engine(url="duckdb://"))
 ```
 
 ##### Prepare SQL clause based on duckdb syntax
@@ -441,10 +441,10 @@ conn._transpile_query(input_sql)
 ##### Prepare connection
 
 ```{code-cell} ipython3
-from sql import connection
+from sql.connection import SQLAlchemyConnection
 from sqlalchemy import create_engine
 
-conn = connection.Connection(engine=create_engine(url="sqlite://"))
+conn = SQLAlchemyConnection(engine=create_engine(url="sqlite://"))
 ```
 
 ##### Prepare SQL clause based on sqlite

doc/user-guide/tables-columns.md

@@ -86,9 +86,9 @@ If the table isn't in the default schema, pass `--schema/-s`. Let's create a new
 :tags: [hide-output]
 
 from sqlalchemy import create_engine
-from sql.connection import Connection
+from sql.connection import SQLAlchemyConnection
 
-conn = Connection(engine=create_engine("sqlite:///my.db"))
+conn = SQLAlchemyConnection(engine=create_engine("sqlite:///my.db"))
 conn.execute("CREATE TABLE numbers (n FLOAT)")
 ```
 

examples/plot_boxplot.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 import urllib.request
-from sql.connection import Connection
 
+from sqlalchemy import create_engine
+
+from sql.connection import SQLAlchemyConnection
 from sql import plot
 
 
@@ -11,6 +13,6 @@
         "iris.csv",
     )
 
-conn = Connection.from_connect_str("duckdb://")
+conn = SQLAlchemyConnection(create_engine("duckdb://"))
 
 plot.boxplot("iris.csv", "petal width", conn=conn)

examples/plot_boxplot_custom.py

@@ -1,8 +1,10 @@
 from pathlib import Path
 import urllib.request
 
-from sql.connection import Connection
+from sqlalchemy import create_engine
 
+
+from sql.connection import SQLAlchemyConnection
 from sql import plot
 
 
@@ -12,7 +14,7 @@
         "iris.csv",
     )
 
-conn = Connection.from_connect_str("duckdb://")
+conn = SQLAlchemyConnection(create_engine("duckdb://"))
 
 # returns matplotlib.Axes object
 ax = plot.boxplot("iris.csv", "petal width", conn=conn)

examples/plot_boxplot_horizontal.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 import urllib.request
 
-from sql.connection import Connection
+from sqlalchemy import create_engine
+
+from sql.connection import SQLAlchemyConnection
 
 from sql import plot
 
@@ -12,6 +14,6 @@
         "iris.csv",
     )
 
-conn = Connection.from_connect_str("duckdb://")
+conn = SQLAlchemyConnection(create_engine("duckdb://"))
 
 plot.boxplot("iris.csv", "petal width", conn=conn, orient="h")

examples/plot_boxplot_many.py

@@ -1,7 +1,9 @@
 from pathlib import Path
 import urllib.request
 
-from sql.connection import Connection
+from sqlalchemy import create_engine
+
+from sql.connection import SQLAlchemyConnection
 
 from sql import plot
 
@@ -12,6 +14,6 @@
         "iris.csv",
     )
 
-conn = Connection.from_connect_str("duckdb://")
+conn = SQLAlchemyConnection(create_engine("duckdb://"))
 
 plot.boxplot("iris.csv", ["petal width", "sepal width"], conn=conn)

examples/plot_histogram.py

@@ -1,7 +1,8 @@
 import urllib.request
 
-from sql.connection import Connection
+from sqlalchemy import create_engine
 
+from sql.connection import SQLAlchemyConnection
 from sql import plot
 
 
@@ -10,6 +11,6 @@
     "iris.csv",
 )
 
-conn = Connection.from_connect_str("duckdb://")
+conn = SQLAlchemyConnection(create_engine("duckdb://"))
 
 plot.histogram("iris.csv", "petal width", bins=50, conn=conn)