diff --git a/CHANGELOG.md b/CHANGELOG.md index c231fd6f8..2273a22cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # CHANGELOG -## 0.10.11dev +## 0.10.13dev + +## 0.10.12 (2024-07-12) + +* [Feature] Remove sqlalchemy upper bound ([#1020](https://github.com/ploomber/jupysql/pull/1020)) + +## 0.10.11 (2024-07-03) + +* [Fix] Fix error when connections.ini contains a `query` value as dictionary ([#1015](https://github.com/ploomber/jupysql/issues/1015)) ## 0.10.10 (2024-02-07) diff --git a/README.md b/README.md index b556ea5b8..495b3d2c0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # JupySQL ![CI](https://github.com/ploomber/jupysql/workflows/CI/badge.svg) -![CI Integration Tests](https://github.com/ploomber/jupysql/workflows/CI%20-%20DB%20Integration/badge.svg) +![CI Integration Tests](https://github.com/ploomber/jupysql/actions/workflows/ci-integration-db.yaml/badge.svg) ![Broken Links](https://github.com/ploomber/jupysql/workflows/check-for-broken-links/badge.svg) [![PyPI version](https://badge.fury.io/py/jupysql.svg)](https://badge.fury.io/py/jupysql) [![Twitter](https://img.shields.io/twitter/follow/edublancas?label=Follow&style=social)](https://twitter.com/intent/user?screen_name=ploomber) diff --git a/doc/user-guide/connection-file.md b/doc/user-guide/connection-file.md index 51b7cf8b3..942b3c03a 100644 --- a/doc/user-guide/connection-file.md +++ b/doc/user-guide/connection-file.md @@ -85,6 +85,19 @@ port = 5432 database = db ``` +Or, to connect to an Oracle database, which might require some query parameters: + +```ini +[ora] +drivername = oracle+oracledb +username = myuser +password = mypass +host = my_oracle_server.example.com +port = 1521 +database = my_oracle_pdb.example.com +query = {"servicename": "my_oracle_db.example.com"} +``` + ```{code-cell} ipython3 from pathlib import Path diff --git a/setup.py b/setup.py index 73cd40348..b9dbb2240 100644 --- a/setup.py +++ b/setup.py @@ -19,8 +19,7 @@ "prettytable", # IPython dropped support for Python 3.8 "ipython<=8.12.0; python_version <= '3.8'", - # sqlalchemy 2.0.29 breaking the CI: https://github.com/ploomber/jupysql/issues/1001 - "sqlalchemy<2.0.29", + "sqlalchemy", "sqlparse", "ipython-genutils>=0.1.0", "jinja2", @@ -41,8 +40,7 @@ "pkgmt", "twine", # tests - # DuckDB 0.10.1 breaking Sqlalchemy v1 tests: https://github.com/ploomber/jupysql/issues/1001 # noqa - "duckdb<0.10.1", + "duckdb", "duckdb-engine", "pyodbc", # sql.plot module tests diff --git a/src/sql/__init__.py b/src/sql/__init__.py index 2a0d0b5fe..ab7b92d2f 100644 --- a/src/sql/__init__.py +++ b/src/sql/__init__.py @@ -1,7 +1,7 @@ from sql.magic import load_ipython_extension -__version__ = "0.10.11dev" +__version__ = "0.10.13dev" __all__ = ["load_ipython_extension"] diff --git a/src/sql/_testing.py b/src/sql/_testing.py index 041994e28..5c0418186 100644 --- a/src/sql/_testing.py +++ b/src/sql/_testing.py @@ -121,7 +121,7 @@ def get_tmp_dir(): "alias": "mySQLTest", "docker_ct": { "name": "mysql", - "image": "mysql", + "image": "mysql:8.0", "ports": {3306: 33306}, }, "query": {}, diff --git a/src/sql/parse.py b/src/sql/parse.py index 29bfbbee4..16abfd138 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -5,6 +5,7 @@ from pathlib import Path import configparser import warnings +import ast from sqlalchemy.engine.url import URL @@ -29,6 +30,25 @@ ] +def _parse_config_section(section): + """Return a given configuration section as a dictionary of keys and values + + If the section contains `query` as key, its value is evaluated such + that a `"{...}"` string is also converted to a dictionary. + + Parameters + ---------- + section : list[tuple[str,str]] + The section object as returned by ConfigParser.items() + """ + url_args = dict(section) + + if "query" in url_args: + url_args["query"] = ast.literal_eval(url_args["query"]) + + return url_args + + class ConnectionsFile: def __init__(self, path_to_file) -> None: self.parser = configparser.ConfigParser() @@ -43,7 +63,7 @@ def get_default_connection_url(self): except configparser.NoSectionError: return None - url = URL.create(**dict(section)) + url = URL.create(**_parse_config_section(section)) return str(url.render_as_string(hide_password=False)) @@ -87,10 +107,8 @@ def connection_str_from_dsn_section(section, config): f"connections file {config.dsn_filename!r}" ) from e - cfg_dict = dict(cfg) - try: - url = URL.create(**cfg_dict) + url = URL.create(**_parse_config_section(cfg)) except TypeError as e: if "unexpected keyword argument" in str(e): raise exceptions.TypeError( @@ -134,8 +152,8 @@ def _connection_string(arg, path_to_file): section = arg.lstrip("[").rstrip("]") parser = configparser.ConfigParser() parser.read(path_to_file) - cfg_dict = dict(parser.items(section)) - url = URL.create(**cfg_dict) + cfg = parser.items(section) + url = URL.create(**_parse_config_section(cfg)) url_ = str(url.render_as_string(hide_password=False)) warnings.warn( diff --git a/src/tests/integration/test_generic_db_operations.py b/src/tests/integration/test_generic_db_operations.py index 987eed02a..415206296 100644 --- a/src/tests/integration/test_generic_db_operations.py +++ b/src/tests/integration/test_generic_db_operations.py @@ -1414,7 +1414,7 @@ def test_autocommit_create_table_multiple_cells( "ip_with_oracle", "mysnip", [ - "table or view does not exist", + 'table or view "PLOOMBER_APP"."MYSNIP" does not exist', ], "RuntimeError", ), diff --git a/src/tests/test_config.py b/src/tests/test_config.py index e7a512ef1..31dafcd2f 100644 --- a/src/tests/test_config.py +++ b/src/tests/test_config.py @@ -91,6 +91,23 @@ def test_start_ini_default_connection_if_any(tmp_empty, ip_no_magics): assert ConnectionManager.current.dialect == "sqlite" +def test_config_loads_query_element_as_url_params(tmp_empty, ip_no_magics): + Path("connections.ini").write_text( + """ +[default] +drivername = sqlite +query = {'param1': 'value1', 'param2': 'value2'} +""" + ) + ip_no_magics.run_cell("%config SqlMagic.dsn_filename = 'connections.ini'") + + load_ipython_extension(ip_no_magics) + + assert set(ConnectionManager.connections) == {"default"} + assert ConnectionManager.current.dialect == "sqlite" + assert ConnectionManager.current.url == "sqlite://?param1=value1¶m2=value2" + + def test_load_home_toml_if_no_pyproject_toml( tmp_empty, ip_no_magics, capsys, monkeypatch ): diff --git a/src/tests/test_dsn_config.ini b/src/tests/test_dsn_config.ini index 29c17282d..8d02cfadc 100644 --- a/src/tests/test_dsn_config.ini +++ b/src/tests/test_dsn_config.ini @@ -11,4 +11,12 @@ drivername = mysql host = 127.0.0.1 database = dolfin username = thefin -password = fishputsfishonthetable \ No newline at end of file +password = fishputsfishonthetable + +[DB_CONFIG_3] +drivername = sqlite +host = 127.0.0.1 +database = dolfin +username = thefin +password = dafish +query = {'sound': 'squeek', 'color': 'grey'} diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index cebacca04..0b0c0af93 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -2296,11 +2296,11 @@ def test_get_query_type(query, query_type): [ ( "%sql select '{\"a\": 1}'::json -> 'a';", - "1", + 1, ), ( '%sql select \'[{"b": "c"}]\'::json -> 0;', - '{"b":"c"}', + {"b": "c"}, ), ( "%sql select '{\"a\": 1}'::json ->> 'a';", @@ -2314,13 +2314,13 @@ def test_get_query_type(query, query_type): """%%sql select '{\"a\": 1}'::json -> 'a';""", - "1", + 1, ), ( """%%sql select '[{\"b\": \"c\"}]'::json -> 0;""", - '{"b":"c"}', + {"b": "c"}, ), ( """%%sql select '{\"a\": 1}'::json @@ -2338,15 +2338,15 @@ def test_get_query_type(query, query_type): ), ( "%sql SELECT '{\"a\": 1}'::json -> 'a';", - "1", + 1, ), ( "%sql SELect '{\"a\": 1}'::json -> 'a';", - "1", + 1, ), ( "%sql SELECT json('{\"a\": 1}') -> 'a';", - "1", + 1, ), ], ids=[ @@ -2377,7 +2377,7 @@ def test_json_arrow_operators(ip, query, expected): """%%sql --save snippet select '{\"a\": 1}'::json -> 'a';""", "%sql select * from snippet", - "1", + 1, ), ( """%sql --save snippet select '[{\"b\": \"c\"}]'::json ->> 0;""", @@ -2390,7 +2390,7 @@ def test_json_arrow_operators(ip, query, expected): -> 2 as number""", "%sql select number from snippet", - "3", + 3, ), ], ids=["cell-magic-key", "line-magic-index", "cell-magic-multi-line-as-column"], @@ -2737,11 +2737,11 @@ def test_var_substitution_section(ip_empty, tmp_empty): [ ( '%sql select json(\'[{"a":1}, {"b":2}]\')', - '[{"a":1},{"b":2}]', + "[{'a': 1}, {'b': 2}]", ), ( '%sql select \'[{"a":1}, {"b":2}]\'::json', - '[{"a":1}, {"b":2}]', + "[{'a': 1}, {'b': 2}]", ), ], ) diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index 3ea46041c..6018b2497 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -170,6 +170,10 @@ def test_parse_connect_shovel_over_newlines(): "DB_CONFIG_2", "mysql://thefin:fishputsfishonthetable@127.0.0.1/dolfin", ), + ( + "DB_CONFIG_3", + "sqlite://thefin:dafish@127.0.0.1/dolfin?color=grey&sound=squeek", + ), ], ) def test_connection_from_dsn_section(section, expected): @@ -192,6 +196,10 @@ def test_connection_from_dsn_section(section, expected): ), ("DB_CONFIG_1", ""), ("not-a-url", ""), + ( + "[DB_CONFIG_3]", + "sqlite://thefin:dafish@127.0.0.1/dolfin?color=grey&sound=squeek", + ), ], ids=[ "empty", @@ -200,6 +208,7 @@ def test_connection_from_dsn_section(section, expected): "section", "not-a-section", "not-a-url", + "section-with-query", ], ) def test_connection_string(input_, expected): diff --git a/src/tests/test_plot.py b/src/tests/test_plot.py index cf5a8fe28..12e1c0cbd 100644 --- a/src/tests/test_plot.py +++ b/src/tests/test_plot.py @@ -47,7 +47,7 @@ def __repr__(self) -> str: def test_boxplot_stats(chinook_db, ip_empty): - # there's some werid behavior in duckdb-engine that will cause the + # there's some weird behavior in duckdb-engine that will cause the # table not to be found if we call commit ip_empty.run_cell("%config SqlMagic.autocommit=False") ip_empty.run_cell("%sql duckdb://") @@ -65,7 +65,7 @@ def test_boxplot_stats(chinook_db, ip_empty): def test_boxplot_stats_exception(chinook_db, ip_empty): - # there's some werid behavior in duckdb-engine that will cause the + # there's some weird behavior in duckdb-engine that will cause the # table not to be found if we call commit ip_empty.run_cell("%config SqlMagic.autocommit=False") ip_empty.run_cell("%sql duckdb://") @@ -101,7 +101,7 @@ def test_summary_stats(chinook_db, ip_empty, tmp_empty): """ ) - # there's some werid behavior in duckdb-engine that will cause the + # there's some weird behavior in duckdb-engine that will cause the # table not to be found if we call commit ip_empty.run_cell("%config SqlMagic.autocommit=False") ip_empty.run_cell("%sql duckdb://") @@ -114,7 +114,7 @@ def test_summary_stats(chinook_db, ip_empty, tmp_empty): def test_summary_stats_missing_file(chinook_db, ip_empty): - # there's some werid behavior in duckdb-engine that will cause the + # there's some weird behavior in duckdb-engine that will cause the # table not to be found if we call commit ip_empty.run_cell("%config SqlMagic.autocommit=False") ip_empty.run_cell("%sql duckdb://")