Merge branch 'ploomber:master' into running-sql-using-sparkconnect-sh…

…ould-not-print-full-stack-trace

CHANGELOG.md

@@ -1,6 +1,14 @@
 # CHANGELOG
 
-## 0.10.11dev
+## 0.10.13dev
+
+## 0.10.12 (2024-07-12)
+
+* [Feature] Remove sqlalchemy upper bound ([#1020](https://github.com/ploomber/jupysql/pull/1020))
+
+## 0.10.11 (2024-07-03)
+
+* [Fix] Fix error when connections.ini contains a `query` value as dictionary ([#1015](https://github.com/ploomber/jupysql/issues/1015))
 
 * [Feature] Disable full stack trace when using spark connect ([#1011](https://github.com/ploomber/jupysql/issues/1011)) (by [@b1ackout](https://github.com/b1ackout))
 

README.md

@@ -1,6 +1,6 @@
 # JupySQL
 ![CI](https://github.com/ploomber/jupysql/workflows/CI/badge.svg)
-![CI Integration Tests](https://github.com/ploomber/jupysql/workflows/CI%20-%20DB%20Integration/badge.svg)
+![CI Integration Tests](https://github.com/ploomber/jupysql/actions/workflows/ci-integration-db.yaml/badge.svg)
 ![Broken Links](https://github.com/ploomber/jupysql/workflows/check-for-broken-links/badge.svg)
 [![PyPI version](https://badge.fury.io/py/jupysql.svg)](https://badge.fury.io/py/jupysql)
 [![Twitter](https://img.shields.io/twitter/follow/edublancas?label=Follow&style=social)](https://twitter.com/intent/user?screen_name=ploomber)

doc/user-guide/connection-file.md

@@ -85,6 +85,19 @@ port = 5432
 database = db
 ```
 
+Or, to connect to an Oracle database, which might require some query parameters:
+
+```ini
+[ora]
+drivername = oracle+oracledb
+username = myuser
+password = mypass
+host = my_oracle_server.example.com
+port = 1521
+database = my_oracle_pdb.example.com
+query = {"servicename": "my_oracle_db.example.com"}
+```
+
 ```{code-cell} ipython3
 from pathlib import Path
 

setup.py

@@ -19,8 +19,7 @@
     "prettytable",
     # IPython dropped support for Python 3.8
     "ipython<=8.12.0; python_version <= '3.8'",
-    # sqlalchemy 2.0.29 breaking the CI: https://github.com/ploomber/jupysql/issues/1001
-    "sqlalchemy<2.0.29",
+    "sqlalchemy",
     "sqlparse",
     "ipython-genutils>=0.1.0",
     "jinja2",
@@ -43,8 +42,7 @@
     "pkgmt",
     "twine",
     # tests
-    # DuckDB 0.10.1 breaking Sqlalchemy v1 tests: https://github.com/ploomber/jupysql/issues/1001 # noqa
-    "duckdb<0.10.1",
+    "duckdb",
     "duckdb-engine",
     "pyodbc",
     # sql.plot module tests

src/sql/__init__.py

@@ -1,7 +1,7 @@
 from sql.magic import load_ipython_extension
 
 
-__version__ = "0.10.11dev"
+__version__ = "0.10.13dev"
 
 
 __all__ = ["load_ipython_extension"]

src/sql/_testing.py

@@ -121,7 +121,7 @@ def get_tmp_dir():
         "alias": "mySQLTest",
         "docker_ct": {
             "name": "mysql",
-            "image": "mysql",
+            "image": "mysql:8.0",
             "ports": {3306: 33306},
         },
         "query": {},

src/sql/parse.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 import configparser
 import warnings
+import ast
 
 from sqlalchemy.engine.url import URL
 
@@ -29,6 +30,25 @@
 ]
 
 
+def _parse_config_section(section):
+    """Return a given configuration section as a dictionary of keys and values
+
+    If the section contains `query` as key, its value is evaluated such
+    that a `"{...}"` string is also converted to a dictionary.
+
+    Parameters
+    ----------
+    section : list[tuple[str,str]]
+        The section object as returned by ConfigParser.items()
+    """
+    url_args = dict(section)
+
+    if "query" in url_args:
+        url_args["query"] = ast.literal_eval(url_args["query"])
+
+    return url_args
+
+
 class ConnectionsFile:
     def __init__(self, path_to_file) -> None:
         self.parser = configparser.ConfigParser()
@@ -43,7 +63,7 @@ def get_default_connection_url(self):
         except configparser.NoSectionError:
             return None
 
-        url = URL.create(**dict(section))
+        url = URL.create(**_parse_config_section(section))
         return str(url.render_as_string(hide_password=False))
 
 
@@ -87,10 +107,8 @@ def connection_str_from_dsn_section(section, config):
             f"connections file {config.dsn_filename!r}"
         ) from e
 
-    cfg_dict = dict(cfg)
-
     try:
-        url = URL.create(**cfg_dict)
+        url = URL.create(**_parse_config_section(cfg))
     except TypeError as e:
         if "unexpected keyword argument" in str(e):
             raise exceptions.TypeError(
@@ -134,8 +152,8 @@ def _connection_string(arg, path_to_file):
         section = arg.lstrip("[").rstrip("]")
         parser = configparser.ConfigParser()
         parser.read(path_to_file)
-        cfg_dict = dict(parser.items(section))
-        url = URL.create(**cfg_dict)
+        cfg = parser.items(section)
+        url = URL.create(**_parse_config_section(cfg))
         url_ = str(url.render_as_string(hide_password=False))
 
         warnings.warn(

src/tests/integration/test_generic_db_operations.py

@@ -1414,7 +1414,7 @@ def test_autocommit_create_table_multiple_cells(
             "ip_with_oracle",
             "mysnip",
             [
-                "table or view does not exist",
+                'table or view "PLOOMBER_APP"."MYSNIP" does not exist',
             ],
             "RuntimeError",
         ),

src/tests/test_config.py

@@ -91,6 +91,23 @@ def test_start_ini_default_connection_if_any(tmp_empty, ip_no_magics):
     assert ConnectionManager.current.dialect == "sqlite"
 
 
+def test_config_loads_query_element_as_url_params(tmp_empty, ip_no_magics):
+    Path("connections.ini").write_text(
+        """
+[default]
+drivername = sqlite
+query = {'param1': 'value1', 'param2': 'value2'}
+"""
+    )
+    ip_no_magics.run_cell("%config SqlMagic.dsn_filename = 'connections.ini'")
+
+    load_ipython_extension(ip_no_magics)
+
+    assert set(ConnectionManager.connections) == {"default"}
+    assert ConnectionManager.current.dialect == "sqlite"
+    assert ConnectionManager.current.url == "sqlite://?param1=value1&param2=value2"
+
+
 def test_load_home_toml_if_no_pyproject_toml(
     tmp_empty, ip_no_magics, capsys, monkeypatch
 ):

src/tests/test_dsn_config.ini

@@ -11,4 +11,12 @@ drivername       = mysql
 host             = 127.0.0.1
 database         = dolfin
 username         = thefin 
-password         = fishputsfishonthetable
+password         = fishputsfishonthetable
+
+[DB_CONFIG_3]
+drivername       = sqlite 
+host             = 127.0.0.1
+database         = dolfin
+username         = thefin 
+password         = dafish
+query            = {'sound': 'squeek', 'color': 'grey'}

src/tests/test_magic.py

@@ -2296,11 +2296,11 @@ def test_get_query_type(query, query_type):
     [
         (
             "%sql select '{\"a\": 1}'::json -> 'a';",
-            "1",
+            1,
         ),
         (
             '%sql select \'[{"b": "c"}]\'::json -> 0;',
-            '{"b":"c"}',
+            {"b": "c"},
         ),
         (
             "%sql select '{\"a\": 1}'::json ->> 'a';",
@@ -2314,13 +2314,13 @@ def test_get_query_type(query, query_type):
             """%%sql select '{\"a\": 1}'::json
             ->
             'a';""",
-            "1",
+            1,
         ),
         (
             """%%sql select '[{\"b\": \"c\"}]'::json
                 ->
             0;""",
-            '{"b":"c"}',
+            {"b": "c"},
         ),
         (
             """%%sql select '{\"a\": 1}'::json
@@ -2338,15 +2338,15 @@ def test_get_query_type(query, query_type):
         ),
         (
             "%sql SELECT '{\"a\": 1}'::json -> 'a';",
-            "1",
+            1,
         ),
         (
             "%sql SELect '{\"a\": 1}'::json -> 'a';",
-            "1",
+            1,
         ),
         (
             "%sql SELECT json('{\"a\": 1}') -> 'a';",
-            "1",
+            1,
         ),
     ],
     ids=[
@@ -2377,7 +2377,7 @@ def test_json_arrow_operators(ip, query, expected):
             """%%sql --save snippet
             select '{\"a\": 1}'::json -> 'a';""",
             "%sql select * from snippet",
-            "1",
+            1,
         ),
         (
             """%sql --save snippet select '[{\"b\": \"c\"}]'::json ->> 0;""",
@@ -2390,7 +2390,7 @@ def test_json_arrow_operators(ip, query, expected):
             -> 2
             as number""",
             "%sql select number from snippet",
-            "3",
+            3,
         ),
     ],
     ids=["cell-magic-key", "line-magic-index", "cell-magic-multi-line-as-column"],
@@ -2737,11 +2737,11 @@ def test_var_substitution_section(ip_empty, tmp_empty):
     [
         (
             '%sql select json(\'[{"a":1}, {"b":2}]\')',
-            '[{"a":1},{"b":2}]',
+            "[{'a': 1}, {'b': 2}]",
         ),
         (
             '%sql select \'[{"a":1}, {"b":2}]\'::json',
-            '[{"a":1}, {"b":2}]',
+            "[{'a': 1}, {'b': 2}]",
         ),
     ],
 )

src/tests/test_parse.py

@@ -170,6 +170,10 @@ def test_parse_connect_shovel_over_newlines():
             "DB_CONFIG_2",
             "mysql://thefin:[email protected]/dolfin",
         ),
+        (
+            "DB_CONFIG_3",
+            "sqlite://thefin:[email protected]/dolfin?color=grey&sound=squeek",
+        ),
     ],
 )
 def test_connection_from_dsn_section(section, expected):
@@ -192,6 +196,10 @@ def test_connection_from_dsn_section(section, expected):
         ),
         ("DB_CONFIG_1", ""),
         ("not-a-url", ""),
+        (
+            "[DB_CONFIG_3]",
+            "sqlite://thefin:[email protected]/dolfin?color=grey&sound=squeek",
+        ),
     ],
     ids=[
         "empty",
@@ -200,6 +208,7 @@ def test_connection_from_dsn_section(section, expected):
         "section",
         "not-a-section",
         "not-a-url",
+        "section-with-query",
     ],
 )
 def test_connection_string(input_, expected):

src/tests/test_plot.py

@@ -47,7 +47,7 @@ def __repr__(self) -> str:
 
 
 def test_boxplot_stats(chinook_db, ip_empty):
-    # there's some werid behavior in duckdb-engine that will cause the
+    # there's some weird behavior in duckdb-engine that will cause the
     # table not to be found if we call commit
     ip_empty.run_cell("%config SqlMagic.autocommit=False")
     ip_empty.run_cell("%sql duckdb://")
@@ -65,7 +65,7 @@ def test_boxplot_stats(chinook_db, ip_empty):
 
 
 def test_boxplot_stats_exception(chinook_db, ip_empty):
-    # there's some werid behavior in duckdb-engine that will cause the
+    # there's some weird behavior in duckdb-engine that will cause the
     # table not to be found if we call commit
     ip_empty.run_cell("%config SqlMagic.autocommit=False")
     ip_empty.run_cell("%sql duckdb://")
@@ -101,7 +101,7 @@ def test_summary_stats(chinook_db, ip_empty, tmp_empty):
 """
     )
 
-    # there's some werid behavior in duckdb-engine that will cause the
+    # there's some weird behavior in duckdb-engine that will cause the
     # table not to be found if we call commit
     ip_empty.run_cell("%config SqlMagic.autocommit=False")
     ip_empty.run_cell("%sql duckdb://")
@@ -114,7 +114,7 @@ def test_summary_stats(chinook_db, ip_empty, tmp_empty):
 
 
 def test_summary_stats_missing_file(chinook_db, ip_empty):
-    # there's some werid behavior in duckdb-engine that will cause the
+    # there's some weird behavior in duckdb-engine that will cause the
     # table not to be found if we call commit
     ip_empty.run_cell("%config SqlMagic.autocommit=False")
     ip_empty.run_cell("%sql duckdb://")