From 53e63ab4a268513aeebbf5f3df959bd9313b4722 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 19 Aug 2024 12:14:21 +0530 Subject: [PATCH 01/18] databricks-sqlalchemy plugin split --- .gitignore | 2 +- README.md | 3 - pyproject.toml | 67 +++ .../README.sqlalchemy.md | 203 +++++++ src/databricks_sqlalchemy/README.tests.md | 44 ++ src/databricks_sqlalchemy/__init__.py | 4 + src/databricks_sqlalchemy/_ddl.py | 100 ++++ src/databricks_sqlalchemy/_parse.py | 385 +++++++++++++ src/databricks_sqlalchemy/_types.py | 325 +++++++++++ src/databricks_sqlalchemy/base.py | 436 ++++++++++++++ src/databricks_sqlalchemy/py.typed | 0 src/databricks_sqlalchemy/requirements.py | 249 ++++++++ src/databricks_sqlalchemy/setup.cfg | 4 + src/databricks_sqlalchemy/test/__init__.py | 0 src/databricks_sqlalchemy/test/_extra.py | 70 +++ src/databricks_sqlalchemy/test/_future.py | 331 +++++++++++ src/databricks_sqlalchemy/test/_regression.py | 311 ++++++++++ .../test/_unsupported.py | 450 +++++++++++++++ src/databricks_sqlalchemy/test/conftest.py | 13 + .../test/overrides/__init__.py | 0 .../overrides/_componentreflectiontest.py | 189 ++++++ .../test/overrides/_ctetest.py | 33 ++ src/databricks_sqlalchemy/test/test_suite.py | 12 + .../test_local/__init__.py | 5 + .../test_local/conftest.py | 44 ++ .../test_local/e2e/MOCK_DATA.xlsx | Bin 0 -> 59837 bytes .../test_local/e2e/__init__.py | 0 .../test_local/e2e/test_basic.py | 543 ++++++++++++++++++ .../test_local/test_ddl.py | 96 ++++ .../test_local/test_parsing.py | 160 ++++++ .../test_local/test_types.py | 161 ++++++ 31 files changed, 4236 insertions(+), 4 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/databricks_sqlalchemy/README.sqlalchemy.md create mode 100644 src/databricks_sqlalchemy/README.tests.md create mode 100644 src/databricks_sqlalchemy/__init__.py create mode 100644 src/databricks_sqlalchemy/_ddl.py create mode 100644 src/databricks_sqlalchemy/_parse.py create mode 100644 src/databricks_sqlalchemy/_types.py create mode 100644 src/databricks_sqlalchemy/base.py create mode 100755 src/databricks_sqlalchemy/py.typed create mode 100644 src/databricks_sqlalchemy/requirements.py create mode 100644 src/databricks_sqlalchemy/setup.cfg create mode 100644 src/databricks_sqlalchemy/test/__init__.py create mode 100644 src/databricks_sqlalchemy/test/_extra.py create mode 100644 src/databricks_sqlalchemy/test/_future.py create mode 100644 src/databricks_sqlalchemy/test/_regression.py create mode 100644 src/databricks_sqlalchemy/test/_unsupported.py create mode 100644 src/databricks_sqlalchemy/test/conftest.py create mode 100644 src/databricks_sqlalchemy/test/overrides/__init__.py create mode 100644 src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py create mode 100644 src/databricks_sqlalchemy/test/overrides/_ctetest.py create mode 100644 src/databricks_sqlalchemy/test/test_suite.py create mode 100644 src/databricks_sqlalchemy/test_local/__init__.py create mode 100644 src/databricks_sqlalchemy/test_local/conftest.py create mode 100644 src/databricks_sqlalchemy/test_local/e2e/MOCK_DATA.xlsx create mode 100644 src/databricks_sqlalchemy/test_local/e2e/__init__.py create mode 100644 src/databricks_sqlalchemy/test_local/e2e/test_basic.py create mode 100644 src/databricks_sqlalchemy/test_local/test_ddl.py create mode 100644 src/databricks_sqlalchemy/test_local/test_parsing.py create mode 100644 src/databricks_sqlalchemy/test_local/test_types.py diff --git a/.gitignore b/.gitignore index 9581b67..4c7e499 100644 --- a/.gitignore +++ b/.gitignore @@ -194,7 +194,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ ### Python Patch ### # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration diff --git a/README.md b/README.md index a8823de..e69de29 100644 --- a/README.md +++ b/README.md @@ -1,3 +0,0 @@ -# SQLAlchemy Dialect for Databricks - -See PECO-1396 for more information about this repository. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c328585 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,67 @@ +[tool.poetry] +name = "databricks-sqlalchemy" +version = "1.0.0" +description = "Databricks SQLAlchemy plugin for Python" +authors = ["Databricks "] +license = "Apache-2.0" +readme = "README.md" +packages = [{ include = "databricks_sqlalchemy", from = "src" }] +include = ["CHANGELOG.md"] + +[tool.poetry.dependencies] +python = "^3.8.0" +#thrift = ">=0.16.0,<0.21.0" +#pandas = [ +# { version = ">=1.2.5,<2.2.0", python = ">=3.8" } +#] +#pyarrow = ">=14.0.1,<17" + +#lz4 = "^4.0.2" +#requests = "^2.18.1" +#oauthlib = "^3.1.0" +#numpy = [ +# { version = "^1.16.6", python = ">=3.8,<3.11" }, +# { version = "^1.23.4", python = ">=3.11" }, +#] +#sqlalchemy = { version = ">=2.0.21", optional = true } +sqlalchemy = { version = ">=2.0.21" } +#openpyxl = "^3.0.10" +#alembic = { version = "^1.0.11", optional = true } +#urllib3 = ">=1.26" + +#[tool.poetry.extras] +#sqlalchemy = ["sqlalchemy"] +#alembic = ["sqlalchemy", "alembic"] + +[tool.poetry.dev-dependencies] +pytest = "^7.1.2" +mypy = "^1.10.1" +pylint = ">=2.12.0" +black = "^22.3.0" +pytest-dotenv = "^0.5.2" + +[tool.poetry.urls] +"Homepage" = "https://github.com/databricks/databricks-sql-python" +"Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues" + +[tool.poetry.plugins."sqlalchemy.dialects"] +"databricks" = "databricks.sqlalchemy:DatabricksDialect" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.mypy] +ignore_missing_imports = "true" +exclude = ['ttypes\.py$', 'TCLIService\.py$'] + +[tool.black] +exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' +# +#[tool.pytest.ini_options] +#markers = {"reviewed" = "Test case has been reviewed by Databricks"} +#minversion = "6.0" +#log_cli = "false" +#log_cli_level = "INFO" +#testpaths = ["tests", "src/databricks/sqlalchemy/test_local"] +#env_files = ["test.env"] diff --git a/src/databricks_sqlalchemy/README.sqlalchemy.md b/src/databricks_sqlalchemy/README.sqlalchemy.md new file mode 100644 index 0000000..8aa5197 --- /dev/null +++ b/src/databricks_sqlalchemy/README.sqlalchemy.md @@ -0,0 +1,203 @@ +## Databricks dialect for SQLALchemy 2.0 + +The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. + +## Usage with SQLAlchemy <= 2.0 +A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. + + +## Installation + +To install the dialect and its dependencies: + +```shell +pip install databricks-sql-connector[sqlalchemy] +``` + +If you also plan to use `alembic` you can alternatively run: + +```shell +pip install databricks-sql-connector[alembic] +``` + +## Connection String + +Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: + +1. Host +2. HTTP Path for a compute resource +3. API access token +4. Initial catalog for the connection +5. Initial schema for the connection + +**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** + +For example: + +```python +import os +from sqlalchemy import create_engine + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") +catalog = os.getenv("DATABRICKS_CATALOG") +schema = os.getenv("DATABRICKS_SCHEMA") + +engine = create_engine( + f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" + ) +``` + +## Types + +The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. + +|SQLAlchemy Type|Databricks SQL Type| +|-|-| +[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) +[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| +[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) +[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) +[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| +[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) +[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| +[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) +[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) +[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| +[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| +[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) +[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) + +In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: + +- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) +- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) +- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) + + +### `LargeBinary()` and `PickleType()` + +Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. + +## `Enum()` and `CHECK` constraints + +Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. + +SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. + +### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` + +Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. + +The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. + +If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. + +_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ + +```python +from sqlalchemy import DateTime +from databricks.sqlalchemy import TIMESTAMP + +class SomeModel(Base): + some_date_without_timezone = DateTime() + some_date_with_timezone = TIMESTAMP() +``` + +### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` + +Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. + +### `Time()` + +Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). + +```python +from sqlalchemy import Time + +class SomeModel(Base): + time_tz = Time(timezone=True) + time_ntz = Time() +``` + + +# Usage Notes + +## `Identity()` and `autoincrement` + +Identity and generated value support is currently limited in this dialect. + +When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. + +Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. + +```python +from sqlalchemy import Identity, String + +class SomeModel(Base): + id = BigInteger(Identity()) + value = String() +``` + +When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). + +## Parameters + +`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. + +## Usage with pandas + +Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. + +### Read from Databricks SQL into pandas +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +with engine.connect() as conn: + # This will read the contents of `main.test.some_table` + df = pd.read_sql("some_table", conn) +``` + +### Write to Databricks SQL from pandas + +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +squares = [(i, i * i) for i in range(100)] +df = pd.DataFrame(data=squares,columns=['x','x_squared']) + +with engine.connect() as conn: + # This will write the contents of `df` to `main.test.squares` + df.to_sql('squares',conn) +``` + +## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) + +Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. + +When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. + +If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. + +```python +from sqlalchemy import Table, Column, ForeignKey, BigInteger, String + +users = Table( + "users", + metadata_obj, + Column("id", BigInteger, primary_key=True), + Column("name", String(), nullable=False), + Column("email", String()), + Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) +) +``` diff --git a/src/databricks_sqlalchemy/README.tests.md b/src/databricks_sqlalchemy/README.tests.md new file mode 100644 index 0000000..3ed92ab --- /dev/null +++ b/src/databricks_sqlalchemy/README.tests.md @@ -0,0 +1,44 @@ +## SQLAlchemy Dialect Compliance Test Suite with Databricks + +The contents of the `test/` directory follow the SQLAlchemy developers' [guidance] for running the reusable dialect compliance test suite. Since not every test in the suite is applicable to every dialect, two options are provided to skip tests: + +- Any test can be skipped by subclassing its parent class, re-declaring the test-case and adding a `pytest.mark.skip` directive. +- Any test that is decorated with a `@requires` decorator can be skipped by marking the indicated requirement as `.closed()` in `requirements.py` + +We prefer to skip test cases directly with the first method wherever possible. We only mark requirements as `closed()` if there is no easier option to avoid a test failure. This principally occurs in test cases where the same test in the suite is parametrized, and some parameter combinations are conditionally skipped depending on `requirements.py`. If we skip the entire test method, then we skip _all_ permutations, not just the combinations we don't support. + +## Regression, Unsupported, and Future test cases + +We maintain three files of test cases that we import from the SQLAlchemy source code: + +* **`_regression.py`** contains all the tests cases with tests that we expect to pass for our dialect. Each one is marked with `pytest.mark.reiewed` to indicate that we've evaluated it for relevance. This file only contains base class declarations. +* **`_unsupported.py`** contains test cases that fail because of missing features in Databricks. We mark them as skipped with a `SkipReason` enumeration. If Databricks comes to support these features, those test or entire classes can be moved to `_regression.py`. +* **`_future.py`** contains test cases that fail because of missing features in the dialect itself, but which _are_ supported by Databricks generally. We mark them as skipped with a `FutureFeature` enumeration. These are features that have not been prioritised or that do not violate our acceptance criteria. All of these test cases will eventually move to either `_regression.py`. + +In some cases, only certain tests in class should be skipped with a `SkipReason` or `FutureFeature` justification. In those cases, we import the class into `_regression.py`, then import it from there into one or both of `_future.py` and `_unsupported.py`. If a class needs to be "touched" by regression, unsupported, and future, the class will be imported in that order. If an entire class should be skipped, then we do not import it into `_regression.py` at all. + +We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dialect test fixtures but which are specific to Databricks (e.g TinyIntegerTest). + +## Running the reusable dialect tests + +``` +poetry shell +cd src/databricks/sqlalchemy/test +python -m pytest test_suite.py --dburi \ + "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" +``` + +Whatever schema you pass in the `dburi` argument should be empty. Some tests also require the presence of an empty schema named `test_schema`. Note that we plan to implement our own `provision.py` which SQLAlchemy can automatically use to create an empty schema for testing. But for now this is a manual process. + +You can run only reviewed tests by appending `-m "reviewed"` to the test runner invocation. + +You can run only the unreviewed tests by appending `-m "not reviewed"` instead. + +Note that because these tests depend on SQLAlchemy's custom pytest plugin, they are not discoverable by IDE-based test runners like VSCode or PyCharm and must be invoked from a CLI. + +## Running local unit and e2e tests + +Apart from the SQLAlchemy reusable suite, we maintain our own unit and e2e tests under the `test_local/` directory. These can be invoked from a VSCode or Pycharm since they don't depend on a custom pytest plugin. Due to pytest's lookup order, the `pytest.ini` which is required for running the reusable dialect tests, also conflicts with VSCode and Pycharm's default pytest implementation and overrides the settings in `pyproject.toml`. So to run these tests, you can delete or rename `pytest.ini`. + + +[guidance]: "https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_22/README.dialects.rst" diff --git a/src/databricks_sqlalchemy/__init__.py b/src/databricks_sqlalchemy/__init__.py new file mode 100644 index 0000000..35ea050 --- /dev/null +++ b/src/databricks_sqlalchemy/__init__.py @@ -0,0 +1,4 @@ +from databricks_sqlalchemy.base import DatabricksDialect +from databricks_sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ + +__all__ = ["TINYINT", "TIMESTAMP", "TIMESTAMP_NTZ"] diff --git a/src/databricks_sqlalchemy/_ddl.py b/src/databricks_sqlalchemy/_ddl.py new file mode 100644 index 0000000..d5d0bf8 --- /dev/null +++ b/src/databricks_sqlalchemy/_ddl.py @@ -0,0 +1,100 @@ +import re +from sqlalchemy.sql import compiler, sqltypes +import logging + +logger = logging.getLogger(__name__) + + +class DatabricksIdentifierPreparer(compiler.IdentifierPreparer): + """https://docs.databricks.com/en/sql/language-manual/sql-ref-identifiers.html""" + + legal_characters = re.compile(r"^[A-Z0-9_]+$", re.I) + + def __init__(self, dialect): + super().__init__(dialect, initial_quote="`") + + +class DatabricksDDLCompiler(compiler.DDLCompiler): + def post_create_table(self, table): + post = [" USING DELTA"] + if table.comment: + comment = self.sql_compiler.render_literal_value( + table.comment, sqltypes.String() + ) + post.append("COMMENT " + comment) + + post.append("TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'enabled')") + return "\n".join(post) + + def visit_unique_constraint(self, constraint, **kw): + logger.warning("Databricks does not support unique constraints") + pass + + def visit_check_constraint(self, constraint, **kw): + logger.warning("This dialect does not support check constraints") + pass + + def visit_identity_column(self, identity, **kw): + """When configuring an Identity() with Databricks, only the always option is supported. + All other options are ignored. + + Note: IDENTITY columns must always be defined as BIGINT. An exception will be raised if INT is used. + + https://www.databricks.com/blog/2022/08/08/identity-columns-to-generate-surrogate-keys-are-now-available-in-a-lakehouse-near-you.html + """ + text = "GENERATED %s AS IDENTITY" % ( + "ALWAYS" if identity.always else "BY DEFAULT", + ) + return text + + def visit_set_column_comment(self, create, **kw): + return "ALTER TABLE %s ALTER COLUMN %s COMMENT %s" % ( + self.preparer.format_table(create.element.table), + self.preparer.format_column(create.element), + self.sql_compiler.render_literal_value( + create.element.comment, sqltypes.String() + ), + ) + + def visit_drop_column_comment(self, create, **kw): + return "ALTER TABLE %s ALTER COLUMN %s COMMENT ''" % ( + self.preparer.format_table(create.element.table), + self.preparer.format_column(create.element), + ) + + def get_column_specification(self, column, **kwargs): + """ + Emit a log message if a user attempts to set autoincrement=True on a column. + See comments in test_suite.py. We may implement implicit IDENTITY using this + feature in the future, similar to the Microsoft SQL Server dialect. + """ + if column is column.table._autoincrement_column or column.autoincrement is True: + logger.warning( + "Databricks dialect ignores SQLAlchemy's autoincrement semantics. Use explicit Identity() instead." + ) + + colspec = super().get_column_specification(column, **kwargs) + if column.comment is not None: + literal = self.sql_compiler.render_literal_value( + column.comment, sqltypes.STRINGTYPE + ) + colspec += " COMMENT " + literal + + return colspec + + +class DatabricksStatementCompiler(compiler.SQLCompiler): + def limit_clause(self, select, **kw): + """Identical to the default implementation of SQLCompiler.limit_clause except it writes LIMIT ALL instead of LIMIT -1, + since Databricks SQL doesn't support the latter. + + https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-limit.html + """ + text = "" + if select._limit_clause is not None: + text += "\n LIMIT " + self.process(select._limit_clause, **kw) + if select._offset_clause is not None: + if select._limit_clause is None: + text += "\n LIMIT ALL" + text += " OFFSET " + self.process(select._offset_clause, **kw) + return text diff --git a/src/databricks_sqlalchemy/_parse.py b/src/databricks_sqlalchemy/_parse.py new file mode 100644 index 0000000..e9498ec --- /dev/null +++ b/src/databricks_sqlalchemy/_parse.py @@ -0,0 +1,385 @@ +from typing import List, Optional, Dict +import re + +import sqlalchemy +from sqlalchemy.engine import CursorResult +from sqlalchemy.engine.interfaces import ReflectedColumn + +from databricks_sqlalchemy import _types as type_overrides + +""" +This module contains helper functions that can parse the contents +of metadata and exceptions received from DBR. These are mostly just +wrappers around regexes. +""" + + +class DatabricksSqlAlchemyParseException(Exception): + pass + + +def _match_table_not_found_string(message: str) -> bool: + """Return True if the message contains a substring indicating that a table was not found""" + + DBR_LTE_12_NOT_FOUND_STRING = "Table or view not found" + DBR_GT_12_NOT_FOUND_STRING = "TABLE_OR_VIEW_NOT_FOUND" + return any( + [ + DBR_LTE_12_NOT_FOUND_STRING in message, + DBR_GT_12_NOT_FOUND_STRING in message, + ] + ) + + +def _describe_table_extended_result_to_dict_list( + result: CursorResult, +) -> List[Dict[str, str]]: + """Transform the CursorResult of DESCRIBE TABLE EXTENDED into a list of Dictionaries""" + + rows_to_return = [] + for row in result.all(): + this_row = {"col_name": row.col_name, "data_type": row.data_type} + rows_to_return.append(this_row) + + return rows_to_return + + +def extract_identifiers_from_string(input_str: str) -> List[str]: + """For a string input resembling (`a`, `b`, `c`) return a list of identifiers ['a', 'b', 'c']""" + + # This matches the valid character list contained in DatabricksIdentifierPreparer + pattern = re.compile(r"`([A-Za-z0-9_]+)`") + matches = pattern.findall(input_str) + return [i for i in matches] + + +def extract_identifier_groups_from_string(input_str: str) -> List[str]: + """For a string input resembling : + + FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_sqlalchemy`.`tb1` (`name`, `id`, `attr`) + + Return ['(`pname`, `pid`, `pattr`)', '(`name`, `id`, `attr`)'] + """ + pattern = re.compile(r"\([`A-Za-z0-9_,\s]*\)") + matches = pattern.findall(input_str) + return [i for i in matches] + + +def extract_three_level_identifier_from_constraint_string(input_str: str) -> dict: + """For a string input resembling : + FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`) + + Return a dict like + { + "catalog": "main", + "schema": "pysql_dialect_compliance", + "table": "users" + } + + Raise a DatabricksSqlAlchemyParseException if a 3L namespace isn't found + """ + pat = re.compile(r"REFERENCES\s+(.*?)\s*\(") + matches = pat.findall(input_str) + + if not matches: + raise DatabricksSqlAlchemyParseException( + "3L namespace not found in constraint string" + ) + + first_match = matches[0] + parts = first_match.split(".") + + def strip_backticks(input: str): + return input.replace("`", "") + + try: + return { + "catalog": strip_backticks(parts[0]), + "schema": strip_backticks(parts[1]), + "table": strip_backticks(parts[2]), + } + except IndexError: + raise DatabricksSqlAlchemyParseException( + "Incomplete 3L namespace found in constraint string: " + ".".join(parts) + ) + + +def _parse_fk_from_constraint_string(constraint_str: str) -> dict: + """Build a dictionary of foreign key constraint information from a constraint string. + + For example: + + ``` + FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_dialect_compliance`.`tb1` (`name`, `id`, `attr`) + ``` + + Return a dictionary like: + + ``` + { + "constrained_columns": ["pname", "pid", "pattr"], + "referred_table": "tb1", + "referred_schema": "pysql_dialect_compliance", + "referred_columns": ["name", "id", "attr"] + } + ``` + + Note that the constraint name doesn't appear in the constraint string so it will not + be present in the output of this function. + """ + + referred_table_dict = extract_three_level_identifier_from_constraint_string( + constraint_str + ) + referred_table = referred_table_dict["table"] + referred_schema = referred_table_dict["schema"] + + # _extracted is a tuple of two lists of identifiers + # we assume the first immediately follows "FOREIGN KEY" and the second + # immediately follows REFERENCES $tableName + _extracted = extract_identifier_groups_from_string(constraint_str) + constrained_columns_str, referred_columns_str = ( + _extracted[0], + _extracted[1], + ) + + constrained_columns = extract_identifiers_from_string(constrained_columns_str) + referred_columns = extract_identifiers_from_string(referred_columns_str) + + return { + "constrained_columns": constrained_columns, + "referred_table": referred_table, + "referred_columns": referred_columns, + "referred_schema": referred_schema, + } + + +def build_fk_dict( + fk_name: str, fk_constraint_string: str, schema_name: Optional[str] +) -> dict: + """ + Given a foriegn key name and a foreign key constraint string, return a dictionary + with the following keys: + + name + the name of the foreign key constraint + constrained_columns + a list of column names that make up the foreign key + referred_table + the name of the table that the foreign key references + referred_columns + a list of column names that are referenced by the foreign key + referred_schema + the name of the schema that the foreign key references. + + referred schema will be None if the schema_name argument is None. + This is required by SQLAlchey's ComponentReflectionTest::test_get_foreign_keys + """ + + # The foreign key name is not contained in the constraint string so we + # need to add it manually + base_fk_dict = _parse_fk_from_constraint_string(fk_constraint_string) + + if not schema_name: + schema_override_dict = dict(referred_schema=None) + else: + schema_override_dict = {} + + # mypy doesn't like this method of conditionally adding a key to a dictionary + # while keeping everything immutable + complete_foreign_key_dict = { + "name": fk_name, + **base_fk_dict, + **schema_override_dict, # type: ignore + } + + return complete_foreign_key_dict + + +def _parse_pk_columns_from_constraint_string(constraint_str: str) -> List[str]: + """Build a list of constrained columns from a constraint string returned by DESCRIBE TABLE EXTENDED + + For example: + + PRIMARY KEY (`id`, `name`, `email_address`) + + Returns a list like + + ["id", "name", "email_address"] + """ + + _extracted = extract_identifiers_from_string(constraint_str) + + return _extracted + + +def build_pk_dict(pk_name: str, pk_constraint_string: str) -> dict: + """Given a primary key name and a primary key constraint string, return a dictionary + with the following keys: + + constrained_columns + A list of string column names that make up the primary key + + name + The name of the primary key constraint + """ + + constrained_columns = _parse_pk_columns_from_constraint_string(pk_constraint_string) + + return {"constrained_columns": constrained_columns, "name": pk_name} + + +def match_dte_rows_by_value(dte_output: List[Dict[str, str]], match: str) -> List[dict]: + """Return a list of dictionaries containing only the col_name:data_type pairs where the `data_type` + value contains the match argument. + + Today, DESCRIBE TABLE EXTENDED doesn't give a deterministic name to the fields + a constraint will be found in its output. So we cycle through its output looking + for a match. This is brittle. We could optionally make two roundtrips: the first + would query information_schema for the name of the constraint on this table, and + a second to DESCRIBE TABLE EXTENDED, at which point we would know the name of the + constraint. But for now we instead assume that Python list comprehension is faster + than a network roundtrip + """ + + output_rows = [] + + for row_dict in dte_output: + if match in row_dict["data_type"]: + output_rows.append(row_dict) + + return output_rows + + +def match_dte_rows_by_key(dte_output: List[Dict[str, str]], match: str) -> List[dict]: + """Return a list of dictionaries containing only the col_name:data_type pairs where the `col_name` + value contains the match argument. + """ + + output_rows = [] + + for row_dict in dte_output: + if match in row_dict["col_name"]: + output_rows.append(row_dict) + + return output_rows + + +def get_fk_strings_from_dte_output(dte_output: List[Dict[str, str]]) -> List[dict]: + """If the DESCRIBE TABLE EXTENDED output contains foreign key constraints, return a list of dictionaries, + one dictionary per defined constraint + """ + + output = match_dte_rows_by_value(dte_output, "FOREIGN KEY") + + return output + + +def get_pk_strings_from_dte_output( + dte_output: List[Dict[str, str]] +) -> Optional[List[dict]]: + """If the DESCRIBE TABLE EXTENDED output contains primary key constraints, return a list of dictionaries, + one dictionary per defined constraint. + + Returns None if no primary key constraints are found. + """ + + output = match_dte_rows_by_value(dte_output, "PRIMARY KEY") + + return output + + +def get_comment_from_dte_output(dte_output: List[Dict[str, str]]) -> Optional[str]: + """Returns the value of the first "Comment" col_name data in dte_output""" + output = match_dte_rows_by_key(dte_output, "Comment") + if not output: + return None + else: + return output[0]["data_type"] + + +# The keys of this dictionary are the values we expect to see in a +# TGetColumnsRequest's .TYPE_NAME attribute. +# These are enumerated in ttypes.py as class TTypeId. +# TODO: confirm that all types in TTypeId are included here. +GET_COLUMNS_TYPE_MAP = { + "boolean": sqlalchemy.types.Boolean, + "smallint": sqlalchemy.types.SmallInteger, + "tinyint": type_overrides.TINYINT, + "int": sqlalchemy.types.Integer, + "bigint": sqlalchemy.types.BigInteger, + "float": sqlalchemy.types.Float, + "double": sqlalchemy.types.Float, + "string": sqlalchemy.types.String, + "varchar": sqlalchemy.types.String, + "char": sqlalchemy.types.String, + "binary": sqlalchemy.types.String, + "array": sqlalchemy.types.String, + "map": sqlalchemy.types.String, + "struct": sqlalchemy.types.String, + "uniontype": sqlalchemy.types.String, + "decimal": sqlalchemy.types.Numeric, + "timestamp": type_overrides.TIMESTAMP, + "timestamp_ntz": type_overrides.TIMESTAMP_NTZ, + "date": sqlalchemy.types.Date, +} + + +def parse_numeric_type_precision_and_scale(type_name_str): + """Return an intantiated sqlalchemy Numeric() type that preserves the precision and scale indicated + in the output from TGetColumnsRequest. + + type_name_str + The value of TGetColumnsReq.TYPE_NAME. + + If type_name_str is "DECIMAL(18,5) returns sqlalchemy.types.Numeric(18,5) + """ + + pattern = re.compile(r"DECIMAL\((\d+,\d+)\)") + match = re.search(pattern, type_name_str) + precision_and_scale = match.group(1) + precision, scale = tuple(precision_and_scale.split(",")) + + return sqlalchemy.types.Numeric(int(precision), int(scale)) + + +def parse_column_info_from_tgetcolumnsresponse(thrift_resp_row) -> ReflectedColumn: + """Returns a dictionary of the ReflectedColumn schema parsed from + a single of the result of a TGetColumnsRequest thrift RPC + """ + + pat = re.compile(r"^\w+") + + # This method assumes a valid TYPE_NAME field in the response. + # TODO: add error handling in case TGetColumnsResponse format changes + + _raw_col_type = re.search(pat, thrift_resp_row.TYPE_NAME).group(0).lower() # type: ignore + _col_type = GET_COLUMNS_TYPE_MAP[_raw_col_type] + + if _raw_col_type == "decimal": + final_col_type = parse_numeric_type_precision_and_scale( + thrift_resp_row.TYPE_NAME + ) + else: + final_col_type = _col_type + + # See comments about autoincrement in test_suite.py + # Since Databricks SQL doesn't currently support inline AUTOINCREMENT declarations + # the autoincrement must be manually declared with an Identity() construct in SQLAlchemy + # Other dialects can perform this extra Identity() step automatically. But that is not + # implemented in the Databricks dialect right now. So autoincrement is currently always False. + # It's not clear what IS_AUTO_INCREMENT in the thrift response actually reflects or whether + # it ever returns a `YES`. + + # Per the guidance in SQLAlchemy's docstrings, we prefer to not even include an autoincrement + # key in this dictionary. + this_column = { + "name": thrift_resp_row.COLUMN_NAME, + "type": final_col_type, + "nullable": bool(thrift_resp_row.NULLABLE), + "default": thrift_resp_row.COLUMN_DEF, + "comment": thrift_resp_row.REMARKS or None, + } + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return this_column # type: ignore diff --git a/src/databricks_sqlalchemy/_types.py b/src/databricks_sqlalchemy/_types.py new file mode 100644 index 0000000..6d32ff5 --- /dev/null +++ b/src/databricks_sqlalchemy/_types.py @@ -0,0 +1,325 @@ +from datetime import datetime, time, timezone +from itertools import product +from typing import Any, Union, Optional + +import sqlalchemy +from sqlalchemy.engine.interfaces import Dialect +from sqlalchemy.ext.compiler import compiles + +# from databricks.sql.utils import ParamEscaper +import databricks.sql +from databricks.sql.utils import ParamEscaper + + +def process_literal_param_hack(value: Any): + """This method is supposed to accept a Python type and return a string representation of that type. + But due to some weirdness in the way SQLAlchemy's literal rendering works, we have to return + the value itself because, by the time it reaches our custom type code, it's already been converted + into a string. + + TimeTest + DateTimeTest + DateTimeTZTest + + This dynamic only seems to affect the literal rendering of datetime and time objects. + + All fail without this hack in-place. I'm not sure why. But it works. + """ + return value + + +@compiles(sqlalchemy.types.Enum, "databricks") +@compiles(sqlalchemy.types.String, "databricks") +@compiles(sqlalchemy.types.Text, "databricks") +@compiles(sqlalchemy.types.Time, "databricks") +@compiles(sqlalchemy.types.Unicode, "databricks") +@compiles(sqlalchemy.types.UnicodeText, "databricks") +@compiles(sqlalchemy.types.Uuid, "databricks") +def compile_string_databricks(type_, compiler, **kw): + """ + We override the default compilation for Enum(), String(), Text(), and Time() because SQLAlchemy + defaults to incompatible / abnormal compiled names + + Enum -> VARCHAR + String -> VARCHAR[LENGTH] + Text -> VARCHAR[LENGTH] + Time -> TIME + Unicode -> VARCHAR[LENGTH] + UnicodeText -> TEXT + Uuid -> CHAR[32] + + But all of these types will be compiled to STRING in Databricks SQL + """ + return "STRING" + + +@compiles(sqlalchemy.types.Integer, "databricks") +def compile_integer_databricks(type_, compiler, **kw): + """ + We need to override the default Integer compilation rendering because Databricks uses "INT" instead of "INTEGER" + """ + return "INT" + + +@compiles(sqlalchemy.types.LargeBinary, "databricks") +def compile_binary_databricks(type_, compiler, **kw): + """ + We need to override the default LargeBinary compilation rendering because Databricks uses "BINARY" instead of "BLOB" + """ + return "BINARY" + + +@compiles(sqlalchemy.types.Numeric, "databricks") +def compile_numeric_databricks(type_, compiler, **kw): + """ + We need to override the default Numeric compilation rendering because Databricks uses "DECIMAL" instead of "NUMERIC" + + The built-in visit_DECIMAL behaviour captures the precision and scale. Here we're just mapping calls to compile Numeric + to the SQLAlchemy Decimal() implementation + """ + return compiler.visit_DECIMAL(type_, **kw) + + +@compiles(sqlalchemy.types.DateTime, "databricks") +def compile_datetime_databricks(type_, compiler, **kw): + """ + We need to override the default DateTime compilation rendering because Databricks uses "TIMESTAMP_NTZ" instead of "DATETIME" + """ + return "TIMESTAMP_NTZ" + + +@compiles(sqlalchemy.types.ARRAY, "databricks") +def compile_array_databricks(type_, compiler, **kw): + """ + SQLAlchemy's default ARRAY can't compile as it's only implemented for Postgresql. + The Postgres implementation works for Databricks SQL, so we duplicate that here. + + :type_: + This is an instance of sqlalchemy.types.ARRAY which always includes an item_type attribute + which is itself an instance of TypeEngine + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.ARRAY + """ + + inner = compiler.process(type_.item_type, **kw) + + return f"ARRAY<{inner}>" + + +class TIMESTAMP_NTZ(sqlalchemy.types.TypeDecorator): + """Represents values comprising values of fields year, month, day, hour, minute, and second. + All operations are performed without taking any time zone into account. + + Our dialect maps sqlalchemy.types.DateTime() to this type, which means that all DateTime() + objects are stored without tzinfo. To read and write timezone-aware datetimes use + databricks.sql.TIMESTAMP instead. + + https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html + """ + + impl = sqlalchemy.types.DateTime + + cache_ok = True + + def process_result_value(self, value: Union[None, datetime], dialect): + if value is None: + return None + return value.replace(tzinfo=None) + + +class TIMESTAMP(sqlalchemy.types.TypeDecorator): + """Represents values comprising values of fields year, month, day, hour, minute, and second, + with the session local time-zone. + + Our dialect maps sqlalchemy.types.DateTime() to TIMESTAMP_NTZ, which means that all DateTime() + objects are stored without tzinfo. To read and write timezone-aware datetimes use + this type instead. + + ```python + # This won't work + `Column(sqlalchemy.DateTime(timezone=True))` + + # But this does + `Column(TIMESTAMP)` + ```` + + https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html + """ + + impl = sqlalchemy.types.DateTime + + cache_ok = True + + def process_result_value(self, value: Union[None, datetime], dialect): + if value is None: + return None + + if not value.tzinfo: + return value.replace(tzinfo=timezone.utc) + return value + + def process_bind_param( + self, value: Union[datetime, None], dialect + ) -> Optional[datetime]: + """pysql can pass datetime.datetime() objects directly to DBR""" + return value + + def process_literal_param( + self, value: Union[datetime, None], dialect: Dialect + ) -> str: + """ """ + return process_literal_param_hack(value) + + +@compiles(TIMESTAMP, "databricks") +def compile_timestamp_databricks(type_, compiler, **kw): + """ + We need to override the default DateTime compilation rendering because Databricks uses "TIMESTAMP_NTZ" instead of "DATETIME" + """ + return "TIMESTAMP" + + +class DatabricksTimeType(sqlalchemy.types.TypeDecorator): + """Databricks has no native TIME type. So we store it as a string.""" + + impl = sqlalchemy.types.Time + cache_ok = True + + BASE_FMT = "%H:%M:%S" + MICROSEC_PART = ".%f" + TIMEZONE_PART = "%z" + + def _generate_fmt_string(self, ms: bool, tz: bool) -> str: + """Return a format string for datetime.strptime() that includes or excludes microseconds and timezone.""" + _ = lambda x, y: x if y else "" + return f"{self.BASE_FMT}{_(self.MICROSEC_PART,ms)}{_(self.TIMEZONE_PART,tz)}" + + @property + def allowed_fmt_strings(self): + """Time strings can be read with or without microseconds and with or without a timezone.""" + + if not hasattr(self, "_allowed_fmt_strings"): + ms_switch = tz_switch = [True, False] + self._allowed_fmt_strings = [ + self._generate_fmt_string(x, y) + for x, y in product(ms_switch, tz_switch) + ] + + return self._allowed_fmt_strings + + def _parse_result_string(self, value: str) -> time: + """Parse a string into a time object. Try all allowed formats until one works.""" + for fmt in self.allowed_fmt_strings: + try: + # We use timetz() here because we want to preserve the timezone information + # Calling .time() will strip the timezone information + return datetime.strptime(value, fmt).timetz() + except ValueError: + pass + + raise ValueError(f"Could not parse time string {value}") + + def _determine_fmt_string(self, value: time) -> str: + """Determine which format string to use to render a time object as a string.""" + ms_bool = value.microsecond > 0 + tz_bool = value.tzinfo is not None + return self._generate_fmt_string(ms_bool, tz_bool) + + def process_bind_param(self, value: Union[time, None], dialect) -> Union[None, str]: + """Values sent to the database are converted to %:H:%M:%S strings.""" + if value is None: + return None + fmt_string = self._determine_fmt_string(value) + return value.strftime(fmt_string) + + # mypy doesn't like this workaround because TypeEngine wants process_literal_param to return a string + def process_literal_param(self, value, dialect) -> time: # type: ignore + """ """ + return process_literal_param_hack(value) + + def process_result_value( + self, value: Union[None, str], dialect + ) -> Union[time, None]: + """Values received from the database are parsed into datetime.time() objects""" + if value is None: + return None + + return self._parse_result_string(value) + + +class DatabricksStringType(sqlalchemy.types.TypeDecorator): + """We have to implement our own String() type because SQLAlchemy's default implementation + wants to escape single-quotes with a doubled single-quote. Databricks uses a backslash for + escaping of literal strings. And SQLAlchemy's default escaping breaks Databricks SQL. + """ + + impl = sqlalchemy.types.String + cache_ok = True + pe = ParamEscaper() + + def process_literal_param(self, value, dialect) -> str: + """SQLAlchemy's default string escaping for backslashes doesn't work for databricks. The logic here + implements the same logic as our legacy inline escaping logic. + """ + + return self.pe.escape_string(value) + + def literal_processor(self, dialect): + """We manually override this method to prevent further processing of the string literal beyond + what happens in the process_literal_param() method. + + The SQLAlchemy docs _specifically_ say to not override this method. + + It appears that any processing that happens from TypeEngine.process_literal_param happens _before_ + and _in addition to_ whatever the class's impl.literal_processor() method does. The String.literal_processor() + method performs a string replacement that doubles any single-quote in the contained string. This raises a syntax + error in Databricks. And it's not necessary because ParamEscaper() already implements all the escaping we need. + + We should consider opening an issue on the SQLAlchemy project to see if I'm using it wrong. + + See type_api.py::TypeEngine.literal_processor: + + ```python + def process(value: Any) -> str: + return fixed_impl_processor( + fixed_process_literal_param(value, dialect) + ) + ``` + + That call to fixed_impl_processor wraps the result of fixed_process_literal_param (which is the + process_literal_param defined in our Databricks dialect) + + https://docs.sqlalchemy.org/en/20/core/custom_types.html#sqlalchemy.types.TypeDecorator.literal_processor + """ + + def process(value): + """This is a copy of the default String.literal_processor() method but stripping away + its double-escaping behaviour for single-quotes. + """ + + _step1 = self.process_literal_param(value, dialect="databricks") + if dialect.identifier_preparer._double_percents: + _step2 = _step1.replace("%", "%%") + else: + _step2 = _step1 + + return "%s" % _step2 + + return process + + +class TINYINT(sqlalchemy.types.TypeDecorator): + """Represents 1-byte signed integers + + Acts like a sqlalchemy SmallInteger() in Python but writes to a TINYINT field in Databricks + + https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html + """ + + impl = sqlalchemy.types.SmallInteger + cache_ok = True + + +@compiles(TINYINT, "databricks") +def compile_tinyint(type_, compiler, **kw): + return "TINYINT" diff --git a/src/databricks_sqlalchemy/base.py b/src/databricks_sqlalchemy/base.py new file mode 100644 index 0000000..0cff681 --- /dev/null +++ b/src/databricks_sqlalchemy/base.py @@ -0,0 +1,436 @@ +from typing import Any, List, Optional, Dict, Union + +import databricks_sqlalchemy._ddl as dialect_ddl_impl +import databricks_sqlalchemy._types as dialect_type_impl +from databricks import sql +from databricks_sqlalchemy._parse import ( + _describe_table_extended_result_to_dict_list, + _match_table_not_found_string, + build_fk_dict, + build_pk_dict, + get_fk_strings_from_dte_output, + get_pk_strings_from_dte_output, + get_comment_from_dte_output, + parse_column_info_from_tgetcolumnsresponse, +) + +import sqlalchemy +from sqlalchemy import DDL, event +from sqlalchemy.engine import Connection, Engine, default, reflection +from sqlalchemy.engine.interfaces import ( + ReflectedForeignKeyConstraint, + ReflectedPrimaryKeyConstraint, + ReflectedColumn, + ReflectedTableComment, +) +from sqlalchemy.engine.reflection import ReflectionDefaults +from sqlalchemy.exc import DatabaseError, SQLAlchemyError + +try: + import alembic +except ImportError: + pass +else: + from alembic.ddl import DefaultImpl + + class DatabricksImpl(DefaultImpl): + __dialect__ = "databricks" + + +import logging + +logger = logging.getLogger(__name__) + + +class DatabricksDialect(default.DefaultDialect): + """This dialect implements only those methods required to pass our e2e tests""" + + # See sqlalchemy.engine.interfaces for descriptions of each of these properties + name: str = "databricks" + driver: str = "databricks" + default_schema_name: str = "default" + preparer = dialect_ddl_impl.DatabricksIdentifierPreparer # type: ignore + ddl_compiler = dialect_ddl_impl.DatabricksDDLCompiler + statement_compiler = dialect_ddl_impl.DatabricksStatementCompiler + supports_statement_cache: bool = True + supports_multivalues_insert: bool = True + supports_native_decimal: bool = True + supports_sane_rowcount: bool = False + non_native_boolean_check_constraint: bool = False + supports_identity_columns: bool = True + supports_schemas: bool = True + default_paramstyle: str = "named" + div_is_floordiv: bool = False + supports_default_values: bool = False + supports_server_side_cursors: bool = False + supports_sequences: bool = False + supports_native_boolean: bool = True + + colspecs = { + sqlalchemy.types.DateTime: dialect_type_impl.TIMESTAMP_NTZ, + sqlalchemy.types.Time: dialect_type_impl.DatabricksTimeType, + sqlalchemy.types.String: dialect_type_impl.DatabricksStringType, + } + + # SQLAlchemy requires that a table with no primary key + # constraint return a dictionary that looks like this. + EMPTY_PK: Dict[str, Any] = {"constrained_columns": [], "name": None} + + # SQLAlchemy requires that a table with no foreign keys + # defined return an empty list. Same for indexes. + EMPTY_FK: List + EMPTY_INDEX: List + EMPTY_FK = EMPTY_INDEX = [] + + @classmethod + def import_dbapi(cls): + return sql + + def _force_paramstyle_to_native_mode(self): + """This method can be removed after databricks-sql-connector wholly switches to NATIVE ParamApproach. + + This is a hack to trick SQLAlchemy into using a different paramstyle + than the one declared by this module in src/databricks/sql/__init__.py + + This method is called _after_ the dialect has been initialised, which is important because otherwise + our users would need to include a `paramstyle` argument in their SQLAlchemy connection string. + + This dialect is written to support NATIVE queries. Although the INLINE approach can technically work, + the same behaviour can be achieved within SQLAlchemy itself using its literal_processor methods. + """ + + self.paramstyle = self.default_paramstyle + + def create_connect_args(self, url): + # TODO: can schema be provided after HOST? + # Expected URI format is: databricks+thrift://token:dapi***@***.cloud.databricks.com?http_path=/sql/*** + + kwargs = { + "server_hostname": url.host, + "access_token": url.password, + "http_path": url.query.get("http_path"), + "catalog": url.query.get("catalog"), + "schema": url.query.get("schema"), + "use_inline_params": False, + } + + self.schema = kwargs["schema"] + self.catalog = kwargs["catalog"] + + self._force_paramstyle_to_native_mode() + + return [], kwargs + + def get_columns( + self, connection, table_name, schema=None, **kwargs + ) -> List[ReflectedColumn]: + """Return information about columns in `table_name`.""" + + with self.get_connection_cursor(connection) as cur: + resp = cur.columns( + catalog_name=self.catalog, + schema_name=schema or self.schema, + table_name=table_name, + ).fetchall() + + if not resp: + # TGetColumnsRequest will not raise an exception if passed a table that doesn't exist + # But Databricks supports tables with no columns. So if the result is an empty list, + # we need to check if the table exists (and raise an exception if not) or simply return + # an empty list. + self._describe_table_extended( + connection, + table_name, + self.catalog, + schema or self.schema, + expect_result=False, + ) + return resp + columns = [] + for col in resp: + row_dict = parse_column_info_from_tgetcolumnsresponse(col) + columns.append(row_dict) + + return columns + + def _describe_table_extended( + self, + connection: Connection, + table_name: str, + catalog_name: Optional[str] = None, + schema_name: Optional[str] = None, + expect_result=True, + ) -> Union[List[Dict[str, str]], None]: + """Run DESCRIBE TABLE EXTENDED on a table and return a list of dictionaries of the result. + + This method is the fastest way to check for the presence of a table in a schema. + + If expect_result is False, this method returns None as the output dict isn't required. + + Raises NoSuchTableError if the table is not present in the schema. + """ + + _target_catalog = catalog_name or self.catalog + _target_schema = schema_name or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`.`{table_name}`" + + # sql injection risk? + # DESCRIBE TABLE EXTENDED in DBR doesn't support parameterised inputs :( + stmt = DDL(f"DESCRIBE TABLE EXTENDED {_target}") + + try: + result = connection.execute(stmt) + except DatabaseError as e: + if _match_table_not_found_string(str(e)): + raise sqlalchemy.exc.NoSuchTableError( + f"No such table {table_name}" + ) from e + raise e + + if not expect_result: + return None + + fmt_result = _describe_table_extended_result_to_dict_list(result) + return fmt_result + + @reflection.cache + def get_pk_constraint( + self, + connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedPrimaryKeyConstraint: + """Fetch information about the primary key constraint on table_name. + + Returns a dictionary with these keys: + constrained_columns + a list of column names that make up the primary key. Results is an empty list + if no PRIMARY KEY is defined. + + name + the name of the primary key constraint + """ + + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + # Type ignore is because mypy knows that self._describe_table_extended *can* + # return None (even though it never will since expect_result defaults to True) + raw_pk_constraints: List = get_pk_strings_from_dte_output(result) # type: ignore + if not any(raw_pk_constraints): + return self.EMPTY_PK # type: ignore + + if len(raw_pk_constraints) > 1: + logger.warning( + "Found more than one primary key constraint in DESCRIBE TABLE EXTENDED output. " + "This is unexpected. Please report this as a bug. " + "Only the first primary key constraint will be returned." + ) + + first_pk_constraint = raw_pk_constraints[0] + pk_name = first_pk_constraint.get("col_name") + pk_constraint_string = first_pk_constraint.get("data_type") + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return build_pk_dict(pk_name, pk_constraint_string) # type: ignore + + def get_foreign_keys( + self, connection, table_name, schema=None, **kw + ) -> List[ReflectedForeignKeyConstraint]: + """Return information about foreign_keys in `table_name`.""" + + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + # Type ignore is because mypy knows that self._describe_table_extended *can* + # return None (even though it never will since expect_result defaults to True) + raw_fk_constraints: List = get_fk_strings_from_dte_output(result) # type: ignore + + if not any(raw_fk_constraints): + return self.EMPTY_FK + + fk_constraints = [] + for constraint_dict in raw_fk_constraints: + fk_name = constraint_dict.get("col_name") + fk_constraint_string = constraint_dict.get("data_type") + this_constraint_dict = build_fk_dict( + fk_name, fk_constraint_string, schema_name=schema + ) + fk_constraints.append(this_constraint_dict) + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return fk_constraints # type: ignore + + def get_indexes(self, connection, table_name, schema=None, **kw): + """SQLAlchemy requires this method. Databricks doesn't support indexes.""" + return self.EMPTY_INDEX + + @reflection.cache + def get_table_names(self, connection: Connection, schema=None, **kwargs): + """Return a list of tables in the current schema.""" + + _target_catalog = self.catalog + _target_schema = schema or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`" + + stmt = DDL(f"SHOW TABLES FROM {_target}") + + tables_result = connection.execute(stmt).all() + views_result = self.get_view_names(connection=connection, schema=schema) + + # In Databricks, SHOW TABLES FROM returns both tables and views. + # Potential optimisation: rewrite this to instead query information_schema + tables_minus_views = [ + row.tableName for row in tables_result if row.tableName not in views_result + ] + + return tables_minus_views + + @reflection.cache + def get_view_names( + self, + connection, + schema=None, + only_materialized=False, + only_temp=False, + **kwargs, + ) -> List[str]: + """Returns a list of string view names contained in the schema, if any.""" + + _target_catalog = self.catalog + _target_schema = schema or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`" + + stmt = DDL(f"SHOW VIEWS FROM {_target}") + result = connection.execute(stmt).all() + + return [ + row.viewName + for row in result + if (not only_materialized or row.isMaterialized) + and (not only_temp or row.isTemporary) + ] + + @reflection.cache + def get_materialized_view_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> List[str]: + """A wrapper around get_view_names that fetches only the names of materialized views""" + return self.get_view_names(connection, schema, only_materialized=True) + + @reflection.cache + def get_temp_view_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> List[str]: + """A wrapper around get_view_names that fetches only the names of temporary views""" + return self.get_view_names(connection, schema, only_temp=True) + + def do_rollback(self, dbapi_connection): + # Databricks SQL Does not support transactions + pass + + @reflection.cache + def has_table( + self, connection, table_name, schema=None, catalog=None, **kwargs + ) -> bool: + """For internal dialect use, check the existence of a particular table + or view in the database. + """ + + try: + self._describe_table_extended( + connection=connection, + table_name=table_name, + catalog_name=catalog, + schema_name=schema, + ) + return True + except sqlalchemy.exc.NoSuchTableError as e: + return False + + def get_connection_cursor(self, connection): + """Added for backwards compatibility with 1.3.x""" + if hasattr(connection, "_dbapi_connection"): + return connection._dbapi_connection.dbapi_connection.cursor() + elif hasattr(connection, "raw_connection"): + return connection.raw_connection().cursor() + elif hasattr(connection, "connection"): + return connection.connection.cursor() + + raise SQLAlchemyError( + "Databricks dialect can't obtain a cursor context manager from the dbapi" + ) + + @reflection.cache + def get_schema_names(self, connection, **kw): + """Return a list of all schema names available in the database.""" + stmt = DDL("SHOW SCHEMAS") + result = connection.execute(stmt) + schema_list = [row[0] for row in result] + return schema_list + + @reflection.cache + def get_table_comment( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedTableComment: + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + if result is None: + return ReflectionDefaults.table_comment() + + comment = get_comment_from_dte_output(result) + + if comment: + return dict(text=comment) + else: + return ReflectionDefaults.table_comment() + + +@event.listens_for(Engine, "do_connect") +def receive_do_connect(dialect, conn_rec, cargs, cparams): + """Helpful for DS on traffic from clients using SQLAlchemy in particular""" + + # Ignore connect invocations that don't use our dialect + if not dialect.name == "databricks": + return + + ua = cparams.get("_user_agent_entry", "") + + def add_sqla_tag_if_not_present(val: str): + if not val: + output = "sqlalchemy" + + if val and "sqlalchemy" in val: + output = val + + else: + output = f"sqlalchemy + {val}" + + return output + + cparams["_user_agent_entry"] = add_sqla_tag_if_not_present(ua) + + if sqlalchemy.__version__.startswith("1.3"): + # SQLAlchemy 1.3.x fails to parse the http_path, catalog, and schema from our connection string + # These should be passed in as connect_args when building the Engine + + if "schema" in cparams: + dialect.schema = cparams["schema"] + + if "catalog" in cparams: + dialect.catalog = cparams["catalog"] diff --git a/src/databricks_sqlalchemy/py.typed b/src/databricks_sqlalchemy/py.typed new file mode 100755 index 0000000..e69de29 diff --git a/src/databricks_sqlalchemy/requirements.py b/src/databricks_sqlalchemy/requirements.py new file mode 100644 index 0000000..5c70c02 --- /dev/null +++ b/src/databricks_sqlalchemy/requirements.py @@ -0,0 +1,249 @@ +""" +The complete list of requirements is provided by SQLAlchemy here: + +https://github.com/sqlalchemy/sqlalchemy/blob/main/lib/sqlalchemy/testing/requirements.py + +When SQLAlchemy skips a test because a requirement is closed() it gives a generic skip message. +To make these failures more actionable, we only define requirements in this file that we wish to +force to be open(). If a test should be skipped on Databricks, it will be specifically marked skip +in test_suite.py with a Databricks-specific reason. + +See the special note about the array_type exclusion below. +See special note about has_temp_table exclusion below. +""" + +import sqlalchemy.testing.requirements +import sqlalchemy.testing.exclusions + + +class Requirements(sqlalchemy.testing.requirements.SuiteRequirements): + @property + def date_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_literals(self): + """target dialect supports rendering of a date, time, or datetime as a + literal string, e.g. via the TypeEngine.literal_processor() method. + + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def timestamp_microseconds(self): + """target dialect supports representation of Python + datetime.datetime() with microsecond objects but only + if TIMESTAMP is used.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_microseconds(self): + """target dialect supports representation of Python + datetime.time() with microsecond objects. + + This requirement declaration isn't needed but I've included it here for completeness. + Since Databricks doesn't have a TIME type, SQLAlchemy will compile Time() columns + as STRING Databricks data types. And we use a custom time type to render those strings + between str() and time.time() representations. Therefore we can store _any_ precision + that SQLAlchemy needs. The time_microseconds requirement defaults to ON for all dialects + except mssql, mysql, mariadb, and oracle. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def infinity_floats(self): + """The Float type can persist and load float('inf'), float('-inf').""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_retains_significant_digits(self): + """A precision numeric type will return empty significant digits, + i.e. a value such as 10.000 will come back in Decimal form with + the .000 maintained.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_many_significant_digits(self): + """target backend supports values with many digits on both sides, + such as 319438950232418390.273596, 87673.594069654243 + + """ + return sqlalchemy.testing.exclusions.open() + + @property + def array_type(self): + """While Databricks does support ARRAY types, pysql cannot bind them. So + we cannot use them with SQLAlchemy + + Due to a bug in SQLAlchemy, we _must_ define this exclusion as closed() here or else the + test runner will crash the pytest process due to an AttributeError + """ + + # TODO: Implement array type using inline? + return sqlalchemy.testing.exclusions.closed() + + @property + def table_ddl_if_exists(self): + """target platform supports IF NOT EXISTS / IF EXISTS for tables.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY""" + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns_standard(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY with a standard syntax. + This is mainly to exclude MSSql. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def has_temp_table(self): + """target dialect supports checking a single temp table name + + unfortunately this is not the same as temp_table_names + + SQLAlchemy's HasTableTest is not normalised in such a way that temp table tests + are separate from temp view and normal table tests. If those tests were split out, + we would just add detailed skip markers in test_suite.py. But since we'd like to + run the HasTableTest group for the features we support, we must set this exclusinon + to closed(). + + It would be ideal if there were a separate requirement for has_temp_view. Without it, + we're in a bind. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def temporary_views(self): + """target database supports temporary views""" + return sqlalchemy.testing.exclusions.open() + + @property + def views(self): + """Target database must support VIEWs.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def temporary_tables(self): + """target database supports temporary tables + + ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def table_reflection(self): + """target database has general support for table reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection(self): + """Indicates if the database support table comment reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection_full_unicode(self): + """Indicates if the database support table comment reflection in the + full unicode range, including emoji etc. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def temp_table_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def index_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def unique_constraint_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + + Databricks doesn't support UNIQUE constraints. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def reflects_pk_names(self): + """Target driver reflects the name of primary key constraints.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_implicit_bound(self): + """target dialect when given a datetime object will bind it such + that the database server knows the object is a date, and not + a plain string. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def tuple_in(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes_with_update_delete(self): + return sqlalchemy.testing.exclusions.open() + + @property + def delete_from(self): + """Target must support DELETE FROM..FROM or DELETE..USING syntax""" + return sqlalchemy.testing.exclusions.open() + + @property + def table_value_constructor(self): + return sqlalchemy.testing.exclusions.open() + + @property + def reflect_tables_no_columns(self): + return sqlalchemy.testing.exclusions.open() + + @property + def denormalized_names(self): + """Target database must have 'denormalized', i.e. + UPPERCASE as case insensitive names.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_timezone(self): + """target dialect supports representation of Python + datetime.time() with tzinfo with Time(timezone=True).""" + + return sqlalchemy.testing.exclusions.open() diff --git a/src/databricks_sqlalchemy/setup.cfg b/src/databricks_sqlalchemy/setup.cfg new file mode 100644 index 0000000..ab89d17 --- /dev/null +++ b/src/databricks_sqlalchemy/setup.cfg @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks.sqlalchemy.requirements:Requirements +profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/test/__init__.py b/src/databricks_sqlalchemy/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/databricks_sqlalchemy/test/_extra.py b/src/databricks_sqlalchemy/test/_extra.py new file mode 100644 index 0000000..2f3e7a7 --- /dev/null +++ b/src/databricks_sqlalchemy/test/_extra.py @@ -0,0 +1,70 @@ +"""Additional tests authored by Databricks that use SQLAlchemy's test fixtures +""" + +import datetime + +from sqlalchemy.testing.suite.test_types import ( + _LiteralRoundTripFixture, + fixtures, + testing, + eq_, + select, + Table, + Column, + config, + _DateFixture, + literal, +) +from databricks.sqlalchemy import TINYINT, TIMESTAMP + + +class TinyIntegerTest(_LiteralRoundTripFixture, fixtures.TestBase): + __backend__ = True + + def test_literal(self, literal_round_trip): + literal_round_trip(TINYINT, [5], [5]) + + @testing.fixture + def integer_round_trip(self, metadata, connection): + def run(datatype, data): + int_table = Table( + "tiny_integer_table", + metadata, + Column( + "id", + TINYINT, + primary_key=True, + test_needs_autoincrement=False, + ), + Column("tiny_integer_data", datatype), + ) + + metadata.create_all(config.db) + + connection.execute(int_table.insert(), {"id": 1, "integer_data": data}) + + row = connection.execute(select(int_table.c.integer_data)).first() + + eq_(row, (data,)) + + assert isinstance(row[0], int) + + return run + + +class DateTimeTZTestCustom(_DateFixture, fixtures.TablesTest): + """This test confirms that when a user uses the TIMESTAMP + type to store a datetime object, it retains its timezone + """ + + __backend__ = True + datatype = TIMESTAMP + data = datetime.datetime(2012, 10, 15, 12, 57, 18, tzinfo=datetime.timezone.utc) + + @testing.requires.datetime_implicit_bound + def test_select_direct(self, connection): + + # We need to pass the TIMESTAMP type to the literal function + # so that the value is processed correctly. + result = connection.scalar(select(literal(self.data, TIMESTAMP))) + eq_(result, self.data) diff --git a/src/databricks_sqlalchemy/test/_future.py b/src/databricks_sqlalchemy/test/_future.py new file mode 100644 index 0000000..2baa3d3 --- /dev/null +++ b/src/databricks_sqlalchemy/test/_future.py @@ -0,0 +1,331 @@ +# type: ignore + +from enum import Enum + +import pytest +from databricks_sqlalchemy.src.sqlalchemy.test._regression import ( + ExpandingBoundInTest, + IdentityAutoincrementTest, + LikeFunctionsTest, + NormalizedNameTest, +) +from databricks.sqlalchemy.test._unsupported import ( + ComponentReflectionTest, + ComponentReflectionTestExtra, + CTETest, + InsertBehaviorTest, +) +from sqlalchemy.testing.suite import ( + ArrayTest, + BinaryTest, + BizarroCharacterFKResolutionTest, + CollateTest, + ComputedColumnTest, + ComputedReflectionTest, + DifficultParametersTest, + FutureWeCanSetDefaultSchemaWEventsTest, + IdentityColumnTest, + IdentityReflectionTest, + JSONLegacyStringCastIndexTest, + JSONTest, + NativeUUIDTest, + QuotedNameArgumentTest, + RowCountTest, + SimpleUpdateDeleteTest, + WeCanSetDefaultSchemaWEventsTest, +) + + +class FutureFeature(Enum): + ARRAY = "ARRAY column type handling" + BINARY = "BINARY column type handling" + CHECK = "CHECK constraint handling" + COLLATE = "COLLATE DDL generation" + CTE_FEAT = "required CTE features" + EMPTY_INSERT = "empty INSERT support" + FK_OPTS = "foreign key option checking" + GENERATED_COLUMNS = "Delta computed / generated columns support" + IDENTITY = "identity reflection" + JSON = "JSON column type handling" + MULTI_PK = "get_multi_pk_constraint method" + PROVISION = "event-driven engine configuration" + REGEXP = "_visit_regexp" + SANE_ROWCOUNT = "sane_rowcount support" + TBL_OPTS = "get_table_options method" + TEST_DESIGN = "required test-fixture overrides" + TUPLE_LITERAL = "tuple-like IN markers completely" + UUID = "native Uuid() type" + VIEW_DEF = "get_view_definition method" + + +def render_future_feature(rsn: FutureFeature, extra=False) -> str: + postfix = " More detail in _future.py" if extra else "" + return f"[FUTURE][{rsn.name}]: This dialect doesn't implement {rsn.value}.{postfix}" + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.BINARY)) +class BinaryTest(BinaryTest): + """Databricks doesn't support binding of BINARY type values. When DBR supports this, we can implement + in this dialect. + """ + + pass + + +class ExpandingBoundInTest(ExpandingBoundInTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_heterogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_heterogeneous_tuples_direct(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_homogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_homogeneous_tuples_direct(self): + pass + + +class NormalizedNameTest(NormalizedNameTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) + def test_get_table_names(self): + """I'm not clear how this test can ever pass given that it's assertion looks like this: + + ```python + eq_(tablenames[0].upper(), tablenames[0].lower()) + eq_(tablenames[1].upper(), tablenames[1].lower()) + ``` + + It's forcibly calling .upper() and .lower() on the same string and expecting them to be equal. + """ + pass + + +class CTETest(CTETest): + @pytest.mark.skip(render_future_feature(FutureFeature.CTE_FEAT, True)) + def test_delete_from_round_trip(self): + """Databricks dialect doesn't implement multiple-table criteria within DELETE""" + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) +class IdentityColumnTest(IdentityColumnTest): + """Identity works. Test needs rewrite for Databricks. See comments in test_suite.py + + The setup for these tests tries to create a table with a DELTA IDENTITY column but has two problems: + 1. It uses an Integer() type for the column. Whereas DELTA IDENTITY columns must be BIGINT. + 2. It tries to set the start == 42, which Databricks doesn't support + + I can get the tests to _run_ by patching the table fixture to use BigInteger(). But it asserts that the + identity of two rows are 42 and 43, which is not possible since they will be rows 1 and 2 instead. + + I'm satisified through manual testing that our implementation of visit_identity_column works but a better test is needed. + """ + + pass + + +class IdentityAutoincrementTest(IdentityAutoincrementTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) + def test_autoincrement_with_identity(self): + """This test has the same issue as IdentityColumnTest.test_select_all in that it creates a table with identity + using an Integer() rather than a BigInteger(). If I override this behaviour to use a BigInteger() instead, the + test passes. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN)) +class BizarroCharacterFKResolutionTest(BizarroCharacterFKResolutionTest): + """Some of the combinations in this test pass. Others fail. Given the esoteric nature of these failures, + we have opted to defer implementing fixes to a later time, guided by customer feedback. Passage of + these tests is not an acceptance criteria for our dialect. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN)) +class DifficultParametersTest(DifficultParametersTest): + """Some of the combinations in this test pass. Others fail. Given the esoteric nature of these failures, + we have opted to defer implementing fixes to a later time, guided by customer feedback. Passage of + these tests is not an acceptance criteria for our dialect. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.IDENTITY, True)) +class IdentityReflectionTest(IdentityReflectionTest): + """It's not clear _how_ to implement this for SQLAlchemy. Columns created with GENERATED ALWAYS AS IDENTITY + are not specially demarked in the output of TGetColumnsResponse or DESCRIBE TABLE EXTENDED. + + We could theoretically parse this from the contents of `SHOW CREATE TABLE` but that feels like a hack. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.JSON)) +class JSONTest(JSONTest): + """Databricks supports JSON path expressions in queries it's just not implemented in this dialect.""" + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.JSON)) +class JSONLegacyStringCastIndexTest(JSONLegacyStringCastIndexTest): + """Same comment applies as JSONTest""" + + pass + + +class LikeFunctionsTest(LikeFunctionsTest): + @pytest.mark.skip(render_future_feature(FutureFeature.REGEXP)) + def test_not_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.REGEXP)) + def test_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.COLLATE)) +class CollateTest(CollateTest): + """This is supported in Databricks. Not implemented here.""" + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.UUID, True)) +class NativeUUIDTest(NativeUUIDTest): + """Type implementation will be straightforward. Since Databricks doesn't have a native UUID type we can use + a STRING field, create a custom TypeDecorator for sqlalchemy.types.Uuid and add it to the dialect's colspecs. + + Then mark requirements.uuid_data_type as open() so this test can run. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.SANE_ROWCOUNT)) +class RowCountTest(RowCountTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.SANE_ROWCOUNT)) +class SimpleUpdateDeleteTest(SimpleUpdateDeleteTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.PROVISION, True)) +class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.PROVISION, True)) +class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + + pass + + +class ComponentReflectionTest(ComponentReflectionTest): + @pytest.mark.skip(reason=render_future_feature(FutureFeature.TBL_OPTS, True)) + def test_multi_get_table_options_tables(self): + """It's not clear what the expected ouput from this method would even _be_. Requires research.""" + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.VIEW_DEF)) + def test_get_view_definition(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.VIEW_DEF)) + def test_get_view_definition_does_not_exist(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.MULTI_PK)) + def test_get_multi_pk_constraint(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.CHECK)) + def test_get_multi_check_constraints(self): + pass + + +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + @pytest.mark.skip(render_future_feature(FutureFeature.CHECK)) + def test_get_check_constraints(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.FK_OPTS)) + def test_get_foreign_key_options(self): + """It's not clear from the test code what the expected output is here. Further research required.""" + pass + + +class InsertBehaviorTest(InsertBehaviorTest): + @pytest.mark.skip(render_future_feature(FutureFeature.EMPTY_INSERT, True)) + def test_empty_insert(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error + """ + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.EMPTY_INSERT, True)) + def test_empty_insert_multiple(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error + """ + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.ARRAY)) +class ArrayTest(ArrayTest): + """While Databricks supports ARRAY types, DBR cannot handle bound parameters of this type. + This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline + the values of these parameters (which risks sql injection). + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) +class QuotedNameArgumentTest(QuotedNameArgumentTest): + """These tests are challenging. The whole test setup depends on a table with a name like `quote ' one` + which will never work on Databricks because table names can't contains spaces. But QuotedNamedArgumentTest + also checks the behaviour of DDL identifier preparation process. We need to override some of IdentifierPreparer + methods because these are the ultimate control for whether or not CHECK and UNIQUE constraints are emitted. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_future_feature(FutureFeature.GENERATED_COLUMNS)) +class ComputedColumnTest(ComputedColumnTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_future_feature(FutureFeature.GENERATED_COLUMNS)) +class ComputedReflectionTest(ComputedReflectionTest): + pass diff --git a/src/databricks_sqlalchemy/test/_regression.py b/src/databricks_sqlalchemy/test/_regression.py new file mode 100644 index 0000000..1cd4c35 --- /dev/null +++ b/src/databricks_sqlalchemy/test/_regression.py @@ -0,0 +1,311 @@ +# type: ignore + +import pytest +from sqlalchemy.testing.suite import ( + ArgSignatureTest, + BooleanTest, + CastTypeDecoratorTest, + ComponentReflectionTestExtra, + CompositeKeyReflectionTest, + CompoundSelectTest, + DateHistoricTest, + DateTest, + DateTimeCoercedToDateTimeTest, + DateTimeHistoricTest, + DateTimeMicrosecondsTest, + DateTimeTest, + DeprecatedCompoundSelectTest, + DistinctOnTest, + EscapingTest, + ExistsTest, + ExpandingBoundInTest, + FetchLimitOffsetTest, + FutureTableDDLTest, + HasTableTest, + IdentityAutoincrementTest, + InsertBehaviorTest, + IntegerTest, + IsOrIsNotDistinctFromTest, + JoinTest, + LikeFunctionsTest, + NormalizedNameTest, + NumericTest, + OrderByLabelTest, + PingTest, + PostCompileParamsTest, + ReturningGuardsTest, + RowFetchTest, + SameNamedSchemaTableTest, + StringTest, + TableDDLTest, + TableNoColumnsTest, + TextTest, + TimeMicrosecondsTest, + TimestampMicrosecondsTest, + TimeTest, + TimeTZTest, + TrueDivTest, + UnicodeTextTest, + UnicodeVarcharTest, + UuidTest, + ValuesExpressionTest, +) + +from databricks.sqlalchemy.test.overrides._ctetest import CTETest +from databricks_sqlalchemy.src.sqlalchemy.test.overrides._componentreflectiontest import ( + ComponentReflectionTest, +) + + +@pytest.mark.reviewed +class NumericTest(NumericTest): + pass + + +@pytest.mark.reviewed +class HasTableTest(HasTableTest): + pass + + +@pytest.mark.reviewed +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + pass + + +@pytest.mark.reviewed +class InsertBehaviorTest(InsertBehaviorTest): + pass + + +@pytest.mark.reviewed +class ComponentReflectionTest(ComponentReflectionTest): + """This test requires two schemas be present in the target Databricks workspace: + - The schema set in --dburi + - A second schema named "test_schema" + + Note that test_get_multi_foreign keys is flaky because DBR does not guarantee the order of data returned in DESCRIBE TABLE EXTENDED + + _Most_ of these tests pass if we manually override the bad test setup. + """ + + pass + + +@pytest.mark.reviewed +class TableDDLTest(TableDDLTest): + pass + + +@pytest.mark.reviewed +class FutureTableDDLTest(FutureTableDDLTest): + pass + + +@pytest.mark.reviewed +class FetchLimitOffsetTest(FetchLimitOffsetTest): + pass + + +@pytest.mark.reviewed +class UuidTest(UuidTest): + pass + + +@pytest.mark.reviewed +class ValuesExpressionTest(ValuesExpressionTest): + pass + + +@pytest.mark.reviewed +class BooleanTest(BooleanTest): + pass + + +@pytest.mark.reviewed +class PostCompileParamsTest(PostCompileParamsTest): + pass + + +@pytest.mark.reviewed +class TimeMicrosecondsTest(TimeMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class TextTest(TextTest): + pass + + +@pytest.mark.reviewed +class StringTest(StringTest): + pass + + +@pytest.mark.reviewed +class DateTimeMicrosecondsTest(DateTimeMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class TimestampMicrosecondsTest(TimestampMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class DateTimeCoercedToDateTimeTest(DateTimeCoercedToDateTimeTest): + pass + + +@pytest.mark.reviewed +class TimeTest(TimeTest): + pass + + +@pytest.mark.reviewed +class DateTimeTest(DateTimeTest): + pass + + +@pytest.mark.reviewed +class DateTimeHistoricTest(DateTimeHistoricTest): + pass + + +@pytest.mark.reviewed +class DateTest(DateTest): + pass + + +@pytest.mark.reviewed +class DateHistoricTest(DateHistoricTest): + pass + + +@pytest.mark.reviewed +class RowFetchTest(RowFetchTest): + pass + + +@pytest.mark.reviewed +class CompositeKeyReflectionTest(CompositeKeyReflectionTest): + pass + + +@pytest.mark.reviewed +class TrueDivTest(TrueDivTest): + pass + + +@pytest.mark.reviewed +class ArgSignatureTest(ArgSignatureTest): + pass + + +@pytest.mark.reviewed +class CompoundSelectTest(CompoundSelectTest): + pass + + +@pytest.mark.reviewed +class DeprecatedCompoundSelectTest(DeprecatedCompoundSelectTest): + pass + + +@pytest.mark.reviewed +class CastTypeDecoratorTest(CastTypeDecoratorTest): + pass + + +@pytest.mark.reviewed +class DistinctOnTest(DistinctOnTest): + pass + + +@pytest.mark.reviewed +class EscapingTest(EscapingTest): + pass + + +@pytest.mark.reviewed +class ExistsTest(ExistsTest): + pass + + +@pytest.mark.reviewed +class IntegerTest(IntegerTest): + pass + + +@pytest.mark.reviewed +class IsOrIsNotDistinctFromTest(IsOrIsNotDistinctFromTest): + pass + + +@pytest.mark.reviewed +class JoinTest(JoinTest): + pass + + +@pytest.mark.reviewed +class OrderByLabelTest(OrderByLabelTest): + pass + + +@pytest.mark.reviewed +class PingTest(PingTest): + pass + + +@pytest.mark.reviewed +class ReturningGuardsTest(ReturningGuardsTest): + pass + + +@pytest.mark.reviewed +class SameNamedSchemaTableTest(SameNamedSchemaTableTest): + pass + + +@pytest.mark.reviewed +class UnicodeTextTest(UnicodeTextTest): + pass + + +@pytest.mark.reviewed +class UnicodeVarcharTest(UnicodeVarcharTest): + pass + + +@pytest.mark.reviewed +class TableNoColumnsTest(TableNoColumnsTest): + pass + + +@pytest.mark.reviewed +class ExpandingBoundInTest(ExpandingBoundInTest): + pass + + +@pytest.mark.reviewed +class CTETest(CTETest): + pass + + +@pytest.mark.reviewed +class NormalizedNameTest(NormalizedNameTest): + pass + + +@pytest.mark.reviewed +class IdentityAutoincrementTest(IdentityAutoincrementTest): + pass + + +@pytest.mark.reviewed +class LikeFunctionsTest(LikeFunctionsTest): + pass + + +@pytest.mark.reviewed +class TimeTZTest(TimeTZTest): + pass diff --git a/src/databricks_sqlalchemy/test/_unsupported.py b/src/databricks_sqlalchemy/test/_unsupported.py new file mode 100644 index 0000000..c1270c2 --- /dev/null +++ b/src/databricks_sqlalchemy/test/_unsupported.py @@ -0,0 +1,450 @@ +# type: ignore + +from enum import Enum + +import pytest +from databricks_sqlalchemy.src.sqlalchemy.test._regression import ( + ComponentReflectionTest, + ComponentReflectionTestExtra, + CTETest, + FetchLimitOffsetTest, + FutureTableDDLTest, + HasTableTest, + InsertBehaviorTest, + NumericTest, + TableDDLTest, + UuidTest, +) + +# These are test suites that are fully skipped with a SkipReason +from sqlalchemy.testing.suite import ( + AutocommitIsolationTest, + DateTimeTZTest, + ExceptionTest, + HasIndexTest, + HasSequenceTest, + HasSequenceTestEmpty, + IsolationLevelTest, + LastrowidTest, + LongNameBlowoutTest, + PercentSchemaNamesTest, + ReturningTest, + SequenceCompilerTest, + SequenceTest, + ServerSideCursorsTest, + UnicodeSchemaTest, +) + + +class SkipReason(Enum): + AUTO_INC = "implicit AUTO_INCREMENT" + CTE_FEAT = "required CTE features" + CURSORS = "server-side cursors" + DECIMAL_FEAT = "required decimal features" + ENFORCE_KEYS = "enforcing primary or foreign key restraints" + FETCH = "fetch clauses" + IDENTIFIER_LENGTH = "identifiers > 255 characters" + IMPL_FLOAT_PREC = "required implicit float precision" + IMPLICIT_ORDER = "deterministic return order if ORDER BY is not present" + INDEXES = "SQL INDEXes" + RETURNING = "INSERT ... RETURNING syntax" + SEQUENCES = "SQL SEQUENCES" + STRING_FEAT = "required STRING type features" + SYMBOL_CHARSET = "symbols expected by test" + TEMP_TBL = "temporary tables" + TIMEZONE_OPT = "timezone-optional TIMESTAMP fields" + TRANSACTIONS = "transactions" + UNIQUE = "UNIQUE constraints" + + +def render_skip_reason(rsn: SkipReason, setup_error=False, extra=False) -> str: + prefix = "[BADSETUP]" if setup_error else "" + postfix = " More detail in _unsupported.py" if extra else "" + return f"[UNSUPPORTED]{prefix}[{rsn.name}]: Databricks does not support {rsn.value}.{postfix}" + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.ENFORCE_KEYS)) +class ExceptionTest(ExceptionTest): + """Per Databricks documentation, primary and foreign key constraints are informational only + and are not enforced. + + https://docs.databricks.com/api/workspace/tableconstraints + """ + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.IDENTIFIER_LENGTH)) +class LongNameBlowoutTest(LongNameBlowoutTest): + """These tests all include assertions that the tested name > 255 characters""" + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class HasSequenceTest(HasSequenceTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class HasSequenceTestEmpty(HasSequenceTestEmpty): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) +class HasIndexTest(HasIndexTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SYMBOL_CHARSET)) +class UnicodeSchemaTest(UnicodeSchemaTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.CURSORS)) +class ServerSideCursorsTest(ServerSideCursorsTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SYMBOL_CHARSET)) +class PercentSchemaNamesTest(PercentSchemaNamesTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.TRANSACTIONS)) +class IsolationLevelTest(IsolationLevelTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.TRANSACTIONS)) +class AutocommitIsolationTest(AutocommitIsolationTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.RETURNING)) +class ReturningTest(ReturningTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class SequenceTest(SequenceTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class SequenceCompilerTest(SequenceCompilerTest): + pass + + +class FetchLimitOffsetTest(FetchLimitOffsetTest): + @pytest.mark.flaky + @pytest.mark.skip(reason=render_skip_reason(SkipReason.IMPLICIT_ORDER, extra=True)) + def test_limit_render_multiple_times(self): + """This test depends on the order that records are inserted into the table. It's passing criteria requires that + a record inserted with id=1 is the first record returned when no ORDER BY clause is specified. But Databricks occasionally + INSERTS in a different order, which makes this test seem to fail. The test is flaky, but the underlying functionality + (can multiple LIMIT clauses be rendered) is not broken. + + Unclear if this is a bug in Databricks, Delta, or some race-condition in the test itself. + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_bound_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_no_order(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_nobinds(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_percent(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_percent_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_expr_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_percent(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_percent_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_ties_exact_number(self): + pass + + +class UuidTest(UuidTest): + @pytest.mark.skip(reason=render_skip_reason(SkipReason.RETURNING)) + def test_uuid_returning(self): + pass + + +class FutureTableDDLTest(FutureTableDDLTest): + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_create_index_if_not_exists(self): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_drop_index_if_exists(self): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + +class TableDDLTest(TableDDLTest): + @pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) + def test_create_index_if_not_exists(self, connection): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) + def test_drop_index_if_exists(self, connection): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + +class ComponentReflectionTest(ComponentReflectionTest): + """This test requires two schemas be present in the target Databricks workspace: + - The schema set in --dburi + - A second schema named "test_schema" + + Note that test_get_multi_foreign keys is flaky because DBR does not guarantee the order of data returned in DESCRIBE TABLE EXTENDED + """ + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.UNIQUE)) + def test_get_multi_unique_constraints(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL, True, True)) + def test_get_temp_view_names(self): + """While Databricks supports temporary views, this test creates a temp view aimed at a temp table. + Databricks doesn't support temp tables. So the test can never pass. + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_columns(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_indexes(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_names(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_unique_constraints(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_reflect_table_temp_table(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_get_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_multi_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def get_noncol_index(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.UNIQUE)) + def test_get_unique_constraints(self): + pass + + +class NumericTest(NumericTest): + @pytest.mark.skip(render_skip_reason(SkipReason.DECIMAL_FEAT)) + def test_enotation_decimal(self): + """This test automatically runs if requirements.precision_numerics_enotation_large is open()""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.DECIMAL_FEAT)) + def test_enotation_decimal_large(self): + """This test automatically runs if requirements.precision_numerics_enotation_large is open()""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.IMPL_FLOAT_PREC, extra=True)) + def test_float_coerce_round_trip(self): + """ + This automatically runs if requirements.literal_float_coercion is open() + + Without additional work, Databricks returns 15.75629997253418 when you SELECT 15.7563. + This is a potential area where we could override the Float literal processor to add a CAST. + Will leave to a PM to decide if we should do so. + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.IMPL_FLOAT_PREC, extra=True)) + def test_float_custom_scale(self): + """This test automatically runs if requirements.precision_generic_float_type is open()""" + pass + + +class HasTableTest(HasTableTest): + """Databricks does not support temporary tables.""" + + @pytest.mark.skip(render_skip_reason(SkipReason.TEMP_TBL)) + def test_has_table_temp_table(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.TEMP_TBL, True, True)) + def test_has_table_temp_view(self): + """Databricks supports temporary views but this test depends on requirements.has_temp_table, which we + explicitly close so that we can run other tests in this group. See the comment under has_temp_table in + requirements.py for details. + + From what I can see, there is no way to run this test since it will fail during setup if we mark has_temp_table + open(). It _might_ be possible to hijack this behaviour by implementing temp_table_keyword_args in our own + provision.py. Doing so would mean creating a real table during this class setup instead of a temp table. Then + we could just skip the temp table tests but run the temp view tests. But this test fixture doesn't cleanup its + temp tables and has no hook to do so. + + It would be ideal for SQLAlchemy to define a separate requirements.has_temp_views. + """ + pass + + +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_reflect_covering_index(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_reflect_expression_based_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.STRING_FEAT, extra=True)) + def test_varchar_reflection(self): + """Databricks doesn't enforce string length limitations like STRING(255).""" + pass + + +class InsertBehaviorTest(InsertBehaviorTest): + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_autoclose_on_insert(self): + """The setup for this test creates a column with implicit autoincrement enabled. + This dialect does not implement implicit autoincrement - users must declare Identity() explicitly. + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_insert_from_select_autoinc(self): + """Implicit autoincrement is not implemented in this dialect.""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_insert_from_select_autoinc_no_rows(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.RETURNING)) + def test_autoclose_on_insert_implicit_returning(self): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, extra=True)) +class LastrowidTest(LastrowidTest): + """SQLAlchemy docs describe that a column without an explicit Identity() may implicitly create one if autoincrement=True. + That is what this method tests. Databricks supports auto-incrementing IDENTITY columns but they must be explicitly + declared. This limitation is present in our dialect as well. Which means that SQLAlchemy's autoincrement setting of a column + is ignored. We emit a logging.WARN message if you try it. + + In the future we could handle this autoincrement by implicitly calling the visit_identity_column() method of our DDLCompiler + when autoincrement=True. There is an example of this in the Microsoft SQL Server dialect: MSSDDLCompiler.get_column_specification + + For now, if you need to create a SQLAlchemy column with an auto-incrementing identity, you must set this explicitly in your column + definition by passing an Identity() to the column constructor. + """ + + pass + + +class CTETest(CTETest): + """During the teardown for this test block, it tries to drop a constraint that it never named which raises + a compilation error. This could point to poor constraint reflection but our other constraint reflection + tests pass. Requires investigation. + """ + + @pytest.mark.skip(render_skip_reason(SkipReason.CTE_FEAT, extra=True)) + def test_select_recursive_round_trip(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.CTE_FEAT, extra=True)) + def test_delete_scalar_subq_round_trip(self): + """Error received is [UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.MUST_AGGREGATE_CORRELATED_SCALAR_SUBQUERY] + + This suggests a limitation of the platform. But a workaround may be possible if customers require it. + """ + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_skip_reason(SkipReason.TIMEZONE_OPT, True)) +class DateTimeTZTest(DateTimeTZTest): + """Test whether the sqlalchemy.DateTime() type can _optionally_ include timezone info. + This dialect maps DateTime() → TIMESTAMP, which _always_ includes tzinfo. + + Users can use databricks.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs + acknowledge this is expected for some dialects. + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime + """ + + pass diff --git a/src/databricks_sqlalchemy/test/conftest.py b/src/databricks_sqlalchemy/test/conftest.py new file mode 100644 index 0000000..ea43e8d --- /dev/null +++ b/src/databricks_sqlalchemy/test/conftest.py @@ -0,0 +1,13 @@ +from sqlalchemy.dialects import registry +import pytest + +registry.register("databricks", "databricks.sqlalchemy", "DatabricksDialect") +# sqlalchemy's dialect-testing machinery wants an entry like this. +# This seems to be based around dialects maybe having multiple drivers +# and wanting to test driver-specific URLs, but doesn't seem to make +# much sense for dialects with only one driver. +registry.register("databricks.databricks", "databricks.sqlalchemy", "DatabricksDialect") + +pytest.register_assert_rewrite("sqlalchemy.testing.assertions") + +from sqlalchemy.testing.plugin.pytestplugin import * diff --git a/src/databricks_sqlalchemy/test/overrides/__init__.py b/src/databricks_sqlalchemy/test/overrides/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py b/src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py new file mode 100644 index 0000000..a1f58fa --- /dev/null +++ b/src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py @@ -0,0 +1,189 @@ +"""The default test setup uses self-referential foreign keys and indexes for a test table. +We override to remove these assumptions. + +Note that test_multi_foreign_keys currently does not pass for all combinations due to +an ordering issue. The dialect returns the expected information. But this test makes assertions +on the order of the returned results. We can't guarantee that order at the moment. + +The test fixture actually tries to sort the outputs, but this sort isn't working. Will need +to follow-up on this later. +""" +import sqlalchemy as sa +from sqlalchemy.testing import config +from sqlalchemy.testing.schema import Column +from sqlalchemy.testing.schema import Table +from sqlalchemy import ForeignKey +from sqlalchemy import testing + +from sqlalchemy.testing.suite.test_reflection import ComponentReflectionTest + + +class ComponentReflectionTest(ComponentReflectionTest): # type: ignore + @classmethod + def define_reflected_tables(cls, metadata, schema): + if schema: + schema_prefix = schema + "." + else: + schema_prefix = "" + + if testing.requires.self_referential_foreign_keys.enabled: + parent_id_args = ( + ForeignKey( + "%susers.user_id" % schema_prefix, name="user_id_fk", use_alter=True + ), + ) + else: + parent_id_args = () + users = Table( + "users", + metadata, + Column("user_id", sa.INT, primary_key=True), + Column("test1", sa.CHAR(5), nullable=False), + Column("test2", sa.Float(), nullable=False), + Column("parent_user_id", sa.Integer, *parent_id_args), + sa.CheckConstraint( + "test2 > 0", + name="zz_test2_gt_zero", + comment="users check constraint", + ), + sa.CheckConstraint("test2 <= 1000"), + schema=schema, + test_needs_fk=True, + ) + + Table( + "dingalings", + metadata, + Column("dingaling_id", sa.Integer, primary_key=True), + Column( + "address_id", + sa.Integer, + ForeignKey( + "%semail_addresses.address_id" % schema_prefix, + name="zz_email_add_id_fg", + comment="di fk comment", + ), + ), + Column( + "id_user", + sa.Integer, + ForeignKey("%susers.user_id" % schema_prefix), + ), + Column("data", sa.String(30), unique=True), + sa.CheckConstraint( + "address_id > 0 AND address_id < 1000", + name="address_id_gt_zero", + ), + sa.UniqueConstraint( + "address_id", + "dingaling_id", + name="zz_dingalings_multiple", + comment="di unique comment", + ), + schema=schema, + test_needs_fk=True, + ) + Table( + "email_addresses", + metadata, + Column("address_id", sa.Integer), + Column("remote_user_id", sa.Integer, ForeignKey(users.c.user_id)), + Column("email_address", sa.String(20)), + sa.PrimaryKeyConstraint( + "address_id", name="email_ad_pk", comment="ea pk comment" + ), + schema=schema, + test_needs_fk=True, + ) + Table( + "comment_test", + metadata, + Column("id", sa.Integer, primary_key=True, comment="id comment"), + Column("data", sa.String(20), comment="data % comment"), + Column( + "d2", + sa.String(20), + comment=r"""Comment types type speedily ' " \ '' Fun!""", + ), + Column("d3", sa.String(42), comment="Comment\nwith\rescapes"), + schema=schema, + comment=r"""the test % ' " \ table comment""", + ) + Table( + "no_constraints", + metadata, + Column("data", sa.String(20)), + schema=schema, + comment="no\nconstraints\rhas\fescaped\vcomment", + ) + + if testing.requires.cross_schema_fk_reflection.enabled: + if schema is None: + Table( + "local_table", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("data", sa.String(20)), + Column( + "remote_id", + ForeignKey("%s.remote_table_2.id" % testing.config.test_schema), + ), + test_needs_fk=True, + schema=config.db.dialect.default_schema_name, + ) + else: + Table( + "remote_table", + metadata, + Column("id", sa.Integer, primary_key=True), + Column( + "local_id", + ForeignKey( + "%s.local_table.id" % config.db.dialect.default_schema_name + ), + ), + Column("data", sa.String(20)), + schema=schema, + test_needs_fk=True, + ) + Table( + "remote_table_2", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("data", sa.String(20)), + schema=schema, + test_needs_fk=True, + ) + + if testing.requires.index_reflection.enabled: + Index("users_t_idx", users.c.test1, users.c.test2, unique=True) + Index("users_all_idx", users.c.user_id, users.c.test2, users.c.test1) + + if not schema: + # test_needs_fk is at the moment to force MySQL InnoDB + noncol_idx_test_nopk = Table( + "noncol_idx_test_nopk", + metadata, + Column("q", sa.String(5)), + test_needs_fk=True, + ) + + noncol_idx_test_pk = Table( + "noncol_idx_test_pk", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("q", sa.String(5)), + test_needs_fk=True, + ) + + if ( + testing.requires.indexes_with_ascdesc.enabled + and testing.requires.reflect_indexes_with_ascdesc.enabled + ): + Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) + Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) + + if testing.requires.view_column_reflection.enabled: + cls.define_views(metadata, schema) + if not schema and testing.requires.temp_table_reflection.enabled: + cls.define_temp_tables(metadata) diff --git a/src/databricks_sqlalchemy/test/overrides/_ctetest.py b/src/databricks_sqlalchemy/test/overrides/_ctetest.py new file mode 100644 index 0000000..3cdae03 --- /dev/null +++ b/src/databricks_sqlalchemy/test/overrides/_ctetest.py @@ -0,0 +1,33 @@ +"""The default test setup uses a self-referential foreign key. With our dialect this requires +`use_alter=True` and the fk constraint to be named. So we override this to make the test pass. +""" + +from sqlalchemy.testing.suite import CTETest + +from sqlalchemy.testing.schema import Column +from sqlalchemy.testing.schema import Table +from sqlalchemy import ForeignKey +from sqlalchemy import Integer +from sqlalchemy import String + + +class CTETest(CTETest): # type: ignore + @classmethod + def define_tables(cls, metadata): + Table( + "some_table", + metadata, + Column("id", Integer, primary_key=True), + Column("data", String(50)), + Column( + "parent_id", ForeignKey("some_table.id", name="fk_test", use_alter=True) + ), + ) + + Table( + "some_other_table", + metadata, + Column("id", Integer, primary_key=True), + Column("data", String(50)), + Column("parent_id", Integer), + ) diff --git a/src/databricks_sqlalchemy/test/test_suite.py b/src/databricks_sqlalchemy/test/test_suite.py new file mode 100644 index 0000000..1900589 --- /dev/null +++ b/src/databricks_sqlalchemy/test/test_suite.py @@ -0,0 +1,12 @@ +""" +The order of these imports is important. Test cases are imported first from SQLAlchemy, +then are overridden by our local skip markers in _regression, _unsupported, and _future. +""" + + +# type: ignore +# fmt: off +from sqlalchemy.testing.suite import * +from databricks_sqlalchemy.src.sqlalchemy.test._regression import * +from databricks.sqlalchemy.test._unsupported import * +from databricks.sqlalchemy.test._future import * diff --git a/src/databricks_sqlalchemy/test_local/__init__.py b/src/databricks_sqlalchemy/test_local/__init__.py new file mode 100644 index 0000000..eca1cf5 --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/__init__.py @@ -0,0 +1,5 @@ +""" +This module contains tests entirely maintained by Databricks. + +These tests do not rely on SQLAlchemy's custom test runner. +""" diff --git a/src/databricks_sqlalchemy/test_local/conftest.py b/src/databricks_sqlalchemy/test_local/conftest.py new file mode 100644 index 0000000..c8b350b --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/conftest.py @@ -0,0 +1,44 @@ +import os +import pytest + + +@pytest.fixture(scope="session") +def host(): + return os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +@pytest.fixture(scope="session") +def http_path(): + return os.getenv("DATABRICKS_HTTP_PATH") + + +@pytest.fixture(scope="session") +def access_token(): + return os.getenv("DATABRICKS_TOKEN") + + +@pytest.fixture(scope="session") +def ingestion_user(): + return os.getenv("DATABRICKS_USER") + + +@pytest.fixture(scope="session") +def catalog(): + return os.getenv("DATABRICKS_CATALOG") + + +@pytest.fixture(scope="session") +def schema(): + return os.getenv("DATABRICKS_SCHEMA", "default") + + +@pytest.fixture(scope="session", autouse=True) +def connection_details(host, http_path, access_token, ingestion_user, catalog, schema): + return { + "host": host, + "http_path": http_path, + "access_token": access_token, + "ingestion_user": ingestion_user, + "catalog": catalog, + "schema": schema, + } diff --git a/src/databricks_sqlalchemy/test_local/e2e/MOCK_DATA.xlsx b/src/databricks_sqlalchemy/test_local/e2e/MOCK_DATA.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e080689a9d978891664c1848474f64401a453165 GIT binary patch literal 59837 zcmZU)WmH^2w=LR0kl+L-xCaUD5FA2qPjIJ$HZ<UI-7eob@4b7^ zeLrgNQ9st$YtFT5R@JK7O0sYtZ~y=R5&)2td1TCMIkWiQ6NLo;P~PAA_NG=~maqSL zmc@1|bhDxco(W2BKc(1wN6I#BCeK^NWWX@^)DW@M6N8=ABJ>B&2#AC(pmqE9vfjD; z^2zK*ELKPw?x+A>an}S7NIid5kx8Gs;LohMR4rc~r!`Yi)2CzU5P5 zq9gLw@!{u%vjHedB{*t(g;0{a0=(=9g6toWb1&0Kx zDwRkk^|+7ggny>=KNvV0%Iz6|0RTS20|03647gaafE`?|Ou=7WtgZ6ZmMvCTF+I;T zkvv6Aty4tshm~Lh#jueF#r5kfU9Bg2zuQ+{-4M!@GmDES%wUvEE`rCxrHm8D##ICz z#auYQ6q9tP-*v}eI-mOgJ1rsf5A@bD#=Wbo!~ciC=(6SeUB;e1R3OKt!cP5Ad3H#^ zfjjpWl}rOEf5*I=l^HoacD42Qhi(o*i7!)}1To&o`|hwaj?(#&W0Ew#rZi10uQlL% zKLRYvKe%niZj#mSee&yTrS+Mrw1gh+w2;G)%Mc`PEXzU#bO8|ijWv`K%Y`f&uMET@ zwa?5xlI&uvVZ(@1VLKc`D{Y05C~6Mu^;k}ts=k;YmA@1!wWPg_<_pdjLt|1zVaGc? zAYJ?mnNidgNbO60v(~3U^@CAR8EjFu;8{eIM(o1kK>{Gf4o~G9e4_Bl>1`n%jRFB( z<)Rs5WR9Nr{=p~TlbJ3>o8(U>Ydo1(8B3nA>F_V!VY;~2%R!_b1l!NNkhn@mk5MCB zSH!vQevMl%zt_9opuCe4#nt;gS2*8qaGe~6RU zxsQ==)n0cjeJrKSueee%Yn#lfyh;floR%9Z9xoC8XUL6RJp|F;L%#dou--#%Vr#5q zZwmsm7~9&L{wM6#G0KYFDf*k)5f89q~^DjC7J7Pk1E$674=vWf+`L^)>YUjcF(E?%} z-1S3*Ml)6hgR;I2#}nHv5}(A&?S26ekLu>6^OWXMN}35_W&p?TeWCf%T0w8>Z0RKH zlzU|4#UB*pM{29_DKc?KO?M5_@&NZg$(v|XER1|7?caOD{6EPv1cCm;U3z2{au+L( zq;q?Yi7hsX#E;aky^GmRiQ9E(Nq1tGOgCo{NKFEIV>_S}o2xscfFxo=#@qN$@L3c` zAe&xC-bA{|gWXSGL`)Ei2fXFVekO@Co3f+av#Apr(_%{s+h^D%aLSK~DDG~;{389! z|6_q*!k4*Qlm1(u+m3gB!bmgAYM}YRx77jt^0ijL|MdNyxQwRh-RH#jz!SZ{-^coY z3!byBy`_<@t>u3J{I~c;%cFGtNBmwxW1EL@aIs48v7F)`;C>`}Zp<<6JNhUqSz!SKE6dg`Rb7m_Fn?rt8^0 zem3*4)O6pWu{5p6`9Du`EQ4j}#S;s1rVl*sUfRYJT7Ei?;URtp`c^EYjsJwvgSYljtr|bke(2E{ z?!6^|LD9PTS;f6~&-4L;mnK0<>5?)-!IxYt9}%o_VgmQiklP)H;U4u{BGkNc=O4TU zv;q1vYEP)ih+WF^@5o;<;+AiI3GE^o9q0lJDZzKO9ij7)^u}%}l-StBV*#FuO&D%L zCY(5*lP{Yc@;>nw$s=RIA3S6=(>fPf=K2u0OMgOd%}3Cc5)&Ij=#$mC)S&3R*)5bl zn#Cp&U}Pm@-e|7~W&1O;0Bd6YfYiG2Ofq%K=)YRl`ZScz>E-)Y8E!;>^%?PDE7LxB zQ=9qB>;KB42op`Fu6Hi!-g(9N?=1R1^Cu!|+~z%X`p(=#2kv6nvqRM7q2$yy0v&KJ z5wf}5cBY66TXCHe&C?N<6=m}xSsQHne-{}zhyH6d_QtP4B;rb)F zUk>xo1Jm}Fv`!F0i%>GY#LJ9-VoOv}&@iXC=EvOOU2K53HDHPRB4#JNu{NHGOKYEw z#i)3R_|uN`mY^GI+=Fkw^cffINY~jekQdz-ekxC8r&%nt9Or z+!j1uu1)RoH&pbhYJ$WOBC)fZ(1EW?)q?RqT$#qFVo%^w`z=<7-MW4aan!FP5CnKD zgYW;ZwEtB1r$cTH00^vr2N3-KeE#3w=CyjpEpyjbJgKWd)$iJsKeU0@llf|C`W39` zzEwG^xNENR!^cMV;7Kxg{D!_9u!z)Q*H%_~t$CKv#x<59s#Hk%x%dPGc)wPR7v46z zbA`*VzZ|(&X?ee;A41w}nqJ!5-o_P$U#Xs6y4xY(` zpIJ2!9b>m3!X*lCqvd1@_EX-(x5D9jqJwUK6izTY-O`^yvGR8>bpdY#ZTS&i4VDab zCnXPgO$SQ@Sz3Gt7y_gzSBV>K!ifzptR8n~-WfZ3o@&$H z88?|uqu0f6S?%W)XU^3mvv=z+=V#up=l4tAZ#6d8%}Kl-JVmSRZQc~GThdi}D61Jp)9pNm>{@RxtN90qua}1Vbn5c5^knOXu{+YUhb)lCii(J!*USFw#D=!# zBf&cP!f)TwhT2FnpJ$`^S;35Num2{FF&;m=6*^7F;lDNzxxGMsCy}+EFRws02k{?= z3vAc-GoxNRm%T6CfQQbHFK>@VWlr`sbXSh@*WPuP`Dcn`jEjeg6~@;u&INvdJ-s26^cj-CVPbMs?pgwSK^xX zIBgs@&!SuI{asTz!TeW>#L>QG>inE<|D5lesHIpbyCQhg7`*wG2Q|TUNeLrO4SPi# zg}<;k`Tf<)>g#K!N0ULkdx|V+s_d(`XN|Cqcbn|l!jR1CQ>M80gUwH`j>pjv@79So zzOSpY?c&qS?Jm<<-s_*y6y9tOySp;6?QA=d08@a$vlB!+U$)T#K~m zXj^~U{oWT>xVEpPvc8hguYlNH7S?w9nERR>t_!x(=#~q>_plNQRPNj<7Szf&qLHXY z&iNZ+d++QaPS@5c?;(E6ybdu0YC+(vUCvDzOSVK~_Ys*u{W=2y9mj? z6qDIt%&^=CBCZcxERVa7Z$+4w#FZX0rr{(`%tt&(JuC8L49BVGZ4vh~e(iZ!rx;7e zH0PlhTjY7pwRLu1R9^oVNGE(KExh>BAm~hp&&qt7s&MOfePh^$t53FDOg>YPaCBd; zNrV7DG9H75yT+wb}dK`v$l8e6x#=-q0 zE$_05-{+O)!#9`R@jtTvr?LCtjJuudMM?J36>(r*>wwPw?;@=Ig8pPRL5Y-OJS5+1 z*G+(XJ&9*YkoMYz7-_nAOTBjYpY^YFP@TexTl$9X9tM}gwwx7QN9L*NJXe24ik2;m z`|q|cb+!x)`FU5rQJ!ry?x=$p@$vW@37fLF!G$$zz6=nJ;)+%VAf-@^QNZH24cvet zMjSd87m_v1qJn1aT=$KK9@mWVKU`qC?g)#02ooMDo?DCTk?N#iSk1tb+ z+l8xuQ+e;-zrVm3IRE^vK2lh~e{Vv>`45e4qRk4|@<8^+>KEu6Gt&`v`E7{y8Mep< z^>0s~$9>&WfC1On?eZiw$D%;!wBc<8!8o?Ck8Y6pYE&I;!Lg4aG!wL=xU!VFVY(k7 zc)${qz1xZLVtVOdMk<(wZ?8X4ok?f=H^##)XT+Fxh*DPbfx&cdEaCa%{A*9y>&La= zwW7zVV;VvegUBGW1}bksZ4uJ1n%C#dLMgVP%DALBhK*Vk*PJZD&`G{ZIaZLo(h6iF zZXPqL!|YpZ6~fBX;WTShT%K4hyp=nofN=%qscpSssK-rRrJbxoMEey4H^D^QD;7n0 z!0@vMreTH^G|Sqk-tD7V5T!0Gk49)m`V0c5atP_bsf-`sU6=E6$rlqXmfgO*KysGc z;Fm5>#V?ZP&Z%|t-Ne#C;xY!d*U$t4UTx17j@Bl6`$~ibTTkKGf3Un<1m`~o9%uHy zaMp=lA#-k`u|&bgpqG=D-t5Qt>v%8=pa!S6X$zpb$Uagl(z%@jY>H;~4+aH!Pw=am z3qynAuQu*idbHSvN=y#KjSberA{MLD;h? z{YJa@c3B?Pn9yz5;T(E2uVTXa7Cf%|iT4d5<}-gH-RnaeX$m@GyF22$H#E@U))Mq; zF=n(@5DaX0|9l!^?${Ou>)Iv6dGErc7{HGBxuuxHO5PdCis09NJ@YZzVS4(J3|{G;yNa_EqO@tgni?WIQ5Y+J|;<2 z9Z6IhT(*t;gZYCjt#N}0CTkVvmlu~juP^n0Q$_DdZ#)dh&1tQVQxfpX0LX4|EzUOJ zN`=9sjjE%KYGci|QFg$SFQBO%w@mxFL9f4L3K~oOFO6b41d8OP_-l^0;fQNo-LidE6Nd)7(|`WHSVD=>m-$FubKsUdwp+%W_z(Sr#PU3Px!(4X22qaI*}jsE}_j$+l#0 z>)9P_ke~k|Q?3Db`RXFc(pw~{JEUG%l-8=5MQDa83L#*EH0DuPR@LjUIq1H7+ly(1 zcnc@wf4)$lUZ@vCNam9UVSI!V6h5fcTYS4)>&2W%XfVF|`IjlIun+hlRomv5@xGR^ zi)93bt@7i8R3WYv;J$f9RW*pYW5|6}2&WL5o>C)>22^_@3Cck874D1C$;apzukx?3tP{o#pOQIgh5Rdx zgJ{CD8N)ZrSOY}}?lL}n?(j@|T+5@1$wsK={Z?(Hdu;KhcIMg~3JCjWx+TeUG^>-u ztc%w%vLlDa4r}i7C=%t+pJi+QUq~a}kQHtQJ%d`{YHGVt@MV`2=+j1L)+j z%}@ms*3Rn~iJAN|-u^FqihAN6|3qi6jgSDsXrsw#moPRwgi0R*tWs4H)^zl)p{+3p zS-%SBkd5o@sBD}_el-z5n+R49jXr!*P^Y6~-@@@p73a8RhK8XGWH;qdavsTBd$aZD zwm!UNv914vtu*cZqF>RqlLe1&BTGc3NtCPoJKQDcvFi8e13CW&IoOeSClZxs{3H)X zQtkQLP%45v(E`_k>NG0R=zieHJ-HR7od0Q-|Nkp}$L&2id&Z)lsgIx$Fnv949YDe8* z3Rp@f`xS%pcoz8>v*}1U@wXxDDoLhf&Qm9^DKls#kGi^rjZ%Sd&CR$+;O{R&b!^XA zY!c2E+ruj1iS#t{8~-aTyx-ZU>OAq{9IkXUSzNXQot@%w$Ia*(eYOi}>jRhl12_Hq zb7WL5&y~u~KbFyiYR|97U;OC6Fg}IN0XV4hf3X^M}pi|C$%w>vwE%v#7cNsO^$SK^%p;gFReF zGbe(XqQZ5e!fk}uVbPt^A7kux2le|^>g$L+Ij}R~^i02!tA1@huSVpn2sC`!Fa@fTN2=D6$9Bvf*0TvFQaq%G0Ta%xG z25qkjqkgitstg*t69~(N%wgfWVd1vS?7Q-Z5gCTl*;37(v-L6EAxtZ^@aLVx3++=z zg=CRIZFhlBuBnj#Y`4le?C2g;@)c_8`eQby6SlUx{-pUL#Y9AYa~P2At`Kw%vGO2` zuZjiI^f%wx9D%dT0q9_1XKIhj4yqQcAD(LLBM=Dq?PC21)hwgy6TDy68QZq)!t-(N z77G=hTrb%q%OC8z2zZ2kgaA+}g`D>J>WWQ!6N*EaHN!*1xEjg{;FeXJ(}{qa%YfT5 zL6=At*z?oI=$tQd3Bkg1;{xqSm8LM%y0;%C`XY%F82ZW?`WhevaOyMTuL*j`aNB1P z4P9{Be%v@sZoIfNP}J(d==jmd<0y=cMX{Z0HWpQ0 zir88ZUB2P&+M1N4Pg;W*1TP$&Xfx#`=U`-}}@S9aZ9@Odg)eO;N7RRjn{MV0P{4rl9gkw7UBv<$^Y{r>zyVWWFl zoDL7kj|eG=7%bzaHFzrLi}}m}a4-34;v>2E$#zp-!TGWv{ zU8RJ05jPohDn2=NR0^`$39&(Mdi*sD6Mxh@$YTP$qU)BpAT;^Jl{*iNOs`Cj5N*ey z`QprmexynrVc6au!f3yA<(PEkbadrx9N9KU5Aun=yTUAJ2n!snH~|uF#6q)!N07=a z)*aw#w<6deW<<==g!60zbqdG61?BX#3NosCsR>=FCTXsx>av>dOR(h{2sp;sL3@W; zEdlZm8sDx@{N4;uec-T-!6}n+#|yL5Ju1Ea^mc9~p&wb~DiB}XSB?ke!9K4PbeUte zFiq`()kX|W7u7gQ(c4b=z=7qOPQyNWo&4l4IplLzmCW*TI`VQhTx^@x2ae*IB3Rs_tZQh%iI4hKn8m;ekQGGFc*7UGe#T5;D8pXyo$}Dv~td6~`P~2|13JTH? z6=w$f8Vk4qo)^>k>B6V?Ds*{KBTgMe-SKi>X7}?Y-jCfXa#pT&*tnG*+l0ad!h&+b zf(ElBFL5n~2ZImf0+VpZ=XVlkVC9D}LSM7zEkwYyhfFQoDoB1?@%o#dB=VsP7G$@BJ`NM`^X=F+Ar)_TuZ|i*l^l`S({_@loX> zVmo10S+O}!-&@6H$fu~gek;v-1J7v{^TVv6!i2GG(eIrm#$Sx{=o?a1Nx5<^#j;N5}p`|Bm#(~RGVrn z8O~x@CJ3gh{;>4TqkRQ;WO{$qYD;QrOImw08hV=cUcJkpb^_Qu#tq^YP>tnc%u;>s z>6MK4ihv1Xdl^>nnZX#K)*0fr8}A=c=^Ih+yVfAgYza)0P7|E{34q2&&@~YGsJBLo zsuYc1I^2>4A)K(zx1DG4X*3X5gw?1Ip;0C0hbNbWFP5>J0Xbp>@1QUX6}hA+`#5SM z8Sj4PHAxT4L@lo=39}&L8c?EZ?h>KZ5K%5DSbsF$MK*R~3@5ae&lIHl#qIMnUSkfA z3!1j-AZJVf=a|Z=%|m_sIzVL9<DF!;#(X-HO9VS}|-69J~`b=W%ChJhyz-VmETs`3p1@qRn=y%gt zyOa3&M38LZ_-l0vOsZRPRG+&V(J1-a^x*#43D z{$mYNjP2@tBHq`cbpJMQW0dbgCM-ZDJOPrg>wowskgf6SXNIQJB)$d)U#po61#-BBATJw%$0U04petN7HRY07A!rOr3cz^@)~#604Dz?E zC*@>5fy1n4Hnd?Z=}%dg#!Y&LV@&7nq_yxg{(}`vQFAqgpDtTMqS<-s2$(|&rpP!= ze{bI7Aay0SZzaC(ScB+2Sq`W3Uy)rkEa4`EM-(bqlwm5ncNmF}Z~RU6uQyJb;XZQi zMDYSD)Gw)20~M*$1a=bycFP5J8%~q17;dgsCi_QW?(AakYV+aFZFB}Ul=ekDZ}o{E zSJI^b2x5xj30NP9Ne7r6M|2%WY-iaJzvP?YP;8BWS=65b!r7xpWJlrUDaJ?mk$~Oq z0*sZtYKjFlJCE8KAu2*xHkN8&{K|`V+UEP% zxZTfENFR}`+E{$#t(mr%2zj{^wat>6&63tr49DE3cFB)Y#^eFp{VW)CFE~Q6%Q`+~ zeHEYy{AP_P`Zal&iP#4B+$C<=Nl@Hm?QZOVy(|V;6XQx-XhfOTT)yDM25_g4F}9Nh zf%}yV+fjJS!q1Dp+O6VTp^AUDgb0js{eEdcydRXy=R|FPdpcJ~6FN+0lj~&}s;Dn9 zQn)iQ)o^Tu`6!@U;2PGL3%%sKk8AJzz}2sIK{_l%rB!y@wiy)olrW}2AtRjmA0ro zm;6$$-*6_JOlqfwa>;V7NSLc4kIDWpCJqywj79-~Ifwh((`c05L~f77UtpK}qB~3b zld`|G(w!-By#Bn>A&-rRYWF443TO2@Rq77T*lvILj4Mv!p{Te~xI>On_}W%H@Qbp` z@_~nJ1)q-j-l7Sl$(w`s(z(YMC-Q0AO-UcwvwBkTc)xk>%Ur|K!?k+Pk0Fm+`4{eB zPg@&Ys;D|!EdI1|E6^9WMYWL-!Dk8&QABC-QDaWM)M-*H8ICH2lceTP408>5kMFkR z*@!BSV*(Ohu}?y}?e<|4W=++ddpV4ZHi3~^m(s-Ffd(2M<%RX_2$zl07h#;4)!L)5 zL3#sTO>||hevKFwMJXiKuff)1QJ`}G3R_nFGu9q-C#7>njeq!I(8xx5g=;cD^g+Jo zPZ;D+IhIdHlzkC8zG4%cui};_|GmoJ(Zqg^A~{9x@^H$jzZ>k`k1C;I%M#dKx5}H3 z`}RinzmGFpLS3>T?)GG8c4TUfg5NK>Kl;z@F@7!&J7m3QsS!J2g+VkiK){blP32l_ zruMIlP{WjBAg01?iEEvGIq3v%elikP0PDJ=^HbdNe(TDvoU=Qyf3LEAJr^XDq_t*d zJ-}SiO2n*>Mi%=XK$QneoMNS&84GI( zZ?Jv0E_4w?cpsY{pJ@Uye}c;v1h0mk{h&b9j6)Iz3sSdSM>K^No)OWjDPeH9J_ zxZqjui@%}LCm>_g1;A)Hg>Gg+3;9QFMfmK(u1VF-;l;CS^EL9*lVd)ukdCKZif#fc z-7K?bA>w0iuk@x;DkV{Iq-801!`!YaK$Yv;P#uwna!qApIacw2fJ0h#$oRRS_W|@( z)*Ij3CWDDm(kk-FbT}5@_N^;HS#+QXWbz*-!d`cUuE3-#W2=kHy{@$-6;?^OyWxI= zYPaRZeg%zLex-}_Tn9gk`SiQWi&3CBq z)_=`6Pe58|a`W&0F+Qk$Kr$vOpu>ZRmr@#Q6<`CMEHIHyQ<~^dp73$5j0oZN#F$2? zA|n+nf%=q;5uqL#2@i{6Fp$(t!D?NO4a1n%f}si?gYqU+HmT+!N#+|#<{eM-6@C^6 zD;^T;gIP!?WHqAV>clDbw^3W;^$H>>FMIh-?ZKK@&0mNX<_Iz82zloSeOl#T;1xS! zZqeiw$A@6Pyvyfl=+QylQDBkq@wiZW$ak=MGW29=Wb}cix8ru)=X|{5VjOTB$Z(jt zN`AbR-R`o>WboO6)1;AQsQyebpQwc=Fe~A?8yTcnV?Zh?>uTItqGZr)F4Abe(P$3% zQ2zUK^~fti42#9$mrGt`N@sh~ZgIN=zhaY7D~#-@@oPJWag%zrr};RC&Vp-8nL0m( zd-IKZa{xE?Q)C-0`X93K2)-yOy-(o_)AvY0@{`mfpYm(L*_g z$~>$dO#h3m7MGJ6hZBTzJy;TaVc33G`WNmTW7f^aODwO23FyMKYe)1~u#9%eUqoP} z2=+Tz_FU!JW%15DD1L%_JeqZUf^9rHvGVPbt`1<&*gT1EpW#u2&uKLZnak6dHdnk* znTc))f>X$&UH338k1?*}sIB9muH|s3<@iv@hPaN!D`E3^$jk6p)PdJ&dJ~VXC`oe@ zrQ;4PffPgUGN#3>?MX7|rfGzn53US+s%SW=sW}~}Ikz2;N39Ahv0upHkNbCiqI$C6 z7-4Kt5j#m@bGG?YPK36%!+ng9UzImxAd@)m0|Nzs-+6)iyujc{4^ry%dyEdAn$xN{ z{y45VEJ5P7DMS71OwH=c-@?G)qiu4od7UNVeJrV{0}r|eW=?7*P6sB=51-5rLGFDH z4v2%VIXLNauoJqTrrlt>!u+C}y>!38=k&C_%kztC^RYyuPuVA#6q}jZTbUG?GCNgX z9j$R!6XCe6u}BM5p8nm!VMC~UFRx^}HK{F$Uvdzj<>>bIxgLZgaB+#lQ7rXgf&O8E zkzs)b!m7zaGW_>IL`MP-fowoo3FBw<-s0Z{D#`hhaVWaVC8-rk;8)Udu-jUa32g+U z*}S&cvZQ#Iqy)iG_z*gG`tgQAm9_-dz*p8>Q=uI_`-wLK%cit1I z_;wkb@m19{Cz;H1UnsS|Q;B^Sm;U}fBN0%eyM^EATmlwZFdQ0TG}Vbv_l$%G-1%g@ z8L)ssAw1-X?pimE)l4mwsdzEWc~}&AF;sb2-+$jI>H!|wmTuGYDp*91Pfe<|If>Ws zcMJ|uxUl0WLcfLZoC=zsvarnxa==7_CF_%<{F9_2lca#cwU#YZ{6@3Tcy)0-s}@O+ zu6KSniuM{2SC6AJ$VM&6BWK%SU^a%rnMS4)m=gOa(_z4a7_#HReMO=;$}TU4waK z6r11;F!rQ*J?IOv`Gz0c2;I#<-t|eaQ)+f>cPggEgjNe3%VtDaW<-2uL>G}}_5cqI zRkhio>%TC7tPF@H`?JyQ;q_l-X9w?rPH6)bT>yg$IBVY$g7PL&cfv_|%xQnjDV89k zE7c;ppfUF3M>N1W&Fs7U$VgJwZrG1ryX&0+%f@91;}-sLG$Pq*!kQ%li6w%VCBlWr zJvYTB3m=6iI=BD1TNPAO0Dq~_?ToOcKwaGDAR zO%2#hArZe?X^l@x0@_a96|;+(#rj|3Bg8y{(K&mCesuQIn|0YZu}heh%kSloE?W>` zTM+SCJZ`J2l~0zRNx9L%WqA{){7w-PAoPh(l;#a>yWk}OPAu=h;QPDnHmdoVH@jPX zwvZ#RI3}$w}+M>EywQ(3}>>Pi7)*GhmDPoZF%nVJ=USwRJiq_1p0D+d1M(vUZ*i%Jk7k zd=Oc5ur*=CN5Tja!U$I~_tl0Bqp;v-=pAhYti5+bmhx@cd|?&>v<=-s&H2i5!??cX zyVWpIZUA^Z034BBq_fWj`&G&dCtSZE2G59c1OMXFxxRumsIX7XPh(&eKDcSU{;UqJ zVE(jBM1)O5BtS%T#eQp%S0N&@V*6rxoz4tfL{x4?9VQF58XNAUTpuB>-NE2i9N#%=)9E zM&cIMNoZhsllGXy>6imyIc2&tZmmUh-~C}HEGM2U`sKH1?xXe(iLIzj6aikQCIt<< zrV$)?cip`2R!?$BPvIm_9+wP8%2fw$l5|cmms_qtehZ{E%4!mPPL)P>eBj_`yN+bk z#2vlRg)}&W_eGbv7|{G5(7elXL7VS^`xWM4*U-wHKMFm%zuQ-;xl6)p2`z#&PTM;a z3mBP913M)sl-La|7=ag1l(RyVM?#b*MwEB?{PZj08Xkn?u{gxC%SxOM57=nFWF$HH z&xIOe#{R15X##x`{0VY=kE~p~TK3+sPgO$;lI1=1lm3;HJynw*7WM@nBEu=GFLYqT zJM5rcl>9gXe20WjGEtmzLEZ%suwzV?LK)$^EQFszPI5nP=7wzLejHdh+502-2o>Fa zf^yYxxedZsb{}9xridw$o3|5XV{D@Ft>NJ=NyxK|KSVFnreM;h@Y1H7Yu(mldlAwEZ_l6&EK3C#M}J z=cfFK<*UyV?}42UIyVMnGFmyz!tTAUo{g}aX;}SZt|Whq?5(ACm4jl1!)}G+&*>t) zzOrnG*97v`uLaocgO;qx;)$4B>8HhyP#>H{H$Fjg2u{^z{r_vQiFTamb^f>d!-l}@ zNuju~OGdolN}c{e5u?Hn`Kv{;Yx6Tt*r^qnzhC(#o@qKM>sC2v{_~})_qB@MW>$)Z zN+B&b(u={^`VEQhGjZk8w+heQQ3v42s}{Jv7t zZs8;;OogVwd*91rU*4 zA@?h{`xS67l&4wA{&(p$`O|AM|7-FkWqr(a=X1S#7y5SH#d@F zFaU*DG3CeC-Agmo?{quOP0U$eD6)#Fvc7z`wm!Z>wc`i?>I(|@iiD@grjoE6kg*HA@bSi{F0rS4Vo%{`Pq~yb`#Q%z`IP1JoOPIe z7at5;Dk$7Fxo0GJr&@DWk>FSG{6d_!+1F{o7lu;qKV*eHii9msg6;XPF{Z=ELc$Xz z%zzgIWDHlEA<0tycT;h;gN(!R1aUIo)^vhN+T?+iFDcpbZzFh<;h2*nSd-yA)z0=n zDK{6G?XU~bfhX51ObOY%Chwz4_=}*L#XN>P7%4r)u1#!|EfDRKv35Hd zem*&JF^PH{I21X;w?ckCt0EiuelZ$Z9fha@<)uOfP3IpmFcgKaNoPi<%NnnBL{!=!2ZlS-N#7|Tm$ z35)VzEz@HaO*2cN6cD%z1R{*NK?cE3#DE)*s_E8nJnxQ-eraCx(5KzJXXsuQ#aS*6 z$gqhzD8-}KAEWUki*Pe5c`J*s!e~t#>f>H~n8CmCKE-Bpnh@hvKZ0!#xJZbsrRm-{ zNoxBem<0(X4j85tCwp{iI;2+TCc-BsMkXgviP6(b`ZTGDvffO+r!7-vMGWBd{2FSs2_U4DJ^O*J43V2Y=rn z#Apk3F1wor0k~*FQ9thm9Nt#1aX7w;s;>Qk2mQ8yJh9wlC&~^dCoS(LWA7#l>?Zr2 z$d%{88tBW#U9Ty9_z4d92ire|$~HR@^}4+KeyywVDh;}53pEt=iG@N>GWa$#pj#Py zX9lvS^!&T@53_kwr=HH(a82J{0{Ts}(O-Jk=QET2QFy_|vTZA|!_q4n<7rCcE6U@< zCqC$&M|?gg6Fk*~bl%d40$K0H0-Y*N24AOVm_0t`Ct8x3703X_UcLLSzx!^a`)-TR zi}U*9;U^e-4n^#AGxQFYc#18e6SVg!qG}gPt#dF&vP%l(bf^(SgRYjtxt3!OcZ4Z% z`T}9raqqB^1D^08_zI-$oxi1P_xn0`dVMg{(-?(ZzH-`983;0#wJ9-IC^A1SG9Qy& zu%Cq5#v+wEmMgw&B&S{#LCURKSemXj&UVIe6gv)hQ?;4ogwldnlLo9WMm}DQkX($o zqPQ~)Wh3CX6uFVtu+|5!tTO{WSZlh>8m{h6&BN%(B@;^V&U8NB8faA3*McAXuurs4 zJRTV+tvnw@zOiBzx#K}if*S=$^1j%FwjP$4Sv=naDv7^>sMvpgU2a0A)7}&Oi6XF8 z4mxzU|A$sPNj#su6%mRq!;WYa*5Sr#;-M+H4|064$|(jt_FcR~&`e>(Bx8 zN(t5Go~9IdG148jfL^#25f>+WpD>@QCSStS2q^6d@HuV5(pBnSu^e7+v}W+|T`H%4 zhYlgmTh=eqnO8<0H#WBwuMdU@ELEx9ZP#G`VF_xZ3mc#SF`*G=wwXo{*X^W2$iE_O z#TTa;L|)Ydl9d_@{42M8M@KofJbz{Z{9BlLwXMC#y@l_1VS-Na{Cdwr<~*O$_((I- zeGPTb2p$>5Vf(3p%`tZC=V&Co&NX4+X_=*{FWIXU#Q~RIAFyyYt3B&@{m5(-Re0Fg zVW&Scy5|0y-XaKvUr9;TU@cGe!?Vn*LHd%YGp<^D47S4tsG3k~_IR+B;)6H;h~F1g z4OcVvjnjz(7Qt`$rna$6&sE-QwDkY3*HZp<8os?u z?C&u^|M@Pg$9r*eQ~go|hHc!wGLpRg;uM%=(|v`0ga&h_ju&t$lUCdt(l&ITA{Tp+ z(H##k30!7vBK%=71KaUH0WFfp=^p-D*-BRvnL9sR%}zmNcVwp1GM64lQNi4M4A;*> zC>^ z2YZokbMfs8W|Z}rDBAMy*#%w7sxZ!+LNCNVWEj=_^|V0=b2L8lTk9Tz%y-zs#e8gW zgcIcSyg}#rjYdG5t$!FH2wk`8qW2Jxhx+-_ZTnt9YD#^6kPG0AUr(@Mm;7?vOLXhu zjA*4$T?YOP*CO=jN#qwc7_G^xl$OG+7Fqije0FJWZ)JI>UfBfGRX^n7`eEH2>1BJ>y zBQ2}J8$!C+)L~R?Z&ntu?TZ?(A6YgOoQdQl3?sp=7%VwbjX(XCB#Vp73H&KC-^f1H zry=;KAxkU#=Olm{-GLI}pO>oR>NLJ2mD>BF^kb;jDo=Il_J((!Z(mCdeg|_e^T>5C zbwQ06(SjEdrWX;f7g3hbK&v?$z-Kx8Ao7~0MY6P%Y?GRapDQ9>&>h^Oq%9|nXoAPI z7YYkr3n=j|M|2Z7^p8b|j74Zkc$q1V`Ia3vpeUr<0dF^&*GbLL_e#e9@73L2NkFI3 zm0i2shAWXVG#aCUIR}O!=MPm542!Yw5%jC>Ek;cH4V~hmu*4b~N=lF9Y{BKykl9GSxrj&H4~E6M>M35Xv81UT*%y&86WF)K6sTEo?)V-Z6tUM2i zP!R#0mOp+e1{GIFhWOmJoJdv}Xh;}n#2EiE<5F_R{OsSFD(sF=00UWKOwE%7Lt54M z-VdG3Y3ZXZ(YK`LgKOqT5dTKipF{x2?%*ikm$$zMlVFrd<=R0W?11RmO5FcTMKR||}Lzs5Q z@^ng_u8H`7y<`^Ae=Q+&dPi1>$12tasiq=HrW;A79jS9An==mA7xw;b8scY*34pvM z7N81VKJhiDTA1g2;wuMUkDr_{k;iadlY8)Jc1;`Uo6y*!?DKW-l~S+IzD-a8bE%r}Q#B!fHQ{C7&6x8O zE3EK5WYk5>uU~Ll~L>plV_5SIOA<*JqZxZNl5*TR`*aB*` zBpb2gQe?pQ5hh8=zDbW42MKuQW2@JKkIE)Oi7ur{(vW6FqvB}xDK`Y5%&8OA zJ!2YMntunH?*q+|A_IpGMQrom4`roJoGkVQ-|r#3+TZ`6Lj>#ZcZaU%xdkj;IJ+qk z&Tm%*1B;`ErBoR~p8309?u?0ttwcXAe3Of})z4bh00Fl7qFG6SI|rwQx^-uq@1t z=K2pj* zQYtc1s^f@=9|qsxv7N>fi;Yro%a_)iWWP#v#0rbEM^FTYkjee=D1Su1i&jmdK32*< zRw^=9YGc7%KOrL-<$n?NRZ(qqUAs6GiWPS&ZiNJQC|2Byw?J_xxJz*>#U;2GC{TiH zaR>=k+={zH@sszv_|F*gc8%<;u`}mdkA;41>;N?|F)$8(@wU3r`nyz|4!}2qM26F( zXY#3_H-*TCrw{rM&sog&y|C?;ur1nqZ9K4N6|U^s=!wc9m6mki@@ASE20+H<{vQCH z*mZ~TkEZw8xK-GN#Iim-S`$931&{Ww&wq#%w{MYIAM-ja-g zjzs7sm8a#qoMl&DgBpg8Bnem~3DhJBAwvRA9+J~>JV;OTAXAM$R?CbaA+rT59{X8A zy=j>0e7;jrh^m^Kg+zbmhf6-G@p%?XLKaF97D|RRD%z7_Dr17yZ^bT}m|4wf4B!1z zcVB03{KW;%q)G#33o^pUj;1Ue<7`!Xuwp`e@=HPh^2WtYmBa(Q;2}upU_|I(V(4H;)YIQev8a}eaP6&xJ>&OY+hHvE zo+VG)Zd#nGw-{DkC5vV+8;33Z5Z}QF-@!!R!B27?aJn3x{f^KdK`7~+tXtxuY6gP7 zY=f6FN&C+gQqvO1WY{&*JK69MDF@o69SWiy3b7rE@a2WFgQ!TZfd-i)l{-VvCf-i! z9=W?T^)BM4r(SP#Li-^)cUQ4JQB!-v%#>$BW97GyYXHj?!0QUIqjm%}nKB4)Cb(P8 zO)DKEWV&)~YuLXx#>@T<+lUi&)g;V8m|zP;#PnkR4@H%s8A*Yrzj#20` zv#qq|lsG*7S!VjDZRF(ISbkO*j&Ngc(~p$$3KSzpP8(G=nXz&TvTN8e#q@?(8BZoztnkpY!u$oWQR> zgJUeguXsQ4j@M||XYUcx^iVyu?tSG#8XZlxGt0g2TEcC7fpE{fAp;eF7ei2rW!R-M zmsYa~R(0vs?-~*KIlt=e!3Lmzp-1s%O7Nk2O<$Bdo~r=86w^t)(ZlGfWQ`Ek!bHbX(7^20PQ`>!+9zMKs({ zcexNV=PKYW`<_f*rzVfm{{iSdGVn={0RRf3T{i8yUXj-OIab_YsLxtLT9uP7b*qq^ z+@iLS?0>K&*2&AsEISdZVG*iP6RPCmhmYiL>4?e}ri|EpxY>K63+4J%i7jBkUjb-kr1t`>ipq?$~D%qBoj6Ck>OK~=V&-wDcalFC6t+c2izBG%dO ze?s}RaWb^K%5FO3aBw&5(j=QZ=Y81;z@Nc6dC@xijdixDb@oNdjsJA>mh%I~H#(U6 znUxgkMUV;n7k8Vr(49(nm~~4n-U*HXt4l`mWfWA$SVc!xMMqIZXB8S$?BM52&@8gY zT4>WqS0Ly4Wm-pzZ=b7x#A_PmXBqfK)G*7DF^CS>^cBb)0rZRjZYLa-Z)|08Uo0!? zw2uJF&a41Dwu>awGI^>J!2;^5@uYC_xdGp)Wy5c*W1S=gNeJfR4NJBtOLiEI+P+91 zN=t^EIR{LgP5O2_P3=MU_EgK${PF2A>q`t$MkZ0goD-v97IPCikQoi=i3Z&Ma)dv7 z4F>FlJfq9=74V9C(O%n2t58L ze7qikT_OwU)$IQz33-GMD^D`v&~N+7Oc(A=Jm-?avB;dkK*dpLjPdvOffGY3NQiYr zh;?F!bw_lkZdqy{SdyVj=IZbnRKyyE#C02~IwYI^BwjPDU{b$wpz`W@p}J5NBQkA#TGJ$aE&Suid`^=g*MVt36$quES=nEQr$1^T^legB zAane^fLE8}`mlIS*tixfUZ`#@r}mOuihgZ+x2QR~UCQoVUei8o)T-%U_|wtv{E&Dk z7MLqT594uvXZ#N|9u6I!guXtqz6swY^?l!U#XN#s%b^yz9Ox!=IydYB{EH4xxg^C{d%Ea zkjQpok9|mZwzuXhmLOS#kSq~M7P5nGm+=^ZFaj;4u-ia8LI}+EZ5lPZ7N?bC3P_xV zH%0=SMzy&V*HQEX`ImrzJ|JKS2>AKMR-F%6K4nmZvcA;%ho+|T794}EF?P=@9>_Lo zy=IU81^XI7UR%C~Wu4`8YItDt?Klnp|6Qrq2>%gKjG#C3_DkJE)P`DBx`{<~^PlEK zLIr!|jY7<DY{0jyg9k1fQVX!-Wp_axxr)v(%Hp7@+N>P4*)+_A?0 zUxt{6MxNtS4p9N0bux0vu)z*Lw#O}Z@a6|+V;l94HYXo#@c8X+^M35%I0{8SHBpt6 zNFizhuGHR&vFVSjB6`DSvsrOsw>%^ zDA`WnER@Ryr&w-|WTT!bwl3ZAb0LW)7zoD2=4pG{@i1JoLP7>xP7-4Y<;UVzmdS_c zx}i>V&{uR&8aimNvZe&&UcUwD-c0?&q|gXzg9NuNX|P<4=0Ao2K1Fo5@aJ@e;6?PL33u4>cE$GAeHMwH2PB$bcn1P^5Y{O}i+U{C25_dgSVF>Q{ThrT{ z(h$aO&u>fa*+lNyV(!_OV2IwAQ)L;hYuP<9)>UqzTHw15&GiQ5^KUd0M zhd5awA>G5n!PVcS1=0aRfW9}zyn*)|eJWo{gU5_vXc&__*~@lC-)Wk$x;VS1vbgXr zoy!%yRvuvEeWLbLF97Khik0h6mIxgwOnP1lbx!+QOhgJ**Q#P+dR7|2U}{0q>NGhm z##Wee)8t!;!!ND^W(atzq6NvKGWW_~1$X+k{HNkAs`^jG%j;mQETv0$t4-5ZR(uO2 zFwRJF)hFRL??e|Exc1V0>)pw3th{4XkxRAr+tU_vrD?`B(4i86s6xLOEx^eFoe3V= zP7n1{{JpViZVd#hp#m+4Sd-CvGhCxAXvEa_xKPUN%W1;STy-)P{>W#cW_pTRriY>R zScjG_lDYD_qOw;zW+t!-{KRinQM|NH@)aj_I60Mn9Z#0c-W+wTytMv_Rbo&@ZyTz? z?|K;WS~hiBZhYi=gH3=$kV5=X^f49lxVZQTr2GUT>}aHcbqo169E-oc=2@+B0z zYI#WCH;1{&BM$rVIvwgZ?*EwOcg12o`4%)%5w_6WOLU?oDH-zGLU^yDb@Yob_0ewqIKOL8Dc1`=kZ&4{Y#g?pCO%x zDzv>Kn{N3un&tbg{J% zd?v-X{B~)(m%}(kaNc#-tHsn8cdq&3Khd!E>ThX-WZCs3cMV!5`DMHE-Xp3B>R$Yw zcn(`3Pv-HRYi+*>V|<*iPBnc^`cLZ3=Rp@7_WK~)8L78{F!8XLWYDYE{dqi;Yl-DCD;-J8N`8NQwOzK(gWkhdEKH*xyR(Z2JTI*_g*GhFVvrGpf8v>V7N zp@lgR;djqS)<4UNYaRR?SSimzyN4`rG|kM+()V*6J74wuXFYW zfwK>?$`mBmRH?FjEyt25df|HMYpf)H$=xnUw=3%ImNJ$@wi_gE^v%Bfiu}7L9UhXb zx$)n*xApYVVQE>&o*t)@-b+k;27Hd59N1Z~U}A#zRK;+()hV0cgXz(e^-cbLed}M@ z5Z9#(3W5s?;R}k80Fx(jCMg8-NCyk|PRD`=bu=slYK^EvtUCvmhGdLY=`c&!slGU) zVEwu)KPvM5v}7YaZ-c-=dW`4m{M(A132ckZGVa4C-F7#aPjK=InUpKogUvDhORTL3 zC*4uv4W9qMP}ZuCqwWIrLh2S%K+S5vIRd=HnG?t}vs+p`UH%z?3QI-BbcD9Be80Gj zjR-w1K;9?_-7G*pwd~IHdjxNwEAJhywo9SSpXeTw3H!g(2CkZWZ;R7BEj$#45c}E- z7N}`S6L=Xbpvo$sDk`8N;RkKqDk>WnU!<9oU=Z?B zL2D~QI}VZZYGwYZ2c*{py6OUv9txb+e7q2vq5XIGHFa>Du63TO~Hz_5X3ym;IP+u0Ry?G>^IF%dcFFU)R6if7sP> zo7tda6;U0x;JN1s2*u20hZ1mCzCS8q{-pcFmgb!JcFxUAfi8F32@cJk|j_XSKA~FG#4Am$&XS4*TSMUm1Mj(FPgd7Ggf?U>lA*eWU6?kU+08_ye?-w~p9 z>D=14(2^7tPizj|xJYJvI;HzHAfb%tkME3m(BStlmzO9=XkqnJ8+Sx2Et8uB!7HZ(I)&di&9}>2D#{DvZuQ6Ak#P3kqur7zzstiwhWBEQ6Ia zq;~6@%gd%b9Q@eHer9Yk6BKN@|lgU3P;q{Zr zWQ`i`+50-4NG6FC>>BB8pUR(@a_8hL@fxaAy`?CD=;~z1izZo+WigBNwt14l;hU-G zvnlxb6l-0PHN)mim9@qyZNhqz5p!UYZV1n{`8bgt$wAz+SWT(gEdNMcfrLvPX0CD( zQ5Jl~OoJZa)e++;mVhOeKrNQwqvF5jPwI`_OmXjaC83AxUq!nSV3P+rSD*<%U9CWs zfI_*&cKX_=Q?s#|mNc3AG?|7pnOs?P@tA2m!4IYo>-d6jywPLCiydHYk}Wg)>?0*6 zPnO&(I=iU?l4#?ZH}tn^w|n1Wx7A|TkF!`;b8w&DwS-k440Pfwk1=yYa}4&4xRm3& z9qYQPDnTdtwRO5Dt@hn2En)`^Dc^;7!Ge;|f>OkS@;n=o*B+ZFw3tJU@tx^C%o@q$ zv}?*&NA=0~s;)OA{Sg^+r0qPVzlGRge4!&_IDI-gd3yNUG}f`ls=t_sMv(39G>{E8 zkHSMQA=P(hdY;AJ8jeskQJY-8;nq`zGQKvi=8Ja}NWc)F8j90a44%)=s=;X)LVNx6XmTmj+yrF&1$6%fL>X!8GI|}d zl-bl*SZXkXyoB(etdd^K8F)WWn-nmg5Mqx51-27)zDta1O#PqQw?_sRe(|2#v2s*h z3Z&2bIS-$J11&hR1?6rk=ZikD;KRseo^u z)=8b!Zv-cVPFLcJMnVy7Z{==uXSax5W0es7*WV<8mV<#A!9;_1`D~CT0Lw3c*Dt^h z?opooAgV+>>P6rhK58I7hV0UG8uWz=VdGL6F;M;^tiWBU%M((r!mZ#D3_JwaAA|We z!1bG8K4JO~yFE?>w>cg0(}i@yQ?HHR&;2&{ktB`^qD>QWAp~xp?u%zcsW@3mwET=~ zt#<=AVLRaye(6qa=ygaS@pw|L0(@0Y>TU zhlBK|V(A}myfT0gJs^uN&`TG%6L_=@zXgY4y!vE%o4=oVY2|AK58x?_Y7|v@X<)RC zjuRe+nJN(^zD{SX>osxGZ zG58}UDMyN4I`ANIC-v42Pq=Rbu@n)5gLDix_!R>;l-@8=j-Wpu~YmW zji-0R69BC(t?y9|AMbFzDGDJiz34NXQ!J@9?v3c9k;^v|5oZ(q=Mxb^KMl$U4XCk` zbHXWE*BE$ustyR#l@X@t&fSqa$78guYySJREXbLFG0~D3QJ)ymkQmWw-xaW4yGxa{ z=Xvo@yMiyC8}dzUaKslQ>YdK2hwqM}gR$V6!jRWRj!kCju@|1kI{(v+5-G|eqzfM{ z_`GYq1Ol~*e@a0*ZC0p5+-;PyNGt)B-~UcTh5b*7i-KSsQYD2}VR(E0AJ!GNs_fnx zdBEJlWFAVPHAZR`SqJ z_cP%((f!y|h z+!h-z3UYtjmLq<9$avq}wQ>jfie!Xqpr1YYqkbSmfizPOG>HcMf>x#Mx^!<^P1j zC^4z{U_%ik^Rdp|3TmXoPKGZDRc}JAGnKLFp&uDl6}l@m`giS0AwgvkL1l?SW$*mc zRkxPuUB%{_cE62;dp9wZaxCtEef7iI)$TlYLP)K>eOvS8-v)Fz9jd-MRHZpol{xtV zE=mPkrQ(K}H-DHUa!1nlmm6J4LBe*HMpe5TA_h?deZ*79i>_K`NMWf2uo*&Fs`on& z?da7|)b*(6!poIdRQN@}m+h{KgHfp;;!9%4SrR82Kdw8H*FLE`ovXe&SEV^u^*uG( z`3maj<%28eUmU3@q5T9?NKRe!7+hiE-aQBlzfdr9N@Qa0Sc4=v`5M0hm?8k45&tQ? zJ6}m?&^yFC5!|`k5-U3sE~MjdhkF=8brfCxy9i)y+U)#-9q>YUf4b0=~4&cHqKx8P~%1tED^w%je?TR zj*`vb!`x6SVCWEG?)emav;;ZN<;UD>X>YjrD1LQ0X^w*(i1reVH@r+-+d;bihOk}j z+rr;0!oOJ}f3waptVug1aAHCN%%6%*B&zV?`?{WyX~#mjt_pO&sX$=B9IRYL&*Fw6 zdtoPwa3@P-C#!Hu5;ZRptEJg5ajq(=zEWa@D}*-*kwB&!KNMA1m#OuXg@@fznkbuE zI}ZBRT_MoO3}ku>^mq&0OgLIG5dizkA>EzB@uyRm)uX`WOXs{MUIW6=@&>&|F#KQm z%lL|B5BGP^#(t6zoFag+5>RK%|P?ti^R0;wVpZwdnxivdc_0QKSZUy3j0y(KM_Tf47{M7r|nXt?ND_V&^? zQt51cU%-75I{JO*Q*P^9%vJQf1%+%vg=`Up>`-Ep%Pa{U^7V$Q>fLbG1Nv9Zh;zAJ$Lz^{MATdYTDzDOi(N)C^Zw*N5FrnqSvDvj_`x-US>T8HAjml z;&4P+{D@~4XfI1)71(>R8ANykB2lKSqO!4VeM1_!-HkOoWHkVa8i36m~F&Q3{$bXw+kY4x#Ea%3>L8r>Eq)`mJUPFKuqL#S&w?j!+yB>|<8fch#w zuwnWw;JBV<`)e?gAt^RiER9tD6K8ut`Z;MTBHl@*(n8ykxpTJQLxqLPIlBmr;*dX4t}?Ik+;rx{fj% zWSR$nto=amexRcJ+&5Ft@Se3Vw<}_^%ahf7NRlgKwE{l_*zi1#S0<65u1UI}cffc1 ztz$ZRXL51JWg;78t(#>c!pr;Y|L}jR1xs>jgz-p!@%*w)miY$P=r1GQ7!kTdeH*Ei zB|?;Kr~c9I)92T~ zt88P8=>LT;l-wue+Fpj;=on7@#z8T^v5JR8JAA-R|EvYsS+hUW*L#n9=P$$vrO4oF zO&`W>acZ}vnLqa`)kWxaU0lgjrV{~o1o9=7tbqrZvj=u`G(N81#s=&Q;$u%X-e#?% zN?79^FzRnjerT^0Sts$h%&?_dob;akPt6(#V!$!Pq4|7T?X>0zcGx8d5Z1d+5*$y9Y2<|lymx2jRDk4V;=wK8;z%d(lwQcmZGdEuG(yJl2*U-?n>h7 zr1Vy+8Jxd2idR!-SuxGTf1IqJ zCW(@(@jv=&#Lq-vCDBOM=PTs}K1+{`nzFoy(_s@AG1buw9)z>-z9#omc*#;>_wAK^ zGW!Gk;V(4&;XA`vN5!~uuuFH48JOqqxGB7vO8~EbP^5A5sLkeAmnW>ej*+x!F=~22 zNMvjB<>9O1PiY>)Zn89U z3Zgne$Q5?T+|P8KWwtOSzISIH(erO1)xym88~B$B-KQEaDl0pBF^~CrF-s^$8m~Eo z7I= zL(j@f9)J*papfrX32MraSGc~`4Uv#yl;(eBvsKiIGO zF(M)x4%Ra(P7dt+I(x#@RFWdN!C?d%8`0(K$A@|n)9vC2mMLR4DxBI#+2)ARM6p`* zB{(DY9sry?14o7DXi zbA3Jk=Y`zgHYK3J-@evN&g9b`&;WVMs(#$!gzk=cz#8GenowL1$wyG zKVxsELhG_yvKBgvYL1p5(>OaxKAmw5->n)$9y2Fn%56Y;)W?PzgMBJZNn~N98cBp2 zFhULdJxfmNY^Jl{@0ls*BbTwb6NS7Cb}w11MPV0G!QHe2S0Er5GO=eRT}s-(EVp8O zXkH14_KT19ON{nw%Un5*XbCr25dnEJ<+>pMQPnuB9ohYuF#t->ekv^dl^JeUlv=6M zOhV{WPvr+m@{3RMOHA@x_xk?aV<5Zid3B;y0sGoBMbO#~<2BX!qrJ*TDIeFY?x!LG zOtLQs$6 z2O|^sK~ntUQ~VNB5WCj74@1o}$I+o~a;lt{qB2G!W%|}Gm{0cIOS4PCyrwkdD!mIE zoYWQ-(dASsX;})kEajCP_FyBkyLeF;+1N##`SV%t&A zj1nBO#kS`dd9t0?_IACU+8Ray#!`U_Qh|z6fu_CBT#HQYqzFM((|*Y`x3pO~`pQ$9 z!A@b85Z3_P+Q*RkEfVzY4IY8_ox-Nf2m7rqIpwg-^3%-nx^}XaX&eL6kVpRI%X5r@ z*ujk;q30zsZd;&JN;qcGTd5#-{|_a+91zfbJm@|Vbl(=xxgQJY7D8J$Xmpr2ogn=- z@T;Jm{WP+0Eh$4GYr8I8U0NGQe(~*ud7dBTHy5#Tm~i>2aCzOs9G&%6Tpi*e0cpr2 zU80^BpDXD_k0ObNZ`{z!Ze@wkuoxKH#zJYFD=`oPf}+5XVFlL+TD zEb+;)!l9Ml7ifOUf*DEplcxnS$%zw@bSpQT2S9ws!`I}8S2TwK=j_%}tCH9;BuJ0C z@%WDBzqlF4oMpV{9v!U?Bc#kSS3-<;1obvA3}XCbY3I8r-gQw3cTt2?!z{M)C7I4( zWlm$EQhg}4Zk_e*r@phQ+uKn-syG{3Xx31BiXg`e0v^i1#_-nkjp=#QXx+I7?D2O^F zh`Q1dv=5!ZTFXXSA!ZG-#A+$a?OjQFG{UcrlsM>=ld~JG5Zgcp$P`SzVi-f^q6(HRxD8+i#;;DO}6# zA0*U@UZW3sO@z2j^teq(ll{b3sq~P{M=8izJPX_=@ZpN#Lanjjq?9u=3~}}Aja_^_ zjptT#ANYyNria{So!q7%Zmuk|>@VY%tRulwC7To$2HLQ%lo>75l7go*(!gK+(+Qk@ z@#@bAy&>~P3QqQcL+o)5@@7uxW)5--izCz*?C@yE{VBs@kZ6qot^Q$!$VA~zcfy=C z7QM4pf#>g@Z)`>+XZ{~|&+I6OTV%W&MxtBSKVol_v8R|-ddKHg31Zm}WFt=@Nc^QS zQSRMo(g$s+N41CoRA*nJ_olA=ATRR86AJzl3egjagEu!#B0wD>UY-C+#)7pRGu!NF zr2}1&;x8$%r3^U-eWv_hD3Raf@XP`yp5>a1D#;QlU5zE>6eQ&oCFL@O$Nq#WEI*7* z5R*YdMP!q%6S#HYs#LfoQY(reg1WslO+gJW&&D^42Ne7V6ru+d2OKwLP75~SRA|lz zMNVok2>AkICT02B9)El1gpudtG|9VU@e!F??m%ztxDTVSDc%gZC26@qX}KY3xf0hW z`JW8ZK?s$^00h3k_{+h?(Qs?7==j&6`oBXJK1CPt&DoSweQXmE;oU@TNnD`xDjninafw+^%|j`dC}0nJvx+c{B~dyoiOLO%$_*Hh|5{K&$`dpUgNN z)!LWb8ioV}S62Bl?`+~_hyM|R0Pj9?(Z+f(NG)N?+3{hXN`>#~6MLI2=n>*N5#mx1 z;?fY}_C9DllapyShUq(24{$TW+R?dh?8X&9s4|#634-&S6-3REK?b_(w)UTqX=E*T#*y6x$KQ^EaM# zZx&W5_*N-IRw)YKFSrev2_t%D?+j|N9CFWvl9i!D+C7}S;0;?JyKk6dzTemW1C^Lz zQdl%0Y?u&+wd|KD$NNP9^$H4dE8h`3PP zR;N5d%e-%vd4$7h+=G9?bxK__N4pkBl$H6^Vdc{RQ`*ahwUMOTUg=Ru^km&A=F)a+ zcraVM<7GVgWjys|Jm}?K|Ekl~1aH_|zPo%BvoE&x_b#HDKwhd`QOGsGXKZUXe;hck z4DL4zh9R)%df0G1OsTm(DM`Y7jUx0kt{Mcw8HSj_8^%uKbSCqyH_6DZ4a@^kQl>^S z-tIc0cPTnyCXiz$P-7;5&~v=!`Z6je0>RCAOzlqdj3Do}`Y$iM%S7?aFgH!#VdFL( zc@zjU$1szSl*?w?{-cRNZ6-i-Lw@!E=8Z7w2bkg9g`kH*c;{0#L`H0g=K-3@WXMkCQ={CVbBm2DMq%j)!_I-MAaRhs`1a^8(_>#lsnRl6E%&Qi5R3)We~d*v&*yG|FCxDJ_|c_bEO+us&ypz$o;DM> zd3}|S50~_pe};V9%4E7T(NgiKBB{G6#IiJ&v{aC^RFt$_={i{i2fB94tmjy#_SJ-! z(KiHS%A0o`pji%wVyNk-xTcM#Xh`H;eO>l~X1YP=-Jl;cYTLcufTvN;j$MzjpRX@T z>#c9mGUkk*V0jH1zD>?3QxQvALsII=GG&s*4(_r-@4~XRMlN8s3-AO z!}TtD)H?jzuPi(sv0Vyv7l)BcK+}6!M}M1fADbf|o7W{yW_nb<<t<&EFQABF_1tyZk&d z4WuAjzaXJ*c`JVtIL9#W_L&^s4QC#aE4X=Uo(G0MX>Xxo)2C$fSIMSmY%X6-OoRV3 zpG;Ttl0e_Ar1Y-SU7!BvE~?04{T0&ZH`;Jdp18{Rv)IJp>m8XG?z~q_c`=Bp{cIi? zW7Lo1VPf;*k#$gfRr4ylN_=&mqwSiDS4|K2K~{=f{ow-504xJ3`paq7GJ4g7U~VEN z;r)|vw71J2IC1*yH^hjx7FS`PNOrH`S^JLIBCL3?-@@E=4cUxpL$!m`DoIo453=4J zWCQUc~4Tj+QiA0VR!yVXQa$jVr1Z%U1BA^=NzrIpg$HF*S@{SR{}2hw}Xr!TSLy(vAjc z*LI7lI_;`b?5a}lB3_i8w{Cx@e;_PEfV>5CyEthb0z;BJPyjoQXSItoHMib6Vwqa?sW#XTY@Q%I!m6(jw_W|)3^KvmSVM!6Q3H4es`ZeiqI zZfiQNs$&k#+o($JTz011N~TRQbjLjPiF+!SE1UsESxl;(kY+aGQZL?pMA9T7`qUNPQM1R&Mv9MyKwRrg7Du3 zPM+}Oee8byx};_oLF}xkX?nEyy42F8l_Q}2`2RusdzHIvC4nfe{`=m)e;hxqQulnh zRyBBd?AqtJ8^I=5Hv80byzBQgBROmQ@b&=f`s4OidiMN}-tWI>(qXCJU^Cxf#2tT1Za(!5q6_E}Hsl~1XaGJ6 z+=-9n{Jk5AC2-L_P>L0*aLj?DY6OGV?gc#TIc-MSY!29LurI#sCqaHDA8xgmWfsp| z1sLcdHAmUBwZY;hj*NU=p6wsdtOTofJ9uyP>-~Z<7cR2+F0w=~?*8h;Pbxumk(z0` zE>~m_xUfEs5zUKysQbIrH!jjCuS)li->aDR1B|5r3Q_HhGsx4bpWTHoyPQ z+{>W{7zY^30Tko_igEzc!PS@p{GOU*qh>_f2V=wiq1SeR^NpIwDaFv;%jg2^lak0u zL6Z&0VS)OgS~aQNsffa|D9f^l0ywI**MKrySN=yFiXWky5E500 zw919swv)S}W=Db&+e_9D1YG%-6aj;ZfFVVIEBaz4YQ8OAk5pWJSG+8@Q;$C4rY*V~ z_kusB_{;ZsbS^)nxP){~B^|@fKGNbBDN(l-T@WS*6VBHu=G`vsE8#YWWQ(Y?p;Thc^_O3pcpwK&xB|_wH}zH@pIYMlaKf=0UL!dA!BR z;AlA15ooXhN3S=%Lo-;rbT7zAj>^Bc4kmRX#$^jmV4Nz(>sv@!Jq;b9ML@-Ah}AmE zh}Bs~jsl0ceUc>G&FB2*5;=q>p7OqCdk5!hkb?g1WMcm_%2cC8ZKD&v)+(KFx`kfv zQqZ%O`q{T>fj%+$_dz@rcDVjUiSQGUWi?$e>yBVC`@l335G;7A#aFaN`amo`@E=y} zH>b9nzZF4i!j~v-E;#+I^R4{ZgNhXI-1nbplM8`QXgQvePo8#FG9B5GFRwpr$rSN; z24KszO>5?8Dd35A=Zxmg(T^4t9qaH{i;yq6(uTy{u6S3TqfRZ z1xvZ7dZ*Sk>Am-(_|p4c_sYb@jIahE_OG@^;V^jdU`&G#le1}N1pk}2&h!QICkR&F zc!GcoPlls>xb68Lt{mP)+XnF1;lWnOe`JAkKbPS7J$5Vo+CNNLRvi?!-^9&;qs9YmL&2 zW}a~-lggneK$Og;Q8bv$47uQ2Dh1x~jCsS$lQ9XIz!s#`FTT_-vDB|eJEt(;JL!m! z++rGGcJo-;P#Lf!8OoG}K%nW;!E~w{P*^0+O_Hrd-DK^zrLlsrK6;`v8Bh^rnHVWj#=g!(W-eGM{5 zyS~qTSP%Zd9VUt9`et3@&OfxAt5zK1hxi>l+>#mrY2A#@wq_uU_Lz|Z__&gGvl5Ib z9((VMa93&69v9RlMcX)G53tPlr@Sg2^|QH^EXKsu=V^!696?}PXcQBSR40j02S%u) zVbJKZ`L!D;zuxZ4w{BVEvHSaVHe-I|nB727{&FG1FT(_9WhVQ)7IA7l^Jhh{QxUCY z5!kYb_UDLvh+4wf*YJ{u-?$15G#5Vc&qzp_S~LO-}hxX8CSr zIp&m8PM0H&A%&9=VqTQVaph_U=b+O6eL zF))Dk?*BkZdV^;2sGD!WpwM|K%EEbSlrc<3sHD`(73kKyKJU$>@Hz=iAA1514%;20 z_AP7Iah~L6UKgUA%}*)osflH~!E|?nfmq_I@3ZA31#G6F6TH@wh2ORe@h&{WubsXq zR~D7F#Xms2?&H1g6TR-+TvnP4Hae(7s5qSbE51_MA>~|T33_$`F3nY#Xcxokw|<7_ zObQ#Fu)PT`-G<9|)nrK)XLTW});K5fIUrjEC170!;%tb2nHc3A*pog@if~X_S zd+QUqua=Bn?2aw$C1qa>IGf?VBoa6%5;!ChX!?6q2_H`QuSsxV-ZVlYZh{o?e2b6% z({mZV$_6(v1%ob32IDeE+PZH)ib_1#w_Xl6Gtbz(o$-DIVAP8ESImm6d@$F+67~TA`#=o)05KC zQ>k8Iya5GjYr+fVSI8X*2t0QdYHtQiyS({3`BO;xx;oju2PqaX#{9e&}XCvX2Kl*be#3W4b9E z*yITcZmZv^>-dq=ApCl_e!jO3h!pAg z|D{N`C3KcEes@emc-1H#B3(>7r;+xPp0j9rY(}{uQDFZubPkelXMx6ZVYGAic3@uJiY{V2FyakG?Zw0yC~u}E)vAQ z-`6oyqC-+J;{O+DLeDU#ioVq37(TpK9n0$$)8kG*CBD`8U81%vQUw7^hYx#DEA3J5 zzB5us>d<__XkrF_dJA-Y3xrD?&H1LkvM~;M)YT2B`+&CU{XW~l?C67Nmnq5PXB=_` z;}Or4suuVdsS%tgkfHe)@aYKPdIW$oA6WvF$71k;3cFk_nQOKRtSA4e=A7PDO2(A$ zAh51T)N9qigLoq50tzkS#lIB%e<_6jBF@0e35b;Hv8(U|kxy%Q8FYOJ*z3|+25Q;G zTXy-F=gZ-DM?*%g8ZWxu@M#2!y&G6mXC@H^35$#mi%blQY)cMojmKR|APF@;ynBc> z*GCdj$FUFE((x76Z1}!Y?5{lZrLdC4&0ibvQ+Uma^7ab%oZ$9pP3e289 zf>!*Cp}VB?yDkfJJB>sa(mgPAHw+;;ApLth-}jt%t^40+&6+3fz3*#( z_O<2u_~i_1+X>A_I;dBe&UHCBi0wD-2>*25w!3RQLzRJT&_U)ZD_DaMSXZGr<%!IB zi8*A7fs4k6ES;-zG7mZFxFu3V_g5Vvk79#R6+v|EMb+3)^}cn&hfpn4V( zhhyW0;c>&Hi_V$*nuFPByIEZtPXaA5YFn3QeOluQK@7viHouIy()jYF{gd_oMW<5tseVn!JnIkZ#gB+=rJ}! zFzO*f`XS&5eW~4yBzC$4y`JE6Z| z)W2YKzhIqWBZJ5Lk5(nq^wnxfosNH>oxeS*@yOER`T96`eGKx_JQed>=KdJjWlh5M z7HBRJ#Fq#XNdyJFo9a7xNoRz9dgz|w>TOOb?b?F9MCGBIzP*(u*7q-kubO_N*NRMz zT`xD@P^!07gjp&^TPij~eUi6=NCb-`7T(=5e2(nju$8%N>2XuDwjiZeCJ^cWshT9F zx@VMGa&;5Ol|sgv633cC_F~S7=dImwYz;C7+qjc4XX&K(&rs_uQ4)A+&h}d3VVCIs zTC3ZII|qZ}+-6a~Oo`)v*Tz*C9u2q3kZE0_EAwP2wL70sR)wCCw^mMW>hZ$p>@xEB zDFQlDcbbgOS*s1fCI(LiY9RK5ZjE;}f~92BD+6q{XqZ19n-q_J6{;VYXqWOJF6fFJa}g zY@6rG?h#~{PNCjH5oVznZK24-SC0&BUuDpOl3qufJd=n+J3t*{9qr#bKnOe|*+TE8 zrXTfth%xXe)mjIwvDasSEz-e`|B{nM##=weorjRD4gcG=@@rXktw->quBauUtEOG< zb(KfF!(1ZfD@L=1h%vm&#K)&|WZEla+9zb{TxQiRZ{hFTSgSgqM7#Kf zA$*bXtQ2&uW7qB8ajNrtHIYBep*h!*Bc(Z_uimZ~X4e~S*W2vw1J-dvUX2J#cO>TV zP2}Kz8bgq`+U?w}r7XXumVm``uCPw&He zK+(I<$5nD)ij&t99)!@A-4e&n+ix0>o1b1o+r|e(j-B&gf21gd_143BVX)q4Snq1p z_FTF(>lK;5R6<%@DLtM_E3vxa$K_ogk~EvVU$yWc7C8UFdzZpBP?s85FS^W8$u!;cVOaN^Kj;J;@4byZsG<74nK}pibVaZrZ1A>^*ol=O;iCb=(e$wX>{DV(3Lb{|xNS+LDLJlp1O$ z-q^=Z(%goeET!0@cBYdOZ%taPem7dJH(KQ`%yVtamSUR_tYwqglaPQLrFOm}IhM8Z zNO>bD?K1K`@6KhzT)D2#!n@y325U}sYfj2vzS&TR?Fel#% zxjr28R5x~h(1W{t)7sgV@T{Lma@1EPS5zfeQYAMWY!^HX(wGIs`qlQda?{Pd`W(56 z1l}^^vu1N^S>BL%|-Q0PXv!o zbdOKS>sD7jN1Tmpku`BNlV7s6Dy+VH>~VbR%Z~36bU)aT<*%=7Q?RcuRU^ODm|q`M z2n%`^9rUlXT*Mb0>$m3{YE?e#n)m@ zV9bK~O97Xc0@#&Oq1KQ&x_Tm6DXN;z(}75TI&}h1JPVHIcO&)-xsjj!wrDjvzn<5o zZ+pJY2$?Ki;7;Y|P8H=&-BY|M5UK9d0g`D(;t6Zw{wgE5++Uu()xez1FiaBFJbedJ zu33(5I>_5Gtq*jU+YGf+l zXdh+26W0b5i3M5hURmuvS#3ZvbM6^Erjqx+^~_8tjN8*eYfR2W%KfPG!s&u=io|O0 z=qayg%}S_~G(&SZOmj+1b2#jUOKqQ-VWXWd-0Ht+vqx{1TA7vm#yG4YxMOjl!TbdC z4wKyh_pvhl<6NCqBji@57T{Oi`yapRrEWpg>ShPx*%Sr9`U! zY8SWE6IQ^KAff!Kj7JXrO%s<2f`6WW&t((5AP@e}Q~0;Htm!*#9nA=a-M;hgIDP2W z4}QNCB@uts{%wTu9s}SKl(rd%FH*{7b)>=i(qN#;NJ$zTo7+Mg*}PUb19%)oUg`TA z>!FEwN#=yU$kIDp2NAL=(N1;}y?^7(dHmzgdwzX!u%b9vNgNzo^o5}%fD5ZbW)~L` z8Tt|}BUD*ZY`xKw(U{fEQ@%r)g(<(}T4=bG1BPi#q53_)USwx3fG{?4CrA-{)C`kMA z8iG&S>d-#YQJup#leV=YXq|Zn?c^R170AfA8}0|t_~f(92nik!06c1t5HJ-Na1s|l zBph76*}ZI+*}SM;QQ?HC5l;m&{5ks=k)K)iiO@p=7hicIYtj13LHY%@uEqVI+VjuuD$NrAlb>4h*l`5eTf@1AcHHEJuuNVe1>$WXfK|n zOy#3Y6`@QGpw;MF7PKL4C@}E3x5bOquFDP;%u;hf3>A?75_YV0v0aW|nLiI`XgZlc z9o@1^|MY+m19Z--gtXIsDJz-H7RBO`p9*GfAK-GhM&mgSn}_%;J4b`=_q~LWz_OEz z1NW9aP~}5OCXZB4@y_Q@s(21Zkq&*+vGXj|CBU6QoHd17Jf8`WWwi>`3q!GwWYx3B zN0hyvik@kwcb<{nKuHmuZvmN;uhNwGY#Ot?FMm(>J;4Ido6jYE&_mTU%TDD0u6Rt^Ag5yY2*5q0vYaqORU2+ z^d^6`J$G>&r98rti8;t_nFjuzRR-|)Y+gkqj_idl+9Tb(O|Q62%Jv>f!miA}#e5kT zK8?VzUcEkhs;(QGn1|53-mv;;C zBce8#VB|eqWBw^_o77!F6nF%s(J~)16Hf1}D*)UVg|n@q#&G^bMfaqz*_8;Vk>^V+ z)%#TE_Mj-DL8f3CT$g~o7Z7h7In&N!95A9qeFb;oIE!EIdK&6#N=CusO+dl8P$fGpP@Mg(mU*2elwPrg-mmw0DGxIQm zVIw1RBZHx_+9twrwyGfmaJ&$>#UZl~{TNUY%IE>UX-nDh^e`}NZ0Qgah!sa@axHq? zUFZg=ZK~I z_zlzrkXySQg8P(l40TkwsSA@f>gb?${*?1;g5K<2Q76r(H6>XU=N zFr(dnl&bgst|R|`2D@BNq6zr0mbK6liS3czk=tH{W6le561{Q~eR2|pgT7p%96-3g zVLvBRT)mSvqu)w0BEBf2r-(h*W6P=3vd+r(jk~T7x`qXL{>x$}A3}pL{(P|ss-a#yf>3iU7Ei|damut(nCny~`sj=KD2n+gi2=y$@;h%G{O+Z?kknxO-z2NQ z#St!v>V19RZEik2<=X3%*zo^GjLeB>esuyxor6&4^r&+e%$q#+g}|2Ud0;!5Js){@ zCSs50TbzY#{!67=2e)5;PiG(hD2c0uqAuI}VBs1GzD9~&BZHhjYBPA8VXYQ&HGijY zre8`Bs$br+&<(r0g7Y?bp86TWesW<6n=?FHzGZyTQ>wJ|UM)*gZADYfShuo8_GL>H zyrG7-`cJJw{u~#?aJw)$>Eg;yQdwjR!tRlYxs~3qCL?S9`236T%6~6pTr||m!Sa0` z{TdvZ`_mNNQ2GYDoO4l+;QsM7I(N3^cz#? z>{~ORFD}i_Z_W()l9EHmh3BbC)C!r~Sh5}}eQVbjej~xZkz&7*n0DebQO<27aP_Te z*pj{E!}(V~gHC8E4~utQG~{mtd*yK|0gmLa1LNf=&^-|M0#{}#XOpvA6(yY-t#0$A6I*un`6xfS7=O2#F77>u*B zG`zAj!m>2Gz8BEq7rqGM&Jnnf*nO7!e@Znob@Md1QLMgM7`Dsc>oQ(>A%y^FKDuds z0I(`7?dBtRm}5F0SX;E^H4iZEydKc>HJ|5NO;jSE%>F&J3sH%xJn)}NK;^s(j!&|%Z=qRNlX?(8K6*Vl=9<5(-4YCnEd?kkrmD-!;xD)8y8 zfc;wmG&{kxuG@^BqY?DxlE|8O-E>vjS6DNBt#pywydz@CHI2qtc>@76O&%!XBn#oa zaFtJAAe|7%Dg=V&2Z0LAJwWxJ%!^!(N6N7ar?4-#X?8c~-QhZ{61 zqfiULv_dj?CmCFtstRzqBmedo{mK%dN%VwyGy7~1^{Ar7;1A4SF>XgNcI=fzfA0Ge z0PPk2`9B#I`#{z5gVO@jaT21Ns!;mVIO)erk0wrCo2_#(fnYkzPibe~h4f51Jdx+p z3-uu0dXR8E$lqq6KDi*M(luySUBRVMaAJ?>6x#BPaTnMeIlm*6>5Qg2wp<$8dp zGwtjkh<6YqJP6v&M3qxt;aK3^N~in{Iu^(KxwAoTkac?P$U6mpqRPX#7`Rs3na8Pa zr$BEvf#&QOA@?7>5wnjUASf5UHr$TmuS!yFT>b68V+KG};icK@%2qL{ur-`K_BEWR zP=XSmBSU%8=QD)Bq94y<7tbQB+O~(E3GZ(5JT!M>M?|{pPwzDA)b`PErf+DCyDfnt zV!h(~nUEz7mlZ{dS2gRmf|5V4WduHJ!%6bk!KiBC(UWlPF z>tDil@pR1o6Jm|R2AUr3cM9(zHx~VWEq4D}6m!mGxj1b*`3T}I88t6`s~=D@46U33 z8)~cPe73nbW#*z*$=ns)xkBD3(VkEL{1wL899r48Q#5!QTz+{A2AD z?Q$;1o@~|a+uGx*R8SueRFJwfw_Oc?>F0W!u~fMQ3AtVgxjqRw!@k$y!B!PPE4^!u z%OIIBG=t16O*QRK2L+Z|ml#pZJ*MIN=UZ~U zu6=MlanN0Ki;j+rpI$*DBPvXK@33*-t%T>R#rCpvmEWX!^68c8 z!xUj*iqT<;%?Wq>qbsJrU(O6XP$<>VRyY^+_8Ii0^*x+MWsRX{y?u})zU3^Suz$A^ z9$|$b=W@}Ev0;m%W{aX@i*gq8`VtJB%iB_xGK|O38#%*l0=-L!S$eJ#+_L=7kdhD6 zKMf9_Q7F|%D8eEXqazfh6;y{3kAQhNOD~VLmW2I{NQCMioh|Xw(VBY^D}KWyei}b9 zJTvrdeILDnUciJ-z>!X1Lg7G;5C2jY&HGFKg=(1iAIjSBQ=_O?f3N1fUUq)-P*fpV zOYqepn>z*Zoq|M8K}d#+EZj9TDtkWxtrZJ-Ny(QG@*Tvnw%w@mXL;Ul;%zS6dEe%c z5icNRC0TLMOfu8VIMYnB#@2%o63Kbd%ck8DHX=q_dZ?OE{hNE+N|w=MU8xhSr)=dK zduyVyUq7(ui?Aw+uquhLW=AV5NS^jo3dLq@C?5T~Y7DxrN02Y(UzG?CKkVn^d?6h3Do_eGhmsjq@ zN1tI9{|AZGV_ZlCffGSuiJ<-8r~wKa-0wsf36~rG2ewCjX+B2Lg~CE47Q}8s=ihJJ zuJ(r1qzmU>6y*L!pBjCSX}51NylZi=Yw@FOMz@Bu(23Q@C2*lP7qp-AWFiDj{4fi- zeSGxrq*Y5AedlY#es;lT)ch0fXn@ED;YQKpM!A3%PCayKyM5Y5`N*%UY{?8%$DFlS z_``4z-zf$nM>(|}(3KQIIHdRnIzO7&q(O$0AqUA2ShW>T-p}yyUp$2jie1K{n6`RZQ@SO^HX3J6SR!1RP1;(!2$m^ev^ zX;PzH3KYD&s<=9dAJ4ThH$vw6pKVu8u^TQElvu~la;Nfg(}-~c8_1%Jjh_?`-z`4L zY-L{|)QYtqmjT{_v}+aFzU+bzv<0t8eRWMI`0#>J-L5Y2)Mw9I-G)z_yO&WabCP;oaej2w^23a`H%AQ!}IbVHC`W( zSoDpR6Iooo^GhcYYW!Dpqan!?$D$lw}@4r$i6Ru<4HyxOf#dIn2+7M=8L z_2TriS!9PD>0k5NN2+OLA_N>cs#)quCjok`7fK2Vose~F;>N9X&h(M^iI3-qq;5e( zFdw3u4}q0p#eATT)_7<90PerPOuHDg5V9pRdXJqmV!vsd9?XV5}BG$~60FFd&-Cvx2~5InP80?l4}I)x zisKXyTHDl7+_5-}v64`Ke?8rE0S`Wbcb~xJF9F%Fz#BZC67p`eIJ%ukCc> zL7CAgxelRJutc4LfVsSYgFKMME@t`6(5z+9p-EB3NadS3`KMm= z3MTov4Pu9APrFHLaZ*aUx0l*&cyZ){no$D zER{A**Sr1>G4HzuyC9A#=%M?&o2oVmp8xG{E>Y(rVD2s8;w>;$2XlqV4!azyBNFT0 zeWd-lfqGbDSbMJ;zR9t&Z-KA1%(Z z0{RRJJnKCcx*ws++$e_$iNnEAE?D2V(swGb)XT5;DTMhHM*9>te{Eai!qf00ggTHW zKF)Z_Kw0ZC%=0c;L7W@6D~y(+v**0mLzRnNyzLljtztR&+H&u;Whw0}P_lh9W{a*Y z2Gu?RDTtRkmJ&R`*lKvfDmW72R)VFAhD={W2##-$W9D0C=A&XhLqmS_66Fq zTrK$d`@R?)DS+axabRaJRIm@x`gV-o&~IV&H$3_DbHUaDsJ2;umSXib8ty944{R1u zhcc>y7***RRe!apfm7cI1EV_h5hNpSlEY=%yoY~d_ie-^e_h^p=-C+gu;TYWH5mW=M^xcZR}1h% z>HUu%ieZ2I2pK5=#Q5!Eu6%Zg?5trvl}vUE>-rpC@4SnFGqGO&9qGyOQ?;A}TvsS- zp#tfB4cU9W8>!CMaW1|UU9Rzg_@^lTp$%>$X{r2q>!XoW7R6&IrQADWwyKx?efD-q8Gj%~M z2KDPmq2hp>cP$e!#jm4j92S(py~^M|WpJ#+g0mP{naUY$neZ_DP^U#+ z%;S9S;Q6SakH~nIGb}>AD)3I|Z~L{b?)WER==Q$Z0hdc-phs4PyyQoHt>)V(@+&`O zB!FM`QWid;c$DM(|eKGX=_C)`b6l$H*2QLMb!A>W}la#XN2fhrxu=< zHW6Rw`%y?SDHM;J= zN+YkgPNqZ>WuX%xetekv15<3eBIBR#_^Mb-;Itf=VRC!=;pa+e*g_Q8-B>%jsF|kQ zf*C?5GM%jti7gLwPWZupxXAV~)HUCbC&Ld$ZUsNBq3-L&yHddO6+2IW^JC?p{`4*j zT?tH@h>FrE-V-b2Q;m|py z$nyIBo+GX**6sifIen{y?W8*y>mKc$ua26HZU&gzF^AlWFY}dhJz1Sh$NN7-wfbF7 zef8isrfTZwg*AM6bVoB52Q1%=n#|TF>Lw8Qdd~nDx2fzOj4Mk#VHk<*rUEZ8#oHD} zVtOSQ%Y$|c3{{90jc2SWuDm4(^R$>0o1dICl zW&9T~68%`$M?YB7$nL-=OK52=_a#rI7k5PC%j~k3(xcfzcF_qG0>>jwf^~b3NHVL2{BTQ^ z;=b3#JFkmz?cU7YLb}^%L%TX9`DD#JKEQ9Z+rJPW{scRoND0KBD&Y4o)b|vDDt%)- z>%BZ-UY^ljp3M
8FQw81_MY4=P}IJ#Iy=ITwyy{ay|K>Ec?nku7uqj54Z9ZZZZPnn~!tBnc%op*GbvS$!ps4pf7 zQPOo{90@-1X(m@L18Z(3u(-|KOzpPkMndVc5lx#bXiO8y*lit~H=h@+$o!VrIM&&T z{9jO&mjuSO8&|!WG!>Y15|~7MJxD6}#>Hs&DJTEcNdH~SJWD-C*6I!JJf~KBB(;-9 zk8Vc=nt(a41!pOBfuGwv_Z}bjo(T6|Kub^G`{d14Pv_;i2H7LaZdV{^+>7GgT?^a6 z6TnpGBut|&OgWP^3G*+8X{n4mm``m8P}Bq{Is%k4>x~O(1SRXSsY8Ob0_%BuTh--? zleQsyoyyCokdTS@&KgZ&6S*bSXj@U%bK~K5L&anL#XJ2#v~qa9HXo{f~d1A&dF6=82!7f{(s}kD`K)l7i3DfFuQL zno^a_>Ws-E!b6zRJwrlG{rE1%+qzj>Y;3MNQ|jrMCvPtQKKh$q9jIYV@Fe+4qE=R% z)=Hd~@l3UlmLe-g2cbFFs~561e3HNVTzprYrn3 z*2i7L;;y6Pt~+0+wg0$MM&kL@c$sm|+mX4y2ot~dFovU>jx!AZY}HJ1iiXFY{!m&{ znBgoX{A)MuzM$#TtLf9H=@SdnAB$`y^~|`PI=`DeR_7v%EYI6%rp_81(;sd@g`>u zjSl731ieDZ6DS`F#D}8iLm6`v`g=*GJxN0Y%!_n+cKJrtQ5uwG-}bkiaQ*D~COVan z&T-59S^;Er7bfcql6zpuJu%6>eZ6{m?p&IH$E`8mnpE97J%kM8FPuHv58@Rz6LfR^ z^VFgx+xT+zkk&J~Pk$;1e3GAZ`NM}PF>41`pW&t=8TZ~jF9e(&>3}Iw;TM5&R>e8>!YwX*5;kBOiW8_ z8n@OfdV#RU5X@)`@GF;(Me#iaz4Px!{53~9HAm3Qrc!ai_)l_4Z}!a}cM%yyF7xbG z4c{P}#o1p3hjoauQpMhR!ESVJ~UBE3Crvw z_iokTmnwWw8acojCFG1EM)zZF1dzN(nsk+}cWz+{7tDD*$V-u%G5 zJH83!%h2^=p01o$mevYbFQ{g5dHQ~3h%JCC5uG(LfHe`_bH8cf5ywma#ngwB66&1{ zY(__V+cFUYEXK=XaS9q`0`Xm@)McYA?P`N;Kdbpv5N|3-I2E)zkNTp?@bzaVY3|OfN{pE`EEqS_V7vu(_%dY2L{23hTt^2jochBr(a!u zL|YpE5@fLATieW7nwH$FADD(Q_UFWLqPl?Akh#I zMwd;YURV?Hz4f)yXLQ`nE8t1@c_<*aART&q=qd1M`=;?3IVh(^KfA>)`#&|dA-9y! z?W`Sg^HOtwRbjAT z;%(-)PsGykYAO6>LdQ0`y^C*QjG=8i)zWRpP-w^}UXef2LacrYb$8oQ5vfab^A#Ze z3Xo_8Xm1_mYV$#9MH&vf`{;fDXmt)fbJsM>E$QN|Va`@|$Gj*yvS2PE^|5B0NL`%C zmhyu2Q|%HvTjpR62u}pFpiiaPvIde&QAg!^+D$guzJ9D zU&FW}xZW!S<`oj{6|#!G`{7)IN5=oQX;s^>9#8xrTs-r%L}aW*;ZPC-mcOKgHmP4} z;)zc4ji7`X7%6O43wEvrPh#5`84!zYmtGE&!YSgP;NIk{U^;mlJmk*-S%W7($9M)QWaX;gD# z%>%or^ra_3$J6Qrut_}FF&;dDeNd61Ho-?Jy<)iC`DiQ%%^EoAxDQ17re?gBEW>8C zGEPM+c(?75+XtMge3Z==EW2kdH)k!e4xDxwJ;(q}<1Zh$WhkynLy^edHHPWFW%sV-=B{Nv?JVONkA(7f`h*^?QPfxU59qr< zsu{C#wq&0BF2N&#Pf6sol2Sw3bG%%TTZcpk5bgfIOM{nzK1$>|dA9K|^6BC}rlXND zo`mGKxtKwzkADB5Ed0Z%Gwvx;8-edBycU2X^2Ur7(oGH7q=w|D%^EJ?dEB6R>x=*4 z_BQtjP6Y^~`~H?(y3hj6NII16_6$a}7V7&J`uZRyt) z67(SZcIAq7s=;vJm!uzpX+`i1?Cp5E!#XeG)I`EZd><5$)iBzQS{_dF7yk@ z)mF6k%EZ|nwNO?H`#BcZ`y8Q0qO<+4fo{~HSCyw7{<`hwSm_O!mZEK&S#A*}%}Oy) z`X~^66g_>E3!4}Ck?x)UP0bZIHvPFp)8jcs50RVQCJ$bxpJFeppV7MpoW$RWFmdcm zN9MAHzCJWd>8_9Mg~j$p$M!ZS`DEwbfBS}UargnxkhcL(;*YHj?an)wGJ&&As>+;S zAM{oO;wysm-<-K4XLA*o6DN}6CT8O%J~UKU7KK1x)XJ`Rp}qq-0W%DT!ypG?kRLa*l_AS$r)0P|!mxx`EA;*wTbYHI zc8o(Zk_hZBz|870yH<&0B7yNYtZB9$o?Hc=t%3s&q$R=*8r@h~luMOFLH`O71&wU= z`w>y-^X@G37v%JJpuuE^#8gLu>bpGKA;@q)hnSQ>eiuUyivOuFA>-dnj?*apZ9`7jYb_Oa zb6TsQBKvXa#dwNjc|^i?$R>L3%RAp6MSAbccVdnFURBWxe5Mm{pc6nRE=A4B9v9_$gL&|Na|!ql zRj@~XwN+J189O|Nv!C52JjQQi&zWdA_nRW}FrkI~riQFjLvp{&F2#6&c=Xz51Or#J z#U$@<$YTqQ;I1*5%z+NIP0{!(6Vc|x?l>>3+3J0t!F-=Z`##&1Jby>K7KoQU`8Pye z9qW@qhTZ%{tZ||2VssE+?Wdrp2G)Hvqv+}-Rs}RrU@V~Gf5rkXwYNXXI4=gp9!099 zE{VEVgxl=zARLkCKnG5wP^l-aX1T9&RTv7pM{qQlZ?v0lR9>jk%_1LvUA9wZ-PS>p z00XPP3wXfK7)Q2#6x8_#Y+@ViY?}w;2fF{FbDugP*B2&N6ed>^CSPi6p9*s4!Ge!S zk2nE>?q-t#{6Z2aSzZ-_U@QQ3*j&)nwQ~{X*$7>DOTF-xZsDyn(o+cLeWs6fFPZ4V zho5FoFF`0&FR^WLuOjhqq*OxANuv@i;(m;jntFh-t7}Fz9i~3oNDbdeee^n;iy>I< zCECPe?qNGPR(#@d74*z=If>HJT*AX8Ltol+pq1WFvTUk_sj&V_A?(Yu=q~{7lqY#R zumZR@@MygeMpF}GS>To$kJ>4&%^^5ChqeN%)k z`9g_B5PYR13@?kbO~FP?{1-IKIBJgLD7f>rV*3TyWX zYcDk}8_gED2Q^-H1@ZoJvDF>>baTXj8<4XJ*{5rR6#5L#8zb4E+WKH^MX%VjU(3)z;) zeiL!OBB*JBwVGhm(%9LEZSQ<>nl|bPh}%RG*}k$`BQ7Rg`60fi>29rJG5FeI@3lqg z!nD^tg;`LekQid{ZUEf?@Rt$W(w<9h7tjHxYR#3lY6${g;~M*Eyk12SVATLEgh30V zp@q$!J~j-0GlW-{#|BM;wH~2C_h}b0>oD{_kwRV2-O_w(^+fTsGh-#kp>uCRb8qSA z-nt;KwP=~=SFLLVHz0pAf9PYyAizsTr=GjRd);?dEpi(W=9vM3kEM*7YbAO z3sXc3Q}(7&mf*1(8GA;ik|>cFDg5>xIoFA~ZBSX4 zL-}UdW@VUq(n-Q04<&7s(5!&C`_!k7hy@KFAL9k&PV00yE1(@n@LI2EA+$s~V-e13Mh-tDK z$=Mx)q;^b5pet#M6TTl1@0%i0u_&|bEw+dBR*cL) zUWwk+wZllF&O@*}r_SHH`TDNVJ-d2I1Rmn859~A%Bzg z{u)481-JP#XeE;vbL{%o?0IE3-e6I+ST;vKNLuATb>jF7k@l!&bsR@L)wj)+sH?GW zzBu^X;+b0UP*MZ{s>{nleQp)V&OpCd;2ngLaE z^|TE~C$cX8%Sv?9mr~?)Rw3}qKNcAAPg*F(8AA(A&dAAT>*QUnmdDDXlCNQkAB2acR@y)Er|Nzb4m@72OnkGyZJ3cU%Ng#MLI} zpRuSRx%4C<&Cv(REtH}UOwYHQBig{GeD4oL&}4<=RK&z3H$^?!?Q9wVcQ4$kX9P5g znQPfQgRKhZ95IlsP|sFVvE_<@!hJuQ-63@cw}%@4n)~qn%Ms^I3B0&VRG=$c+ZSK< zK_#|4#wfN>gPp{szuHG~bqLjFHQ*;tQz{-fvVgn!cCsK4!?+HkigDy2nJTI3Hj<>3 zc#J@$snak)X*psft`+E6CByw1daFMO2JAI3Gw0FWM!<#-2ys9I|M(8SomPC^m+N}% zvq?*x4VhP?U!d4`A;ey%&y&R*nOZiwRun9#ppKM&r&MVdDO%HYsJC4F#Hrc@B z$-uj9YCIm(A%$GXyWZT2rxz=S!QVU~VpWI6deOR1hIwpRdJ6jO&5a_PMnxX7{d%u~ z=J>Q+kG#$!O|&NQAkO?=`>;PrZuSJr^rueHf8y&=Ws4gEX@uZH=fr)rK0vxYX3`XEtvrfEcOT86pY#CE& zA_1Om+P<_@kWE6N%CXPS<^`NRKAb%foV|detQai^QovGUUR#0&NkO$Xh3xQl8|FNN zQ`|LC_be(1WK;TOcFMku%>BIi6{i*3u;`s`rfv4rd!Lg3ABPp2EhS@(B|D8sK*$}! z>`25sxGNBk(8Hgt_YG|6IK4Bs>2aZh<^ACFKi{z%WkapV8E3bb=St?~wgaoeoN<@< zLs*mAy+y@@8d+?guT0B{v5vVyo|OY`L$+^qZf3Toyo=ssaXZxpHy_0Nd=!ah-%=5me9OQ(xgz zH=d~+GwoXGWv<{FwUqpV7h8k~bk(hE;RURY1wF}WG&Gf)Sx%=2DOUv5viQ_i_|zDm z!YO|rlCA<)y-V(7)3)>yMs_xr5em3i})yt z_$Z0^{PV)YwF(Iu%Q)M%E{Lo(rYVsifIa=9exW;F9>Z#97b1qz~O13&%;=} z-z6I^oE-rAjeNgymh_$?<;BhaoF)P)J|9_fkd1ypzp_ex&QW>e|Dy%_@Oy`yxJXP} zP=~*8pA5Qza!g)k*yN7lO2)z-h<6Vpya(DPzJQW~htaRH)>w3^rFXARdM~X0uNvL) zG)3jJp9>8)Qxc8So94+>#P?7PeQ8WZX-p+)%;R7u_Ajgk`9OjgdbO`*5InoTdmu-! zVA&)xL^3qayLwCs_Z9Yw3L%>co>1v8$~nwq-pC8s$OH5iy5FM~Pg`ZNHjhkKYHu+x z`>j8~+x!0&7PK$~s1*#1(RR!m^tP>9Bc_hv@M(DLI2=9>*PyBf8sBMhEI}qKl!#mC zhm!}mzPxTr$BA!Jn#aDdbCf|CHBr$^e*b*m(dAaZf!_)D@ocpT0%q|7&hY}1BL{69 zv$UB+7*c|FRpv+A4S0rwDPuOq+X}k3)ZKqLAxvEBq984=_P?(6#geo5P(~sUBN06# z(fG}nYagR4^Up@LmrV+Q3J)=s=xK@7Ps+*J#KswbH0NelYR8s8ndheA=cWP3d z)dak1LPMx#MX9Uf2ijt#v&mVPL4mx8Kgs38*3Df~=ej?`)47*8u4vIt?!(uTSjbQt-tMyuiw{@8-{hE#8Ia&~Gm89q6SnEW94D$QGMCQEM9&hLX?OaL`aC} zX0$MRNrI3FqSs-JQAe*MdJlpaF-n9GC3;H`1PNnCiQb}&PLvVdH+kRxTHpWoe(!f@ zow;+@I?tZ7&%W#4bNAi**=>zEkIdzCspJyDazySI*iG39fp_-A+`yJmd?;@wAD1`4 zm>zrJF0k?$85Z%A{MDn@s!FpxD4!MY-HJl~h(dmJA%FW8KR+{vyG(Nhvy{}In9ty{ zRb2X_y>o7C3#&MbJcfjITqgeT0Uy0|R-^wTa|vB430(yVU3<0DM+ETkXP@ZaKPP^7 z{+tG?hpZWSt@cs=Bn;WfiQ{hmD!q}nVIWsyRy$L-d}pxBa5zGLxXWNzJ)`zdGTG&m z8=r2K(&Mu~K|FCJ(*(oN_tE*E%O{f3p`1=mBq6Xg)-C2CpZN9II~Do-5&8V+eE#Z! zFB{Kwtuje}r@6LCl4O2ewEwOWpBO#J^Q;ykX*y+yG7)Vf=$Z5BQt|03@afv$JT(z1 zc36O&ksO^v>?eEp_A1Nqw#+pp3noN3{ zOzwuI%{?APe^vlrvPSgN47`&6R+NlJOdR&m?BrW>3a(1BSJGn}R&})(;-IQA(1lni z@5gZgySe@X-tgR=#==m*lbXBHPQgOPx|-LYA-QxTRrl6)6aA+{E*@c&aDBv7e}I9PzA+Q6otbX_kJEFaHIF0#{Bpy(nWbD&;=3g`g^4;Vh zi?Um!lRYJ5EoaM82D9vq{KUP+;YNmWHwrgPOY-MDNPgEyw>b09QRP|Th2*Cz{k^q* ziL)l$F=148DN$;1rcO<4l+m-G*QKV{1=H)Q3~W19izsOE(~#j#4`sz4U+FEi9Q>89 zaJZNn919z+(cwkxxio7Tl<8!$Hl(A^j zg}&a8Y)hoDqY6}4W|7{ot&yHi`ev6gVu*dMj~y|NG-X zODZ>izcAV|uhPH|%AOriTvLA+lve1&OyA(yZb7C?O{NPb(}hq67K`R&&GhS%BQJl7 zS`osJmAr5&iPJrf0RX*(cIBWE4@iNOAb7G@@^$lJ*7d`zfWxeUz^;<>jtEX>{!$RZ z);78Q!urt{a+7B38%|GCzFRw+->Va3J^a?h<~jy8=l~mF!3KUQa$>L#Z#2xKLTgOU z{v2!pl|P;eeGK0lqTeJZXVZw2^W$l*VrE`|@?F(mFLiwU!EwFB@v3F?69=G9y^PjU zkHK)A-8<1x+%AQldXHNpUEjRq2#N8J>j_(jT+*6Yh%MZ6mDHa#$mVi3a&MK={W)2)HLL7+Iz|bMY z^CbUNIwx*6iLYAXs^>EU-E4-u;igZA%F5Com!Z>Mon;yb~mLvzi@|>M%CaSr7%h;NXxX0)lo8^pV zw?St)qS+fS;X7t04phj~q9#XEK|u9K`0I5vOPKR{EH{K>27P`4 z`9^^IBS5kSGzK@_fOpf=BATGVafVXr)l36pP>^4dw`g%SGOVz%&pyN@hG~4L(RSQ| z1VcrFQ6Rw}zpTZ=`D)eX>0vO}I)~N=ka{KX*XDa4pIEJ%WrKP`eOrmK-mDHcv*Vn* ztP~nB=I8xX=lu%j{kUuV=i`WI<@~Z+JoIDLA*SWYh~tNCl9w#*&Qd8Iv%yjIRWI3G z#uR}aia@L)&^^o_#$OHSqf1c&;s0YaFQ#C6{H^VaPvzpdqQFnR)H4#JuB8c zw>OcrF6230TUmSpeajV5_=qSxIto93f4_?R&1-~m2z<+g*XSYcI&VirH{L=uyRzb$X{cM{i*)-#1?j<|-F&1x3&DokV|fw#x*B;F!|hQH`$? zr&DY>zh(#bp7qjNT=1}6eX*-zG1kPYtG$IE#l(*aaxRp!DQt2qY!ZgF!HQb%&R=+Ec@dVo zMAq}P`WAuZE!D%!Z5^K{HAvXYzUS3%4L4idu-l%5J{gB@Pe4U!CgdTvIwk+8BhO>M z98k{h_64lXWwOpZVr8~w3rPz1Z2E3m2E|OLc5Xd}K6wb;egvg^KEyvMlR%v>-wPwM z>9}?QlnzQu(SG39b+DG6V*ZTj_xISIqIMr@#p~+dajpUCU-{zhUCZMI`r}0mf{CGd zfWOCOohwSuxN-g(r+(aWGd^~D?D`oK9ebR8x+q=70T=6X+4~wOVt&|x*4m|sH)QFsnVGI z2-zfLM)@Dp;%@%TntM>)fCISEhwhySkFk=gQ9IM{Y%$lmaC5jVAu2k&hhDl5bs&aF zna}gvG)R2xqyq@$_zV{IZF+1ix_@CWI+zU8bC#|V4{)?|xhXup|4rS}+Wf8v)m;&V zyCTTc`};1Bi_HLg;UW+E&4)?6)+)=;+Ecm_$$x}~JxMa7vI|~VWmA^;5v&?jO&L^8 z{ZvhRRSzB5M#2JzNXO|stend>RNnwQ%|Wu)=cGY+4{t?XKGyAiD4+j|iY+O3H}6TxM9EaD z9OI_j)5d3}49`sco|z)=%#`Ffnrbta5PB`N$!be0%38$&rE*8|7k3_h)gQ8_v~0$V zHWx^fs3kNBy(ho@K2qR4IfFLF+yXEVL}n~dqcCVP2)Gzh8a~Odb+k-mv{PKKY`slh z=(6pIScMO1^N2}SG)0eO1injW}uJGwdBy z%qgV7%#nAGac@7x4n#eE5+0ajaI#o1_MZ*&;HiX?ocYx1s*jSp^9K(mo0OD@7U>>Q+`-TDyzr%dChz0INm)2#E`u-i1hS( zs-3m;%6Sm`i(Re7hcq-5InKk)&uW}gp2f$xY<5pI;A~HgX@ZXhZFSLfrLN2-w>heI z2xJz%zR3?~%2(Mkf?M1(FQE(}0pSh#hp+CM<|=9XBECs`gm1}zIF_^g^wjiY%68Fv zL#nU2JE$qM8d9L4I>RhBbkFX2X{($Us!AV}0UCx#wn*Qo@?I&d`DUBlT=hp}H2cBSvTFR6yvw7 zmWhN6RERb&;?w7qi;*{AtDpJC}`Wz9XRboHF|Ct6clH8b0{ZTfh-A z>#ZRyQ*G-jeR_;HGx|Q?JISER&+U1HnO308EZ9%y8kUs&cz>pY{91>eXI9OGr%yDS zYZw2@w|aM?lEr_OOnJbaY(K$$(JRjzfzmztkz``Rg@0lk2kz2Pd$e!HpH_S| zBmZomfXe)sHt1Nm9vwtnqj-ozoEJT<@{nJg5?WtLP+EI`T6&Y&$#?pSD)BavkD*@g z31j67KBZBP)_jGQYK2x|g|?jwT1-)0L`oX^g|>8k(rmAt^DE51sHdO|CS6XR*eQAs zX{NnM3{ZhEJSu*nF@NHqeB>~H>;R~Cnp#(9?N(V@F&KUe?V&83U@`G}zFb~Ptef)L z?Pbal@?BixZntH}L|@46=XxshIa;bYT7@}Uq!g@1H=p^|s~v&@&~b?>r&&mz`$u$) z(9lbYU++W^PR)Y#&UT!;;&WvZmprZZlQ|G4c-0P-9Fe;)wbjnU?o14|!37P2rE?>zrKgi%R4>=ij!;4|B{HQ(V4Wy5$&bEh-3TGM*XB7*L($ zt1r);uM&}$63aGwFmKIvdmc%`6I=pT^|(R$109n5wvw!nipchsK<%VvotM`p2&W<(VadQ(PXb#wClQ{WSmn}kL z^(yos6?ByvdM|#w4F$2m%U#zuV+&I>y0W@>p3a4Z%&dp$ZD^4_x=kt9Fj4nz;>7)- z!qim@l55l?*T5v#zQ*gHKlFj9O!s_e@hIAf0tB#CZpx+yPjRD+t{n!%w0Kgy+uca$ zFZcDp?kK;!q-QnS!@IsjXrP#X9>Z1dzm9Jm7)%sFNODJtM7< zcnaHay2js=j)A&5uiavKjg|ORmH70O_)Iz!Wen6BUaKu-mkiT;eO3YUnkg;XnD<}& zmaQ?)vu3cdASoe^Cz-XS zS=im5glrGU^$_a$C6|Wu^p)y#uX;t4W<-=GI!cphi&-D9s9$s`xm!%8laYX9@yb;{ zEAGO0Iq|EvT(fG#&|7KDr`1+;^&2^izj9Jma#A*}l|Gg@eQ4XHOOdfa*f=!N&T^HwNbG0P|tNd?wu?z?%fEZJHZL zj>Cu^Q{?AdW3Mx^JF_os3JIKdX1m9_rg*1%={TC`I2h?T{OLHp8~F@RBYsgND^0Br zoXzg$Wkzn(#c9gEwWO6$Eco3vq)qfKJvR55vd!&@{p$M##QPbv_ZgI$zPWGv4hy%y zd9QX_be65;J=^xs=&6L1Z-wt$>iS;0w5Cbm*9~|T?L2+&vtl4c+U(E47~Ej*@ZiJ4 zYMJ4o1FtEOh>KCS3`KvE!Yws>+`dtnf0E`eM>+kcW?~~p&^x~R_jrxf`Bc^U^wjw# zdJZwZ^j`a&T&i>Z#o_4HK1bx}`VTbOh%R?xm)6;w-xZj)@p!}ev+u8XeZU}o1<;HcmqVi+jHmUHp9_CKW5a3x!@fgI*dDa4MB=fADlxb5XllBI_bWbZCA0@JiZQcm(6`{a^zEXBPO=uAf z#R3`Kk44R}UMV^$6XNr9nj$&5QSh%aBF{lsN5rvB~@?^p^u;2DK%<3)g zaXgw7D{vM0#eYq@%4PUI7GbPnr>bJ7r(#Dm__V{*Zr9q0`N3ilWEs6sfYI`F?&v~e z9E-j=leSf`O( zjVhj!$0aM`X!1QTW9DX;HzvXz(}lpOOIPF`wBOKHUz(~Yp5b=4;vhqB?BR5Lb;lH{ zdyI*G$7#u|H}3Dp>&D_ts^Uy~;!Gwrhc3F#bn}Y7tUjldxPaE`m8rd1SdvK5 zw_MhjUke54QsFW-ei*jv8T0W8o%B196+4@s0GQC&j@_YPVv@8n+Ck z$_|cSz`PwZnWgcSwc73Fv8q0&ne4@xcH2K7s=bhf?+{-3(I>jWeF_YutD#nt+e!Yf z$#LsJ5NvU^k(SHon}8LpQ)sD5x*C~@FD z%@En2uEX@}1mvOOEFA6i4Kox%!&aiK--fRLa-&@QZZ9~z!${;XoE+eZ6oHCz=r87lt&(!hwk0AO#N> z;B7c?7Y+UQv?q_gMx$)$QZ(NB3&$x5DJZ_XAs_gbF}V0)w=ZUKOfdZ?i(tJd$ch3Cv6 zYZ_df4Oxx?l)TA7cdvj5S3u-DQhEWreD!Key6<2WSLu{9JJZ1%o*}i%_c`hzoE&k< za_gihOW-{?Z^2odg0~3ht}uuo3`)1yH26rtKg{w<9z0Wjt>F`LQtc;G5O<#sT8veZ z7$&5zJUQfXxE3KY#Txtgbu$O*ItMC%165!)5N8zH9<1NV>Z(h^`u02yjct4>eFk*j z)ntA94fW~Sm2TDLohQaB2C6CsdMXC)Z6#}|y7E^-;K^E|^Yfkv^TGY+q5+?ME(?5ER|Le=jUD>Nz zI|@*<<_t+9Tj>w4^EYZtsi}T8CTq|kYk-wCm}pk~B=rK&XuQ?VjG%y8SxxLe8(4`) zq5PjSyD3|Pc7G~bzne!*!0DTocJ%&bGy^?31C#byAFkmiw$V7>lZ*|q1RUNEf@S4H z!{i?I9SeP9a`zz?oNsbztANs=n}Y1%kY7$m=0dRS_wT_S=wClo&~x{ z86@?9?Cnk;OqNQ%+c=Ypw9ti}H9GsVa&~4v^I!tQ;0V(oYi_~Ixzq-F`75(JrNsMA zr(U~Jw7}7szGzx`{)eMZbK*;zWfI3*9TR0{r}spe-Bf{Ns=y9aAXXJP(RZPQsG|Qw zWC7P@`ZLbbZguK-a;D|K*~6FZ9)EEI-u~6_gN0>9WHhPVIS>H>@!5m+?LlNEHJMJg z=NPvc{D&ug{!$hbMSdX$a<24vzDjccDjU?L(&6$J4oB6@eC%1skU;2y_;f(~Iv_Gd zr`8_smMAB0vU%)!Sk|JXF@{1Rlo=@0&d|Y2*DKy;Bw2Hf@!PpAo&&1M2F0^ODRiqH zT9rtS`!0{?h*@)Or>zDKV_!K@;kN^VPu2CNLRWY>E-V^<_q}zoa@|+LC~BhDE0P@| zk{!^=4$49H8I?U(Tq^Bwx%g9AJLlgF>U>z z8rLL>Y;|t^WpLt_Q4LV5X1eA6wa)M%Rp3i}1qvU5!lP06>i5C03?}eC$&rfTEfhY< zWS&HUP`Yj$@Z(2#xtMIW$W~tD6D;(h%^sQLtZQ|U-RQCEXGx5OHilXo1J=estfsOm zWdm+fx8JODkM%~D(Fr|R&Z}f+!za2=(&(FoI%iLWn0eM(s9~tpFkm$z;IwIr9y21b z1^`@)zd8T26)6`-@_Xh;M^8ge%3Es9-yzwK=p|F2jBTl}huAD|7iXI*kn~yDVW{me zU^@)N?9>}Z(cDN1P^v`ps<{wO*98)kb_)5@ zMn)6wTqI{he78~5R0ef}6=8w%vn8|e<5fp7fMJi2yi{WzV}J4Ps{4Nj*KuQR+x>7y_53LrP~a!*HRmw;F_i9$d2z)N_e9=s=y5KpZIQbN&_ znO-)j$i0wAp?vb&`H=$ok#K97!-VvY67=Keh0o>xyzbm)4E&HgzxG_Q7V&MsOFZdQ z1wfK2RcbY5OGKbzIr3vJd46tWaW46%hcn#lmsghPP3H#by-56IO*~=R5Gy}jj3;bG zh4A?xe0TgVh5K$@$uUaxKE+HO?&T+3M-5Ij&$gVjOXLwWQ4$4JFmW1kaS zpSUyOP>2R^yT;hcinN40y%0M}$uW2GOMpL;#X( ztF{skPuI`p5)r#s;mZ=?%Yyc0na^JZNr(XeM|!&J>)akN@%fR(m$JnBj~vNB7R8t) z^P}dJj||Ke-YgN`ENE|*hZ99apBWNL=q{t#?|5wrXf)&@bm^&?c%M9*^MPH@$TYS} zAe9uCyy2M5OFEZR5yTP^#DWfDnGb_@626kp2SP5iRQc&Q8BKkSeTL@0^Rj7OmC@Im zx<5Fp-^5L|~K}R~GBR6e!MxN8_Fl!)Njp?VQKs?DaaZl|0KW4UM9t4J zd)czM?A4{bW!SN9@!s4&5fWPn{fTW0N{|heS_p=3?mT-MeUylbnuwA2*Crv793|+j zc)FrY1c7rU2B!_~!XeFSUcnC?`GeoWJ)OcN$jHSmt{w5iLNhKam8GlaP_dh`_2I;o z2aCg~+Tn$JWN`^R&>N0VE8TvhneQ$xan!5jQtw)QculZ3t)dNETx4^vJLn&iz=&Z*#^s*b`y1667?A$yd{|%07)hkXS!WM{);U93xEB^~__7&_|r*cbkc* zOyZw&`~$sBL=(sTZ=HSJJe_Ub+?@Z4_^P6S1*XG zm(_EBcL>7bzYQeO-dMPU2oxX+q5p>f5aquH^1p}hpHnz_FC2y-g5rsd?;l`3qFsqv z`~@SR5QrDze-!%HfyDn;o$GaB?K-mnKvX3;fbAaw$NnFI|5ud0HaW%L&eX)u6Qb=* Ki0m!^!2bbc3S|iZ literal 0 HcmV?d00001 diff --git a/src/databricks_sqlalchemy/test_local/e2e/__init__.py b/src/databricks_sqlalchemy/test_local/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/databricks_sqlalchemy/test_local/e2e/test_basic.py b/src/databricks_sqlalchemy/test_local/e2e/test_basic.py new file mode 100644 index 0000000..ce0b5d8 --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/e2e/test_basic.py @@ -0,0 +1,543 @@ +import datetime +import decimal +from typing import Tuple, Union, List +from unittest import skipIf + +import pytest +from sqlalchemy import ( + Column, + MetaData, + Table, + Text, + create_engine, + insert, + select, + text, +) +from sqlalchemy.engine import Engine +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column +from sqlalchemy.schema import DropColumnComment, SetColumnComment +from sqlalchemy.types import BOOLEAN, DECIMAL, Date, Integer, String + +try: + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base + + +USER_AGENT_TOKEN = "PySQL e2e Tests" + + +def sqlalchemy_1_3(): + import sqlalchemy + + return sqlalchemy.__version__.startswith("1.3") + + +def version_agnostic_select(object_to_select, *args, **kwargs): + """ + SQLAlchemy==1.3.x requires arguments to select() to be a Python list + + https://docs.sqlalchemy.org/en/20/changelog/migration_14.html#orm-query-is-internally-unified-with-select-update-delete-2-0-style-execution-available + """ + + if sqlalchemy_1_3(): + return select([object_to_select], *args, **kwargs) + else: + return select(object_to_select, *args, **kwargs) + + +def version_agnostic_connect_arguments(connection_details) -> Tuple[str, dict]: + HOST = connection_details["host"] + HTTP_PATH = connection_details["http_path"] + ACCESS_TOKEN = connection_details["access_token"] + CATALOG = connection_details["catalog"] + SCHEMA = connection_details["schema"] + + ua_connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} + + if sqlalchemy_1_3(): + conn_string = f"databricks://token:{ACCESS_TOKEN}@{HOST}" + connect_args = { + **ua_connect_args, + "http_path": HTTP_PATH, + "server_hostname": HOST, + "catalog": CATALOG, + "schema": SCHEMA, + } + + return conn_string, connect_args + else: + return ( + f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}", + ua_connect_args, + ) + + +@pytest.fixture +def db_engine(connection_details) -> Engine: + conn_string, connect_args = version_agnostic_connect_arguments(connection_details) + return create_engine(conn_string, connect_args=connect_args) + + +def run_query(db_engine: Engine, query: Union[str, Text]): + if not isinstance(query, Text): + _query = text(query) # type: ignore + else: + _query = query # type: ignore + with db_engine.begin() as conn: + return conn.execute(_query).fetchall() + + +@pytest.fixture +def samples_engine(connection_details) -> Engine: + details = connection_details.copy() + details["catalog"] = "samples" + details["schema"] = "nyctaxi" + conn_string, connect_args = version_agnostic_connect_arguments(details) + return create_engine(conn_string, connect_args=connect_args) + + +@pytest.fixture() +def base(db_engine): + return declarative_base() + + +@pytest.fixture() +def session(db_engine): + return Session(db_engine) + + +@pytest.fixture() +def metadata_obj(db_engine): + return MetaData() + + +def test_can_connect(db_engine): + simple_query = "SELECT 1" + result = run_query(db_engine, simple_query) + assert len(result) == 1 + + +def test_connect_args(db_engine): + """Verify that extra connect args passed to sqlalchemy.create_engine are passed to DBAPI + + This will most commonly happen when partners supply a user agent entry + """ + + conn = db_engine.connect() + connection_headers = conn.connection.thrift_backend._transport._headers + user_agent = connection_headers["User-Agent"] + + expected = f"(sqlalchemy + {USER_AGENT_TOKEN})" + assert expected in user_agent + + +@pytest.mark.skipif(sqlalchemy_1_3(), reason="Pandas requires SQLAlchemy >= 1.4") +@pytest.mark.skip( + reason="DBR is currently limited to 256 parameters per call to .execute(). Test cannot pass." +) +def test_pandas_upload(db_engine, metadata_obj): + import pandas as pd + + SCHEMA = "default" + try: + df = pd.read_excel( + "src/databricks/sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" + ) + df.to_sql( + "mock_data", + db_engine, + schema=SCHEMA, + index=False, + method="multi", + if_exists="replace", + ) + + df_after = pd.read_sql_table("mock_data", db_engine, schema=SCHEMA) + assert len(df) == len(df_after) + except Exception as e: + raise e + finally: + db_engine.execute("DROP TABLE mock_data") + + +def test_create_table_not_null(db_engine, metadata_obj: MetaData): + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + SampleTable = Table( + table_name, + metadata_obj, + Column("name", String(255)), + Column("episodes", Integer), + Column("some_bool", BOOLEAN, nullable=False), + ) + + metadata_obj.create_all(db_engine) + + columns = db_engine.dialect.get_columns( + connection=db_engine.connect(), table_name=table_name + ) + + name_column_description = columns[0] + some_bool_column_description = columns[2] + + assert name_column_description.get("nullable") is True + assert some_bool_column_description.get("nullable") is False + + metadata_obj.drop_all(db_engine) + + +def test_column_comment(db_engine, metadata_obj: MetaData): + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + column = Column("name", String(255), comment="some comment") + SampleTable = Table(table_name, metadata_obj, column) + + metadata_obj.create_all(db_engine) + connection = db_engine.connect() + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == "some comment" + + column.comment = "other comment" + connection.execute(SetColumnComment(column)) + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == "other comment" + + connection.execute(DropColumnComment(column)) + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == None + + metadata_obj.drop_all(db_engine) + + +def test_bulk_insert_with_core(db_engine, metadata_obj, session): + import random + + # Maximum number of parameter is 256. 256/4 == 64 + num_to_insert = 64 + + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + names = ["Bim", "Miki", "Sarah", "Ira"] + + SampleTable = Table( + table_name, metadata_obj, Column("name", String(255)), Column("number", Integer) + ) + + rows = [ + {"name": names[i % 3], "number": random.choice(range(64))} + for i in range(num_to_insert) + ] + + metadata_obj.create_all(db_engine) + with db_engine.begin() as conn: + conn.execute(insert(SampleTable).values(rows)) + + with db_engine.begin() as conn: + rows = conn.execute(version_agnostic_select(SampleTable)).fetchall() + + assert len(rows) == num_to_insert + + +def test_create_insert_drop_table_core(base, db_engine, metadata_obj: MetaData): + """ """ + + SampleTable = Table( + "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), + metadata_obj, + Column("name", String(255)), + Column("episodes", Integer), + Column("some_bool", BOOLEAN), + Column("dollars", DECIMAL(10, 2)), + ) + + metadata_obj.create_all(db_engine) + + insert_stmt = insert(SampleTable).values( + name="Bim Adewunmi", episodes=6, some_bool=True, dollars=decimal.Decimal(125) + ) + + with db_engine.connect() as conn: + conn.execute(insert_stmt) + + select_stmt = version_agnostic_select(SampleTable) + with db_engine.begin() as conn: + resp = conn.execute(select_stmt) + + result = resp.fetchall() + + assert len(result) == 1 + + metadata_obj.drop_all(db_engine) + + +# ORM tests are made following this tutorial +# https://docs.sqlalchemy.org/en/14/orm/quickstart.html + + +@skipIf(False, "Unity catalog must be supported") +def test_create_insert_drop_table_orm(db_engine): + """ORM classes built on the declarative base class must have a primary key. + This is restricted to Unity Catalog. + """ + + class Base(DeclarativeBase): + pass + + class SampleObject(Base): + __tablename__ = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + name: Mapped[str] = mapped_column(String(255), primary_key=True) + episodes: Mapped[int] = mapped_column(Integer) + some_bool: Mapped[bool] = mapped_column(BOOLEAN) + + Base.metadata.create_all(db_engine) + + sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) + sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) + + session = Session(db_engine) + session.add(sample_object_1) + session.add(sample_object_2) + session.flush() + + stmt = version_agnostic_select(SampleObject).where( + SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"]) + ) + + if sqlalchemy_1_3(): + output = [i for i in session.execute(stmt)] + else: + output = [i for i in session.scalars(stmt)] + + assert len(output) == 2 + + Base.metadata.drop_all(db_engine) + + +def test_dialect_type_mappings(db_engine, metadata_obj: MetaData): + """Confirms that we get back the same time we declared in a model and inserted using Core""" + + class Base(DeclarativeBase): + pass + + SampleTable = Table( + "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), + metadata_obj, + Column("string_example", String(255)), + Column("integer_example", Integer), + Column("boolean_example", BOOLEAN), + Column("decimal_example", DECIMAL(10, 2)), + Column("date_example", Date), + ) + + string_example = "" + integer_example = 100 + boolean_example = True + decimal_example = decimal.Decimal(125) + date_example = datetime.date(2013, 1, 1) + + metadata_obj.create_all(db_engine) + + insert_stmt = insert(SampleTable).values( + string_example=string_example, + integer_example=integer_example, + boolean_example=boolean_example, + decimal_example=decimal_example, + date_example=date_example, + ) + + with db_engine.connect() as conn: + conn.execute(insert_stmt) + + select_stmt = version_agnostic_select(SampleTable) + with db_engine.begin() as conn: + resp = conn.execute(select_stmt) + + result = resp.fetchall() + this_row = result[0] + + assert this_row.string_example == string_example + assert this_row.integer_example == integer_example + assert this_row.boolean_example == boolean_example + assert this_row.decimal_example == decimal_example + assert this_row.date_example == date_example + + metadata_obj.drop_all(db_engine) + + +def test_inspector_smoke_test(samples_engine: Engine): + """It does not appear that 3L namespace is supported here""" + + schema, table = "nyctaxi", "trips" + + try: + inspector = Inspector.from_engine(samples_engine) + except Exception as e: + assert False, f"Could not build inspector: {e}" + + # Expect six columns + columns = inspector.get_columns(table, schema=schema) + + # Expect zero views, but the method should return + views = inspector.get_view_names(schema=schema) + + assert ( + len(columns) == 6 + ), "Dialect did not find the expected number of columns in samples.nyctaxi.trips" + assert len(views) == 0, "Views could not be fetched" + + +@pytest.mark.skip(reason="engine.table_names has been removed in sqlalchemy verison 2") +def test_get_table_names_smoke_test(samples_engine: Engine): + with samples_engine.connect() as conn: + _names = samples_engine.table_names(schema="nyctaxi", connection=conn) # type: ignore + _names is not None, "get_table_names did not succeed" + + +def test_has_table_across_schemas( + db_engine: Engine, samples_engine: Engine, catalog: str, schema: str +): + """For this test to pass these conditions must be met: + - Table samples.nyctaxi.trips must exist + - Table samples.tpch.customer must exist + - The `catalog` and `schema` environment variables must be set and valid + """ + + with samples_engine.connect() as conn: + # 1) Check for table within schema declared at engine creation time + assert samples_engine.dialect.has_table(connection=conn, table_name="trips") + + # 2) Check for table within another schema in the same catalog + assert samples_engine.dialect.has_table( + connection=conn, table_name="customer", schema="tpch" + ) + + # 3) Check for a table within a different catalog + # Create a table in a different catalog + with db_engine.connect() as conn: + conn.execute(text("CREATE TABLE test_has_table (numbers_are_cool INT);")) + + try: + # Verify that this table is not found in the samples catalog + assert not samples_engine.dialect.has_table( + connection=conn, table_name="test_has_table" + ) + # Verify that this table is found in a separate catalog + assert samples_engine.dialect.has_table( + connection=conn, + table_name="test_has_table", + schema=schema, + catalog=catalog, + ) + finally: + conn.execute(text("DROP TABLE test_has_table;")) + + +def test_user_agent_adjustment(db_engine): + # If .connect() is called multiple times on an engine, don't keep pre-pending the user agent + # https://github.com/databricks/databricks-sql-python/issues/192 + c1 = db_engine.connect() + c2 = db_engine.connect() + + def get_conn_user_agent(conn): + return conn.connection.dbapi_connection.thrift_backend._transport._headers.get( + "User-Agent" + ) + + ua1 = get_conn_user_agent(c1) + ua2 = get_conn_user_agent(c2) + same_ua = ua1 == ua2 + + c1.close() + c2.close() + + assert same_ua, f"User agents didn't match \n {ua1} \n {ua2}" + + +@pytest.fixture +def sample_table(metadata_obj: MetaData, db_engine: Engine): + """This fixture creates a sample table and cleans it up after the test is complete.""" + from databricks.sqlalchemy._parse import GET_COLUMNS_TYPE_MAP + + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + args: List[Column] = [ + Column(colname, coltype) for colname, coltype in GET_COLUMNS_TYPE_MAP.items() + ] + + SampleTable = Table(table_name, metadata_obj, *args) + + metadata_obj.create_all(db_engine) + + yield table_name + + metadata_obj.drop_all(db_engine) + + +def test_get_columns(db_engine, sample_table: str): + """Created after PECO-1297 and Github Issue #295 to verify that get_columsn behaves like it should for all known SQLAlchemy types""" + + inspector = Inspector.from_engine(db_engine) + + # this raises an exception if `parse_column_info_from_tgetcolumnsresponse` fails a lookup + columns = inspector.get_columns(sample_table) + + assert True + + +class TestCommentReflection: + @pytest.fixture(scope="class") + def engine(self, connection_details: dict): + HOST = connection_details["host"] + HTTP_PATH = connection_details["http_path"] + ACCESS_TOKEN = connection_details["access_token"] + CATALOG = connection_details["catalog"] + SCHEMA = connection_details["schema"] + + connection_string = f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}" + connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} + + engine = create_engine(connection_string, connect_args=connect_args) + return engine + + @pytest.fixture + def inspector(self, engine: Engine) -> Inspector: + return Inspector.from_engine(engine) + + @pytest.fixture(scope="class") + def table(self, engine): + md = MetaData() + tbl = Table( + "foo", + md, + Column("bar", String, comment="column comment"), + comment="table comment", + ) + md.create_all(bind=engine) + + yield tbl + + md.drop_all(bind=engine) + + def test_table_comment_reflection(self, inspector: Inspector, table: Table): + comment = inspector.get_table_comment(table.name) + assert comment == {"text": "table comment"} + + def test_column_comment(self, inspector: Inspector, table: Table): + result = inspector.get_columns(table.name)[0].get("comment") + assert result == "column comment" diff --git a/src/databricks_sqlalchemy/test_local/test_ddl.py b/src/databricks_sqlalchemy/test_local/test_ddl.py new file mode 100644 index 0000000..f596dff --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/test_ddl.py @@ -0,0 +1,96 @@ +import pytest +from sqlalchemy import Column, MetaData, String, Table, create_engine +from sqlalchemy.schema import ( + CreateTable, + DropColumnComment, + DropTableComment, + SetColumnComment, + SetTableComment, +) + + +class DDLTestBase: + engine = create_engine( + "databricks://token:****@****?http_path=****&catalog=****&schema=****" + ) + + def compile(self, stmt): + return str(stmt.compile(bind=self.engine)) + + +class TestColumnCommentDDL(DDLTestBase): + @pytest.fixture + def metadata(self) -> MetaData: + """Assemble a metadata object with one table containing one column.""" + metadata = MetaData() + + column = Column("foo", String, comment="bar") + table = Table("foobar", metadata, column) + + return metadata + + @pytest.fixture + def table(self, metadata) -> Table: + return metadata.tables.get("foobar") + + @pytest.fixture + def column(self, table) -> Column: + return table.columns[0] + + def test_create_table_with_column_comment(self, table): + stmt = CreateTable(table) + output = self.compile(stmt) + + # output is a CREATE TABLE statement + assert "foo STRING COMMENT 'bar'" in output + + def test_alter_table_add_column_comment(self, column): + stmt = SetColumnComment(column) + output = self.compile(stmt) + assert output == "ALTER TABLE foobar ALTER COLUMN foo COMMENT 'bar'" + + def test_alter_table_drop_column_comment(self, column): + stmt = DropColumnComment(column) + output = self.compile(stmt) + assert output == "ALTER TABLE foobar ALTER COLUMN foo COMMENT ''" + + +class TestTableCommentDDL(DDLTestBase): + @pytest.fixture + def metadata(self) -> MetaData: + """Assemble a metadata object with one table containing one column.""" + metadata = MetaData() + + col1 = Column("foo", String) + col2 = Column("foo", String) + tbl_w_comment = Table("martin", metadata, col1, comment="foobar") + tbl_wo_comment = Table("prs", metadata, col2) + + return metadata + + @pytest.fixture + def table_with_comment(self, metadata) -> Table: + return metadata.tables.get("martin") + + @pytest.fixture + def table_without_comment(self, metadata) -> Table: + return metadata.tables.get("prs") + + def test_create_table_with_comment(self, table_with_comment): + stmt = CreateTable(table_with_comment) + output = self.compile(stmt) + assert "USING DELTA" in output + assert "COMMENT 'foobar'" in output + + def test_alter_table_add_comment(self, table_without_comment: Table): + table_without_comment.comment = "wireless mechanical keyboard" + stmt = SetTableComment(table_without_comment) + output = self.compile(stmt) + + assert output == "COMMENT ON TABLE prs IS 'wireless mechanical keyboard'" + + def test_alter_table_drop_comment(self, table_with_comment): + """The syntax for COMMENT ON is here: https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-comment.html""" + stmt = DropTableComment(table_with_comment) + output = self.compile(stmt) + assert output == "COMMENT ON TABLE martin IS NULL" diff --git a/src/databricks_sqlalchemy/test_local/test_parsing.py b/src/databricks_sqlalchemy/test_local/test_parsing.py new file mode 100644 index 0000000..c8ab443 --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/test_parsing.py @@ -0,0 +1,160 @@ +import pytest +from databricks.sqlalchemy._parse import ( + extract_identifiers_from_string, + extract_identifier_groups_from_string, + extract_three_level_identifier_from_constraint_string, + build_fk_dict, + build_pk_dict, + match_dte_rows_by_value, + get_comment_from_dte_output, + DatabricksSqlAlchemyParseException, +) + + +# These are outputs from DESCRIBE TABLE EXTENDED +@pytest.mark.parametrize( + "input, expected", + [ + ("PRIMARY KEY (`pk1`, `pk2`)", ["pk1", "pk2"]), + ("PRIMARY KEY (`a`, `b`, `c`)", ["a", "b", "c"]), + ("PRIMARY KEY (`name`, `id`, `attr`)", ["name", "id", "attr"]), + ], +) +def test_extract_identifiers(input, expected): + assert ( + extract_identifiers_from_string(input) == expected + ), "Failed to extract identifiers from string" + + +@pytest.mark.parametrize( + "input, expected", + [ + ( + "FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_sqlalchemy`.`tb1` (`name`, `id`, `attr`)", + [ + "(`pname`, `pid`, `pattr`)", + "(`name`, `id`, `attr`)", + ], + ) + ], +) +def test_extract_identifer_batches(input, expected): + assert ( + extract_identifier_groups_from_string(input) == expected + ), "Failed to extract identifier groups from string" + + +def test_extract_3l_namespace_from_constraint_string(): + input = "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)" + expected = { + "catalog": "main", + "schema": "pysql_dialect_compliance", + "table": "users", + } + + assert ( + extract_three_level_identifier_from_constraint_string(input) == expected + ), "Failed to extract 3L namespace from constraint string" + + +def test_extract_3l_namespace_from_bad_constraint_string(): + input = "FOREIGN KEY (`parent_user_id`) REFERENCES `pysql_dialect_compliance`.`users` (`user_id`)" + + with pytest.raises(DatabricksSqlAlchemyParseException): + extract_three_level_identifier_from_constraint_string(input) + + +@pytest.mark.parametrize("tschema", [None, "some_schema"]) +def test_build_fk_dict(tschema): + fk_constraint_string = "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`some_schema`.`users` (`user_id`)" + + result = build_fk_dict("some_fk_name", fk_constraint_string, schema_name=tschema) + + assert result == { + "name": "some_fk_name", + "constrained_columns": ["parent_user_id"], + "referred_schema": tschema, + "referred_table": "users", + "referred_columns": ["user_id"], + } + + +def test_build_pk_dict(): + pk_constraint_string = "PRIMARY KEY (`id`, `name`, `email_address`)" + pk_name = "pk1" + + result = build_pk_dict(pk_name, pk_constraint_string) + + assert result == { + "constrained_columns": ["id", "name", "email_address"], + "name": "pk1", + } + + +# This is a real example of the output from DESCRIBE TABLE EXTENDED as of 15 October 2023 +RAW_SAMPLE_DTE_OUTPUT = [ + ["id", "int"], + ["name", "string"], + ["", ""], + ["# Detailed Table Information", ""], + ["Catalog", "main"], + ["Database", "pysql_sqlalchemy"], + ["Table", "exampleexampleexample"], + ["Created Time", "Sun Oct 15 21:12:54 UTC 2023"], + ["Last Access", "UNKNOWN"], + ["Created By", "Spark "], + ["Type", "MANAGED"], + ["Location", "s3://us-west-2-****-/19a85dee-****/tables/ccb7***"], + ["Provider", "delta"], + ["Comment", "some comment"], + ["Owner", "some.user@example.com"], + ["Is_managed_location", "true"], + ["Predictive Optimization", "ENABLE (inherited from CATALOG main)"], + [ + "Table Properties", + "[delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.minReaderVersion=1,delta.minWriterVersion=2]", + ], + ["", ""], + ["# Constraints", ""], + ["exampleexampleexample_pk", "PRIMARY KEY (`id`)"], + [ + "exampleexampleexample_fk", + "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)", + ], +] + +FMT_SAMPLE_DT_OUTPUT = [ + {"col_name": i[0], "data_type": i[1]} for i in RAW_SAMPLE_DTE_OUTPUT +] + + +@pytest.mark.parametrize( + "match, output", + [ + ( + "PRIMARY KEY", + [ + { + "col_name": "exampleexampleexample_pk", + "data_type": "PRIMARY KEY (`id`)", + } + ], + ), + ( + "FOREIGN KEY", + [ + { + "col_name": "exampleexampleexample_fk", + "data_type": "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)", + } + ], + ), + ], +) +def test_filter_dict_by_value(match, output): + result = match_dte_rows_by_value(FMT_SAMPLE_DT_OUTPUT, match) + assert result == output + + +def test_get_comment_from_dte_output(): + assert get_comment_from_dte_output(FMT_SAMPLE_DT_OUTPUT) == "some comment" diff --git a/src/databricks_sqlalchemy/test_local/test_types.py b/src/databricks_sqlalchemy/test_local/test_types.py new file mode 100644 index 0000000..b91217e --- /dev/null +++ b/src/databricks_sqlalchemy/test_local/test_types.py @@ -0,0 +1,161 @@ +import enum + +import pytest +import sqlalchemy + +from databricks.sqlalchemy.base import DatabricksDialect +from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ + + +class DatabricksDataType(enum.Enum): + """https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html""" + + BIGINT = enum.auto() + BINARY = enum.auto() + BOOLEAN = enum.auto() + DATE = enum.auto() + DECIMAL = enum.auto() + DOUBLE = enum.auto() + FLOAT = enum.auto() + INT = enum.auto() + INTERVAL = enum.auto() + VOID = enum.auto() + SMALLINT = enum.auto() + STRING = enum.auto() + TIMESTAMP = enum.auto() + TIMESTAMP_NTZ = enum.auto() + TINYINT = enum.auto() + ARRAY = enum.auto() + MAP = enum.auto() + STRUCT = enum.auto() + + +# Defines the way that SQLAlchemy CamelCase types are compiled into Databricks SQL types. +# Note: I wish I could define this within the TestCamelCaseTypesCompilation class, but pytest doesn't like that. +camel_case_type_map = { + sqlalchemy.types.BigInteger: DatabricksDataType.BIGINT, + sqlalchemy.types.LargeBinary: DatabricksDataType.BINARY, + sqlalchemy.types.Boolean: DatabricksDataType.BOOLEAN, + sqlalchemy.types.Date: DatabricksDataType.DATE, + sqlalchemy.types.DateTime: DatabricksDataType.TIMESTAMP_NTZ, + sqlalchemy.types.Double: DatabricksDataType.DOUBLE, + sqlalchemy.types.Enum: DatabricksDataType.STRING, + sqlalchemy.types.Float: DatabricksDataType.FLOAT, + sqlalchemy.types.Integer: DatabricksDataType.INT, + sqlalchemy.types.Interval: DatabricksDataType.TIMESTAMP_NTZ, + sqlalchemy.types.Numeric: DatabricksDataType.DECIMAL, + sqlalchemy.types.PickleType: DatabricksDataType.BINARY, + sqlalchemy.types.SmallInteger: DatabricksDataType.SMALLINT, + sqlalchemy.types.String: DatabricksDataType.STRING, + sqlalchemy.types.Text: DatabricksDataType.STRING, + sqlalchemy.types.Time: DatabricksDataType.STRING, + sqlalchemy.types.Unicode: DatabricksDataType.STRING, + sqlalchemy.types.UnicodeText: DatabricksDataType.STRING, + sqlalchemy.types.Uuid: DatabricksDataType.STRING, +} + + +def dict_as_tuple_list(d: dict): + """Return a list of [(key, value), ...] from a dictionary.""" + return [(key, value) for key, value in d.items()] + + +class CompilationTestBase: + dialect = DatabricksDialect() + + def _assert_compiled_value( + self, type_: sqlalchemy.types.TypeEngine, expected: DatabricksDataType + ): + """Assert that when type_ is compiled for the databricks dialect, it renders the DatabricksDataType name. + + This method initialises the type_ with no arguments. + """ + compiled_result = type_().compile(dialect=self.dialect) # type: ignore + assert compiled_result == expected.name + + def _assert_compiled_value_explicit( + self, type_: sqlalchemy.types.TypeEngine, expected: str + ): + """Assert that when type_ is compiled for the databricks dialect, it renders the expected string. + + This method expects an initialised type_ so that we can test how a TypeEngine created with arguments + is compiled. + """ + compiled_result = type_.compile(dialect=self.dialect) + assert compiled_result == expected + + +class TestCamelCaseTypesCompilation(CompilationTestBase): + """Per the sqlalchemy documentation[^1] here, the camel case members of sqlalchemy.types are + are expected to work across all dialects. These tests verify that the types compile into valid + Databricks SQL type strings. For example, the sqlalchemy.types.Integer() should compile as "INT". + + Truly custom types like STRUCT (notice the uppercase) are not expected to work across all dialects. + We test these separately. + + Note that these tests have to do with type **name** compiliation. Which is separate from actually + mapping values between Python and Databricks. + + Note: SchemaType and MatchType are not tested because it's not used in table definitions + + [1]: https://docs.sqlalchemy.org/en/20/core/type_basics.html#generic-camelcase-types + """ + + @pytest.mark.parametrize("type_, expected", dict_as_tuple_list(camel_case_type_map)) + def test_bare_camel_case_types_compile(self, type_, expected): + self._assert_compiled_value(type_, expected) + + def test_numeric_renders_as_decimal_with_precision(self): + self._assert_compiled_value_explicit( + sqlalchemy.types.Numeric(10), "DECIMAL(10)" + ) + + def test_numeric_renders_as_decimal_with_precision_and_scale(self): + self._assert_compiled_value_explicit( + sqlalchemy.types.Numeric(10, 2), "DECIMAL(10, 2)" + ) + + +uppercase_type_map = { + sqlalchemy.types.ARRAY: DatabricksDataType.ARRAY, + sqlalchemy.types.BIGINT: DatabricksDataType.BIGINT, + sqlalchemy.types.BINARY: DatabricksDataType.BINARY, + sqlalchemy.types.BOOLEAN: DatabricksDataType.BOOLEAN, + sqlalchemy.types.DATE: DatabricksDataType.DATE, + sqlalchemy.types.DECIMAL: DatabricksDataType.DECIMAL, + sqlalchemy.types.DOUBLE: DatabricksDataType.DOUBLE, + sqlalchemy.types.FLOAT: DatabricksDataType.FLOAT, + sqlalchemy.types.INT: DatabricksDataType.INT, + sqlalchemy.types.SMALLINT: DatabricksDataType.SMALLINT, + sqlalchemy.types.TIMESTAMP: DatabricksDataType.TIMESTAMP, + TINYINT: DatabricksDataType.TINYINT, + TIMESTAMP: DatabricksDataType.TIMESTAMP, + TIMESTAMP_NTZ: DatabricksDataType.TIMESTAMP_NTZ, +} + + +class TestUppercaseTypesCompilation(CompilationTestBase): + """Per the sqlalchemy documentation[^1], uppercase types are considered to be specific to some + database backends. These tests verify that the types compile into valid Databricks SQL type strings. + + [1]: https://docs.sqlalchemy.org/en/20/core/type_basics.html#backend-specific-uppercase-datatypes + """ + + @pytest.mark.parametrize("type_, expected", dict_as_tuple_list(uppercase_type_map)) + def test_bare_uppercase_types_compile(self, type_, expected): + if isinstance(type_, type(sqlalchemy.types.ARRAY)): + # ARRAY cannot be initialised without passing an item definition so we test separately + # I preserve it in the uppercase_type_map for clarity + assert True + else: + self._assert_compiled_value(type_, expected) + + def test_array_string_renders_as_array_of_string(self): + """SQLAlchemy's ARRAY type requires an item definition. And their docs indicate that they've only tested + it with Postgres since that's the only first-class dialect with support for ARRAY. + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.ARRAY + """ + self._assert_compiled_value_explicit( + sqlalchemy.types.ARRAY(sqlalchemy.types.String), "ARRAY" + ) From 91d6cd2c9aa9a2893ad1e9800b03da398dafbe85 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Thu, 22 Aug 2024 22:30:28 +0530 Subject: [PATCH 02/18] Refractored code and added pytest.ini for sqla_testing issue --- pyproject.toml | 14 +++++++------- src/databricks_sqlalchemy/pytest.ini | 4 ++++ src/databricks_sqlalchemy/setup.cfg | 2 +- src/databricks_sqlalchemy/test/_extra.py | 2 +- src/databricks_sqlalchemy/test/_future.py | 4 ++-- src/databricks_sqlalchemy/test/_regression.py | 4 ++-- src/databricks_sqlalchemy/test/_unsupported.py | 4 ++-- src/databricks_sqlalchemy/test/conftest.py | 4 ++-- src/databricks_sqlalchemy/test/test_suite.py | 6 +++--- 9 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 src/databricks_sqlalchemy/pytest.ini diff --git a/pyproject.toml b/pyproject.toml index c328585..9b05222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,10 +58,10 @@ exclude = ['ttypes\.py$', 'TCLIService\.py$'] [tool.black] exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' # -#[tool.pytest.ini_options] -#markers = {"reviewed" = "Test case has been reviewed by Databricks"} -#minversion = "6.0" -#log_cli = "false" -#log_cli_level = "INFO" -#testpaths = ["tests", "src/databricks/sqlalchemy/test_local"] -#env_files = ["test.env"] +[tool.pytest.ini_options] +markers = {"reviewed" = "Test case has been reviewed by Databricks"} +minversion = "6.0" +log_cli = "false" +log_cli_level = "INFO" +testpaths = ["tests", "databricks_sqlalchemy/src/databricks_sqlalchemy/test_local"] +env_files = ["test.env"] diff --git a/src/databricks_sqlalchemy/pytest.ini b/src/databricks_sqlalchemy/pytest.ini new file mode 100644 index 0000000..4ce8d29 --- /dev/null +++ b/src/databricks_sqlalchemy/pytest.ini @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks_sqlalchemy.requirements:Requirements +profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/setup.cfg b/src/databricks_sqlalchemy/setup.cfg index ab89d17..4ce8d29 100644 --- a/src/databricks_sqlalchemy/setup.cfg +++ b/src/databricks_sqlalchemy/setup.cfg @@ -1,4 +1,4 @@ [sqla_testing] -requirement_cls=databricks.sqlalchemy.requirements:Requirements +requirement_cls=databricks_sqlalchemy.requirements:Requirements profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/test/_extra.py b/src/databricks_sqlalchemy/test/_extra.py index 2f3e7a7..48cc0ec 100644 --- a/src/databricks_sqlalchemy/test/_extra.py +++ b/src/databricks_sqlalchemy/test/_extra.py @@ -15,7 +15,7 @@ _DateFixture, literal, ) -from databricks.sqlalchemy import TINYINT, TIMESTAMP +from databricks_sql_connector_core.sqlalchemy import TINYINT, TIMESTAMP class TinyIntegerTest(_LiteralRoundTripFixture, fixtures.TestBase): diff --git a/src/databricks_sqlalchemy/test/_future.py b/src/databricks_sqlalchemy/test/_future.py index 2baa3d3..76b4f7a 100644 --- a/src/databricks_sqlalchemy/test/_future.py +++ b/src/databricks_sqlalchemy/test/_future.py @@ -3,13 +3,13 @@ from enum import Enum import pytest -from databricks_sqlalchemy.src.sqlalchemy.test._regression import ( +from databricks_sqlalchemy.test._regression import ( ExpandingBoundInTest, IdentityAutoincrementTest, LikeFunctionsTest, NormalizedNameTest, ) -from databricks.sqlalchemy.test._unsupported import ( +from databricks_sqlalchemy.test._unsupported import ( ComponentReflectionTest, ComponentReflectionTestExtra, CTETest, diff --git a/src/databricks_sqlalchemy/test/_regression.py b/src/databricks_sqlalchemy/test/_regression.py index 1cd4c35..1d2e6d0 100644 --- a/src/databricks_sqlalchemy/test/_regression.py +++ b/src/databricks_sqlalchemy/test/_regression.py @@ -51,8 +51,8 @@ ValuesExpressionTest, ) -from databricks.sqlalchemy.test.overrides._ctetest import CTETest -from databricks_sqlalchemy.src.sqlalchemy.test.overrides._componentreflectiontest import ( +from databricks_sqlalchemy.test.overrides._ctetest import CTETest +from databricks_sqlalchemy.test.overrides._componentreflectiontest import ( ComponentReflectionTest, ) diff --git a/src/databricks_sqlalchemy/test/_unsupported.py b/src/databricks_sqlalchemy/test/_unsupported.py index c1270c2..c40c597 100644 --- a/src/databricks_sqlalchemy/test/_unsupported.py +++ b/src/databricks_sqlalchemy/test/_unsupported.py @@ -3,7 +3,7 @@ from enum import Enum import pytest -from databricks_sqlalchemy.src.sqlalchemy.test._regression import ( +from databricks_sqlalchemy.test._regression import ( ComponentReflectionTest, ComponentReflectionTestExtra, CTETest, @@ -441,7 +441,7 @@ class DateTimeTZTest(DateTimeTZTest): """Test whether the sqlalchemy.DateTime() type can _optionally_ include timezone info. This dialect maps DateTime() → TIMESTAMP, which _always_ includes tzinfo. - Users can use databricks.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs + Users can use databricks_sql_connector_core.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs acknowledge this is expected for some dialects. https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime diff --git a/src/databricks_sqlalchemy/test/conftest.py b/src/databricks_sqlalchemy/test/conftest.py index ea43e8d..81e9479 100644 --- a/src/databricks_sqlalchemy/test/conftest.py +++ b/src/databricks_sqlalchemy/test/conftest.py @@ -1,12 +1,12 @@ from sqlalchemy.dialects import registry import pytest -registry.register("databricks", "databricks.sqlalchemy", "DatabricksDialect") +registry.register("databricks", "databricks_sqlalchemy", "DatabricksDialect") # sqlalchemy's dialect-testing machinery wants an entry like this. # This seems to be based around dialects maybe having multiple drivers # and wanting to test driver-specific URLs, but doesn't seem to make # much sense for dialects with only one driver. -registry.register("databricks.databricks", "databricks.sqlalchemy", "DatabricksDialect") +registry.register("databricks.databricks", "databricks_sqlalchemy", "DatabricksDialect") pytest.register_assert_rewrite("sqlalchemy.testing.assertions") diff --git a/src/databricks_sqlalchemy/test/test_suite.py b/src/databricks_sqlalchemy/test/test_suite.py index 1900589..02a4208 100644 --- a/src/databricks_sqlalchemy/test/test_suite.py +++ b/src/databricks_sqlalchemy/test/test_suite.py @@ -7,6 +7,6 @@ # type: ignore # fmt: off from sqlalchemy.testing.suite import * -from databricks_sqlalchemy.src.sqlalchemy.test._regression import * -from databricks.sqlalchemy.test._unsupported import * -from databricks.sqlalchemy.test._future import * +from databricks_sqlalchemy.test._regression import * +from databricks_sqlalchemy.test._unsupported import * +from databricks_sqlalchemy.test._future import * From 4bc939bdc3e0f77b8dd58eb0f6814c991e04f8e0 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Sun, 25 Aug 2024 23:07:50 +0530 Subject: [PATCH 03/18] Changed imports from databricks.sqlalchemy to databricks_sqlalchemy --- src/databricks_sqlalchemy/test_local/e2e/test_basic.py | 4 ++-- src/databricks_sqlalchemy/test_local/test_parsing.py | 2 +- src/databricks_sqlalchemy/test_local/test_types.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/databricks_sqlalchemy/test_local/e2e/test_basic.py b/src/databricks_sqlalchemy/test_local/e2e/test_basic.py index ce0b5d8..215d3eb 100644 --- a/src/databricks_sqlalchemy/test_local/e2e/test_basic.py +++ b/src/databricks_sqlalchemy/test_local/e2e/test_basic.py @@ -144,7 +144,7 @@ def test_pandas_upload(db_engine, metadata_obj): SCHEMA = "default" try: df = pd.read_excel( - "src/databricks/sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" + "databricks_sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" ) df.to_sql( "mock_data", @@ -472,7 +472,7 @@ def get_conn_user_agent(conn): @pytest.fixture def sample_table(metadata_obj: MetaData, db_engine: Engine): """This fixture creates a sample table and cleans it up after the test is complete.""" - from databricks.sqlalchemy._parse import GET_COLUMNS_TYPE_MAP + from databricks_sqlalchemy._parse import GET_COLUMNS_TYPE_MAP table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) diff --git a/src/databricks_sqlalchemy/test_local/test_parsing.py b/src/databricks_sqlalchemy/test_local/test_parsing.py index c8ab443..7616142 100644 --- a/src/databricks_sqlalchemy/test_local/test_parsing.py +++ b/src/databricks_sqlalchemy/test_local/test_parsing.py @@ -1,5 +1,5 @@ import pytest -from databricks.sqlalchemy._parse import ( +from databricks_sqlalchemy._parse import ( extract_identifiers_from_string, extract_identifier_groups_from_string, extract_three_level_identifier_from_constraint_string, diff --git a/src/databricks_sqlalchemy/test_local/test_types.py b/src/databricks_sqlalchemy/test_local/test_types.py index b91217e..4d61c55 100644 --- a/src/databricks_sqlalchemy/test_local/test_types.py +++ b/src/databricks_sqlalchemy/test_local/test_types.py @@ -3,8 +3,8 @@ import pytest import sqlalchemy -from databricks.sqlalchemy.base import DatabricksDialect -from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ +from databricks_sqlalchemy.base import DatabricksDialect +from databricks_sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ class DatabricksDataType(enum.Enum): From dc0a25b7afaf2f316e52c785451e1069ede711d7 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Sun, 25 Aug 2024 23:25:47 +0530 Subject: [PATCH 04/18] Added the integration.yml workflow --- .github/workflows/integration.yml | 63 +++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..afcd3ee --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,63 @@ +name: Integration + +on: + workflow_dispatch: + +jobs: + build_and_test: + runs-on: ubuntu-latest + + env: + DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_CATALOG: peco + DATABRICKS_SCHEMA : main + DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} + + steps: + # Checkout your own repository + - name: Checkout Repository + uses: actions/checkout@v3 + + # Checkout the other repository + - name: Checkout Dependency Repository + uses: actions/checkout@v3 + with: + repository: 'jprakash-db/databricks-sql-python' + path: 'databricks_sql_connector_core' + ref : 'jprakash-db/PECO-1803' + + # Set up Python + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + # Install Poetry + - name: Install Poetry + run: | + python -m pip install --upgrade pip + pip3 install poetry + + # Build the .whl file in the dependency repository + - name: Build Dependency Package + run: | + poetry build -C databricks_sql_connector_core + + # Install the .whl file using pip + - name: Install Dependency Package + run: | + pip3 install databricks_sql_connector_core/dist/*.whl + + # Install the requirements of your repository + - name: Install Dependencies + run: | + poetry install + + # Run pytest to execute tests in your repository + - name: Run Tests + run: | + python -m pytest src/databricks_sqlalchemy --dburi \ + "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" + From 433c172f0f33908820128cc61a9b7b9748a9f9e1 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 26 Aug 2024 09:46:21 +0530 Subject: [PATCH 05/18] Changed screts name --- .github/workflows/integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index afcd3ee..7eb1c83 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -11,8 +11,8 @@ jobs: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_CATALOG: peco - DATABRICKS_SCHEMA : main + DATABRICKS_CATALOG: ${{ secrets.DATABRICKS_CATALOG }} + DATABRICKS_SCHEMA : ${{ secrets.DATABRICKS_SCHEMA }} DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: From 68fa020f25ce4992beff76762464304eb571a823 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 26 Aug 2024 10:02:14 +0530 Subject: [PATCH 06/18] Added workflow to run on commit changes --- .github/workflows/integration.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7eb1c83..e68ed81 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,6 +1,9 @@ name: Integration on: + pull_request: + types: [ opened, synchronize, reopened ] + branches: [ main, PECO-1803 ] workflow_dispatch: jobs: @@ -24,9 +27,9 @@ jobs: - name: Checkout Dependency Repository uses: actions/checkout@v3 with: - repository: 'jprakash-db/databricks-sql-python' - path: 'databricks_sql_connector_core' - ref : 'jprakash-db/PECO-1803' + repository: jprakash-db/databricks-sql-python + path: databricks_sql_connector_core + ref : jprakash-db/PECO-1803 # Set up Python - name: Set up Python @@ -58,6 +61,5 @@ jobs: # Run pytest to execute tests in your repository - name: Run Tests run: | - python -m pytest src/databricks_sqlalchemy --dburi \ - "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" + python -m pytest src/databricks_sqlalchemy --dburi "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" From 296c5a3da25c41ce28dbb6f5d666ec43580173fc Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 26 Aug 2024 10:12:41 +0530 Subject: [PATCH 07/18] Fixed issue of failing tests in github actions --- .github/workflows/integration.yml | 61 +++++++++++++++++++++-------- pyproject.toml | 2 +- src/databricks_sqlalchemy/_types.py | 7 ++-- src/databricks_sqlalchemy/base.py | 2 +- 4 files changed, 50 insertions(+), 22 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e68ed81..115f935 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,14 +9,14 @@ on: jobs: build_and_test: runs-on: ubuntu-latest - + environment: azure-prod env: - DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} + DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME }} + DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_CATALOG: ${{ secrets.DATABRICKS_CATALOG }} DATABRICKS_SCHEMA : ${{ secrets.DATABRICKS_SCHEMA }} - DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} + DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }} steps: # Checkout your own repository @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v3 with: repository: jprakash-db/databricks-sql-python - path: databricks_sql_connector_core + path: databricks_sql_python ref : jprakash-db/PECO-1803 # Set up Python @@ -37,29 +37,58 @@ jobs: with: python-version: '3.9' +# #---------------------------------------------- +# # ----- install & configure poetry ----- +# #---------------------------------------------- +# - name: Install Poetry +# uses: snok/install-poetry@v1 +# with: +# virtualenvs-create: true +# virtualenvs-in-project: true +# installer-parallel: true + # Install Poetry - name: Install Poetry run: | python -m pip install --upgrade pip pip3 install poetry + python3 -m venv venv + ls databricks_sql_python/databricks_sql_connector_core - # Build the .whl file in the dependency repository - - name: Build Dependency Package + # Install the requirements of your repository + - name: Install Dependencies run: | - poetry build -C databricks_sql_connector_core + source venv/bin/activate + poetry build + pip3 install dist/*.whl - # Install the .whl file using pip - - name: Install Dependency Package + # Build the .whl file in the dependency repository + - name: Build Dependency Package run: | - pip3 install databricks_sql_connector_core/dist/*.whl + source venv/bin/activate + pip3 install databricks_sql_python/databricks_sql_connector_core/dist/*.whl - # Install the requirements of your repository - - name: Install Dependencies - run: | - poetry install +# # Install the .whl file using pip +# - name: Install Dependency Package +# run: | +# pip3 install databricks_sql_python/databricks_sql_connector_core/dist/*.whl # Run pytest to execute tests in your repository - name: Run Tests run: | - python -m pytest src/databricks_sqlalchemy --dburi "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" + source venv/bin/activate + pip3 list + pip3 install pytest + + - name : Main Tests + run: | + source venv/bin/activate + pytest src/databricks_sqlalchemy/test_local +# cd src/databricks_sqlalchemy +# python -m pytest test_local --dburi "databricks://token:dfafadfvscsd@e2-dogfood.staging.cloud.databricks.com?http_path=/sql/1.0/warehouses/dd43ee29fedd958d&catalog=peco&schema=default" +# DBURI="databricks://token:${DATABRICKS_TOKEN}@${DATABRICKS_SERVER_HOSTNAME}?http_path=${DATABRICKS_HTTP_PATH}&catalog=${DATABRICKS_CATALOG}&schema=${DATABRICKS_SCHEMA}" +# python -m pytest test_local --dburi "$DBURI" +# poetry run python -m pytest test_local --dburi "databricks://token:${DATABRICKS_TOKEN}@${DATABRICKS_SERVER_HOSTNAME}?http_path=${DATABRICKS_HTTP_PATH}&catalog=${DATABRICKS_CATALOG}&schema=${DATABRICKS_SCHEMA}" + +# python -m pytest src/databricks_sqlalchemy/test_local --dburi "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" diff --git a/pyproject.toml b/pyproject.toml index 9b05222..91dd04d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ pytest-dotenv = "^0.5.2" "Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues" [tool.poetry.plugins."sqlalchemy.dialects"] -"databricks" = "databricks.sqlalchemy:DatabricksDialect" +"databricks" = "databricks_sqlalchemy:DatabricksDialect" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/src/databricks_sqlalchemy/_types.py b/src/databricks_sqlalchemy/_types.py index 6d32ff5..9424f4c 100644 --- a/src/databricks_sqlalchemy/_types.py +++ b/src/databricks_sqlalchemy/_types.py @@ -6,9 +6,8 @@ from sqlalchemy.engine.interfaces import Dialect from sqlalchemy.ext.compiler import compiles -# from databricks.sql.utils import ParamEscaper -import databricks.sql -from databricks.sql.utils import ParamEscaper +from databricks_sql_connector_core.sql.utils import ParamEscaper + def process_literal_param_hack(value: Any): @@ -112,7 +111,7 @@ class TIMESTAMP_NTZ(sqlalchemy.types.TypeDecorator): Our dialect maps sqlalchemy.types.DateTime() to this type, which means that all DateTime() objects are stored without tzinfo. To read and write timezone-aware datetimes use - databricks.sql.TIMESTAMP instead. + databricks_sql_connector_core.sql.TIMESTAMP instead. https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html """ diff --git a/src/databricks_sqlalchemy/base.py b/src/databricks_sqlalchemy/base.py index 0cff681..94bce80 100644 --- a/src/databricks_sqlalchemy/base.py +++ b/src/databricks_sqlalchemy/base.py @@ -2,7 +2,7 @@ import databricks_sqlalchemy._ddl as dialect_ddl_impl import databricks_sqlalchemy._types as dialect_type_impl -from databricks import sql +from databricks_sql_connector_core import sql from databricks_sqlalchemy._parse import ( _describe_table_extended_result_to_dict_list, _match_table_not_found_string, From 7abe28ed93a5f23720e7206a4201abeeda7e670e Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Thu, 29 Aug 2024 15:56:21 +0530 Subject: [PATCH 08/18] Added dependency tests --- dependency_test/test_dependency.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 dependency_test/test_dependency.py diff --git a/dependency_test/test_dependency.py b/dependency_test/test_dependency.py new file mode 100644 index 0000000..644e4a0 --- /dev/null +++ b/dependency_test/test_dependency.py @@ -0,0 +1,22 @@ +import pytest + +class DatabricksImportError(Exception): + pass + +class TestLibraryDependencySuite: + + @pytest.mark.skipif(pytest.importorskip("databricks_sql_connector_core"), reason="databricks_sql_connector_core is present") + def test_sql_core(self): + with pytest.raises(DatabricksImportError, match="databricks_sql_connector_core module is not available"): + try: + import databricks_sql_connector_core + except ImportError: + raise DatabricksImportError("databricks_sql_connector_core module is not available") + + @pytest.mark.skipif(pytest.importorskip("sqlalchemy"), reason="SQLAlchemy is present") + def test_sqlalchemy(self): + with pytest.raises(DatabricksImportError, match="sqlalchemy module is not available"): + try: + import sqlalchemy + except ImportError: + raise DatabricksImportError("sqlalchemy module is not available") \ No newline at end of file From e12abcb7b49d454d6311886534cf5512cd3770e8 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Tue, 3 Sep 2024 22:03:22 +0530 Subject: [PATCH 09/18] Cleaned up the code --- .github/workflows/integration.yml | 23 ----------------------- pyproject.toml | 21 --------------------- 2 files changed, 44 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 115f935..01e09eb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -37,16 +37,6 @@ jobs: with: python-version: '3.9' -# #---------------------------------------------- -# # ----- install & configure poetry ----- -# #---------------------------------------------- -# - name: Install Poetry -# uses: snok/install-poetry@v1 -# with: -# virtualenvs-create: true -# virtualenvs-in-project: true -# installer-parallel: true - # Install Poetry - name: Install Poetry run: | @@ -68,11 +58,6 @@ jobs: source venv/bin/activate pip3 install databricks_sql_python/databricks_sql_connector_core/dist/*.whl -# # Install the .whl file using pip -# - name: Install Dependency Package -# run: | -# pip3 install databricks_sql_python/databricks_sql_connector_core/dist/*.whl - # Run pytest to execute tests in your repository - name: Run Tests run: | @@ -84,11 +69,3 @@ jobs: run: | source venv/bin/activate pytest src/databricks_sqlalchemy/test_local -# cd src/databricks_sqlalchemy -# python -m pytest test_local --dburi "databricks://token:dfafadfvscsd@e2-dogfood.staging.cloud.databricks.com?http_path=/sql/1.0/warehouses/dd43ee29fedd958d&catalog=peco&schema=default" -# DBURI="databricks://token:${DATABRICKS_TOKEN}@${DATABRICKS_SERVER_HOSTNAME}?http_path=${DATABRICKS_HTTP_PATH}&catalog=${DATABRICKS_CATALOG}&schema=${DATABRICKS_SCHEMA}" -# python -m pytest test_local --dburi "$DBURI" -# poetry run python -m pytest test_local --dburi "databricks://token:${DATABRICKS_TOKEN}@${DATABRICKS_SERVER_HOSTNAME}?http_path=${DATABRICKS_HTTP_PATH}&catalog=${DATABRICKS_CATALOG}&schema=${DATABRICKS_SCHEMA}" - -# python -m pytest src/databricks_sqlalchemy/test_local --dburi "databricks://token:$DATABRICKS_TOKEN@$DATABRICKS_SERVER_HOSTNAME?http_path=$DATABRICKS_HTTP_PATH&catalog=$DATABRICKS_CATALOG&schema=$DATABRICKS_SCHEMA" - diff --git a/pyproject.toml b/pyproject.toml index 91dd04d..85ecbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,28 +10,7 @@ include = ["CHANGELOG.md"] [tool.poetry.dependencies] python = "^3.8.0" -#thrift = ">=0.16.0,<0.21.0" -#pandas = [ -# { version = ">=1.2.5,<2.2.0", python = ">=3.8" } -#] -#pyarrow = ">=14.0.1,<17" - -#lz4 = "^4.0.2" -#requests = "^2.18.1" -#oauthlib = "^3.1.0" -#numpy = [ -# { version = "^1.16.6", python = ">=3.8,<3.11" }, -# { version = "^1.23.4", python = ">=3.11" }, -#] -#sqlalchemy = { version = ">=2.0.21", optional = true } sqlalchemy = { version = ">=2.0.21" } -#openpyxl = "^3.0.10" -#alembic = { version = "^1.0.11", optional = true } -#urllib3 = ">=1.26" - -#[tool.poetry.extras] -#sqlalchemy = ["sqlalchemy"] -#alembic = ["sqlalchemy", "alembic"] [tool.poetry.dev-dependencies] pytest = "^7.1.2" From 8d312672c0f4f794fe0db5556a7e9427e2b5d4a7 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Fri, 6 Sep 2024 12:09:14 +0530 Subject: [PATCH 10/18] Changed README.md --- README.md | 203 ++++++++++++++++++ .../README.tests.md => README.tests.md | 0 .../README.sqlalchemy.md | 203 ------------------ 3 files changed, 203 insertions(+), 203 deletions(-) rename src/databricks_sqlalchemy/README.tests.md => README.tests.md (100%) delete mode 100644 src/databricks_sqlalchemy/README.sqlalchemy.md diff --git a/README.md b/README.md index e69de29..8aa5197 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,203 @@ +## Databricks dialect for SQLALchemy 2.0 + +The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. + +## Usage with SQLAlchemy <= 2.0 +A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. + + +## Installation + +To install the dialect and its dependencies: + +```shell +pip install databricks-sql-connector[sqlalchemy] +``` + +If you also plan to use `alembic` you can alternatively run: + +```shell +pip install databricks-sql-connector[alembic] +``` + +## Connection String + +Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: + +1. Host +2. HTTP Path for a compute resource +3. API access token +4. Initial catalog for the connection +5. Initial schema for the connection + +**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** + +For example: + +```python +import os +from sqlalchemy import create_engine + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") +catalog = os.getenv("DATABRICKS_CATALOG") +schema = os.getenv("DATABRICKS_SCHEMA") + +engine = create_engine( + f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" + ) +``` + +## Types + +The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. + +|SQLAlchemy Type|Databricks SQL Type| +|-|-| +[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) +[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| +[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) +[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) +[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| +[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) +[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| +[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) +[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) +[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| +[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| +[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) +[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) + +In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: + +- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) +- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) +- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) + + +### `LargeBinary()` and `PickleType()` + +Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. + +## `Enum()` and `CHECK` constraints + +Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. + +SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. + +### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` + +Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. + +The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. + +If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. + +_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ + +```python +from sqlalchemy import DateTime +from databricks.sqlalchemy import TIMESTAMP + +class SomeModel(Base): + some_date_without_timezone = DateTime() + some_date_with_timezone = TIMESTAMP() +``` + +### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` + +Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. + +### `Time()` + +Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). + +```python +from sqlalchemy import Time + +class SomeModel(Base): + time_tz = Time(timezone=True) + time_ntz = Time() +``` + + +# Usage Notes + +## `Identity()` and `autoincrement` + +Identity and generated value support is currently limited in this dialect. + +When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. + +Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. + +```python +from sqlalchemy import Identity, String + +class SomeModel(Base): + id = BigInteger(Identity()) + value = String() +``` + +When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). + +## Parameters + +`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. + +## Usage with pandas + +Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. + +### Read from Databricks SQL into pandas +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +with engine.connect() as conn: + # This will read the contents of `main.test.some_table` + df = pd.read_sql("some_table", conn) +``` + +### Write to Databricks SQL from pandas + +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +squares = [(i, i * i) for i in range(100)] +df = pd.DataFrame(data=squares,columns=['x','x_squared']) + +with engine.connect() as conn: + # This will write the contents of `df` to `main.test.squares` + df.to_sql('squares',conn) +``` + +## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) + +Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. + +When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. + +If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. + +```python +from sqlalchemy import Table, Column, ForeignKey, BigInteger, String + +users = Table( + "users", + metadata_obj, + Column("id", BigInteger, primary_key=True), + Column("name", String(), nullable=False), + Column("email", String()), + Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) +) +``` diff --git a/src/databricks_sqlalchemy/README.tests.md b/README.tests.md similarity index 100% rename from src/databricks_sqlalchemy/README.tests.md rename to README.tests.md diff --git a/src/databricks_sqlalchemy/README.sqlalchemy.md b/src/databricks_sqlalchemy/README.sqlalchemy.md deleted file mode 100644 index 8aa5197..0000000 --- a/src/databricks_sqlalchemy/README.sqlalchemy.md +++ /dev/null @@ -1,203 +0,0 @@ -## Databricks dialect for SQLALchemy 2.0 - -The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. - -## Usage with SQLAlchemy <= 2.0 -A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. - - -## Installation - -To install the dialect and its dependencies: - -```shell -pip install databricks-sql-connector[sqlalchemy] -``` - -If you also plan to use `alembic` you can alternatively run: - -```shell -pip install databricks-sql-connector[alembic] -``` - -## Connection String - -Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: - -1. Host -2. HTTP Path for a compute resource -3. API access token -4. Initial catalog for the connection -5. Initial schema for the connection - -**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** - -For example: - -```python -import os -from sqlalchemy import create_engine - -host = os.getenv("DATABRICKS_SERVER_HOSTNAME") -http_path = os.getenv("DATABRICKS_HTTP_PATH") -access_token = os.getenv("DATABRICKS_TOKEN") -catalog = os.getenv("DATABRICKS_CATALOG") -schema = os.getenv("DATABRICKS_SCHEMA") - -engine = create_engine( - f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" - ) -``` - -## Types - -The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. - -|SQLAlchemy Type|Databricks SQL Type| -|-|-| -[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) -[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| -[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) -[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) -[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| -[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) -[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| -[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) -[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) -[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| -[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| -[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) -[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) - -In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: - -- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) -- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) -- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) - - -### `LargeBinary()` and `PickleType()` - -Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. - -## `Enum()` and `CHECK` constraints - -Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. - -SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. - -### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` - -Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. - -The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. - -If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. - -_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ - -```python -from sqlalchemy import DateTime -from databricks.sqlalchemy import TIMESTAMP - -class SomeModel(Base): - some_date_without_timezone = DateTime() - some_date_with_timezone = TIMESTAMP() -``` - -### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` - -Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. - -### `Time()` - -Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). - -```python -from sqlalchemy import Time - -class SomeModel(Base): - time_tz = Time(timezone=True) - time_ntz = Time() -``` - - -# Usage Notes - -## `Identity()` and `autoincrement` - -Identity and generated value support is currently limited in this dialect. - -When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. - -Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. - -```python -from sqlalchemy import Identity, String - -class SomeModel(Base): - id = BigInteger(Identity()) - value = String() -``` - -When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). - -## Parameters - -`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. - -## Usage with pandas - -Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. - -### Read from Databricks SQL into pandas -```python -from sqlalchemy import create_engine -import pandas as pd - -engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") -with engine.connect() as conn: - # This will read the contents of `main.test.some_table` - df = pd.read_sql("some_table", conn) -``` - -### Write to Databricks SQL from pandas - -```python -from sqlalchemy import create_engine -import pandas as pd - -engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") -squares = [(i, i * i) for i in range(100)] -df = pd.DataFrame(data=squares,columns=['x','x_squared']) - -with engine.connect() as conn: - # This will write the contents of `df` to `main.test.squares` - df.to_sql('squares',conn) -``` - -## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) - -Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. - -When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. - -If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. - -```python -from sqlalchemy import Table, Column, ForeignKey, BigInteger, String - -users = Table( - "users", - metadata_obj, - Column("id", BigInteger, primary_key=True), - Column("name", String(), nullable=False), - Column("email", String()), - Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) -) -``` From 786c0be3f2ea37e23daf1b316a11630b241211eb Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Sat, 7 Sep 2024 10:28:03 +0530 Subject: [PATCH 11/18] README.md edits --- README.md | 2 +- README.tests.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8aa5197..4f4e477 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4 To install the dialect and its dependencies: ```shell -pip install databricks-sql-connector[sqlalchemy] +pip install databricks-sql-connector[databricks_sqlalchemy] ``` If you also plan to use `alembic` you can alternatively run: diff --git a/README.tests.md b/README.tests.md index 3ed92ab..5d72711 100644 --- a/README.tests.md +++ b/README.tests.md @@ -23,7 +23,7 @@ We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dia ``` poetry shell -cd src/databricks/sqlalchemy/test +cd src/databricks_sqlalchemy/test python -m pytest test_suite.py --dburi \ "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" ``` From a7bbc5aa277bfbae433ee929006d57abf6bd9a39 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Fri, 13 Sep 2024 09:31:23 +0530 Subject: [PATCH 12/18] Renamed the imports to databricks --- .github/workflows/integration.yml | 1 - dependency_test/test_dependency.py | 2 +- src/databricks_sqlalchemy/_types.py | 4 ++-- src/databricks_sqlalchemy/base.py | 2 +- src/databricks_sqlalchemy/test/_extra.py | 2 +- src/databricks_sqlalchemy/test/_unsupported.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 01e09eb..ead1751 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -43,7 +43,6 @@ jobs: python -m pip install --upgrade pip pip3 install poetry python3 -m venv venv - ls databricks_sql_python/databricks_sql_connector_core # Install the requirements of your repository - name: Install Dependencies diff --git a/dependency_test/test_dependency.py b/dependency_test/test_dependency.py index 644e4a0..7cb59e8 100644 --- a/dependency_test/test_dependency.py +++ b/dependency_test/test_dependency.py @@ -9,7 +9,7 @@ class TestLibraryDependencySuite: def test_sql_core(self): with pytest.raises(DatabricksImportError, match="databricks_sql_connector_core module is not available"): try: - import databricks_sql_connector_core + import databricks except ImportError: raise DatabricksImportError("databricks_sql_connector_core module is not available") diff --git a/src/databricks_sqlalchemy/_types.py b/src/databricks_sqlalchemy/_types.py index 9424f4c..f24c12a 100644 --- a/src/databricks_sqlalchemy/_types.py +++ b/src/databricks_sqlalchemy/_types.py @@ -6,7 +6,7 @@ from sqlalchemy.engine.interfaces import Dialect from sqlalchemy.ext.compiler import compiles -from databricks_sql_connector_core.sql.utils import ParamEscaper +from databricks.sql.utils import ParamEscaper @@ -111,7 +111,7 @@ class TIMESTAMP_NTZ(sqlalchemy.types.TypeDecorator): Our dialect maps sqlalchemy.types.DateTime() to this type, which means that all DateTime() objects are stored without tzinfo. To read and write timezone-aware datetimes use - databricks_sql_connector_core.sql.TIMESTAMP instead. + databricks.sql.TIMESTAMP instead. https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html """ diff --git a/src/databricks_sqlalchemy/base.py b/src/databricks_sqlalchemy/base.py index 94bce80..0cff681 100644 --- a/src/databricks_sqlalchemy/base.py +++ b/src/databricks_sqlalchemy/base.py @@ -2,7 +2,7 @@ import databricks_sqlalchemy._ddl as dialect_ddl_impl import databricks_sqlalchemy._types as dialect_type_impl -from databricks_sql_connector_core import sql +from databricks import sql from databricks_sqlalchemy._parse import ( _describe_table_extended_result_to_dict_list, _match_table_not_found_string, diff --git a/src/databricks_sqlalchemy/test/_extra.py b/src/databricks_sqlalchemy/test/_extra.py index 48cc0ec..2f3e7a7 100644 --- a/src/databricks_sqlalchemy/test/_extra.py +++ b/src/databricks_sqlalchemy/test/_extra.py @@ -15,7 +15,7 @@ _DateFixture, literal, ) -from databricks_sql_connector_core.sqlalchemy import TINYINT, TIMESTAMP +from databricks.sqlalchemy import TINYINT, TIMESTAMP class TinyIntegerTest(_LiteralRoundTripFixture, fixtures.TestBase): diff --git a/src/databricks_sqlalchemy/test/_unsupported.py b/src/databricks_sqlalchemy/test/_unsupported.py index c40c597..5d362d5 100644 --- a/src/databricks_sqlalchemy/test/_unsupported.py +++ b/src/databricks_sqlalchemy/test/_unsupported.py @@ -441,7 +441,7 @@ class DateTimeTZTest(DateTimeTZTest): """Test whether the sqlalchemy.DateTime() type can _optionally_ include timezone info. This dialect maps DateTime() → TIMESTAMP, which _always_ includes tzinfo. - Users can use databricks_sql_connector_core.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs + Users can use databricks.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs acknowledge this is expected for some dialects. https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime From 073781eaa60d66b9dda1950e191f48d64ea8fe00 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Tue, 24 Sep 2024 13:31:03 +0530 Subject: [PATCH 13/18] Minor refractors --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f4e477..6e0467c 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4 To install the dialect and its dependencies: ```shell -pip install databricks-sql-connector[databricks_sqlalchemy] +pip install databricks_sqlalchemy ``` If you also plan to use `alembic` you can alternatively run: ```shell -pip install databricks-sql-connector[alembic] +pip install alembic ``` ## Connection String From 52c0a29c1f26a8f088582f4f195af3acfc57012d Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Wed, 25 Sep 2024 12:15:43 +0530 Subject: [PATCH 14/18] Removed the Integration.yml file --- .github/workflows/integration.yml | 70 ------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index ead1751..0000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Integration - -on: - pull_request: - types: [ opened, synchronize, reopened ] - branches: [ main, PECO-1803 ] - workflow_dispatch: - -jobs: - build_and_test: - runs-on: ubuntu-latest - environment: azure-prod - env: - DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME }} - DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_CATALOG: ${{ secrets.DATABRICKS_CATALOG }} - DATABRICKS_SCHEMA : ${{ secrets.DATABRICKS_SCHEMA }} - DATABRICKS_USER: ${{ secrets.DATABRICKS_USER }} - - steps: - # Checkout your own repository - - name: Checkout Repository - uses: actions/checkout@v3 - - # Checkout the other repository - - name: Checkout Dependency Repository - uses: actions/checkout@v3 - with: - repository: jprakash-db/databricks-sql-python - path: databricks_sql_python - ref : jprakash-db/PECO-1803 - - # Set up Python - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - # Install Poetry - - name: Install Poetry - run: | - python -m pip install --upgrade pip - pip3 install poetry - python3 -m venv venv - - # Install the requirements of your repository - - name: Install Dependencies - run: | - source venv/bin/activate - poetry build - pip3 install dist/*.whl - - # Build the .whl file in the dependency repository - - name: Build Dependency Package - run: | - source venv/bin/activate - pip3 install databricks_sql_python/databricks_sql_connector_core/dist/*.whl - - # Run pytest to execute tests in your repository - - name: Run Tests - run: | - source venv/bin/activate - pip3 list - pip3 install pytest - - - name : Main Tests - run: | - source venv/bin/activate - pytest src/databricks_sqlalchemy/test_local From ef0ee615339eba5ce8fbc59c383e2f97ecfd9177 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Wed, 25 Sep 2024 22:33:51 +0530 Subject: [PATCH 15/18] Changed the folder structure to have databricks as the common namespace --- pyproject.toml | 15 +- .../sqlalchemy/README.sqlalchemy.md | 203 ++++++++++++++++++ src/databricks/sqlalchemy/README.tests.md | 44 ++++ src/databricks/sqlalchemy/__init__.py | 4 + .../sqlalchemy}/_ddl.py | 0 .../sqlalchemy}/_parse.py | 2 +- .../sqlalchemy}/_types.py | 1 - .../sqlalchemy}/base.py | 6 +- .../sqlalchemy}/py.typed | 0 src/databricks/sqlalchemy/pytest.ini | 4 + .../sqlalchemy}/requirements.py | 0 src/databricks/sqlalchemy/setup.cfg | 4 + .../sqlalchemy}/test/_extra.py | 0 .../sqlalchemy}/test/_future.py | 4 +- .../sqlalchemy}/test/_regression.py | 4 +- .../sqlalchemy}/test/_unsupported.py | 2 +- .../sqlalchemy}/test/conftest.py | 4 +- .../overrides/_componentreflectiontest.py | 0 .../sqlalchemy}/test/overrides/_ctetest.py | 0 .../sqlalchemy}/test/test_suite.py | 7 +- .../sqlalchemy}/test_local/__init__.py | 0 .../sqlalchemy}/test_local/conftest.py | 0 .../sqlalchemy}/test_local/e2e/MOCK_DATA.xlsx | Bin .../sqlalchemy}/test_local/e2e/test_basic.py | 4 +- .../sqlalchemy}/test_local/test_ddl.py | 0 .../sqlalchemy}/test_local/test_parsing.py | 2 +- .../sqlalchemy}/test_local/test_types.py | 4 +- src/databricks_sqlalchemy/__init__.py | 4 - src/databricks_sqlalchemy/pytest.ini | 4 - src/databricks_sqlalchemy/setup.cfg | 4 - src/databricks_sqlalchemy/test/__init__.py | 0 .../test/overrides/__init__.py | 0 .../test_local/e2e/__init__.py | 0 33 files changed, 285 insertions(+), 41 deletions(-) create mode 100644 src/databricks/sqlalchemy/README.sqlalchemy.md create mode 100644 src/databricks/sqlalchemy/README.tests.md create mode 100644 src/databricks/sqlalchemy/__init__.py rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/_ddl.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/_parse.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/_types.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/base.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/py.typed (100%) create mode 100644 src/databricks/sqlalchemy/pytest.ini rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/requirements.py (100%) create mode 100644 src/databricks/sqlalchemy/setup.cfg rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/_extra.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/_future.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/_regression.py (97%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/_unsupported.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/conftest.py (78%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/overrides/_componentreflectiontest.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/overrides/_ctetest.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test/test_suite.py (51%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/__init__.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/conftest.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/e2e/MOCK_DATA.xlsx (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/e2e/test_basic.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/test_ddl.py (100%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/test_parsing.py (99%) rename src/{databricks_sqlalchemy => databricks/sqlalchemy}/test_local/test_types.py (98%) delete mode 100644 src/databricks_sqlalchemy/__init__.py delete mode 100644 src/databricks_sqlalchemy/pytest.ini delete mode 100644 src/databricks_sqlalchemy/setup.cfg delete mode 100644 src/databricks_sqlalchemy/test/__init__.py delete mode 100644 src/databricks_sqlalchemy/test/overrides/__init__.py delete mode 100644 src/databricks_sqlalchemy/test_local/e2e/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 85ecbbe..a87460a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,11 +5,12 @@ description = "Databricks SQLAlchemy plugin for Python" authors = ["Databricks "] license = "Apache-2.0" readme = "README.md" -packages = [{ include = "databricks_sqlalchemy", from = "src" }] +packages = [{ include = "databricks", from = "src" }] include = ["CHANGELOG.md"] [tool.poetry.dependencies] python = "^3.8.0" +databricks_sql_connector_core = { version = ">=1.0.0"} sqlalchemy = { version = ">=2.0.21" } [tool.poetry.dev-dependencies] @@ -20,20 +21,16 @@ black = "^22.3.0" pytest-dotenv = "^0.5.2" [tool.poetry.urls] -"Homepage" = "https://github.com/databricks/databricks-sql-python" -"Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues" +"Homepage" = "https://github.com/databricks/databricks-sqlalchemy" +"Bug Tracker" = "https://github.com/databricks/databricks-sqlalchemy/issues" [tool.poetry.plugins."sqlalchemy.dialects"] -"databricks" = "databricks_sqlalchemy:DatabricksDialect" +"databricks" = "databricks.sqlalchemy:DatabricksDialect" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" -[tool.mypy] -ignore_missing_imports = "true" -exclude = ['ttypes\.py$', 'TCLIService\.py$'] - [tool.black] exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' # @@ -42,5 +39,5 @@ markers = {"reviewed" = "Test case has been reviewed by Databricks"} minversion = "6.0" log_cli = "false" log_cli_level = "INFO" -testpaths = ["tests", "databricks_sqlalchemy/src/databricks_sqlalchemy/test_local"] +testpaths = ["tests", "src/databricks/sqlalchemy/test_local"] env_files = ["test.env"] diff --git a/src/databricks/sqlalchemy/README.sqlalchemy.md b/src/databricks/sqlalchemy/README.sqlalchemy.md new file mode 100644 index 0000000..8aa5197 --- /dev/null +++ b/src/databricks/sqlalchemy/README.sqlalchemy.md @@ -0,0 +1,203 @@ +## Databricks dialect for SQLALchemy 2.0 + +The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. + +## Usage with SQLAlchemy <= 2.0 +A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. + + +## Installation + +To install the dialect and its dependencies: + +```shell +pip install databricks-sql-connector[sqlalchemy] +``` + +If you also plan to use `alembic` you can alternatively run: + +```shell +pip install databricks-sql-connector[alembic] +``` + +## Connection String + +Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: + +1. Host +2. HTTP Path for a compute resource +3. API access token +4. Initial catalog for the connection +5. Initial schema for the connection + +**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** + +For example: + +```python +import os +from sqlalchemy import create_engine + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") +catalog = os.getenv("DATABRICKS_CATALOG") +schema = os.getenv("DATABRICKS_SCHEMA") + +engine = create_engine( + f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" + ) +``` + +## Types + +The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. + +|SQLAlchemy Type|Databricks SQL Type| +|-|-| +[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) +[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| +[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) +[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) +[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| +[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) +[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| +[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) +[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) +[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| +[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| +[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) +[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) + +In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: + +- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) +- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) +- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) + + +### `LargeBinary()` and `PickleType()` + +Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. + +## `Enum()` and `CHECK` constraints + +Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. + +SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. + +### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` + +Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. + +The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. + +If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. + +_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ + +```python +from sqlalchemy import DateTime +from databricks.sqlalchemy import TIMESTAMP + +class SomeModel(Base): + some_date_without_timezone = DateTime() + some_date_with_timezone = TIMESTAMP() +``` + +### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` + +Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. + +### `Time()` + +Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). + +```python +from sqlalchemy import Time + +class SomeModel(Base): + time_tz = Time(timezone=True) + time_ntz = Time() +``` + + +# Usage Notes + +## `Identity()` and `autoincrement` + +Identity and generated value support is currently limited in this dialect. + +When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. + +Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. + +```python +from sqlalchemy import Identity, String + +class SomeModel(Base): + id = BigInteger(Identity()) + value = String() +``` + +When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). + +## Parameters + +`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. + +## Usage with pandas + +Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. + +### Read from Databricks SQL into pandas +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +with engine.connect() as conn: + # This will read the contents of `main.test.some_table` + df = pd.read_sql("some_table", conn) +``` + +### Write to Databricks SQL from pandas + +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +squares = [(i, i * i) for i in range(100)] +df = pd.DataFrame(data=squares,columns=['x','x_squared']) + +with engine.connect() as conn: + # This will write the contents of `df` to `main.test.squares` + df.to_sql('squares',conn) +``` + +## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) + +Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. + +When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. + +If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. + +```python +from sqlalchemy import Table, Column, ForeignKey, BigInteger, String + +users = Table( + "users", + metadata_obj, + Column("id", BigInteger, primary_key=True), + Column("name", String(), nullable=False), + Column("email", String()), + Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) +) +``` diff --git a/src/databricks/sqlalchemy/README.tests.md b/src/databricks/sqlalchemy/README.tests.md new file mode 100644 index 0000000..3ed92ab --- /dev/null +++ b/src/databricks/sqlalchemy/README.tests.md @@ -0,0 +1,44 @@ +## SQLAlchemy Dialect Compliance Test Suite with Databricks + +The contents of the `test/` directory follow the SQLAlchemy developers' [guidance] for running the reusable dialect compliance test suite. Since not every test in the suite is applicable to every dialect, two options are provided to skip tests: + +- Any test can be skipped by subclassing its parent class, re-declaring the test-case and adding a `pytest.mark.skip` directive. +- Any test that is decorated with a `@requires` decorator can be skipped by marking the indicated requirement as `.closed()` in `requirements.py` + +We prefer to skip test cases directly with the first method wherever possible. We only mark requirements as `closed()` if there is no easier option to avoid a test failure. This principally occurs in test cases where the same test in the suite is parametrized, and some parameter combinations are conditionally skipped depending on `requirements.py`. If we skip the entire test method, then we skip _all_ permutations, not just the combinations we don't support. + +## Regression, Unsupported, and Future test cases + +We maintain three files of test cases that we import from the SQLAlchemy source code: + +* **`_regression.py`** contains all the tests cases with tests that we expect to pass for our dialect. Each one is marked with `pytest.mark.reiewed` to indicate that we've evaluated it for relevance. This file only contains base class declarations. +* **`_unsupported.py`** contains test cases that fail because of missing features in Databricks. We mark them as skipped with a `SkipReason` enumeration. If Databricks comes to support these features, those test or entire classes can be moved to `_regression.py`. +* **`_future.py`** contains test cases that fail because of missing features in the dialect itself, but which _are_ supported by Databricks generally. We mark them as skipped with a `FutureFeature` enumeration. These are features that have not been prioritised or that do not violate our acceptance criteria. All of these test cases will eventually move to either `_regression.py`. + +In some cases, only certain tests in class should be skipped with a `SkipReason` or `FutureFeature` justification. In those cases, we import the class into `_regression.py`, then import it from there into one or both of `_future.py` and `_unsupported.py`. If a class needs to be "touched" by regression, unsupported, and future, the class will be imported in that order. If an entire class should be skipped, then we do not import it into `_regression.py` at all. + +We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dialect test fixtures but which are specific to Databricks (e.g TinyIntegerTest). + +## Running the reusable dialect tests + +``` +poetry shell +cd src/databricks/sqlalchemy/test +python -m pytest test_suite.py --dburi \ + "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" +``` + +Whatever schema you pass in the `dburi` argument should be empty. Some tests also require the presence of an empty schema named `test_schema`. Note that we plan to implement our own `provision.py` which SQLAlchemy can automatically use to create an empty schema for testing. But for now this is a manual process. + +You can run only reviewed tests by appending `-m "reviewed"` to the test runner invocation. + +You can run only the unreviewed tests by appending `-m "not reviewed"` instead. + +Note that because these tests depend on SQLAlchemy's custom pytest plugin, they are not discoverable by IDE-based test runners like VSCode or PyCharm and must be invoked from a CLI. + +## Running local unit and e2e tests + +Apart from the SQLAlchemy reusable suite, we maintain our own unit and e2e tests under the `test_local/` directory. These can be invoked from a VSCode or Pycharm since they don't depend on a custom pytest plugin. Due to pytest's lookup order, the `pytest.ini` which is required for running the reusable dialect tests, also conflicts with VSCode and Pycharm's default pytest implementation and overrides the settings in `pyproject.toml`. So to run these tests, you can delete or rename `pytest.ini`. + + +[guidance]: "https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_22/README.dialects.rst" diff --git a/src/databricks/sqlalchemy/__init__.py b/src/databricks/sqlalchemy/__init__.py new file mode 100644 index 0000000..2a17ac3 --- /dev/null +++ b/src/databricks/sqlalchemy/__init__.py @@ -0,0 +1,4 @@ +from databricks.sqlalchemy.base import DatabricksDialect +from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ + +__all__ = ["TINYINT", "TIMESTAMP", "TIMESTAMP_NTZ"] diff --git a/src/databricks_sqlalchemy/_ddl.py b/src/databricks/sqlalchemy/_ddl.py similarity index 100% rename from src/databricks_sqlalchemy/_ddl.py rename to src/databricks/sqlalchemy/_ddl.py diff --git a/src/databricks_sqlalchemy/_parse.py b/src/databricks/sqlalchemy/_parse.py similarity index 99% rename from src/databricks_sqlalchemy/_parse.py rename to src/databricks/sqlalchemy/_parse.py index e9498ec..6d38e1e 100644 --- a/src/databricks_sqlalchemy/_parse.py +++ b/src/databricks/sqlalchemy/_parse.py @@ -5,7 +5,7 @@ from sqlalchemy.engine import CursorResult from sqlalchemy.engine.interfaces import ReflectedColumn -from databricks_sqlalchemy import _types as type_overrides +from databricks.sqlalchemy import _types as type_overrides """ This module contains helper functions that can parse the contents diff --git a/src/databricks_sqlalchemy/_types.py b/src/databricks/sqlalchemy/_types.py similarity index 99% rename from src/databricks_sqlalchemy/_types.py rename to src/databricks/sqlalchemy/_types.py index f24c12a..5fc14a7 100644 --- a/src/databricks_sqlalchemy/_types.py +++ b/src/databricks/sqlalchemy/_types.py @@ -9,7 +9,6 @@ from databricks.sql.utils import ParamEscaper - def process_literal_param_hack(value: Any): """This method is supposed to accept a Python type and return a string representation of that type. But due to some weirdness in the way SQLAlchemy's literal rendering works, we have to return diff --git a/src/databricks_sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py similarity index 99% rename from src/databricks_sqlalchemy/base.py rename to src/databricks/sqlalchemy/base.py index 0cff681..9148de7 100644 --- a/src/databricks_sqlalchemy/base.py +++ b/src/databricks/sqlalchemy/base.py @@ -1,9 +1,9 @@ from typing import Any, List, Optional, Dict, Union -import databricks_sqlalchemy._ddl as dialect_ddl_impl -import databricks_sqlalchemy._types as dialect_type_impl +import databricks.sqlalchemy._ddl as dialect_ddl_impl +import databricks.sqlalchemy._types as dialect_type_impl from databricks import sql -from databricks_sqlalchemy._parse import ( +from databricks.sqlalchemy._parse import ( _describe_table_extended_result_to_dict_list, _match_table_not_found_string, build_fk_dict, diff --git a/src/databricks_sqlalchemy/py.typed b/src/databricks/sqlalchemy/py.typed similarity index 100% rename from src/databricks_sqlalchemy/py.typed rename to src/databricks/sqlalchemy/py.typed diff --git a/src/databricks/sqlalchemy/pytest.ini b/src/databricks/sqlalchemy/pytest.ini new file mode 100644 index 0000000..ab89d17 --- /dev/null +++ b/src/databricks/sqlalchemy/pytest.ini @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks.sqlalchemy.requirements:Requirements +profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py similarity index 100% rename from src/databricks_sqlalchemy/requirements.py rename to src/databricks/sqlalchemy/requirements.py diff --git a/src/databricks/sqlalchemy/setup.cfg b/src/databricks/sqlalchemy/setup.cfg new file mode 100644 index 0000000..ab89d17 --- /dev/null +++ b/src/databricks/sqlalchemy/setup.cfg @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks.sqlalchemy.requirements:Requirements +profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/test/_extra.py b/src/databricks/sqlalchemy/test/_extra.py similarity index 100% rename from src/databricks_sqlalchemy/test/_extra.py rename to src/databricks/sqlalchemy/test/_extra.py diff --git a/src/databricks_sqlalchemy/test/_future.py b/src/databricks/sqlalchemy/test/_future.py similarity index 99% rename from src/databricks_sqlalchemy/test/_future.py rename to src/databricks/sqlalchemy/test/_future.py index 76b4f7a..6e470f6 100644 --- a/src/databricks_sqlalchemy/test/_future.py +++ b/src/databricks/sqlalchemy/test/_future.py @@ -3,13 +3,13 @@ from enum import Enum import pytest -from databricks_sqlalchemy.test._regression import ( +from databricks.sqlalchemy.test._regression import ( ExpandingBoundInTest, IdentityAutoincrementTest, LikeFunctionsTest, NormalizedNameTest, ) -from databricks_sqlalchemy.test._unsupported import ( +from databricks.sqlalchemy.test._unsupported import ( ComponentReflectionTest, ComponentReflectionTestExtra, CTETest, diff --git a/src/databricks_sqlalchemy/test/_regression.py b/src/databricks/sqlalchemy/test/_regression.py similarity index 97% rename from src/databricks_sqlalchemy/test/_regression.py rename to src/databricks/sqlalchemy/test/_regression.py index 1d2e6d0..4dbc5ec 100644 --- a/src/databricks_sqlalchemy/test/_regression.py +++ b/src/databricks/sqlalchemy/test/_regression.py @@ -51,8 +51,8 @@ ValuesExpressionTest, ) -from databricks_sqlalchemy.test.overrides._ctetest import CTETest -from databricks_sqlalchemy.test.overrides._componentreflectiontest import ( +from databricks.sqlalchemy.test.overrides._ctetest import CTETest +from databricks.sqlalchemy.test.overrides._componentreflectiontest import ( ComponentReflectionTest, ) diff --git a/src/databricks_sqlalchemy/test/_unsupported.py b/src/databricks/sqlalchemy/test/_unsupported.py similarity index 99% rename from src/databricks_sqlalchemy/test/_unsupported.py rename to src/databricks/sqlalchemy/test/_unsupported.py index 5d362d5..c1f8120 100644 --- a/src/databricks_sqlalchemy/test/_unsupported.py +++ b/src/databricks/sqlalchemy/test/_unsupported.py @@ -3,7 +3,7 @@ from enum import Enum import pytest -from databricks_sqlalchemy.test._regression import ( +from databricks.sqlalchemy.test._regression import ( ComponentReflectionTest, ComponentReflectionTestExtra, CTETest, diff --git a/src/databricks_sqlalchemy/test/conftest.py b/src/databricks/sqlalchemy/test/conftest.py similarity index 78% rename from src/databricks_sqlalchemy/test/conftest.py rename to src/databricks/sqlalchemy/test/conftest.py index 81e9479..ea43e8d 100644 --- a/src/databricks_sqlalchemy/test/conftest.py +++ b/src/databricks/sqlalchemy/test/conftest.py @@ -1,12 +1,12 @@ from sqlalchemy.dialects import registry import pytest -registry.register("databricks", "databricks_sqlalchemy", "DatabricksDialect") +registry.register("databricks", "databricks.sqlalchemy", "DatabricksDialect") # sqlalchemy's dialect-testing machinery wants an entry like this. # This seems to be based around dialects maybe having multiple drivers # and wanting to test driver-specific URLs, but doesn't seem to make # much sense for dialects with only one driver. -registry.register("databricks.databricks", "databricks_sqlalchemy", "DatabricksDialect") +registry.register("databricks.databricks", "databricks.sqlalchemy", "DatabricksDialect") pytest.register_assert_rewrite("sqlalchemy.testing.assertions") diff --git a/src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py b/src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py similarity index 100% rename from src/databricks_sqlalchemy/test/overrides/_componentreflectiontest.py rename to src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py diff --git a/src/databricks_sqlalchemy/test/overrides/_ctetest.py b/src/databricks/sqlalchemy/test/overrides/_ctetest.py similarity index 100% rename from src/databricks_sqlalchemy/test/overrides/_ctetest.py rename to src/databricks/sqlalchemy/test/overrides/_ctetest.py diff --git a/src/databricks_sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py similarity index 51% rename from src/databricks_sqlalchemy/test/test_suite.py rename to src/databricks/sqlalchemy/test/test_suite.py index 02a4208..2b40a43 100644 --- a/src/databricks_sqlalchemy/test/test_suite.py +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -7,6 +7,7 @@ # type: ignore # fmt: off from sqlalchemy.testing.suite import * -from databricks_sqlalchemy.test._regression import * -from databricks_sqlalchemy.test._unsupported import * -from databricks_sqlalchemy.test._future import * +from databricks.sqlalchemy.test._regression import * +from databricks.sqlalchemy.test._unsupported import * +from databricks.sqlalchemy.test._future import * +from databricks.sqlalchemy.test._extra import TinyIntegerTest, DateTimeTZTestCustom diff --git a/src/databricks_sqlalchemy/test_local/__init__.py b/src/databricks/sqlalchemy/test_local/__init__.py similarity index 100% rename from src/databricks_sqlalchemy/test_local/__init__.py rename to src/databricks/sqlalchemy/test_local/__init__.py diff --git a/src/databricks_sqlalchemy/test_local/conftest.py b/src/databricks/sqlalchemy/test_local/conftest.py similarity index 100% rename from src/databricks_sqlalchemy/test_local/conftest.py rename to src/databricks/sqlalchemy/test_local/conftest.py diff --git a/src/databricks_sqlalchemy/test_local/e2e/MOCK_DATA.xlsx b/src/databricks/sqlalchemy/test_local/e2e/MOCK_DATA.xlsx similarity index 100% rename from src/databricks_sqlalchemy/test_local/e2e/MOCK_DATA.xlsx rename to src/databricks/sqlalchemy/test_local/e2e/MOCK_DATA.xlsx diff --git a/src/databricks_sqlalchemy/test_local/e2e/test_basic.py b/src/databricks/sqlalchemy/test_local/e2e/test_basic.py similarity index 99% rename from src/databricks_sqlalchemy/test_local/e2e/test_basic.py rename to src/databricks/sqlalchemy/test_local/e2e/test_basic.py index 215d3eb..ce0b5d8 100644 --- a/src/databricks_sqlalchemy/test_local/e2e/test_basic.py +++ b/src/databricks/sqlalchemy/test_local/e2e/test_basic.py @@ -144,7 +144,7 @@ def test_pandas_upload(db_engine, metadata_obj): SCHEMA = "default" try: df = pd.read_excel( - "databricks_sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" + "src/databricks/sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" ) df.to_sql( "mock_data", @@ -472,7 +472,7 @@ def get_conn_user_agent(conn): @pytest.fixture def sample_table(metadata_obj: MetaData, db_engine: Engine): """This fixture creates a sample table and cleans it up after the test is complete.""" - from databricks_sqlalchemy._parse import GET_COLUMNS_TYPE_MAP + from databricks.sqlalchemy._parse import GET_COLUMNS_TYPE_MAP table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) diff --git a/src/databricks_sqlalchemy/test_local/test_ddl.py b/src/databricks/sqlalchemy/test_local/test_ddl.py similarity index 100% rename from src/databricks_sqlalchemy/test_local/test_ddl.py rename to src/databricks/sqlalchemy/test_local/test_ddl.py diff --git a/src/databricks_sqlalchemy/test_local/test_parsing.py b/src/databricks/sqlalchemy/test_local/test_parsing.py similarity index 99% rename from src/databricks_sqlalchemy/test_local/test_parsing.py rename to src/databricks/sqlalchemy/test_local/test_parsing.py index 7616142..c8ab443 100644 --- a/src/databricks_sqlalchemy/test_local/test_parsing.py +++ b/src/databricks/sqlalchemy/test_local/test_parsing.py @@ -1,5 +1,5 @@ import pytest -from databricks_sqlalchemy._parse import ( +from databricks.sqlalchemy._parse import ( extract_identifiers_from_string, extract_identifier_groups_from_string, extract_three_level_identifier_from_constraint_string, diff --git a/src/databricks_sqlalchemy/test_local/test_types.py b/src/databricks/sqlalchemy/test_local/test_types.py similarity index 98% rename from src/databricks_sqlalchemy/test_local/test_types.py rename to src/databricks/sqlalchemy/test_local/test_types.py index 4d61c55..b91217e 100644 --- a/src/databricks_sqlalchemy/test_local/test_types.py +++ b/src/databricks/sqlalchemy/test_local/test_types.py @@ -3,8 +3,8 @@ import pytest import sqlalchemy -from databricks_sqlalchemy.base import DatabricksDialect -from databricks_sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ +from databricks.sqlalchemy.base import DatabricksDialect +from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ class DatabricksDataType(enum.Enum): diff --git a/src/databricks_sqlalchemy/__init__.py b/src/databricks_sqlalchemy/__init__.py deleted file mode 100644 index 35ea050..0000000 --- a/src/databricks_sqlalchemy/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from databricks_sqlalchemy.base import DatabricksDialect -from databricks_sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ - -__all__ = ["TINYINT", "TIMESTAMP", "TIMESTAMP_NTZ"] diff --git a/src/databricks_sqlalchemy/pytest.ini b/src/databricks_sqlalchemy/pytest.ini deleted file mode 100644 index 4ce8d29..0000000 --- a/src/databricks_sqlalchemy/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ - -[sqla_testing] -requirement_cls=databricks_sqlalchemy.requirements:Requirements -profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/setup.cfg b/src/databricks_sqlalchemy/setup.cfg deleted file mode 100644 index 4ce8d29..0000000 --- a/src/databricks_sqlalchemy/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ - -[sqla_testing] -requirement_cls=databricks_sqlalchemy.requirements:Requirements -profile_file=profiles.txt diff --git a/src/databricks_sqlalchemy/test/__init__.py b/src/databricks_sqlalchemy/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/databricks_sqlalchemy/test/overrides/__init__.py b/src/databricks_sqlalchemy/test/overrides/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/databricks_sqlalchemy/test_local/e2e/__init__.py b/src/databricks_sqlalchemy/test_local/e2e/__init__.py deleted file mode 100644 index e69de29..0000000 From a317b3990cbd3eb35b1eb9b2253d7dd952adccc7 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Wed, 25 Sep 2024 22:36:31 +0530 Subject: [PATCH 16/18] Minor Fix --- README.tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.tests.md b/README.tests.md index 5d72711..3ed92ab 100644 --- a/README.tests.md +++ b/README.tests.md @@ -23,7 +23,7 @@ We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dia ``` poetry shell -cd src/databricks_sqlalchemy/test +cd src/databricks/sqlalchemy/test python -m pytest test_suite.py --dburi \ "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" ``` From 5ac49fe712735e119fc6fff0f16b2de863f5ec73 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Wed, 25 Sep 2024 22:43:41 +0530 Subject: [PATCH 17/18] Removed the README.md from within the sqlalchemy folder as it is present globally --- .../sqlalchemy/README.sqlalchemy.md | 203 ------------------ src/databricks/sqlalchemy/README.tests.md | 44 ---- 2 files changed, 247 deletions(-) delete mode 100644 src/databricks/sqlalchemy/README.sqlalchemy.md delete mode 100644 src/databricks/sqlalchemy/README.tests.md diff --git a/src/databricks/sqlalchemy/README.sqlalchemy.md b/src/databricks/sqlalchemy/README.sqlalchemy.md deleted file mode 100644 index 8aa5197..0000000 --- a/src/databricks/sqlalchemy/README.sqlalchemy.md +++ /dev/null @@ -1,203 +0,0 @@ -## Databricks dialect for SQLALchemy 2.0 - -The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. - -## Usage with SQLAlchemy <= 2.0 -A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. - - -## Installation - -To install the dialect and its dependencies: - -```shell -pip install databricks-sql-connector[sqlalchemy] -``` - -If you also plan to use `alembic` you can alternatively run: - -```shell -pip install databricks-sql-connector[alembic] -``` - -## Connection String - -Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: - -1. Host -2. HTTP Path for a compute resource -3. API access token -4. Initial catalog for the connection -5. Initial schema for the connection - -**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** - -For example: - -```python -import os -from sqlalchemy import create_engine - -host = os.getenv("DATABRICKS_SERVER_HOSTNAME") -http_path = os.getenv("DATABRICKS_HTTP_PATH") -access_token = os.getenv("DATABRICKS_TOKEN") -catalog = os.getenv("DATABRICKS_CATALOG") -schema = os.getenv("DATABRICKS_SCHEMA") - -engine = create_engine( - f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" - ) -``` - -## Types - -The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. - -|SQLAlchemy Type|Databricks SQL Type| -|-|-| -[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) -[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| -[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) -[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) -[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| -[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) -[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| -[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) -[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) -[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| -[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| -[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) -[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| -[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) - -In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: - -- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) -- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) -- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) - - -### `LargeBinary()` and `PickleType()` - -Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. - -## `Enum()` and `CHECK` constraints - -Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. - -SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. - -### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` - -Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. - -The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. - -If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. - -_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ - -```python -from sqlalchemy import DateTime -from databricks.sqlalchemy import TIMESTAMP - -class SomeModel(Base): - some_date_without_timezone = DateTime() - some_date_with_timezone = TIMESTAMP() -``` - -### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` - -Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. - -### `Time()` - -Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). - -```python -from sqlalchemy import Time - -class SomeModel(Base): - time_tz = Time(timezone=True) - time_ntz = Time() -``` - - -# Usage Notes - -## `Identity()` and `autoincrement` - -Identity and generated value support is currently limited in this dialect. - -When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. - -Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. - -```python -from sqlalchemy import Identity, String - -class SomeModel(Base): - id = BigInteger(Identity()) - value = String() -``` - -When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). - -## Parameters - -`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. - -## Usage with pandas - -Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. - -### Read from Databricks SQL into pandas -```python -from sqlalchemy import create_engine -import pandas as pd - -engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") -with engine.connect() as conn: - # This will read the contents of `main.test.some_table` - df = pd.read_sql("some_table", conn) -``` - -### Write to Databricks SQL from pandas - -```python -from sqlalchemy import create_engine -import pandas as pd - -engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") -squares = [(i, i * i) for i in range(100)] -df = pd.DataFrame(data=squares,columns=['x','x_squared']) - -with engine.connect() as conn: - # This will write the contents of `df` to `main.test.squares` - df.to_sql('squares',conn) -``` - -## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) - -Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. - -When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. - -If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. - -```python -from sqlalchemy import Table, Column, ForeignKey, BigInteger, String - -users = Table( - "users", - metadata_obj, - Column("id", BigInteger, primary_key=True), - Column("name", String(), nullable=False), - Column("email", String()), - Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) -) -``` diff --git a/src/databricks/sqlalchemy/README.tests.md b/src/databricks/sqlalchemy/README.tests.md deleted file mode 100644 index 3ed92ab..0000000 --- a/src/databricks/sqlalchemy/README.tests.md +++ /dev/null @@ -1,44 +0,0 @@ -## SQLAlchemy Dialect Compliance Test Suite with Databricks - -The contents of the `test/` directory follow the SQLAlchemy developers' [guidance] for running the reusable dialect compliance test suite. Since not every test in the suite is applicable to every dialect, two options are provided to skip tests: - -- Any test can be skipped by subclassing its parent class, re-declaring the test-case and adding a `pytest.mark.skip` directive. -- Any test that is decorated with a `@requires` decorator can be skipped by marking the indicated requirement as `.closed()` in `requirements.py` - -We prefer to skip test cases directly with the first method wherever possible. We only mark requirements as `closed()` if there is no easier option to avoid a test failure. This principally occurs in test cases where the same test in the suite is parametrized, and some parameter combinations are conditionally skipped depending on `requirements.py`. If we skip the entire test method, then we skip _all_ permutations, not just the combinations we don't support. - -## Regression, Unsupported, and Future test cases - -We maintain three files of test cases that we import from the SQLAlchemy source code: - -* **`_regression.py`** contains all the tests cases with tests that we expect to pass for our dialect. Each one is marked with `pytest.mark.reiewed` to indicate that we've evaluated it for relevance. This file only contains base class declarations. -* **`_unsupported.py`** contains test cases that fail because of missing features in Databricks. We mark them as skipped with a `SkipReason` enumeration. If Databricks comes to support these features, those test or entire classes can be moved to `_regression.py`. -* **`_future.py`** contains test cases that fail because of missing features in the dialect itself, but which _are_ supported by Databricks generally. We mark them as skipped with a `FutureFeature` enumeration. These are features that have not been prioritised or that do not violate our acceptance criteria. All of these test cases will eventually move to either `_regression.py`. - -In some cases, only certain tests in class should be skipped with a `SkipReason` or `FutureFeature` justification. In those cases, we import the class into `_regression.py`, then import it from there into one or both of `_future.py` and `_unsupported.py`. If a class needs to be "touched" by regression, unsupported, and future, the class will be imported in that order. If an entire class should be skipped, then we do not import it into `_regression.py` at all. - -We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dialect test fixtures but which are specific to Databricks (e.g TinyIntegerTest). - -## Running the reusable dialect tests - -``` -poetry shell -cd src/databricks/sqlalchemy/test -python -m pytest test_suite.py --dburi \ - "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" -``` - -Whatever schema you pass in the `dburi` argument should be empty. Some tests also require the presence of an empty schema named `test_schema`. Note that we plan to implement our own `provision.py` which SQLAlchemy can automatically use to create an empty schema for testing. But for now this is a manual process. - -You can run only reviewed tests by appending `-m "reviewed"` to the test runner invocation. - -You can run only the unreviewed tests by appending `-m "not reviewed"` instead. - -Note that because these tests depend on SQLAlchemy's custom pytest plugin, they are not discoverable by IDE-based test runners like VSCode or PyCharm and must be invoked from a CLI. - -## Running local unit and e2e tests - -Apart from the SQLAlchemy reusable suite, we maintain our own unit and e2e tests under the `test_local/` directory. These can be invoked from a VSCode or Pycharm since they don't depend on a custom pytest plugin. Due to pytest's lookup order, the `pytest.ini` which is required for running the reusable dialect tests, also conflicts with VSCode and Pycharm's default pytest implementation and overrides the settings in `pyproject.toml`. So to run these tests, you can delete or rename `pytest.ini`. - - -[guidance]: "https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_22/README.dialects.rst" From 144907c782abdaa497d256023fa0facb6b6492e1 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Tue, 8 Oct 2024 11:26:20 +0530 Subject: [PATCH 18/18] Fixed LICENSE and other issues --- CHANGELOG.md | 274 ++++++++++++++++++ CONTRIBUTING.md | 142 +++++++++ LICENSE | 201 +++++++++++++ README.md | 4 +- .../dependency_test}/test_dependency.py | 0 5 files changed, 619 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE rename {dependency_test => src/databricks/sqlalchemy/dependency_test}/test_dependency.py (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4fca204 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,274 @@ +# Release History + +# 3.3.0 (2024-07-18) + +- Don't retry requests that fail with HTTP code 401 (databricks/databricks-sql-python#408 by @Hodnebo) +- Remove username/password (aka "basic") auth option (databricks/databricks-sql-python#409 by @jackyhu-db) +- Refactor CloudFetch handler to fix numerous issues with it (databricks/databricks-sql-python#405 by @kravets-levko) +- Add option to disable SSL verification for CloudFetch links (databricks/databricks-sql-python#414 by @kravets-levko) + +Databricks-managed passwords reached end of life on July 10, 2024. Therefore, Basic auth support was removed from +the library. See https://docs.databricks.com/en/security/auth-authz/password-deprecation.html + +The existing option `_tls_no_verify=True` of `sql.connect(...)` will now also disable SSL cert verification +(but not the SSL itself) for CloudFetch links. This option should be used as a workaround only, when other ways +to fix SSL certificate errors didn't work. + +# 3.2.0 (2024-06-06) + +- Update proxy authentication (databricks/databricks-sql-python#354 by @amir-haroun) +- Relax `pyarrow` pin (databricks/databricks-sql-python#389 by @dhirschfeld) +- Fix error logging in OAuth manager (databricks/databricks-sql-python#269 by @susodapop) +- SQLAlchemy: enable delta.feature.allowColumnDefaults for all tables (databricks/databricks-sql-python#343 by @dhirschfeld) +- Update `thrift` dependency (databricks/databricks-sql-python#397 by @m1n0) + +# 3.1.2 (2024-04-18) + +- Remove broken cookie code (#379) +- Small typing fixes (#382, #384 thanks @wyattscarpenter) + +# 3.1.1 (2024-03-19) + +- Don't retry requests that fail with code 403 (#373) +- Assume a default retry-after for 429/503 (#371) +- Fix boolean literals (#357) + +# 3.1.0 (2024-02-16) + +- Revert retry-after behavior to be exponential backoff (#349) +- Support Databricks OAuth on Azure (#351) +- Support Databricks OAuth on GCP (#338) + +# 3.0.3 (2024-02-02) + +- Revised docstrings and examples for OAuth (#339) +- Redact the URL query parameters from the urllib3.connectionpool logs (#341) + +# 3.0.2 (2024-01-25) + +- SQLAlchemy dialect now supports table and column comments (thanks @cbornet!) +- Fix: SQLAlchemy dialect now correctly reflects TINYINT types (thanks @TimTheinAtTabs!) +- Fix: `server_hostname` URIs that included `https://` would raise an exception +- Other: pinned to `pandas<=2.1` and `urllib3>=1.26` to avoid runtime errors in dbt-databricks (#330) + +## 3.0.1 (2023-12-01) + +- Other: updated docstring comment about default parameterization approach (#287) +- Other: added tests for reading complex types and revised docstrings and type hints (#293) +- Fix: SQLAlchemy dialect raised DeprecationWarning due to `dbapi` classmethod (#294) +- Fix: SQLAlchemy dialect could not reflect TIMESTAMP_NTZ columns (#296) + +## 3.0.0 (2023-11-17) + +- Remove support for Python 3.7 +- Add support for native parameterized SQL queries. Requires DBR 14.2 and above. See docs/parameters.md for more info. +- Completely rewritten SQLAlchemy dialect + - Adds support for SQLAlchemy >= 2.0 and drops support for SQLAlchemy 1.x + - Full e2e test coverage of all supported features + - Detailed usage notes in `README.sqlalchemy.md` + - Adds support for: + - New types: `TIME`, `TIMESTAMP`, `TIMESTAMP_NTZ`, `TINYINT` + - `Numeric` type scale and precision, like `Numeric(10,2)` + - Reading and writing `PrimaryKeyConstraint` and `ForeignKeyConstraint` + - Reading and writing composite keys + - Reading and writing from views + - Writing `Identity` to tables (i.e. autoincrementing primary keys) + - `LIMIT` and `OFFSET` for paging through results + - Caching metadata calls +- Enable cloud fetch by default. To disable, set `use_cloud_fetch=False` when building `databricks.sql.client`. +- Add integration tests for Databricks UC Volumes ingestion queries +- Retries: + - Add `_retry_max_redirects` config + - Set `_enable_v3_retries=True` and warn if users override it +- Security: bump minimum pyarrow version to 14.0.1 (CVE-2023-47248) + +## 2.9.3 (2023-08-24) + +- Fix: Connections failed when urllib3~=1.0.0 is installed (#206) + +## 2.9.2 (2023-08-17) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed. The log changes are incorporated into version 2.9.3 and greater.** + +- Other: Add `examples/v3_retries_query_execute.py` (#199) +- Other: suppress log message when `_enable_v3_retries` is not `True` (#199) +- Other: make this connector backwards compatible with `urllib3>=1.0.0` (#197) + +## 2.9.1 (2023-08-11) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed.** + +- Other: Explicitly pin urllib3 to ^2.0.0 (#191) + +## 2.9.0 (2023-08-10) + +- Replace retry handling with DatabricksRetryPolicy. This is disabled by default. To enable, set `_enable_v3_retries=True` when creating `databricks.sql.client` (#182) +- Other: Fix typo in README quick start example (#186) +- Other: Add autospec to Client mocks and tidy up `make_request` (#188) + +## 2.8.0 (2023-07-21) + +- Add support for Cloud Fetch. Disabled by default. Set `use_cloud_fetch=True` when building `databricks.sql.client` to enable it (#146, #151, #154) +- SQLAlchemy has_table function now honours schema= argument and adds catalog= argument (#174) +- SQLAlchemy set non_native_boolean_check_constraint False as it's not supported by Databricks (#120) +- Fix: Revised SQLAlchemy dialect and examples for compatibility with SQLAlchemy==1.3.x (#173) +- Fix: oauth would fail if expired credentials appeared in ~/.netrc (#122) +- Fix: Python HTTP proxies were broken after switch to urllib3 (#158) +- Other: remove unused import in SQLAlchemy dialect +- Other: Relax pandas dependency constraint to allow ^2.0.0 (#164) +- Other: Connector now logs operation handle guids as hexadecimal instead of bytes (#170) +- Other: test_socket_timeout_user_defined e2e test was broken (#144) + +## 2.7.0 (2023-06-26) + +- Fix: connector raised exception when calling close() on a closed Thrift session +- Improve e2e test development ergonomics +- Redact logged thrift responses by default +- Add support for OAuth on Databricks Azure + +## 2.6.2 (2023-06-14) + +- Fix: Retry GetOperationStatus requests for http errors + +## 2.6.1 (2023-06-08) + +- Fix: http.client would raise a BadStatusLine exception in some cases + +## 2.6.0 (2023-06-07) + +- Add support for HTTP 1.1 connections (connection pools) +- Add a default socket timeout for thrift RPCs + +## 2.5.2 (2023-05-08) + +- Fix: SQLAlchemy adapter could not reflect TIMESTAMP or DATETIME columns +- Other: Relax pandas and alembic dependency specifications + +## 2.5.1 (2023-04-28) + +- Other: Relax sqlalchemy required version as it was unecessarily strict. + +## 2.5.0 (2023-04-14) + +- Add support for External Auth providers +- Fix: Python HTTP proxies were broken +- Other: All Thrift requests that timeout during connection will be automatically retried + +## 2.4.1 (2023-03-21) + +- Less strict numpy and pyarrow dependencies +- Update examples in README to use security best practices +- Update docstring for client.execute() for clarity + +## 2.4.0 (2023-02-21) + +- Improve compatibility when installed alongside other Databricks namespace Python packages +- Add SQLAlchemy dialect + +## 2.3.0 (2023-01-10) + +- Support staging ingestion commands for DBR 12+ + +## 2.2.2 (2023-01-03) + +- Support custom oauth client id and redirect port +- Fix: Add none check on \_oauth_persistence in DatabricksOAuthProvider + +## 2.2.1 (2022-11-29) + +- Add support for Python 3.11 + +## 2.2.0 (2022-11-15) + +- Bump thrift version to address https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-13949 +- Add support for lz4 compression + +## 2.1.0 (2022-09-30) + +- Introduce experimental OAuth support while Bring Your Own IDP is in Public Preview on AWS +- Add functional examples + +## 2.0.5 (2022-08-23) + +- Fix: closing a connection now closes any open cursors from that connection at the server +- Other: Add project links to pyproject.toml (helpful for visitors from PyPi) + +## 2.0.4 (2022-08-17) + +- Add support for Python 3.10 +- Add unit test matrix for supported Python versions + +Huge thanks to @dbaxa for contributing this change! + +## 2.0.3 (2022-08-05) + +- Add retry logic for `GetOperationStatus` requests that fail with an `OSError` +- Reorganised code to use Poetry for dependency management. + +## 2.0.2 (2022-05-04) + +- Better exception handling in automatic connection close + +## 2.0.1 (2022-04-21) + +- Fixed Pandas dependency in setup.cfg to be >= 1.2.0 + +## 2.0.0 (2022-04-19) + +- Initial stable release of V2 +- Added better support for complex types, so that in Databricks runtime 10.3+, Arrays, Maps and Structs will get + deserialized as lists, lists of tuples and dicts, respectively. +- Changed the name of the metadata arg to http_headers + +## 2.0.b2 (2022-04-04) + +- Change import of collections.Iterable to collections.abc.Iterable to make the library compatible with Python 3.10 +- Fixed bug with .tables method so that .tables works as expected with Unity-Catalog enabled endpoints + +## 2.0.0b1 (2022-03-04) + +- Fix packaging issue (dependencies were not being installed properly) +- Fetching timestamp results will now return aware instead of naive timestamps +- The client will now default to using simplified error messages + +## 2.0.0b (2022-02-08) + +- Initial beta release of V2. V2 is an internal re-write of large parts of the connector to use Databricks edge features. All public APIs from V1 remain. +- Added Unity Catalog support (pass catalog and / or schema key word args to the .connect method to select initial schema and catalog) + +--- + +**Note**: The code for versions prior to `v2.0.0b` is not contained in this repository. The below entries are included for reference only. + +--- + +## 1.0.0 (2022-01-20) + +- Add operations for retrieving metadata +- Add the ability to access columns by name on result rows +- Add the ability to provide configuration settings on connect + +## 0.9.4 (2022-01-10) + +- Improved logging and error messages. + +## 0.9.3 (2021-12-08) + +- Add retries for 429 and 503 HTTP responses. + +## 0.9.2 (2021-12-02) + +- (Bug fix) Increased Thrift requirement from 0.10.0 to 0.13.0 as 0.10.0 was in fact incompatible +- (Bug fix) Fixed error message after query execution failed -SQLSTATE and Error message were misplaced + +## 0.9.1 (2021-09-01) + +- Public Preview release, Experimental tag removed +- minor updates in internal build/packaging +- no functional changes + +## 0.9.0 (2021-08-04) + +- initial (Experimental) release of pyhive-forked connector +- Python DBAPI 2.0 (PEP-0249), thrift based +- see docs for more info: https://docs.databricks.com/dev-tools/python-sql-connector.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..3836030 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,142 @@ +# Contributing Guide + +We happily welcome contributions to the `databricks-sqlalchemy` package. We use [GitHub Issues](https://github.com/databricks/databricks-sqlalchemy/issues) to track community reported issues and [GitHub Pull Requests](https://github.com/databricks/databricks-sqlalchemy/pulls) for accepting changes. + +Contributions are licensed on a license-in/license-out basis. + +## Communication +Before starting work on a major feature, please reach out to us via GitHub, Slack, email, etc. We will make sure no one else is already working on it and ask you to open a GitHub issue. +A "major feature" is defined as any change that is > 100 LOC altered (not including tests), or changes any user-facing behavior. +We will use the GitHub issue to discuss the feature and come to agreement. +This is to prevent your time being wasted, as well as ours. +The GitHub review process for major features is also important so that organizations with commit access can come to agreement on design. +If it is appropriate to write a design document, the document must be hosted either in the GitHub tracking issue, or linked to from the issue and hosted in a world-readable location. +Specifically, if the goal is to add a new extension, please read the extension policy. +Small patches and bug fixes don't need prior communication. + +## Coding Style +We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) with one exception: lines can be up to 100 characters in length, not 79. + +## Sign your work +The sign-off is a simple line at the end of the explanation for the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify the below (from developercertificate.org): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +Then you just add a line to every git commit message: + +``` +Signed-off-by: Joe Smith +Use your real name (sorry, no pseudonyms or anonymous contributions.) +``` + +If you set your `user.name` and `user.email` git configs, you can sign your commit automatically with `git commit -s`. + +## Set up your environment + +This project uses [Poetry](https://python-poetry.org/) for dependency management, tests, and linting. + +1. Clone this respository +2. Run `poetry install` + +### Run tests + +We use [Pytest](https://docs.pytest.org/en/7.1.x/) as our test runner. Invoke it with `poetry run python -m pytest`, all other arguments are passed directly to `pytest`. + +#### Unit tests + +Unit tests do not require a Databricks account. + +```bash +poetry run python -m pytest src/databricks/sqlalchemy/test +``` +#### Only a specific test file + +```bash +poetry run python -m pytest src/databricks/sqlalchemy/test/test_suite.py +``` + +#### e2e Tests + +End-to-end tests require a Databricks account. Before you can run them, you must set connection details for a Databricks SQL endpoint in your environment: + +```bash +export host="" +export http_path="" +export access_token="" +export catalog="" +export schema="" +``` + +Or you can write these into a file called `test.env` in the root of the repository: + +``` +host="****.cloud.databricks.com" +http_path="/sql/1.0/warehouses/***" +access_token="dapi***" +staging_ingestion_user="***@example.com" +``` + +To see logging output from pytest while running tests, set `log_cli = "true"` under `tool.pytest.ini_options` in `pyproject.toml`. You can also set `log_cli_level` to any of the default Python log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` + +#### SQLAlchemy dialect tests + +See README.tests.md for details. + +### Code formatting + +This project uses [Black](https://pypi.org/project/black/). + +``` +poetry run python3 -m black src --check +``` + +Remove the `--check` flag to write reformatted files to disk. + +To simplify reviews you can format your changes in a separate commit. + +### Change a pinned dependency version + +Modify the dependency specification (syntax can be found [here](https://python-poetry.org/docs/dependency-specification/)) in `pyproject.toml` and run one of the following in your terminal: + +- `poetry update` +- `rm poetry.lock && poetry install` + +Sometimes `poetry update` can freeze or run forever. Deleting the `poetry.lock` file and calling `poetry install` is guaranteed to update everything but is usually _slower_ than `poetry update` **if `poetry update` works at all**. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..23356f3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Databricks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 6e0467c..64c20bb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Databricks dialect for SQLALchemy 2.0 -The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. +The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. ## Usage with SQLAlchemy <= 2.0 A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. @@ -11,7 +11,7 @@ A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4 To install the dialect and its dependencies: ```shell -pip install databricks_sqlalchemy +pip install databricks-sqlalchemy ``` If you also plan to use `alembic` you can alternatively run: diff --git a/dependency_test/test_dependency.py b/src/databricks/sqlalchemy/dependency_test/test_dependency.py similarity index 100% rename from dependency_test/test_dependency.py rename to src/databricks/sqlalchemy/dependency_test/test_dependency.py