diff --git a/poetry.lock b/poetry.lock index 3db965376b..5b153c3400 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1321,6 +1321,45 @@ files = [ [package.dependencies] python-dateutil = ">=2.7" +[[package]] +name = "fsspec" +version = "2024.10.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, + {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dev = ["pre-commit", "ruff"] +doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] +test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +tqdm = ["tqdm"] + [[package]] name = "funcy" version = "1.13" @@ -3156,23 +3195,25 @@ pyasn1 = ">=0.4.6,<0.6.0" [[package]] name = "pyathena" -version = "1.11.5" +version = "2.25.2" description = "Python DB API 2.0 (PEP 249) client for Amazon Athena" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7.1,<4.0.0" files = [ - {file = "PyAthena-1.11.5-py2.py3-none-any.whl", hash = "sha256:8cc5d40236993fe5241bb625e78d0a0a149e629b74569a9636b49168448a7ac8"}, - {file = "PyAthena-1.11.5.tar.gz", hash = "sha256:86c0f4d10528de44fcd63222506949b010dff36ad57116e4c1274c1cfa9477d0"}, + {file = "pyathena-2.25.2-py3-none-any.whl", hash = "sha256:df7855fec5cc675511431d7c72b814346ebd7e51ed32181ec95847154f79210b"}, + {file = "pyathena-2.25.2.tar.gz", hash = "sha256:aebb8254dd7b2a450841ee3552bf443002a2deaed93fae0ae6f4258b5eb2d367"}, ] [package.dependencies] -boto3 = ">=1.4.4" -botocore = ">=1.5.52" -future = "*" +boto3 = ">=1.26.4" +botocore = ">=1.29.4" +fsspec = "*" tenacity = ">=4.1.0" [package.extras] -pandas = ["pandas (>=0.24.0)", "pyarrow (>=0.15.0)"] +arrow = ["pyarrow (>=7.0.0)"] +fastparquet = ["fastparquet (>=0.4.0)"] +pandas = ["pandas (>=1.3.0)"] sqlalchemy = ["sqlalchemy (>=1.0.0,<2.0.0)"] [[package]] @@ -5456,4 +5497,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "f674647ac7245ed97ec3fb18dd104d8e6cee18eaa46f0ad1295780fc0b381afd" +content-hash = "9721b662687ba50e1844929c1e566c39097ded91d79dc30d5d2b03dbccacd892" diff --git a/pyproject.toml b/pyproject.toml index 456376f819..18351db7c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ pandas = "1.3.4" phoenixdb = "0.7" pinotdb = ">=0.4.5" protobuf = "3.20.2" -pyathena = ">=1.5.0,<=1.11.5" +pyathena = "2.25.2" pydgraph = "2.0.2" pydruid = "0.5.7" pyexasol = "0.12.0" diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index 8ff9b3d1af..957e8bf983 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -90,15 +90,26 @@ def configuration_schema(cls): "title": "Athena cost per Tb scanned (USD)", "default": 5, }, + "result_reuse_enable": { + "type": "boolean", + "title": "Reuse Athena query results", + }, + "result_reuse_minutes": { + "type": "number", + "title": "Minutes to reuse Athena query results", + "default": 60, + }, }, "required": ["region", "s3_staging_dir"], - "extra_options": ["glue", "catalog_ids", "cost_per_tb"], + "extra_options": ["glue", "catalog_ids", "cost_per_tb", "result_reuse_enable", "result_reuse_minutes"], "order": [ "region", "s3_staging_dir", "schema", "work_group", "cost_per_tb", + "result_reuse_enable", + "result_reuse_minutes", ], "secret": ["aws_secret_key"], } @@ -247,6 +258,8 @@ def run_query(self, query, user): kms_key=self.configuration.get("kms_key", None), work_group=self.configuration.get("work_group", "primary"), formatter=SimpleFormatter(), + result_reuse_enable=self.configuration.get("result_reuse_enable", False), + result_reuse_minutes=self.configuration.get("result_reuse_minutes", 60), **self._get_iam_credentials(user=user), ).cursor()