From a0f5c706ffa1825045cbc2ed0d014895cd93c4a4 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Wed, 3 Apr 2024 17:52:18 -0400 Subject: [PATCH] Remove Qubole query runner (#6848) The qds-sdk-py package along with the rest of the Qubole project is no longer maintained: https://github.com/qubole/qds-sdk-py/commit/3c6a34ce336561dffe8ab5ed44ddbc151d3f81bf Removing this eliminates these warnings when running Redash management commands: ./qds_sdk/commands.py:1124: SyntaxWarning: "is" with a literal. Did you mean "=="? if options.mode is "1": ./qds_sdk/commands.py:1137: SyntaxWarning: "is" with a literal. Did you mean "=="? if options.db_update_mode is "updateonly": ./qds_sdk/commands.py:1424: SyntaxWarning: "is" with a literal. Did you mean "=="? if (total is 0) or (downloaded == total): Co-authored-by: github-actions --- client/app/assets/images/db-logos/qubole.png | Bin 2428 -> 0 bytes poetry.lock | 40 +---- pyproject.toml | 1 - redash/query_runner/qubole.py | 169 ------------------- redash/settings/__init__.py | 1 - 5 files changed, 1 insertion(+), 210 deletions(-) delete mode 100644 client/app/assets/images/db-logos/qubole.png delete mode 100644 redash/query_runner/qubole.py diff --git a/client/app/assets/images/db-logos/qubole.png b/client/app/assets/images/db-logos/qubole.png deleted file mode 100644 index dfdc2fa2e20f2ce14da46f0f71d406c70c1b46a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2428 zcmcJRhf@>E7Kb+kDbjn9>a)tUc zhA09T9_=bMK;RM(5a}feA>;+`%=-)8+nL=xyEA9^%$(nMc7JztxFB#?_AmfIz{dKl z695pl2?0D@?8zhibpQaoRyJqPINu{JPYMM;nUm*UgVHH`?_Z0_Ajfk58X#?B59Rtt zgaiah`n&a950K2$z!*Kw26GvrkrD3i8ejqL&lWr$AR`d-3u(?c70dd^vI}Q|Fb?_5OAv2X0q8Qsmn@+ z6P^u(0S`FN9&LdVcmjwiXj@PQb>sl@Xi=~LIMD#kt@NvxJu^Y;8fpuNcg42V>84qJi$3=JQ?uxgNcIr}WTHQ5)PZ<|6D$VO~aj|E)^ELdc;C8EraX!?? zuSQwoRARE*SqV`P;qJxj>G{%qFU73oR-h53!#`0Ed|ljnvAerl>)gH7<)`PJX7I^U zZ4=NweH|Vk|0T0)C93B>6%8wjZ2Njf3Lw|x13xG6t_}_krdKB|>ZA5(N3JhdVX=Kz z?fhlI{NzWqW;%I)pfpSPYW>dRFw%s7q8gaFo^OVn`66sJ?Zcn&`y9$BlM^0?EyfLx z3=Q!ng!7|~aoaG3J3q%@N-&{?i{O|9i(ssk+V5-$j`icq%YCEb2=iB+CB#ATdU4}` z+6Y)!@V>?1CLu~)XeLAv(Q7o~_ z^6*IVe26)MU-!CypK0GNh?dsyeoP&6A)@L!5fi@aKY-d%5OacQRaaN{N{;f%*o+){ z7uSg?4|l_HC|RUrNa;$i;?Pk*LIjvkeP~kOo?;>L5xy?%xKgd5QJh$QQFnLTuCYXk=?Y_gW$ci&8wZ*)DAH2E| z2UQ~9LFTq)Cq_4)2n}3JCj=h&Z@HmoudI5gf9!eeMjm3w0i>^o40S%sY^lJB8@@+X zUvDTXq%4IefB}f}7b=+fQsV3b;`F9&iFA5S2tn2|X4j7AG1+3ZaQO!`qrGUaKxIAv zDT5M~(QLOes4y@Akq4PPFrhjil_p^GYx{stHnjf*RPA4YaGM<#Cvn=8Z!PFVc7qNb z6>pKVN1b_*r4N;SCkO}5#9Q!Rhbe)582AzDImTq{MW{lQfQk!)8vhgbYs=h)c)s{iVje$ zj6u+#Dt(1gUR6~kdrLV_2b`bWnt$88{bpcjC|)^}v(H5c%bc>b(;70M?@L);xDaU5 zi5MSIz{dC>F4qc3SDQQRx6WQey5Hub7o*qLHF1)&Ztje7=%@>EKgxBYBay8%#k$Ok z(!i$pG!^%@DNm@^*0=jvl<&=4X8y)NV;07AwYJt@9$yP_^!8?r8L3jAtl>99YAP$~ z*4`XtoKh+c8dKmAwMm7%AHut{sF0_;lme`Oh`~^1HL=g%cymnuEj7h+BgA6kL8)RR z>K3glRp6XrDC-j@Y)R}0AHnQiKaP=ubRu}CwwJ~sPmJexEhBaJnv&AuQMuDy+l@e< zvdl=O#WWYN(F+xk5(k=_;l>E+%kfg)+U1EW8(<+l<;dL{lAJ@q=BNlG(?gY(+sSep zYOBpHoqYj?~w2p0ZaqMYtGC1vO(6Qw9j6WcYH@%Ayvcq~L%U@_X$ zByBAw(tpWX#g{wmHP(~nG&=mT_GTRwzM3K#@pL2*(Um>E;>C=)+~G}bH0LiIqs~vI z0HZjz!X$QSts$1UR0T zhdZ5Q7g+^2yyhTtrt3yYh7@oNkSF@6Kn~dA0Bn`R*|I+{InnGN$UjTz`h$we3385r zUcB3&F&F&CW<+Hu59t21^*yv*tE}zw%2HGO#lWCz--5+b^cF#G@gG6KjRu5j1fIwc z2Hy%5?CqnEA!Zyc*iZAlpzdc+2L~@`W5o7Ehc4QC#`*prF&qqi!T+=Q=+J}=@ZUqD TBKuz+++%aj;VjnD=fQsfma;V1 diff --git a/poetry.lock b/poetry.lock index 300e5d79b7..7ac8188042 100644 --- a/poetry.lock +++ b/poetry.lock @@ -419,17 +419,6 @@ files = [ {file = "blinker-1.6.2.tar.gz", hash = "sha256:4afd3de66ef3a9f8067559fb7a1cbe555c17dcbe15971b05d1b625c3e7abe213"}, ] -[[package]] -name = "boto" -version = "2.49.0" -description = "Amazon Web Services Library" -optional = false -python-versions = "*" -files = [ - {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, - {file = "boto-2.49.0.tar.gz", hash = "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a"}, -] - [[package]] name = "boto3" version = "1.28.8" @@ -1897,16 +1886,6 @@ thriftpy2 = {version = ">=0.4.0,<0.5.0", markers = "python_version >= \"3.0\""} [package.extras] kerberos = ["thrift_sasl (==0.2.1)"] -[[package]] -name = "inflection" -version = "0.3.1" -description = "A port of Ruby on Rails inflector to Python" -optional = false -python-versions = "*" -files = [ - {file = "inflection-0.3.1.tar.gz", hash = "sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"}, -] - [[package]] name = "influxdb" version = "5.2.3" @@ -3875,23 +3854,6 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] -[[package]] -name = "qds-sdk" -version = "1.16.1" -description = "Python SDK for coding to the Qubole Data Service API" -optional = false -python-versions = "*" -files = [ - {file = "qds_sdk-1.16.1.tar.gz", hash = "sha256:28850682afcf3ab0f2a74a9fd442715519db3ee2ba91c7ecb0b1a56773748ffd"}, -] - -[package.dependencies] -boto = ">=2.45.0" -inflection = "0.3.1" -requests = ">=1.0.3" -six = ">=1.2.0" -urllib3 = ">=1.0.2" - [[package]] name = "rdflib" version = "6.3.2" @@ -5333,4 +5295,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "db3de74fd61befac80d3a2d3351ebffb719aae21686e2f22a5c07b01fea590ce" +content-hash = "c12875e9f500f6563ea6ccdfc7f816de4bb742fb5faa1cd375a972ef6b64deb8" diff --git a/pyproject.toml b/pyproject.toml index cbeb9ae919..bdb0b47bd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,6 @@ pymssql = "2.2.8" pyodbc = "4.0.28" python-arango = "6.1.0" python-rapidjson = "1.1.0" -qds-sdk = ">=1.9.6" requests-aws-sign = "0.1.5" sasl = ">=0.1.3" simple-salesforce = "0.74.3" diff --git a/redash/query_runner/qubole.py b/redash/query_runner/qubole.py deleted file mode 100644 index 717b2cdc48..0000000000 --- a/redash/query_runner/qubole.py +++ /dev/null @@ -1,169 +0,0 @@ -import logging -import time -from io import StringIO - -import requests - -from redash.query_runner import ( - TYPE_STRING, - BaseQueryRunner, - JobTimeoutException, - register, -) - -try: - import qds_sdk # noqa: F401 - from qds_sdk.commands import ( - Command, - HiveCommand, - PrestoCommand, - SqlCommand, - ) - from qds_sdk.qubole import Qubole as qbol - - enabled = True -except ImportError: - enabled = False - - -class Qubole(BaseQueryRunner): - should_annotate_query = False - - @classmethod - def configuration_schema(cls): - return { - "type": "object", - "properties": { - "query_type": { - "type": "string", - "title": "Query Type (quantum / presto / hive)", - "default": "hive", - }, - "endpoint": { - "type": "string", - "title": "API Endpoint", - "default": "https://api.qubole.com", - }, - "token": {"type": "string", "title": "Auth Token"}, - "cluster": { - "type": "string", - "title": "Cluster Label", - "default": "default", - }, - }, - "order": ["query_type", "endpoint", "token", "cluster"], - "required": ["endpoint", "token"], - "secret": ["token"], - } - - @classmethod - def type(cls): - return "qubole" - - @classmethod - def name(cls): - return "Qubole" - - @classmethod - def enabled(cls): - return enabled - - def test_connection(self): - headers = self._get_header() - r = requests.head("%s/api/latest/users" % self.configuration.get("endpoint"), headers=headers) - r.status_code == 200 - - def run_query(self, query, user): - qbol.configure( - api_token=self.configuration.get("token"), - api_url="%s/api" % self.configuration.get("endpoint"), - ) - - try: - query_type = self.configuration.get("query_type", "hive") - - if query_type == "quantum": - cmd = SqlCommand.create(query=query) - elif query_type == "hive": - cmd = HiveCommand.create(query=query, label=self.configuration.get("cluster")) - elif query_type == "presto": - cmd = PrestoCommand.create(query=query, label=self.configuration.get("cluster")) - else: - raise Exception( - "Invalid Query Type:%s.\ - It must be : hive / presto / quantum." - % self.configuration.get("query_type") - ) - - logging.info("Qubole command created with Id: %s and Status: %s", cmd.id, cmd.status) - - while not Command.is_done(cmd.status): - time.sleep(qbol.poll_interval) - cmd = Command.find(cmd.id) - logging.info("Qubole command Id: %s and Status: %s", cmd.id, cmd.status) - - rows = [] - columns = [] - error = None - - if cmd.status == "done": - fp = StringIO() - cmd.get_results( - fp=fp, - inline=True, - delim="\t", - fetch=False, - qlog=None, - arguments=["true"], - ) - - results = fp.getvalue() - fp.close() - - data = results.split("\r\n") - columns = self.fetch_columns([(i, TYPE_STRING) for i in data.pop(0).split("\t")]) - rows = [dict(zip((column["name"] for column in columns), row.split("\t"))) for row in data] - - data = {"columns": columns, "rows": rows} - except (KeyboardInterrupt, JobTimeoutException): - logging.info("Sending KILL signal to Qubole Command Id: %s", cmd.id) - cmd.cancel() - raise - - return data, error - - def get_schema(self, get_stats=False): - schemas = {} - try: - headers = self._get_header() - content = requests.get( - "%s/api/latest/hive?describe=true&per_page=10000" % self.configuration.get("endpoint"), - headers=headers, - ) - data = content.json() - - for schema in data["schemas"]: - tables = data["schemas"][schema] - for table in tables: - table_name = list(table.keys())[0] - columns = [f["name"] for f in table[table_name]["columns"]] - - if schema != "default": - table_name = "{}.{}".format(schema, table_name) - - schemas[table_name] = {"name": table_name, "columns": columns} - - except Exception as e: - logging.error("Failed to get schema information from Qubole. Error {}".format(str(e))) - - return list(schemas.values()) - - def _get_header(self): - return { - "Content-type": "application/json", - "Accept": "application/json", - "X-AUTH-TOKEN": self.configuration.get("token"), - } - - -register(Qubole) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index f62ba611e0..e43a952c59 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -312,7 +312,6 @@ def email_server_is_configured(): "redash.query_runner.salesforce", "redash.query_runner.query_results", "redash.query_runner.prometheus", - "redash.query_runner.qubole", "redash.query_runner.db2", "redash.query_runner.druid", "redash.query_runner.kylin",