Skip to content

Commit

Permalink
Revert changes
Browse files Browse the repository at this point in the history
  • Loading branch information
b1ackout committed May 23, 2024
1 parent d7e0e8f commit 11bedfa
Showing 1 changed file with 8 additions and 25 deletions.
33 changes: 8 additions & 25 deletions src/sql/run/sparkdataframe.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,26 @@
try:
from pyspark.sql import DataFrame
from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
from pyspark.sql.utils import AnalysisException
except ModuleNotFoundError:
DataFrame = None
CDataFrame = None
AnalysisException = None
# try:
from pyspark.sql import DataFrame
from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
# except ModuleNotFoundError:
# DataFrame = None
# CDataFrame = None

from sql import exceptions


def handle_spark_dataframe(dataframe, should_cache=False):
"""Execute a ResultSet sqlaproxy using pysark module."""
if not DataFrame and not CDataFrame and not AnalysisException:
if not DataFrame and not CDataFrame:
raise exceptions.MissingPackageError("pysark not installed")

try:
return SparkResultProxy(dataframe, dataframe.columns, should_cache)
except AnalysisException as e:
print(e)
except Exception as e:
print(e)
raise (e)
return SparkResultProxy(dataframe, dataframe.columns, should_cache)


class SparkResultProxy(object):
"""A fake class that pretends to behave like the ResultProxy from
SqlAlchemy.
"""

dataframe = None

def __init__(self, dataframe, headers, should_cache):
self.dataframe = dataframe
self.fetchall = dataframe.collect
Expand All @@ -40,21 +30,14 @@ def __init__(self, dataframe, headers, should_cache):
self.returns_rows = True
if should_cache:
self.dataframe.cache()

def fetchmany(self, size):
return self.dataframe.take(size)

def fetchone(self):
return self.dataframe.head()

def close(self):
self.dataframe.unpersist()


class SparkCursor(object):
"""Class to extend to give SqlAlchemy Cursor like behaviour"""

description = None

def __init__(self, headers) -> None:
self.description = headers

0 comments on commit 11bedfa

Please sign in to comment.