From 3d2de543603d05838bd143b48e89b80fd079cebc Mon Sep 17 00:00:00 2001 From: Orellabac Date: Mon, 30 Jan 2023 12:02:03 -0600 Subject: [PATCH] 0.0.21 (#22) * 0.0.20 * notebook updates * adjustment for displayHTML * typo * adjust for datediff * 0.0.19 * 0.0.21 removing unused functions --- CHANGE_LOG.txt | 7 ++++++- README.md | 8 ++++---- setup.py | 4 ++-- snowpark_extensions/functions_extensions.py | 13 ------------- tests/test_functions.py | 3 +-- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/CHANGE_LOG.txt b/CHANGE_LOG.txt index 6f127ea..de8c3b4 100644 --- a/CHANGE_LOG.txt +++ b/CHANGE_LOG.txt @@ -107,4 +107,9 @@ Fixing some test_cases Version 0.0.20 -------------- -Changing the implementation for notebook integration for dataframe to a more standard approach +Fixing issue with bround + +Version 0.0.21 +-------------- +Removing function that are now available in snowpark +Adjust to use snowpark-python>=1.1.0 \ No newline at end of file diff --git a/README.md b/README.md index 37434f8..4963744 100644 --- a/README.md +++ b/README.md @@ -198,8 +198,8 @@ df.group_by("ID").applyInPandas( | Name | Description | |------------------------------|-------------------------------------------------------------------------------------| | functions.array_sort | sorts the input array in ascending order or descending order. The elements of the input array must be orderable. Null elements will be placed at the end of the returned array. | -| functions.unix_timestamp | returns the UNIX timestamp of current time. | -| functions.from_unixtimestamp | can be used to convert UNIX time to Snowflake timestamp | +| ~~functions.unix_timestamp~~ | ~~returns the UNIX timestamp of current time.~~ **Available in snowpark-python >= 1.1.0** | +| ~~functions.from_unixtimestamp~~ | ~~can be used to convert UNIX time to Snowflake timestamp~~ **Available in snowpark-python >= 1.1.0** | | functions.format_number | formats numbers using the specified number of decimal places | | functions.reverse | returns a reversed string | | functions.explode | returns a new row for each element in the given array @@ -213,8 +213,8 @@ df.group_by("ID").applyInPandas( | functions.date_add | returns the date that is n days days after | | functions.date_sub | returns the date that is n days before | | functions.regexp_extract | extract a specific group matched by a regex, from the specified string column. | -| functions.asc | returns a sort expression based on the ascending order of the given column name. | -| functions.desc | returns a sort expression based on the descending order of the given column name. | +| ~~functions.asc~~ | ~~returns a sort expression based on the ascending order of the given column name.~~ **Available in snowpark-python >=1.1.0** | +| ~~functions.desc~~ | ~~returns a sort expression based on the descending order of the given column name.~~ **Available in snowpark-python >=1.1.0** | | functions.flatten | creates a single array from an array of arrays | functions.sort_array | sorts the input array in ascending or descending order according to the natural ordering of the array elements. Null elements will be placed at the beginning of the returned array in ascending order or at the end of the returned array in descending order | functions.map_values | Returns an unordered array containing the values of the map. | diff --git a/setup.py b/setup.py index 2c4c178..ffed380 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ this_directory = Path(__file__).parent long_description = (this_directory / "README.md").read_text() -VERSION = '0.0.20' +VERSION = '0.0.21' setup(name='snowpark_extensions', version=VERSION, @@ -14,7 +14,7 @@ long_description_content_type='text/markdown', url='http://github.com/MobilizeNet/snowpark-extensions-py', author='mauricio.rojas', - install_requires=['snowflake-snowpark-python[pandas]', + install_requires=['snowflake-snowpark-python[pandas]>=1.1.0', 'shortuuid', 'nest_asyncio', 'jinja2', 'rich'], author_email='mauricio.rojas@mobilize.net', packages=['snowpark_extensions'], diff --git a/snowpark_extensions/functions_extensions.py b/snowpark_extensions/functions_extensions.py index b37ec8a..3064a55 100644 --- a/snowpark_extensions/functions_extensions.py +++ b/snowpark_extensions/functions_extensions.py @@ -51,13 +51,6 @@ def regexp_extract(value:ColumnOrLiteralStr,regexp:ColumnOrLiteralStr,idx:int) - # we add .* to the expression if needed return coalesce(call_builtin('regexp_substr',value,regexp,lit(1),lit(1),lit('e'),idx),lit('')) - def unix_timestamp(col): - return call_builtin("DATE_PART","epoch_second",col) - - def from_unixtime(col): - col = _to_col_if_str(col,"from_unixtime") - return F.to_timestamp(col).alias('ts') - def format_number(col,d): col = _to_col_if_str(col,"format_number") return F.to_varchar(col,'999,999,999,999,999.' + '0'*d) @@ -295,17 +288,11 @@ def _bround(col: Column, scale: int = 0): F.array_distinct = array_distinct F.regexp_extract = regexp_extract F.create_map = create_map - F.unix_timestamp = unix_timestamp - F.from_unixtime = from_unixtime F.format_number = format_number F.reverse = reverse F.daydiff = daydiff F.date_add = date_add F.date_sub = date_sub - F.asc = lambda col: _to_col_if_str(col, "asc").asc() - F.desc = lambda col: _to_col_if_str(col, "desc").desc() - F.asc_nulls_first = lambda col: _to_col_if_str(col, "asc_nulls_first").asc() - F.desc_nulls_first = lambda col: _to_col_if_str(col, "desc_nulls_first").asc() F.sort_array = _sort_array F.array_sort = _array_sort F.struct = _struct diff --git a/tests/test_functions.py b/tests/test_functions.py index 1198d66..eb5b0fb 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -224,13 +224,12 @@ def test_struct(): assert re.sub(r"\s","",res[0].STRUCT) == '{"A":80,"B":"Bob"}' assert re.sub(r"\s","",res[1].STRUCT) == '{"A":null,"B":"Alice"}' -def test_datediff(): +def test_daydiff(): session = Session.builder.from_snowsql().getOrCreate() df = session.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2']) res = df.select(F.daydiff(F.to_date(df.d2), F.to_date(df.d1)).alias('diff')).collect() assert res[0].DIFF == 32 - def test_bround(): session = Session.builder.from_snowsql().getOrCreate() data0 = [(1.5,0),