From 0f6d8aa90122b3a0f752a001470c26d7f50becd6 Mon Sep 17 00:00:00 2001 From: Yun Wu Date: Mon, 30 Sep 2024 09:59:17 +0800 Subject: [PATCH 1/5] format since and until timestamp --- ecoscope/io/earthranger.py | 9 ++++++--- ecoscope/io/earthranger_utils.py | 7 +++++++ tests/test_earthranger_io.py | 27 +++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index f2cdec46..8ce96907 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -13,7 +13,7 @@ from tqdm.auto import tqdm import ecoscope -from ecoscope.io.earthranger_utils import clean_kwargs, clean_time_cols, dataframe_to_dict, to_gdf +from ecoscope.io.earthranger_utils import clean_kwargs, clean_time_cols, dataframe_to_dict, format_iso_time, to_gdf from ecoscope.io.utils import pack_columns, to_hex @@ -634,9 +634,9 @@ def get_patrols(self, since=None, until=None, patrol_type=None, status=None, **a Parameters ---------- since: - lower date range + lower time range in isoformat until: - upper date range + upper time range in isoformat patrol_type: Comma-separated list of type of patrol UUID status @@ -654,6 +654,9 @@ def get_patrols(self, since=None, until=None, patrol_type=None, status=None, **a return_data=True, ) + since = format_iso_time(since) + until = format_iso_time(until) + filter = {"date_range": {}, "patrol_type": []} if since is not None: diff --git a/ecoscope/io/earthranger_utils.py b/ecoscope/io/earthranger_utils.py index 5ab77c0d..f2b2ffc0 100644 --- a/ecoscope/io/earthranger_utils.py +++ b/ecoscope/io/earthranger_utils.py @@ -43,3 +43,10 @@ def clean_time_cols(df): # convert x is not None to pd.isna(x) is False df[col] = df[col].apply(lambda x: pd.to_datetime(parser.parse(x)) if not pd.isna(x) else None) return df + + +def format_iso_time(date_string: str) -> str: + try: + return pd.to_datetime(date_string).isoformat() + except ValueError: + raise ValueError(f"Failed to parse timestamp'{date_string}'") diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index 5a374c78..20e26cb7 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -4,6 +4,7 @@ import geopandas as gpd import pandas as pd import pytest +import pytz from shapely.geometry import Point import ecoscope @@ -81,10 +82,32 @@ def test_das_client_method(er_io): er_io.get_me() -def test_get_patrols(er_io): - patrols = er_io.get_patrols() +def test_get_patrols_datestr(er_io): + # patrols = er_io.get_patrols( + # since=pd.Timestamp("2017-01-01").isoformat(), + # until=pd.Timestamp("2017-04-01").isoformat(), + # ) + since_str = "2017-01-01" + since_time = pd.to_datetime(since_str).replace(tzinfo=pytz.UTC) + until_str = "2017-04-01" + until_time = pd.to_datetime(until_str).replace(tzinfo=pytz.UTC) + patrols = er_io.get_patrols(since=since_str, until=until_str) + assert len(patrols) > 0 + time_ranges = [ + segment["time_range"] + for segments in patrols["patrol_segments"] + for segment in segments + if "time_range" in segment + ] + + for time_range in time_ranges: + start = pd.to_datetime(time_range["start_time"]) + end = pd.to_datetime(time_range["end_time"]) + + assert start <= until_time and end >= since_time + def test_get_patrol_events(er_io): events = er_io.get_patrol_events( From 88ea866352cd994fa08c84c45d3e72d70cdcf5a6 Mon Sep 17 00:00:00 2001 From: Yun Wu Date: Mon, 30 Sep 2024 10:10:58 +0800 Subject: [PATCH 2/5] add invalid format --- tests/test_earthranger_io.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index 20e26cb7..85c71318 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -83,10 +83,6 @@ def test_das_client_method(er_io): def test_get_patrols_datestr(er_io): - # patrols = er_io.get_patrols( - # since=pd.Timestamp("2017-01-01").isoformat(), - # until=pd.Timestamp("2017-04-01").isoformat(), - # ) since_str = "2017-01-01" since_time = pd.to_datetime(since_str).replace(tzinfo=pytz.UTC) until_str = "2017-04-01" @@ -109,6 +105,11 @@ def test_get_patrols_datestr(er_io): assert start <= until_time and end >= since_time +def test_get_patrols_datestr_invalid_format(er_io): + with pytest.raises(ValueError): + er_io.get_patrols(since="not a date") + + def test_get_patrol_events(er_io): events = er_io.get_patrol_events( since=pd.Timestamp("2017-01-01").isoformat(), @@ -119,6 +120,7 @@ def test_get_patrol_events(er_io): assert "geometry" in events assert "patrol_id" in events assert "patrol_segment_id" in events + assert "time" in events def test_post_observations(er_io): From 7d4620b075e6e7981bb62ada3e347fda7718dd88 Mon Sep 17 00:00:00 2001 From: Yun Wu Date: Mon, 30 Sep 2024 11:26:59 +0800 Subject: [PATCH 3/5] Support patrol type values --- ecoscope/io/earthranger.py | 24 ++++++++++++++++++------ tests/test_earthranger_io.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 8ce96907..4a795428 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -629,7 +629,7 @@ def get_patrol_types(self): df = pd.DataFrame(self._get("activity/patrols/types")) return df.set_index("id") - def get_patrols(self, since=None, until=None, patrol_type=None, status=None, **addl_kwargs): + def get_patrols(self, since=None, until=None, patrol_type=None, patrol_type_value=None, status=None, **addl_kwargs): """ Parameters ---------- @@ -639,6 +639,10 @@ def get_patrols(self, since=None, until=None, patrol_type=None, status=None, **a upper time range in isoformat patrol_type: Comma-separated list of type of patrol UUID + !!!! Either an array or a UUID!!!! + !!!! does not accept comma!!!!! + patrol_type_value: + The value of a patrol type status Comma-separated list of 'scheduled'/'active'/'overdue'/'done'/'cancelled' Returns @@ -647,24 +651,32 @@ def get_patrols(self, since=None, until=None, patrol_type=None, status=None, **a DataFrame of queried patrols """ + patrol_type_value_list = [patrol_type_value] if isinstance(patrol_type_value, str) else patrol_type_value params = clean_kwargs( addl_kwargs, status=status, patrol_type=[patrol_type] if isinstance(patrol_type, str) else patrol_type, + patrol_type_value=patrol_type_value_list, return_data=True, ) - since = format_iso_time(since) - until = format_iso_time(until) - filter = {"date_range": {}, "patrol_type": []} if since is not None: - filter["date_range"]["lower"] = since + filter["date_range"]["lower"] = format_iso_time(since) if until is not None: - filter["date_range"]["upper"] = until + filter["date_range"]["upper"] = format_iso_time(until) if patrol_type is not None: filter["patrol_type"] = params["patrol_type"] + if patrol_type_value_list is not None: + patrol_types = self.get_patrol_types() + matching_rows = patrol_types[patrol_types["value"].isin(patrol_type_value_list)] + missing_values = set(patrol_type_value_list) - set(matching_rows["value"]) + if missing_values: + raise ValueError(f"Failed to find IDs for values: {missing_values}") + + filter["patrol_type"] = matching_rows.index.tolist() + params["filter"] = json.dumps(filter) df = pd.DataFrame( diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index 85c71318..add3d6c0 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -110,6 +110,36 @@ def test_get_patrols_datestr_invalid_format(er_io): er_io.get_patrols(since="not a date") +def test_get_patrols_with_type_value(er_io): + patrols = er_io.get_patrols(since="2017-01-01", until="2017-04-01", patrol_type_value="ecoscope_patrol") + + patrol_types = [ + segment["patrol_type"] + for segments in patrols["patrol_segments"] + for segment in segments + if "patrol_type" in segment + ] + assert all(value == "ecoscope_patrol" for value in patrol_types) + + +def test_get_patrols_with_type_value_list(er_io): + patrol_type_value_list = ["ecoscope_patrol", "MEP_Distance_Survey_Patrol"] + patrols = er_io.get_patrols(since="2024-01-01", until="2024-04-01", patrol_type_value=patrol_type_value_list) + + patrol_types = [ + segment["patrol_type"] + for segments in patrols["patrol_segments"] + for segment in segments + if "patrol_type" in segment + ] + assert all(value in patrol_type_value_list for value in patrol_types) + + +def test_get_patrols_with_invalid_type_value(er_io): + with pytest.raises(ValueError): + er_io.get_patrols(since="2017-01-01", until="2017-04-01", patrol_type_value="invalid") + + def test_get_patrol_events(er_io): events = er_io.get_patrol_events( since=pd.Timestamp("2017-01-01").isoformat(), From 8cfce3ac11a4f7fe81c8225dd859d837f8f09dfa Mon Sep 17 00:00:00 2001 From: Yun Wu Date: Mon, 30 Sep 2024 11:37:57 +0800 Subject: [PATCH 4/5] comment --- ecoscope/io/earthranger.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 4a795428..29a7c470 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -634,17 +634,16 @@ def get_patrols(self, since=None, until=None, patrol_type=None, patrol_type_valu Parameters ---------- since: - lower time range in isoformat + Lower time range until: - upper time range in isoformat + Upper time range patrol_type: - Comma-separated list of type of patrol UUID - !!!! Either an array or a UUID!!!! - !!!! does not accept comma!!!!! + A patrol type UUID or a list of UUIDs patrol_type_value: - The value of a patrol type + A patrol type value or a list of patrol type values status - Comma-separated list of 'scheduled'/'active'/'overdue'/'done'/'cancelled' + 'scheduled'/'active'/'overdue'/'done'/'cancelled' + Accept a status string or a list of statuses Returns ------- patrols : pd.DataFrame From 04fdadd52afe9c07608da78053338c1abf6aa03a Mon Sep 17 00:00:00 2001 From: Yun Wu Date: Mon, 30 Sep 2024 11:45:04 +0800 Subject: [PATCH 5/5] update other patrol related calls --- ecoscope/io/earthranger.py | 45 ++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 29a7c470..ca2b3d50 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -13,7 +13,13 @@ from tqdm.auto import tqdm import ecoscope -from ecoscope.io.earthranger_utils import clean_kwargs, clean_time_cols, dataframe_to_dict, format_iso_time, to_gdf +from ecoscope.io.earthranger_utils import ( + clean_kwargs, + clean_time_cols, + dataframe_to_dict, + format_iso_time, + to_gdf, +) from ecoscope.io.utils import pack_columns, to_hex @@ -691,18 +697,23 @@ def get_patrols(self, since=None, until=None, patrol_type=None, patrol_type_valu df = clean_time_cols(df) return df - def get_patrol_events(self, since=None, until=None, patrol_type=None, status=None, **addl_kwargs): + def get_patrol_events( + self, since=None, until=None, patrol_type=None, patrol_type_value=None, status=None, **addl_kwargs + ): """ Parameters ---------- since: - lower date range + Lower time range until: - upper date range + Upper time range patrol_type: - Comma-separated list of type of patrol UUID + A patrol type UUID or a list of UUIDs + patrol_type_value: + A patrol type value or a list of patrol type values status - Comma-separated list of 'scheduled'/'active'/'overdue'/'done'/'cancelled' + 'scheduled'/'active'/'overdue'/'done'/'cancelled' + Accept a status string or a list of statuses Returns ------- events : pd.DataFrame @@ -712,6 +723,7 @@ def get_patrol_events(self, since=None, until=None, patrol_type=None, status=Non since=since, until=until, patrol_type=patrol_type, + patrol_type_value=patrol_type_value, status=status, **addl_kwargs, ) @@ -771,6 +783,7 @@ def get_patrol_observations_with_patrol_filter( since=None, until=None, patrol_type=None, + patrol_type_value=None, status=None, include_patrol_details=False, **kwargs, @@ -781,13 +794,16 @@ def get_patrol_observations_with_patrol_filter( Parameters ---------- since: - lower date range + Lower time range until: - upper date range + Upper time range patrol_type: - Comma-separated list of type of patrol UUID + A patrol type UUID or a list of UUIDs + patrol_type_value: + A patrol type value or a list of patrol type values status - Comma-separated list of 'scheduled'/'active'/'overdue'/'done'/'cancelled' + 'scheduled'/'active'/'overdue'/'done'/'cancelled' + Accept a status string or a list of statuses include_patrol_details : bool, optional Whether to merge patrol details into dataframe kwargs @@ -798,7 +814,14 @@ def get_patrol_observations_with_patrol_filter( relocations : ecoscope.base.Relocations """ - patrols_df = self.get_patrols(since=since, until=until, patrol_type=patrol_type, status=status, **kwargs) + patrols_df = self.get_patrols( + since=since, + until=until, + patrol_type=patrol_type, + patrol_type_value=patrol_type_value, + status=status, + **kwargs, + ) return self.get_patrol_observations(patrols_df, include_patrol_details=include_patrol_details, **kwargs) def get_patrol_observations(self, patrols_df, include_patrol_details=False, **kwargs):