From b4b11bfdc15a01689a98a0feae8e211495cfa22d Mon Sep 17 00:00:00 2001 From: Alex Morling Date: Wed, 27 Nov 2024 20:47:25 +0200 Subject: [PATCH 1/3] use patrol_id as groupby in get_patrol_obs --- ecoscope/io/async_earthranger.py | 1 + ecoscope/io/earthranger.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ecoscope/io/async_earthranger.py b/ecoscope/io/async_earthranger.py index b78c1d53..7395ac3c 100644 --- a/ecoscope/io/async_earthranger.py +++ b/ecoscope/io/async_earthranger.py @@ -431,6 +431,7 @@ async def get_patrol_observations_with_patrol_filter( observations = await asyncio.gather(*tasks) observations = pd.concat(observations) + observations["groupby_col"] = "patrol_id" return observations diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 4149e321..4fae6bea 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -904,6 +904,7 @@ def get_patrol_observations(self, patrols_df, include_patrol_details=False, **kw df = pd.concat(observations) df = clean_time_cols(df) + df["groupby_col"] = "patrol_id" df = ecoscope.base.Relocations(df) if include_patrol_details: return df.set_index("id") From 675befc1841e9de590537bd5a26efd8d954040eb Mon Sep 17 00:00:00 2001 From: Alex Morling Date: Wed, 27 Nov 2024 21:10:05 +0200 Subject: [PATCH 2/3] test and fix --- ecoscope/io/async_earthranger.py | 2 +- ecoscope/io/earthranger.py | 2 +- tests/test_asyncearthranger_io.py | 1 + tests/test_earthranger_io.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ecoscope/io/async_earthranger.py b/ecoscope/io/async_earthranger.py index 7395ac3c..db1b5951 100644 --- a/ecoscope/io/async_earthranger.py +++ b/ecoscope/io/async_earthranger.py @@ -431,7 +431,7 @@ async def get_patrol_observations_with_patrol_filter( observations = await asyncio.gather(*tasks) observations = pd.concat(observations) - observations["groupby_col"] = "patrol_id" + observations["groupby_col"] = observations["patrol_id"] return observations diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 4fae6bea..ac3254dc 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -904,7 +904,7 @@ def get_patrol_observations(self, patrols_df, include_patrol_details=False, **kw df = pd.concat(observations) df = clean_time_cols(df) - df["groupby_col"] = "patrol_id" + df["groupby_col"] = df["patrol_id"] df = ecoscope.base.Relocations(df) if include_patrol_details: return df.set_index("id") diff --git a/tests/test_asyncearthranger_io.py b/tests/test_asyncearthranger_io.py index c0961d1f..caa06937 100644 --- a/tests/test_asyncearthranger_io.py +++ b/tests/test_asyncearthranger_io.py @@ -241,6 +241,7 @@ async def test_get_patrol_observations_with_patrol_details( assert not observations.empty assert set(observations.columns) == set(get_patrol_observations_fields).union(get_patrol_details_fields) assert type(observations["fixtime"] == pd.Timestamp) + pd.testing.assert_series_equal(observations["patrol_id"], observations["groupby_col"], check_names=False) @pytest.mark.asyncio diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index 13cb1fc2..e04d0acb 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -297,6 +297,7 @@ def test_get_patrol_observations_with_patrol_details(er_io): assert not observations.empty assert "patrol_id" in observations.columns assert "patrol_title" in observations.columns + pd.testing.assert_series_equal(observations["patrol_id"], observations["groupby_col"], check_names=False) def test_users(er_io): From 0c9fc24c5125b3d5a9094b2b50ccc613d2238edb Mon Sep 17 00:00:00 2001 From: Alex Morling Date: Wed, 27 Nov 2024 21:44:50 +0200 Subject: [PATCH 3/3] properly handle include_details=False --- ecoscope/io/async_earthranger.py | 9 ++++----- ecoscope/io/earthranger.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ecoscope/io/async_earthranger.py b/ecoscope/io/async_earthranger.py index db1b5951..4c02fc1e 100644 --- a/ecoscope/io/async_earthranger.py +++ b/ecoscope/io/async_earthranger.py @@ -419,10 +419,7 @@ async def get_patrol_observations_with_patrol_filter( relocations : ecoscope.base.Relocations """ observations = ecoscope.base.Relocations() - df_pt = None - - if include_patrol_details: - df_pt = await self.get_patrol_types_dataframe() + df_pt = await self.get_patrol_types_dataframe() if include_patrol_details else None tasks = [] async for patrol in self.get_patrols(since=since, until=until, patrol_type=patrol_type, status=status): @@ -431,7 +428,9 @@ async def get_patrol_observations_with_patrol_filter( observations = await asyncio.gather(*tasks) observations = pd.concat(observations) - observations["groupby_col"] = observations["patrol_id"] + + if include_patrol_details: + observations["groupby_col"] = observations["patrol_id"] return observations diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index ac3254dc..fa30d8d3 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -895,6 +895,7 @@ def get_patrol_observations(self, patrols_df, include_patrol_details=False, **kw ) ) if len(observation) > 0: + observation["groupby_col"] = patrol["id"] observations.append(observation) except Exception as e: print( @@ -904,7 +905,6 @@ def get_patrol_observations(self, patrols_df, include_patrol_details=False, **kw df = pd.concat(observations) df = clean_time_cols(df) - df["groupby_col"] = df["patrol_id"] df = ecoscope.base.Relocations(df) if include_patrol_details: return df.set_index("id")