Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix get_events time parse #353

Merged
merged 2 commits into from
Dec 4, 2024
Merged

Fix get_events time parse #353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
031f1c8
fix get_events time parse
atmorling Dec 4, 2024
3bca41f
make tests better
atmorling Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ecoscope/io/earthranger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,8 +626,8 @@ def get_events(
)
gdf = gpd.GeoDataFrame(df)

if not df.empty:
df = clean_time_cols(df)
if not gdf.empty:
gdf = clean_time_cols(gdf)
if gdf.loc[0, "location"] is not None:
gdf.loc[~gdf["geojson"].isna(), "geometry"] = gpd.GeoDataFrame.from_features(
gdf.loc[~gdf["geojson"].isna(), "geojson"]
Expand Down
5 changes: 3 additions & 2 deletions ecoscope/io/earthranger_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import pandas as pd
from dateutil import parser

TIME_COLS = ["time", "created_at", "updated_at", "end_time", "last_position_date", "recorded_at", "fixtime"]


def clean_kwargs(addl_kwargs={}, **kwargs):
for k in addl_kwargs.keys():
Expand Down Expand Up @@ -38,8 +40,7 @@ def to_gdf(df):


def clean_time_cols(df):
time_cols = ["time", "created_at", "updated_at", "end_time", "last_position_date", "recorded_at", "fixtime"]
for col in time_cols:
for col in TIME_COLS:
if col in df.columns and not pd.api.types.is_datetime64_ns_dtype(df[col]):
# convert x is not None to pd.isna(x) is False
df[col] = df[col].apply(lambda x: pd.to_datetime(parser.parse(x), utc=True) if not pd.isna(x) else None)
Expand Down
18 changes: 18 additions & 0 deletions tests/test_earthranger_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,17 @@
import ecoscope
from erclient import ERClientException

from ecoscope.io.earthranger_utils import TIME_COLS

pytestmark = pytest.mark.io


def check_time_is_parsed(df):
for col in TIME_COLS:
if col in df.columns:
assert pd.api.types.is_datetime64_ns_dtype(df[col]) or df[col].isna().all()


def test_get_subject_observations(er_io):
relocations = er_io.get_subject_observations(
subject_ids=er_io.SUBJECT_IDS,
Expand All @@ -26,6 +34,7 @@ def test_get_subject_observations(er_io):
assert "groupby_col" in relocations
assert "fixtime" in relocations
assert "extra__source" in relocations
check_time_is_parsed(relocations)


def test_get_source_observations(er_io):
Expand All @@ -36,6 +45,7 @@ def test_get_source_observations(er_io):
assert isinstance(relocations, ecoscope.base.Relocations)
assert "fixtime" in relocations
assert "groupby_col" in relocations
check_time_is_parsed(relocations)


def test_get_source_no_observations(er_io):
Expand All @@ -54,6 +64,7 @@ def test_get_subjectsource_observations(er_io):
assert isinstance(relocations, ecoscope.base.Relocations)
assert "fixtime" in relocations
assert "groupby_col" in relocations
check_time_is_parsed(relocations) # TODO Check this


def test_get_subjectsource_no_observations(er_io):
Expand All @@ -73,6 +84,7 @@ def test_get_subjectgroup_observations(er_io):
def test_get_events(er_events_io):
events = er_events_io.get_events(event_type=["e00ce1f6-f9f1-48af-93c9-fb89ec493b8a"])
assert not events.empty
check_time_is_parsed(events)


def test_das_client_method(er_io):
Expand All @@ -88,6 +100,7 @@ def test_get_patrols_datestr(er_io):
patrols = er_io.get_patrols(since=since_str, until=until_str)

assert len(patrols) > 0
check_time_is_parsed(patrols)

time_ranges = [
segment["time_range"]
Expand Down Expand Up @@ -118,6 +131,7 @@ def test_get_patrols_with_type_value(er_io):
if "patrol_type" in segment
]
assert all(value == "ecoscope_patrol" for value in patrol_types)
check_time_is_parsed(patrols)


def test_get_patrols_with_type_value_list(er_io):
Expand All @@ -131,6 +145,7 @@ def test_get_patrols_with_type_value_list(er_io):
if "patrol_type" in segment
]
assert all(value in patrol_type_value_list for value in patrol_types)
check_time_is_parsed(patrols)


def test_get_patrols_with_invalid_type_value(er_io):
Expand All @@ -149,6 +164,7 @@ def test_get_patrol_events(er_io):
assert "patrol_id" in events
assert "patrol_segment_id" in events
assert "time" in events
check_time_is_parsed(events)


@patch("ecoscope.io.EarthRangerIO.get_patrols")
Expand Down Expand Up @@ -275,6 +291,7 @@ def test_get_patrol_observations(er_io):
include_subjectsource_details=False,
)
assert not observations.empty
check_time_is_parsed(observations)


def test_get_patrol_observations_with_patrol_details(er_io):
Expand All @@ -294,6 +311,7 @@ def test_get_patrol_observations_with_patrol_details(er_io):
assert "patrol_id" in observations.columns
assert "patrol_title" in observations.columns
pd.testing.assert_series_equal(observations["patrol_id"], observations["groupby_col"], check_names=False)
check_time_is_parsed(observations)


def test_users(er_io):
Expand Down
Loading