From 27dd22e84f2c226c309996b5d6d09438cf5e453f Mon Sep 17 00:00:00 2001 From: perhalvorsen <31341520+pmhalvor@users.noreply.github.com> Date: Mon, 14 Oct 2024 00:27:13 +0200 Subject: [PATCH] Update writes (#20) * update LADR w/ table olumns and path specifcs * update path templates to match LADR_0005 * add write to local and bigquery for geo search * add write to local and cloud for raw audio * add write to local and cloud for sift audio * add write to local and cloud for classify * standardize postprocess table * clean up comments * add deduplicate script after pipeline run * fix postprocess tests * hide local tables from git commits * revert is local * add audio and detections paths to final output * fix tests --- .gitignore | 3 + .../monterey_bay_10km_240701_240911.csv | 26 - .../monterey_bay_50km-2016-12-21.csv | 3 - .../monterey_bay_50km-20161221.geojson | 83 -- ...ey_bay_50kmr_2024-09-01_2024-09-02.geojson | 1037 ----------------- .../monterey_bay_humpback_240901_240911.csv | 155 --- data/geo/monterey_bay.geojson | 36 - data/postprocess/output.json | 15 + .../geofile=monterey_bay_50km/encounters.csv | 3 + data/table/raw_audio/metadata.json | 1 + data/table/sifted_audio/metadata.json | 1 + .../LADR_0005_persist_intermediate_outputs.md | 149 ++- {src => examples}/create_table.py | 0 examples/geometery_search.py | 2 +- examples/inference.py | 2 - examples/model.py | 1 - examples/quantized_inference.py | 1 - examples/write_to_bigquery.py | 1 - makefile | 9 +- src/config.py | 16 +- src/config/common.yaml | 115 +- src/config/gcp.yaml | 13 +- src/config/local.yaml | 12 - src/gcp.py | 99 ++ src/pipeline.py | 32 +- src/stages/audio.py | 226 ++-- src/stages/classify.py | 226 ++-- src/stages/postprocess.py | 111 +- src/stages/search.py | 132 ++- src/stages/sift.py | 311 +++-- tests/test_audio.py | 56 +- tests/test_classify.py | 46 +- tests/test_config.py | 3 +- tests/test_postprocess.py | 41 +- tests/test_search.py | 63 +- tests/test_sift.py | 102 +- 36 files changed, 1286 insertions(+), 1846 deletions(-) delete mode 100644 data/encounters/monterey_bay_10km_240701_240911.csv delete mode 100644 data/encounters/monterey_bay_50km-2016-12-21.csv delete mode 100644 data/encounters/monterey_bay_50km-20161221.geojson delete mode 100644 data/encounters/monterey_bay_50kmr_2024-09-01_2024-09-02.geojson delete mode 100644 data/encounters/monterey_bay_humpback_240901_240911.csv delete mode 100644 data/geo/monterey_bay.geojson create mode 100644 data/postprocess/output.json create mode 100644 data/table/geometry_search/geofile=monterey_bay_50km/encounters.csv create mode 100644 data/table/raw_audio/metadata.json create mode 100644 data/table/sifted_audio/metadata.json rename {src => examples}/create_table.py (100%) create mode 100644 src/gcp.py diff --git a/.gitignore b/.gitignore index 51de7ff..f2f108c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ # Repo specific ignores audio/ +classifications/ plots/ model/ +hide/ +table/ # Python basic ignores # Byte-compiled / optimized / DLL files diff --git a/data/encounters/monterey_bay_10km_240701_240911.csv b/data/encounters/monterey_bay_10km_240701_240911.csv deleted file mode 100644 index a1966b0..0000000 --- a/data/encounters/monterey_bay_10km_240701_240911.csv +++ /dev/null @@ -1,26 +0,0 @@ -latitude,longitude,approved,attrs,startDate,startTime,endDate,endTime,timezone,displayImgId,displayThumbUrl,displayImgType,displayImgUrl,id,accuracy,precision,displayImgLicense,maxCount,minCount,orgIds,public,region,species,system:time_start -36.727764,-122.068258,True,,2024-07-08,15:17:52,,15:17:52,-07:00,1110237,https://au-hw-media-t.happywhale.com/c9fe66f6-f55e-40aa-9c50-c31f3fa735a4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c9fe66f6-f55e-40aa-9c50-c31f3fa735a4.jpg,471768,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720389600000 -36.809,-122.077,True,,2024-07-08,11:47:23,,11:47:23,America/Los_Angeles,1119694,https://au-hw-media-t.happywhale.com/da8d1064-2371-4818-ae67-186e1735c742.jpg,IMAGE,https://au-hw-media-m.happywhale.com/da8d1064-2371-4818-ae67-186e1735c742.jpg,475956,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720389600000 -36.809,-122.077,True,,2024-07-08,12:03:43,,12:03:43,America/Los_Angeles,1119696,https://au-hw-media-t.happywhale.com/3f31c73f-bc08-42b4-9add-b871d78b30cc.jpg,IMAGE,https://au-hw-media-m.happywhale.com/3f31c73f-bc08-42b4-9add-b871d78b30cc.jpg,475957,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720389600000 -36.809,-122.077,True,,2024-07-08,12:54:28,,12:56:25,America/Los_Angeles,1119698,https://au-hw-media-t.happywhale.com/46dc42ed-98f0-413a-9936-b7fed9c5ff84.jpg,IMAGE,https://au-hw-media-m.happywhale.com/46dc42ed-98f0-413a-9936-b7fed9c5ff84.jpg,475958,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720389600000 -36.809,-122.077,True,,2024-07-08,12:56:25,,12:56:25,America/Los_Angeles,1119697,https://au-hw-media-t.happywhale.com/cd2376a9-08cd-4552-a6b2-2d951d5ed20a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/cd2376a9-08cd-4552-a6b2-2d951d5ed20a.jpg,475959,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720389600000 -36.706389,-122.077222,True,,2024-07-11,,,,America/Los_Angeles,1114896,https://au-hw-media-t.happywhale.com/958e49b5-c73c-48f0-94fc-f76272c912e7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/958e49b5-c73c-48f0-94fc-f76272c912e7.jpg,474801,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720648800000 -36.701667,-122.072222,True,,2024-07-11,,,,America/Los_Angeles,1114912,https://au-hw-media-t.happywhale.com/7f7097de-bea1-4167-aa3e-eede6b59323a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7f7097de-bea1-4167-aa3e-eede6b59323a.jpg,474802,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720648800000 -36.748477,-122.066232,True,,2024-07-15,10:34:50,,10:34:50,-07:00,1116796,https://au-hw-media-t.happywhale.com/c811516e-0bcd-4d92-a1ba-40987ad3adb2.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c811516e-0bcd-4d92-a1ba-40987ad3adb2.jpg,475105,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720994400000 -36.747752,-122.068655,True,,2024-07-15,10:34:50,,10:35:22,-07:00,1116481,https://au-hw-media-t.happywhale.com/f1b741fd-2cb6-41da-9422-5bfc0ea059c4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f1b741fd-2cb6-41da-9422-5bfc0ea059c4.jpg,475104,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720994400000 -36.743868,-122.083552,True,,2024-07-15,11:07:18,,11:08:06,-07:00,1116797,https://au-hw-media-t.happywhale.com/02508ec5-4baa-45a2-9ac0-d363cd62132c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/02508ec5-4baa-45a2-9ac0-d363cd62132c.jpg,475106,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720994400000 -36.742577,-122.082528,True,,2024-07-15,11:07:18,,11:07:18,-07:00,1116482,https://au-hw-media-t.happywhale.com/4426bb25-ea58-45dc-8d57-928c7651be18.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4426bb25-ea58-45dc-8d57-928c7651be18.jpg,475107,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720994400000 -36.680844,-122.10115,True,,2024-07-15,11:43:53,,11:44:01,-07:00,1161673,https://au-hw-media-t.happywhale.com/521993f3-9d86-4f9a-bb37-89971cee1646.jpg,IMAGE,https://au-hw-media-m.happywhale.com/521993f3-9d86-4f9a-bb37-89971cee1646.jpg,486253,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1720994400000 -36.804363,-122.070625,True,,2024-07-17,11:28:51,,11:28:51,-07:00,1118558,https://au-hw-media-t.happywhale.com/b8372dfd-1c09-4961-a02c-feb790188f6a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b8372dfd-1c09-4961-a02c-feb790188f6a.jpg,475704,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721167200000 -36.638867,-122.068261,True,,2024-07-19,10:46:15,,10:46:28,-07:00,1121117,https://au-hw-media-t.happywhale.com/6d06505d-4937-41aa-aa1f-ef86ee8c5a4a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/6d06505d-4937-41aa-aa1f-ef86ee8c5a4a.jpg,476165,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721340000000 -36.640082,-122.078522,True,,2024-07-20,11:14:47,,11:14:51,-07:00,1120163,https://au-hw-media-t.happywhale.com/a5994c75-9e3b-4a18-85e4-d6db653a68d8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a5994c75-9e3b-4a18-85e4-d6db653a68d8.jpg,476392,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721426400000 -36.640725,-122.081438,True,,2024-07-20,11:14:58,,11:14:58,-07:00,1121417,https://au-hw-media-t.happywhale.com/5fe40322-5a86-47a6-be82-355c84477324.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5fe40322-5a86-47a6-be82-355c84477324.jpg,476393,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721426400000 -36.642222,-122.132222,True,,2024-07-23,12:32:37,,12:32:38,-08:00,1133745,https://au-hw-media-t.happywhale.com/f2a46168-a56e-43a7-b15d-cc52612c109e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f2a46168-a56e-43a7-b15d-cc52612c109e.jpg,479165,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721685600000 -36.640556,-122.106944,True,,2024-07-23,12:02:53,,12:02:53,-08:00,1133758,https://au-hw-media-t.happywhale.com/d8bc0414-68fb-4167-aabd-c3ea75e123c2.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d8bc0414-68fb-4167-aabd-c3ea75e123c2.jpg,479166,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721685600000 -36.673889,-122.118889,True,,2024-07-23,12:39:04,,12:39:04,-08:00,1128468,https://au-hw-media-t.happywhale.com/5dd2cf7a-6426-4e38-afaa-d942b5e17cde.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5dd2cf7a-6426-4e38-afaa-d942b5e17cde.jpg,478203,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1721685600000 -36.774127,-122.080078,True,,2024-07-31,13:46:55,,13:46:55,America/Los_Angeles,1133761,https://au-hw-media-t.happywhale.com/86e9dd8b-b9d0-49e7-8164-b75227ef4fa7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/86e9dd8b-b9d0-49e7-8164-b75227ef4fa7.jpg,479168,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1722376800000 -36.774127,-122.080078,True,,2024-07-31,15:41:56,,15:41:56,America/Los_Angeles,1135454,https://au-hw-media-t.happywhale.com/2a6250d9-1585-48d6-ac6e-d994906eabff.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2a6250d9-1585-48d6-ac6e-d994906eabff.jpg,479170,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1722376800000 -36.661978,-122.139416,True,,2024-08-04,12:22:03,,12:22:03,America/Los_Angeles,1143511,https://au-hw-media-t.happywhale.com/893dcaad-2f4e-4a3c-b244-d52358df20b6.jpg,IMAGE,https://au-hw-media-m.happywhale.com/893dcaad-2f4e-4a3c-b244-d52358df20b6.jpg,482262,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1722722400000 -36.661978,-122.139416,True,,2024-08-04,12:30:41,,12:30:41,America/Los_Angeles,1143512,https://au-hw-media-t.happywhale.com/df66fd0d-4b56-4bdd-96d8-ff49ddde373a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/df66fd0d-4b56-4bdd-96d8-ff49ddde373a.jpg,482263,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1722722400000 -36.661978,-122.139416,True,,2024-08-04,12:37:42,,12:37:42,America/Los_Angeles,1137862,https://au-hw-media-t.happywhale.com/eab1773c-706d-4f33-9b7a-60b623c6c54c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/eab1773c-706d-4f33-9b7a-60b623c6c54c.jpg,482264,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1722722400000 -36.789629,-122.13501,True,,2024-08-24,10:13:59,,10:13:59,-07:00,1165385,https://au-hw-media-t.happywhale.com/fdf812db-2626-41e1-aa04-1cd1e0ddf631.jpg,IMAGE,https://au-hw-media-m.happywhale.com/fdf812db-2626-41e1-aa04-1cd1e0ddf631.jpg,487205,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1724450400000 diff --git a/data/encounters/monterey_bay_50km-2016-12-21.csv b/data/encounters/monterey_bay_50km-2016-12-21.csv deleted file mode 100644 index 164f665..0000000 --- a/data/encounters/monterey_bay_50km-2016-12-21.csv +++ /dev/null @@ -1,3 +0,0 @@ -latitude,longitude,approved,attrs,startDate,startTime,endDate,endTime,timezone,displayImgId,displayThumbUrl,displayImgType,displayImgUrl,id,accuracy,precision,displayImgLicense,maxCount,minCount,orgIds,public,region,species,system:time_start -36.91,-122.02,True,,2016-12-21,13:50:00,,13:50:00,America/Los_Angeles,38770,https://au-hw-media-t.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg,9182,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1482274800000 -36.91,-122.02,True,,2016-12-21,13:21:00,,13:21:00,America/Los_Angeles,45042,https://au-hw-media-t.happywhale.com/c5522187-058e-4a1a-83d7-893560ba6b2c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c5522187-058e-4a1a-83d7-893560ba6b2c.jpg,11486,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1482274800000 diff --git a/data/encounters/monterey_bay_50km-20161221.geojson b/data/encounters/monterey_bay_50km-20161221.geojson deleted file mode 100644 index a87bbbe..0000000 --- a/data/encounters/monterey_bay_50km-20161221.geojson +++ /dev/null @@ -1,83 +0,0 @@ -{ - "features": [ - { - "geometry": { - "coordinates": [ - -122.02, - 36.91 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "13:50:00", - "startDate": "2016-12-21", - "startTime": "13:50:00", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 38770, - "thumbUrl": "https://au-hw-media-t.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg" - }, - "id": 9182, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1482274800000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.02, - 36.91 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "13:21:00", - "startDate": "2016-12-21", - "startTime": "13:21:00", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 45042, - "thumbUrl": "https://au-hw-media-t.happywhale.com/c5522187-058e-4a1a-83d7-893560ba6b2c.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/c5522187-058e-4a1a-83d7-893560ba6b2c.jpg" - }, - "id": 11486, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1482274800000 - }, - "type": "Feature" - } - ], - "type": "FeatureCollection" -} \ No newline at end of file diff --git a/data/encounters/monterey_bay_50kmr_2024-09-01_2024-09-02.geojson b/data/encounters/monterey_bay_50kmr_2024-09-01_2024-09-02.geojson deleted file mode 100644 index 71a12b4..0000000 --- a/data/encounters/monterey_bay_50kmr_2024-09-01_2024-09-02.geojson +++ /dev/null @@ -1,1037 +0,0 @@ -{ - "features": [ - { - "geometry": { - "coordinates": [ - -122.0, - 36.8 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "13:51:46", - "startDate": "2024-09-01", - "startTime": "13:51:46", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1169917, - "thumbUrl": "https://au-hw-media-t.happywhale.com/5b254846-e6eb-4cd0-a366-c686514cb1ca.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/5b254846-e6eb-4cd0-a366-c686514cb1ca.jpg" - }, - "id": 489723, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0135, - 36.6646 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "12:00:00", - "startDate": "2024-09-01", - "startTime": "10:30:00", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1169955, - "thumbUrl": "https://au-hw-media-t.happywhale.com/0d7f7344-71dc-47d8-81e5-abd5bac9dba9.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/0d7f7344-71dc-47d8-81e5-abd5bac9dba9.jpg" - }, - "id": 489848, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MANUAL", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0135, - 36.6646 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "12:00:00", - "startDate": "2024-09-01", - "startTime": "10:30:00", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1169956, - "thumbUrl": "https://au-hw-media-t.happywhale.com/2a9e8162-a970-4b87-aca6-bee7c0017ed2.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/2a9e8162-a970-4b87-aca6-bee7c0017ed2.jpg" - }, - "id": 489847, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MANUAL", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0, - 36.8 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "13:51:25", - "startDate": "2024-09-01", - "startTime": "13:51:25", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1169918, - "thumbUrl": "https://au-hw-media-t.happywhale.com/9db42f6e-8539-43de-aada-fc71ddcbd57e.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/9db42f6e-8539-43de-aada-fc71ddcbd57e.jpg" - }, - "id": 489724, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.9787, - 36.7073 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "12:00:00", - "startDate": "2024-09-01", - "startTime": "10:30:00", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1169954, - "thumbUrl": "https://au-hw-media-t.happywhale.com/9ed7e72a-4462-484c-a7b4-3ff8499ee8d6.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/9ed7e72a-4462-484c-a7b4-3ff8499ee8d6.jpg" - }, - "id": 489846, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MANUAL", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.849 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "11:39:49", - "startDate": "2024-09-01", - "startTime": "11:39:49", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170092, - "thumbUrl": "https://au-hw-media-t.happywhale.com/e8b1d44d-4b5d-445f-afe6-680658e9cf4c.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/e8b1d44d-4b5d-445f-afe6-680658e9cf4c.jpg" - }, - "id": 489941, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.849 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "12:25:34", - "startDate": "2024-09-01", - "startTime": "12:25:34", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170093, - "thumbUrl": "https://au-hw-media-t.happywhale.com/c4ca6192-f3b5-478e-8e36-95a32d6f2f14.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/c4ca6192-f3b5-478e-8e36-95a32d6f2f14.jpg" - }, - "id": 489942, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.849 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "12:32:48", - "startDate": "2024-09-01", - "startTime": "12:32:48", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170094, - "thumbUrl": "https://au-hw-media-t.happywhale.com/806679f1-9c91-4402-8539-28088a8fd72b.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/806679f1-9c91-4402-8539-28088a8fd72b.jpg" - }, - "id": 489943, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725141600000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.955152, - 36.668539 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "09:52:12", - "startDate": "2024-09-02", - "startTime": "09:52:03", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1169914, - "thumbUrl": "https://au-hw-media-t.happywhale.com/ebeba8c9-5934-4247-bc87-b1c2c25705d0.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/ebeba8c9-5934-4247-bc87-b1c2c25705d0.jpg" - }, - "id": 489716, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.955152, - 36.668539 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "09:52:12", - "startDate": "2024-09-02", - "startTime": "09:52:12", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167530, - "thumbUrl": "https://au-hw-media-t.happywhale.com/a5fd6f1f-14b4-4139-80f5-e1b77e756bb1.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/a5fd6f1f-14b4-4139-80f5-e1b77e756bb1.jpg" - }, - "id": 489717, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.955152, - 36.668539 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "09:59:40", - "startDate": "2024-09-02", - "startTime": "09:59:40", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167531, - "thumbUrl": "https://au-hw-media-t.happywhale.com/ec170011-b391-4c34-9023-2b9d72871625.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/ec170011-b391-4c34-9023-2b9d72871625.jpg" - }, - "id": 489718, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.040152, - 36.6641 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "10:35:54", - "startDate": "2024-09-02", - "startTime": "10:35:50", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167552, - "thumbUrl": "https://au-hw-media-t.happywhale.com/2bff4305-b38d-4271-8d12-e763de98ed8f.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/2bff4305-b38d-4271-8d12-e763de98ed8f.jpg" - }, - "id": 489719, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.040152, - 36.6641 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "10:35:54", - "startDate": "2024-09-02", - "startTime": "10:35:54", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167553, - "thumbUrl": "https://au-hw-media-t.happywhale.com/195de9de-b48f-4cda-a037-fc7c6156c741.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/195de9de-b48f-4cda-a037-fc7c6156c741.jpg" - }, - "id": 489720, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "MOBILE_DEVICE", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0, - 36.8 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "19:23:55.88", - "startDate": "2024-09-02", - "startTime": "19:23:55.88", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170258, - "thumbUrl": "https://au-hw-media-t.happywhale.com/1ae620f8-fb06-4399-8649-ff1abdfef6e7.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/1ae620f8-fb06-4399-8649-ff1abdfef6e7.jpg" - }, - "id": 490030, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0, - 36.8 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": null, - "startDate": "2024-09-02", - "startTime": null, - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170152, - "thumbUrl": "https://au-hw-media-t.happywhale.com/fc627e4a-885a-4659-8f0e-ebda7f4fe80c.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/fc627e4a-885a-4659-8f0e-ebda7f4fe80c.jpg" - }, - "id": 489984, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.839 - ], - "type": "Point" - }, - "properties": { - "accuracy": "APPROX", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "14:28:51", - "startDate": "2024-09-02", - "startTime": "14:23:57", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170130, - "thumbUrl": "https://au-hw-media-t.happywhale.com/70e0b92e-e72a-4ac4-aed9-f70edac3e5e4.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/70e0b92e-e72a-4ac4-aed9-f70edac3e5e4.jpg" - }, - "id": 489969, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.839 - ], - "type": "Point" - }, - "properties": { - "accuracy": "APPROX", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "14:24:05", - "startDate": "2024-09-02", - "startTime": "14:24:05", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170131, - "thumbUrl": "https://au-hw-media-t.happywhale.com/c1b3f5f0-302b-4103-9b4c-eeb37450c96a.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/c1b3f5f0-302b-4103-9b4c-eeb37450c96a.jpg" - }, - "id": 489970, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.993, - 36.839 - ], - "type": "Point" - }, - "properties": { - "accuracy": "APPROX", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "14:51:14.28", - "startDate": "2024-09-02", - "startTime": "14:12:16.5", - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1170133, - "thumbUrl": "https://au-hw-media-t.happywhale.com/37d2a564-93a3-445f-81db-923f5293c377.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/37d2a564-93a3-445f-81db-923f5293c377.jpg" - }, - "id": 489971, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.85, - 36.8 - ], - "type": "Point" - }, - "properties": { - "accuracy": "GENERAL", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": null, - "startDate": "2024-09-02", - "startTime": null, - "timezone": "America/Los_Angeles" - }, - "displayImage": { - "id": 1173136, - "thumbUrl": "https://au-hw-media-t.happywhale.com/3d7cf73a-5ccc-4484-ab35-61e36027e35f.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/3d7cf73a-5ccc-4484-ab35-61e36027e35f.jpg" - }, - "id": 490517, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.859728, - 36.777235 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "09:43:48", - "startDate": "2024-09-02", - "startTime": "09:43:48", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167911, - "thumbUrl": "https://au-hw-media-t.happywhale.com/ef2ea0fd-b9ac-4abc-961b-47d03c74f46c.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/ef2ea0fd-b9ac-4abc-961b-47d03c74f46c.jpg" - }, - "id": 488922, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.960648, - 36.835212 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "10:39:43", - "startDate": "2024-09-02", - "startTime": "10:39:34", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167913, - "thumbUrl": "https://au-hw-media-t.happywhale.com/7cd35652-0c06-4f8b-b0cf-d540a49a4f20.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/7cd35652-0c06-4f8b-b0cf-d540a49a4f20.jpg" - }, - "id": 488923, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.960648, - 36.835212 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "10:39:34", - "startDate": "2024-09-02", - "startTime": "10:39:34", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167912, - "thumbUrl": "https://au-hw-media-t.happywhale.com/52dc16d3-1a26-458e-b576-2cba8075487f.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/52dc16d3-1a26-458e-b576-2cba8075487f.jpg" - }, - "id": 488924, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.976618, - 36.834488 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "11:01:03", - "startDate": "2024-09-02", - "startTime": "11:00:54", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167915, - "thumbUrl": "https://au-hw-media-t.happywhale.com/4e248ceb-018d-4295-80bf-77d2b525ce1e.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/4e248ceb-018d-4295-80bf-77d2b525ce1e.jpg" - }, - "id": 488925, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -121.976618, - 36.834488 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "11:00:54", - "startDate": "2024-09-02", - "startTime": "11:00:54", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167914, - "thumbUrl": "https://au-hw-media-t.happywhale.com/e80b323e-b740-4186-b173-03dacec0db76.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/e80b323e-b740-4186-b173-03dacec0db76.jpg" - }, - "id": 488926, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.0249, - 36.825895 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "11:29:29", - "startDate": "2024-09-02", - "startTime": "11:29:29", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167916, - "thumbUrl": "https://au-hw-media-t.happywhale.com/931f1b60-ec14-439d-a103-8e0dd5af485a.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/931f1b60-ec14-439d-a103-8e0dd5af485a.jpg" - }, - "id": 488927, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - }, - { - "geometry": { - "coordinates": [ - -122.023032, - 36.827832 - ], - "type": "Point" - }, - "properties": { - "accuracy": "PRECISE", - "approved": true, - "attrs": null, - "dateRange": { - "endDate": null, - "endTime": "11:40:29", - "startDate": "2024-09-02", - "startTime": "11:40:29", - "timezone": "-07:00" - }, - "displayImage": { - "id": 1167917, - "thumbUrl": "https://au-hw-media-t.happywhale.com/7664235e-340d-444c-bd5d-83dabf20290d.jpg", - "type": "IMAGE", - "url": "https://au-hw-media-m.happywhale.com/7664235e-340d-444c-bd5d-83dabf20290d.jpg" - }, - "id": 488928, - "license": "PUBLIC_DOMAIN", - "maxCount": 1, - "minCount": 1, - "orgIds": [ - null - ], - "precision": "CAMERA", - "public": true, - "region": "California, United States", - "species": "humpback_whale", - "system:time_start": 1725228000000 - }, - "type": "Feature" - } - ], - "type": "FeatureCollection" -} \ No newline at end of file diff --git a/data/encounters/monterey_bay_humpback_240901_240911.csv b/data/encounters/monterey_bay_humpback_240901_240911.csv deleted file mode 100644 index 32a239b..0000000 --- a/data/encounters/monterey_bay_humpback_240901_240911.csv +++ /dev/null @@ -1,155 +0,0 @@ -latitude,longitude,approved,attrs,startDate,startTime,endDate,endTime,timezone,displayImgId,displayThumbUrl,displayImgType,displayImgUrl,id,accuracy,precision,displayImgLicense,maxCount,minCount,orgIds,public,region,species,system:time_start -36.8,-122.0,True,,2024-09-01,13:51:46,,13:51:46,America/Los_Angeles,1169917,https://au-hw-media-t.happywhale.com/5b254846-e6eb-4cd0-a366-c686514cb1ca.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5b254846-e6eb-4cd0-a366-c686514cb1ca.jpg,489723,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.6646,-122.0135,True,,2024-09-01,10:30:00,,12:00:00,America/Los_Angeles,1169955,https://au-hw-media-t.happywhale.com/0d7f7344-71dc-47d8-81e5-abd5bac9dba9.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0d7f7344-71dc-47d8-81e5-abd5bac9dba9.jpg,489848,PRECISE,MANUAL,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.6646,-122.0135,True,,2024-09-01,10:30:00,,12:00:00,America/Los_Angeles,1169956,https://au-hw-media-t.happywhale.com/2a9e8162-a970-4b87-aca6-bee7c0017ed2.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2a9e8162-a970-4b87-aca6-bee7c0017ed2.jpg,489847,PRECISE,MANUAL,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.8,-122.0,True,,2024-09-01,13:51:25,,13:51:25,America/Los_Angeles,1169918,https://au-hw-media-t.happywhale.com/9db42f6e-8539-43de-aada-fc71ddcbd57e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9db42f6e-8539-43de-aada-fc71ddcbd57e.jpg,489724,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.7073,-121.9787,True,,2024-09-01,10:30:00,,12:00:00,America/Los_Angeles,1169954,https://au-hw-media-t.happywhale.com/9ed7e72a-4462-484c-a7b4-3ff8499ee8d6.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9ed7e72a-4462-484c-a7b4-3ff8499ee8d6.jpg,489846,PRECISE,MANUAL,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.849,-121.993,True,,2024-09-01,11:39:49,,11:39:49,America/Los_Angeles,1170092,https://au-hw-media-t.happywhale.com/e8b1d44d-4b5d-445f-afe6-680658e9cf4c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e8b1d44d-4b5d-445f-afe6-680658e9cf4c.jpg,489941,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.849,-121.993,True,,2024-09-01,12:25:34,,12:25:34,America/Los_Angeles,1170093,https://au-hw-media-t.happywhale.com/c4ca6192-f3b5-478e-8e36-95a32d6f2f14.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c4ca6192-f3b5-478e-8e36-95a32d6f2f14.jpg,489942,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.849,-121.993,True,,2024-09-01,12:32:48,,12:32:48,America/Los_Angeles,1170094,https://au-hw-media-t.happywhale.com/806679f1-9c91-4402-8539-28088a8fd72b.jpg,IMAGE,https://au-hw-media-m.happywhale.com/806679f1-9c91-4402-8539-28088a8fd72b.jpg,489943,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725141600000 -36.668539,-121.955152,True,,2024-09-02,09:52:03,,09:52:12,-07:00,1169914,https://au-hw-media-t.happywhale.com/ebeba8c9-5934-4247-bc87-b1c2c25705d0.jpg,IMAGE,https://au-hw-media-m.happywhale.com/ebeba8c9-5934-4247-bc87-b1c2c25705d0.jpg,489716,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.668539,-121.955152,True,,2024-09-02,09:52:12,,09:52:12,-07:00,1167530,https://au-hw-media-t.happywhale.com/a5fd6f1f-14b4-4139-80f5-e1b77e756bb1.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a5fd6f1f-14b4-4139-80f5-e1b77e756bb1.jpg,489717,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.668539,-121.955152,True,,2024-09-02,09:59:40,,09:59:40,-07:00,1167531,https://au-hw-media-t.happywhale.com/ec170011-b391-4c34-9023-2b9d72871625.jpg,IMAGE,https://au-hw-media-m.happywhale.com/ec170011-b391-4c34-9023-2b9d72871625.jpg,489718,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.6641,-122.040152,True,,2024-09-02,10:35:50,,10:35:54,-07:00,1167552,https://au-hw-media-t.happywhale.com/2bff4305-b38d-4271-8d12-e763de98ed8f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2bff4305-b38d-4271-8d12-e763de98ed8f.jpg,489719,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.6641,-122.040152,True,,2024-09-02,10:35:54,,10:35:54,-07:00,1167553,https://au-hw-media-t.happywhale.com/195de9de-b48f-4cda-a037-fc7c6156c741.jpg,IMAGE,https://au-hw-media-m.happywhale.com/195de9de-b48f-4cda-a037-fc7c6156c741.jpg,489720,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.8,-122.0,True,,2024-09-02,19:23:55.88,,19:23:55.88,America/Los_Angeles,1170258,https://au-hw-media-t.happywhale.com/1ae620f8-fb06-4399-8649-ff1abdfef6e7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/1ae620f8-fb06-4399-8649-ff1abdfef6e7.jpg,490030,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.8,-122.0,True,,2024-09-02,,,,America/Los_Angeles,1170152,https://au-hw-media-t.happywhale.com/fc627e4a-885a-4659-8f0e-ebda7f4fe80c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/fc627e4a-885a-4659-8f0e-ebda7f4fe80c.jpg,489984,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.839,-121.993,True,,2024-09-02,14:23:57,,14:28:51,America/Los_Angeles,1170130,https://au-hw-media-t.happywhale.com/70e0b92e-e72a-4ac4-aed9-f70edac3e5e4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/70e0b92e-e72a-4ac4-aed9-f70edac3e5e4.jpg,489969,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.839,-121.993,True,,2024-09-02,14:24:05,,14:24:05,America/Los_Angeles,1170131,https://au-hw-media-t.happywhale.com/c1b3f5f0-302b-4103-9b4c-eeb37450c96a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c1b3f5f0-302b-4103-9b4c-eeb37450c96a.jpg,489970,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.839,-121.993,True,,2024-09-02,14:12:16.5,,14:51:14.28,America/Los_Angeles,1170133,https://au-hw-media-t.happywhale.com/37d2a564-93a3-445f-81db-923f5293c377.jpg,IMAGE,https://au-hw-media-m.happywhale.com/37d2a564-93a3-445f-81db-923f5293c377.jpg,489971,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.8,-121.85,True,,2024-09-02,,,,America/Los_Angeles,1173136,https://au-hw-media-t.happywhale.com/3d7cf73a-5ccc-4484-ab35-61e36027e35f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/3d7cf73a-5ccc-4484-ab35-61e36027e35f.jpg,490517,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.777235,-121.859728,True,,2024-09-02,09:43:48,,09:43:48,-07:00,1167911,https://au-hw-media-t.happywhale.com/ef2ea0fd-b9ac-4abc-961b-47d03c74f46c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/ef2ea0fd-b9ac-4abc-961b-47d03c74f46c.jpg,488922,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.835212,-121.960648,True,,2024-09-02,10:39:34,,10:39:43,-07:00,1167913,https://au-hw-media-t.happywhale.com/7cd35652-0c06-4f8b-b0cf-d540a49a4f20.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7cd35652-0c06-4f8b-b0cf-d540a49a4f20.jpg,488923,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.835212,-121.960648,True,,2024-09-02,10:39:34,,10:39:34,-07:00,1167912,https://au-hw-media-t.happywhale.com/52dc16d3-1a26-458e-b576-2cba8075487f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/52dc16d3-1a26-458e-b576-2cba8075487f.jpg,488924,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.834488,-121.976618,True,,2024-09-02,11:00:54,,11:01:03,-07:00,1167915,https://au-hw-media-t.happywhale.com/4e248ceb-018d-4295-80bf-77d2b525ce1e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4e248ceb-018d-4295-80bf-77d2b525ce1e.jpg,488925,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.834488,-121.976618,True,,2024-09-02,11:00:54,,11:00:54,-07:00,1167914,https://au-hw-media-t.happywhale.com/e80b323e-b740-4186-b173-03dacec0db76.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e80b323e-b740-4186-b173-03dacec0db76.jpg,488926,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.825895,-122.0249,True,,2024-09-02,11:29:29,,11:29:29,-07:00,1167916,https://au-hw-media-t.happywhale.com/931f1b60-ec14-439d-a103-8e0dd5af485a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/931f1b60-ec14-439d-a103-8e0dd5af485a.jpg,488927,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.827832,-122.023032,True,,2024-09-02,11:40:29,,11:40:29,-07:00,1167917,https://au-hw-media-t.happywhale.com/7664235e-340d-444c-bd5d-83dabf20290d.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7664235e-340d-444c-bd5d-83dabf20290d.jpg,488928,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725228000000 -36.789702,-121.832227,True,,2024-09-03,09:25:48,,09:25:48,-07:00,1169046,https://au-hw-media-t.happywhale.com/56987e69-fecb-4772-afb4-cf145312bbf8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/56987e69-fecb-4772-afb4-cf145312bbf8.jpg,489746,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.790385,-121.831647,True,,2024-09-03,09:32:15,,09:32:15,-07:00,1169047,https://au-hw-media-t.happywhale.com/96119369-1a5c-458c-b7c4-b458abf07044.jpg,IMAGE,https://au-hw-media-m.happywhale.com/96119369-1a5c-458c-b7c4-b458abf07044.jpg,489747,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.78258,-121.831715,True,,2024-09-03,09:45:05,,09:46:30,-07:00,1169049,https://au-hw-media-t.happywhale.com/b7f57c33-2efe-4bb3-8db2-197fe41bb7c6.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b7f57c33-2efe-4bb3-8db2-197fe41bb7c6.jpg,489748,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.782912,-121.831762,True,,2024-09-03,09:45:05,,09:45:05,-07:00,1169048,https://au-hw-media-t.happywhale.com/ff406c34-2745-43f8-a2fa-c8032fdad7c1.jpg,IMAGE,https://au-hw-media-m.happywhale.com/ff406c34-2745-43f8-a2fa-c8032fdad7c1.jpg,489749,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.83955,-121.978307,True,,2024-09-03,10:46:56,,10:51:45,-07:00,1169940,https://au-hw-media-t.happywhale.com/a061f1f7-291c-4ae8-b436-e41d29fcb04e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a061f1f7-291c-4ae8-b436-e41d29fcb04e.jpg,489750,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.831568,-121.996183,True,,2024-09-03,11:31:41,,11:31:41,-07:00,1169056,https://au-hw-media-t.happywhale.com/055b216d-9584-4164-841c-d1ead0551a4a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/055b216d-9584-4164-841c-d1ead0551a4a.jpg,489757,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.831005,-121.960287,True,,2024-09-03,10:46:56,,10:46:56,-07:00,1169050,https://au-hw-media-t.happywhale.com/68f617b9-253a-4c5b-8f58-95bac887e695.jpg,IMAGE,https://au-hw-media-m.happywhale.com/68f617b9-253a-4c5b-8f58-95bac887e695.jpg,489751,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.831005,-121.960287,True,,2024-09-03,10:47:00,,10:47:00,-07:00,1169051,https://au-hw-media-t.happywhale.com/0792af70-d9d0-4f84-acd3-5ff57423d05a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0792af70-d9d0-4f84-acd3-5ff57423d05a.jpg,489752,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.83193,-121.995492,True,,2024-09-03,11:26:09,,11:33:42,-07:00,1169057,https://au-hw-media-t.happywhale.com/9fb643f2-0f21-49d3-a492-ef04e349dd0c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9fb643f2-0f21-49d3-a492-ef04e349dd0c.jpg,489754,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.831038,-121.994558,True,,2024-09-03,11:26:09,,11:26:09,-07:00,1169941,https://au-hw-media-t.happywhale.com/01f975ae-3e25-4d30-b9e1-1b2d092355d0.jpg,IMAGE,https://au-hw-media-m.happywhale.com/01f975ae-3e25-4d30-b9e1-1b2d092355d0.jpg,489755,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.738617,-121.892053,True,,2024-09-03,15:13:14,,15:13:14,-07:00,1170295,https://au-hw-media-t.happywhale.com/f05dc309-bcee-4113-ae02-a6fb2aaa9507.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f05dc309-bcee-4113-ae02-a6fb2aaa9507.jpg,490032,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.8,-122.0,True,,2024-09-03,12:27:50,,12:27:50,-07:00,1173889,https://au-hw-media-t.happywhale.com/6ce79791-fa36-4133-aa74-d939407faf4a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/6ce79791-fa36-4133-aa74-d939407faf4a.jpg,490863,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.8,-122.0,True,,2024-09-03,12:53:20,,12:53:20,-07:00,1173890,https://au-hw-media-t.happywhale.com/9b9ba5ec-9329-4973-a88a-961f1da18523.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9b9ba5ec-9329-4973-a88a-961f1da18523.jpg,490864,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.827432,-121.976743,True,,2024-09-03,11:40:53,,11:40:53,-07:00,1169942,https://au-hw-media-t.happywhale.com/811259a6-ebb0-4d8d-bfb7-b182dd5aaea8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/811259a6-ebb0-4d8d-bfb7-b182dd5aaea8.jpg,489758,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725314400000 -36.785608,-121.846963,True,,2024-09-04,09:51:18,,09:51:18,-07:00,1170442,https://au-hw-media-t.happywhale.com/549f3fc1-6eab-49bc-a24c-aedf1cb3b0a9.jpg,IMAGE,https://au-hw-media-m.happywhale.com/549f3fc1-6eab-49bc-a24c-aedf1cb3b0a9.jpg,490118,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.835588,-121.984318,True,,2024-09-04,11:29:49,,11:29:49,-07:00,1170444,https://au-hw-media-t.happywhale.com/2cfe4fef-e8b9-4476-8c18-b7a4d502ba9c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2cfe4fef-e8b9-4476-8c18-b7a4d502ba9c.jpg,490119,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.82755,-121.993143,True,,2024-09-04,11:52:48,,11:53:00,-07:00,1170446,https://au-hw-media-t.happywhale.com/48ca9f05-65ab-42c7-9cd6-b3726a414896.jpg,IMAGE,https://au-hw-media-m.happywhale.com/48ca9f05-65ab-42c7-9cd6-b3726a414896.jpg,490120,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.827183,-121.993425,True,,2024-09-04,11:52:48,,11:52:48,-07:00,1171094,https://au-hw-media-t.happywhale.com/47290403-63dd-4a4d-9219-65746712b514.jpg,IMAGE,https://au-hw-media-m.happywhale.com/47290403-63dd-4a4d-9219-65746712b514.jpg,490121,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.787162,-121.843682,True,,2024-09-04,09:41:39,,09:41:39,-07:00,1170440,https://au-hw-media-t.happywhale.com/afaedaa6-11a8-468f-b54e-ea0d5d460a01.jpg,IMAGE,https://au-hw-media-m.happywhale.com/afaedaa6-11a8-468f-b54e-ea0d5d460a01.jpg,490116,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.785608,-121.846963,True,,2024-09-04,09:51:18,,09:51:31,-07:00,1170443,https://au-hw-media-t.happywhale.com/bc50ed90-5667-4d24-83a8-40a5932ff783.jpg,IMAGE,https://au-hw-media-m.happywhale.com/bc50ed90-5667-4d24-83a8-40a5932ff783.jpg,490117,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.654939,-121.914064,True,,2024-09-04,12:42:31,,12:42:31,-07:00,1171743,https://au-hw-media-t.happywhale.com/addb30cc-0c57-4165-a624-660c38291512.jpg,IMAGE,https://au-hw-media-m.happywhale.com/addb30cc-0c57-4165-a624-660c38291512.jpg,490289,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.715076,-121.918002,True,,2024-09-04,,,,America/Los_Angeles,1171599,https://au-hw-media-t.happywhale.com/d92e7a76-293a-4434-96a1-7309383766fb.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d92e7a76-293a-4434-96a1-7309383766fb.jpg,490537,APPROX,,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.711896,-121.893265,True,,2024-09-04,,,,America/Los_Angeles,1171600,https://au-hw-media-t.happywhale.com/7471f8b8-b5d3-4345-b143-4d1a9f8034bc.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7471f8b8-b5d3-4345-b143-4d1a9f8034bc.jpg,490538,APPROX,,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.81997,-121.970215,True,,2024-09-04,08:57:42,,08:57:42,-07:00,1171486,https://au-hw-media-t.happywhale.com/71ce4d25-e121-4687-bf92-dd0bd84daa4e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/71ce4d25-e121-4687-bf92-dd0bd84daa4e.jpg,490521,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.81997,-121.970215,True,,2024-09-04,10:00:42,,10:00:42,-07:00,1171487,https://au-hw-media-t.happywhale.com/65359156-7dd1-4039-a05d-3b0cc7280c79.jpg,IMAGE,https://au-hw-media-m.happywhale.com/65359156-7dd1-4039-a05d-3b0cc7280c79.jpg,490522,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.857,-121.986,True,,2024-09-04,13:58:17,,13:58:18,America/Los_Angeles,1173780,https://au-hw-media-t.happywhale.com/d243a5c5-3f89-43d6-997b-70679bd17d23.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d243a5c5-3f89-43d6-997b-70679bd17d23.jpg,490804,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.857,-121.986,True,,2024-09-04,14:08:51,,14:08:51,America/Los_Angeles,1173781,https://au-hw-media-t.happywhale.com/b9245235-b25c-4cdb-ba1e-cfd600e66db4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b9245235-b25c-4cdb-ba1e-cfd600e66db4.jpg,490805,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.857,-121.986,True,,2024-09-04,14:31:47,,14:34:01,America/Los_Angeles,1173782,https://au-hw-media-t.happywhale.com/e61d54c7-585c-4fa6-ae30-820a78fefa3c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e61d54c7-585c-4fa6-ae30-820a78fefa3c.jpg,490806,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.857,-121.986,True,,2024-09-04,15:15:12,,15:15:12,America/Los_Angeles,1173784,https://au-hw-media-t.happywhale.com/2f84b0bb-f01e-4b29-b483-0ec2b0ffd923.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2f84b0bb-f01e-4b29-b483-0ec2b0ffd923.jpg,490807,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725400800000 -36.83982,-121.979355,True,,2024-09-05,10:11:07,,10:11:07,America/Los_Angeles,1171403,https://au-hw-media-t.happywhale.com/0e7708e3-c403-4d6b-9895-31993e4318b4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0e7708e3-c403-4d6b-9895-31993e4318b4.jpg,490184,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.84069,-121.980018,True,,2024-09-05,10:18:42,,10:18:42,America/Los_Angeles,1171404,https://au-hw-media-t.happywhale.com/0028ec21-56fd-48ef-aafd-7769a67781c5.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0028ec21-56fd-48ef-aafd-7769a67781c5.jpg,490185,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85656,-122.029035,True,,2024-09-05,10:51:55,,10:51:55,America/Los_Angeles,1171405,https://au-hw-media-t.happywhale.com/94770549-0d19-4e0f-815a-3d04f82cb01c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/94770549-0d19-4e0f-815a-3d04f82cb01c.jpg,490186,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85638,-122.029232,True,,2024-09-05,10:56:04,,10:56:04,America/Los_Angeles,1171406,https://au-hw-media-t.happywhale.com/a893e810-ce71-4dd2-b486-3fb23db5af80.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a893e810-ce71-4dd2-b486-3fb23db5af80.jpg,490187,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85638,-122.029232,True,,2024-09-05,10:56:07,,10:56:07,America/Los_Angeles,1171398,https://au-hw-media-t.happywhale.com/bdc2527f-a562-4c7a-bbd4-92d97dc14fd7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/bdc2527f-a562-4c7a-bbd4-92d97dc14fd7.jpg,490188,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85638,-122.029232,True,,2024-09-05,10:56:10,,10:56:10,America/Los_Angeles,1171399,https://au-hw-media-t.happywhale.com/8ca800a7-f0a2-496f-a9ed-79f2140a4e37.jpg,IMAGE,https://au-hw-media-m.happywhale.com/8ca800a7-f0a2-496f-a9ed-79f2140a4e37.jpg,490189,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85638,-122.029232,True,,2024-09-05,10:56:17,,10:56:17,America/Los_Angeles,1171400,https://au-hw-media-t.happywhale.com/baf1eb47-192e-4614-b977-037bc54dc395.jpg,IMAGE,https://au-hw-media-m.happywhale.com/baf1eb47-192e-4614-b977-037bc54dc395.jpg,490190,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.857237,-122.030117,True,,2024-09-05,11:02:09,,11:04:18,America/Los_Angeles,1171402,https://au-hw-media-t.happywhale.com/41a57391-12b9-4a29-84b2-2cfda4c20a9a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/41a57391-12b9-4a29-84b2-2cfda4c20a9a.jpg,490191,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.856827,-122.029802,True,,2024-09-05,11:02:09,,11:02:09,America/Los_Angeles,1171401,https://au-hw-media-t.happywhale.com/5ef39932-974d-41d4-acde-470e2e053d70.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5ef39932-974d-41d4-acde-470e2e053d70.jpg,490192,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.858293,-122.02781,True,,2024-09-05,11:13:09,,11:13:09,America/Los_Angeles,1171397,https://au-hw-media-t.happywhale.com/9492a84c-530a-4fb4-ad62-1f373f38f197.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9492a84c-530a-4fb4-ad62-1f373f38f197.jpg,490193,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85947,-122.028692,True,,2024-09-05,11:17:40,,11:17:40,America/Los_Angeles,1171392,https://au-hw-media-t.happywhale.com/e0a28423-7c84-4b30-bc7f-d81a51962fef.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e0a28423-7c84-4b30-bc7f-d81a51962fef.jpg,490194,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.858835,-122.027523,True,,2024-09-05,11:21:03,,11:21:03,America/Los_Angeles,1171393,https://au-hw-media-t.happywhale.com/38198bfb-4846-4515-a75d-751bfc0d33f9.jpg,IMAGE,https://au-hw-media-m.happywhale.com/38198bfb-4846-4515-a75d-751bfc0d33f9.jpg,490195,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.858982,-122.02771,True,,2024-09-05,11:21:27,,11:21:27,America/Los_Angeles,1171394,https://au-hw-media-t.happywhale.com/2869540a-2ad0-4c42-8b3c-3f48380d8765.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2869540a-2ad0-4c42-8b3c-3f48380d8765.jpg,490196,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.842288,-121.978315,True,,2024-09-05,10:34:33,,10:34:38,-07:00,1171462,https://au-hw-media-t.happywhale.com/c1cf0417-6773-4587-bf35-c36cc48056c2.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c1cf0417-6773-4587-bf35-c36cc48056c2.jpg,490216,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.858288,-122.027612,True,,2024-09-05,11:13:32,,11:13:32,-07:00,1171440,https://au-hw-media-t.happywhale.com/e5f7e6fc-b049-43d5-9b45-72306675cc2c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e5f7e6fc-b049-43d5-9b45-72306675cc2c.jpg,490237,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.859657,-122.029187,True,,2024-09-05,11:24:26,,11:24:26,-07:00,1171446,https://au-hw-media-t.happywhale.com/416f181d-caf6-40c6-ae6a-964e955f2b44.jpg,IMAGE,https://au-hw-media-m.happywhale.com/416f181d-caf6-40c6-ae6a-964e955f2b44.jpg,490238,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.859575,-122.029043,True,,2024-09-05,11:26:13,,11:26:13,America/Los_Angeles,1171395,https://au-hw-media-t.happywhale.com/f4bb9cd8-c053-4bfd-be42-89b6e18ef4d3.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f4bb9cd8-c053-4bfd-be42-89b6e18ef4d3.jpg,490197,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.85945,-122.028792,True,,2024-09-05,11:27:14,,11:27:14,America/Los_Angeles,1171396,https://au-hw-media-t.happywhale.com/0bd6c434-6aa7-48c5-8599-d1508c10e8d8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0bd6c434-6aa7-48c5-8599-d1508c10e8d8.jpg,490198,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.846352,-121.990948,True,,2024-09-05,11:50:49,,11:50:49,America/Los_Angeles,1171391,https://au-hw-media-t.happywhale.com/9bf2ae47-e910-4e8f-bed7-f771fdb04df6.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9bf2ae47-e910-4e8f-bed7-f771fdb04df6.jpg,490199,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.8,-122.0,True,,2024-09-05,09:00:00,,12:30:00,+01:00,1173164,https://au-hw-media-t.happywhale.com/abe6fb60-0288-4167-ba58-deca299cd7af.jpg,IMAGE,https://au-hw-media-m.happywhale.com/abe6fb60-0288-4167-ba58-deca299cd7af.jpg,490531,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.8,-122.0,True,,2024-09-05,09:00:00,,12:30:00,+01:00,1173165,https://au-hw-media-t.happywhale.com/7279f08d-d157-4b5e-8192-b495c3488a5b.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7279f08d-d157-4b5e-8192-b495c3488a5b.jpg,490532,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725487200000 -36.782358,-121.861243,True,,2024-09-06,11:36:02,,11:37:03,-07:00,1171703,https://au-hw-media-t.happywhale.com/fb1a1c71-b069-4193-a4ff-825854e25055.jpg,IMAGE,https://au-hw-media-m.happywhale.com/fb1a1c71-b069-4193-a4ff-825854e25055.jpg,490550,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.7913,-121.845777,True,,2024-09-06,10:56:35,,10:56:35,-07:00,1171702,https://au-hw-media-t.happywhale.com/5c2ac9d3-0103-487f-9cc9-f5351309a889.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5c2ac9d3-0103-487f-9cc9-f5351309a889.jpg,490549,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.782677,-121.860468,True,,2024-09-06,11:37:03,,11:37:03,-07:00,1171704,https://au-hw-media-t.happywhale.com/10961cd8-5b32-412f-8b45-509b6729ec6d.jpg,IMAGE,https://au-hw-media-m.happywhale.com/10961cd8-5b32-412f-8b45-509b6729ec6d.jpg,490551,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.789948,-121.84232,True,,2024-09-06,10:52:48,,10:56:35,-07:00,1171701,https://au-hw-media-t.happywhale.com/d4c645fc-443b-4d38-911e-575469456f9a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d4c645fc-443b-4d38-911e-575469456f9a.jpg,490548,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.787,-121.858982,True,,2024-09-06,11:44:02,,11:46:07,-07:00,1171705,https://au-hw-media-t.happywhale.com/f8aaa791-b113-4cab-8a91-90a8b12b5f87.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f8aaa791-b113-4cab-8a91-90a8b12b5f87.jpg,490552,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.78764,-121.860068,True,,2024-09-06,11:46:07,,11:46:07,-07:00,1171706,https://au-hw-media-t.happywhale.com/f379513b-7970-4798-966c-9b1d017e35a5.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f379513b-7970-4798-966c-9b1d017e35a5.jpg,490553,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725573600000 -36.714688,-121.982682,True,,2024-09-07,10:27:06,,10:27:06,-07:00,1172058,https://au-hw-media-t.happywhale.com/e24ea764-8818-4399-8295-1e4d657cfb53.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e24ea764-8818-4399-8295-1e4d657cfb53.jpg,490640,PRECISE,MANUAL,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.714688,-121.982682,True,,2024-09-07,10:38:32,,10:38:33,-07:00,1172060,https://au-hw-media-t.happywhale.com/78ea31f5-443f-45db-b2af-6a668075bc0e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/78ea31f5-443f-45db-b2af-6a668075bc0e.jpg,490641,PRECISE,MANUAL,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.656622,-121.980381,True,,2024-09-07,16:35:10,,16:35:10,-07:00,1172194,https://au-hw-media-t.happywhale.com/9c6ee668-45e7-4339-abc7-c3d1af542bca.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9c6ee668-45e7-4339-abc7-c3d1af542bca.jpg,490629,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.656622,-121.980381,True,,2024-09-07,16:34:27,,16:34:27,-07:00,1172193,https://au-hw-media-t.happywhale.com/164408ee-cc65-4803-a15a-e0debfefaf21.jpg,IMAGE,https://au-hw-media-m.happywhale.com/164408ee-cc65-4803-a15a-e0debfefaf21.jpg,490628,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.656622,-121.980381,True,,2024-09-07,16:37:01,,16:37:01,-07:00,1172195,https://au-hw-media-t.happywhale.com/37a9437c-e7d2-4b81-b5d8-b3b719440d5c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/37a9437c-e7d2-4b81-b5d8-b3b719440d5c.jpg,490630,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.67662,-121.960602,True,,2024-09-07,11:10:28,,11:10:28,-07:00,1172184,https://au-hw-media-t.happywhale.com/d88b795f-6c8e-4364-bf74-71c47a9be5bb.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d88b795f-6c8e-4364-bf74-71c47a9be5bb.jpg,490627,PRECISE,MOBILE_DEVICE,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.8,-122.0,True,,2024-09-07,11:55:25,,11:55:25,-07:00,1173428,https://au-hw-media-t.happywhale.com/a4fe3410-ae25-40a5-9db3-4066457278de.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a4fe3410-ae25-40a5-9db3-4066457278de.jpg,490714,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.799346,-121.826019,True,,2024-09-07,09:54:19,,09:54:19,-08:00,1173627,https://au-hw-media-t.happywhale.com/d8547433-9647-4173-939f-1e4d624fc0d7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/d8547433-9647-4173-939f-1e4d624fc0d7.jpg,490932,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.781517,-121.882324,True,,2024-09-07,09:54:31,,09:54:31,-08:00,1173628,https://au-hw-media-t.happywhale.com/f051292c-5684-4a0a-a940-851c3423bfea.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f051292c-5684-4a0a-a940-851c3423bfea.jpg,490933,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.857,-121.986,True,,2024-09-07,14:49:35.39,,14:49:35.72,America/Los_Angeles,1173786,https://au-hw-media-t.happywhale.com/2de6a5e0-2eca-4286-8731-18652a9096bf.jpg,IMAGE,https://au-hw-media-m.happywhale.com/2de6a5e0-2eca-4286-8731-18652a9096bf.jpg,490808,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.8,-122.0,True,,2024-09-07,09:00:00,,12:00:00,America/Los_Angeles,1174780,https://au-hw-media-t.happywhale.com/74bf80aa-bd25-45b7-b35a-2e45c5ae81e4.jpg,IMAGE,https://au-hw-media-m.happywhale.com/74bf80aa-bd25-45b7-b35a-2e45c5ae81e4.jpg,491089,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.8,-122.0,True,,2024-09-07,11:54:49,,11:58:36,-07:00,1173429,https://au-hw-media-t.happywhale.com/84727ede-a569-4fcf-8d13-1cd3a0cb8e49.jpg,IMAGE,https://au-hw-media-m.happywhale.com/84727ede-a569-4fcf-8d13-1cd3a0cb8e49.jpg,490716,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725660000000 -36.668167,-121.993452,True,,2024-09-08,12:52:28,,12:52:31,-07:00,1172845,https://au-hw-media-t.happywhale.com/fa82d9af-8387-4c50-b941-a901f71c7a61.jpg,IMAGE,https://au-hw-media-m.happywhale.com/fa82d9af-8387-4c50-b941-a901f71c7a61.jpg,490433,PRECISE,CAMERA,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.69579,-121.973745,True,,2024-09-08,12:15:25,,12:15:25,-07:00,1173330,https://au-hw-media-t.happywhale.com/3d9a7546-2769-403a-874d-ca3e9eac5adc.jpg,IMAGE,https://au-hw-media-m.happywhale.com/3d9a7546-2769-403a-874d-ca3e9eac5adc.jpg,490634,PRECISE,CAMERA,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.655036,-121.988556,True,,2024-09-08,10:39:11,,10:39:11,-07:00,1172660,https://au-hw-media-t.happywhale.com/6009cda7-74e0-4dba-8d05-272aef9539dc.jpg,IMAGE,https://au-hw-media-m.happywhale.com/6009cda7-74e0-4dba-8d05-272aef9539dc.jpg,490655,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.788872,-121.827155,True,,2024-09-08,09:28:12,,09:28:12,-07:00,1173760,https://au-hw-media-t.happywhale.com/245ec9b6-c399-433c-8e65-c7dbca47fc24.jpg,IMAGE,https://au-hw-media-m.happywhale.com/245ec9b6-c399-433c-8e65-c7dbca47fc24.jpg,490752,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.793868,-121.839638,True,,2024-09-08,09:53:54,,09:53:54,-07:00,1173434,https://au-hw-media-t.happywhale.com/74b50b5c-585e-4ba1-a387-ddd247830924.jpg,IMAGE,https://au-hw-media-m.happywhale.com/74b50b5c-585e-4ba1-a387-ddd247830924.jpg,490753,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.859148,-122.007592,True,,2024-09-08,11:12:57,,11:12:57,-07:00,1173435,https://au-hw-media-t.happywhale.com/8776737e-3b98-4217-969d-b581b55ea349.jpg,IMAGE,https://au-hw-media-m.happywhale.com/8776737e-3b98-4217-969d-b581b55ea349.jpg,490754,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.850405,-122.006808,True,,2024-09-08,11:35:26,,11:35:26,-07:00,1173436,https://au-hw-media-t.happywhale.com/c5014c0a-ab70-41ce-86a0-2d3fa5eb7d16.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c5014c0a-ab70-41ce-86a0-2d3fa5eb7d16.jpg,490755,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.844965,-121.963655,True,,2024-09-08,11:49:07,,11:49:40,-07:00,1173438,https://au-hw-media-t.happywhale.com/bf258a2a-ee73-4ad0-a32a-7dcb1003b2cf.jpg,IMAGE,https://au-hw-media-m.happywhale.com/bf258a2a-ee73-4ad0-a32a-7dcb1003b2cf.jpg,490756,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.844965,-121.963655,True,,2024-09-08,11:49:07,,11:49:07,-07:00,1173437,https://au-hw-media-t.happywhale.com/5d3128b4-4e32-4217-9986-bcd6c19796b1.jpg,IMAGE,https://au-hw-media-m.happywhale.com/5d3128b4-4e32-4217-9986-bcd6c19796b1.jpg,490757,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.841272,-121.9434,True,,2024-09-08,11:55:43,,11:58:29,-07:00,1173440,https://au-hw-media-t.happywhale.com/89ef855e-791d-46d6-b0ae-99f7c4f2284f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/89ef855e-791d-46d6-b0ae-99f7c4f2284f.jpg,490758,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.843222,-121.94623,True,,2024-09-08,11:55:43,,11:55:43,-07:00,1173439,https://au-hw-media-t.happywhale.com/04ef618e-3e8b-4be0-8854-c3477e14776f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/04ef618e-3e8b-4be0-8854-c3477e14776f.jpg,490759,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.835807,-121.940262,True,,2024-09-08,12:09:29,,12:11:42,-07:00,1173762,https://au-hw-media-t.happywhale.com/f2db3848-5e6d-489d-8756-128d0aef75bf.jpg,IMAGE,https://au-hw-media-m.happywhale.com/f2db3848-5e6d-489d-8756-128d0aef75bf.jpg,490760,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.836835,-121.939883,True,,2024-09-08,12:09:29,,12:09:29,-07:00,1173761,https://au-hw-media-t.happywhale.com/9157c806-c620-44ba-8fad-0abc87b60d6b.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9157c806-c620-44ba-8fad-0abc87b60d6b.jpg,490761,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.835807,-121.940262,True,,2024-09-08,12:11:24,,12:11:24,-07:00,1173443,https://au-hw-media-t.happywhale.com/1dbb0baf-78c2-4c20-a8e0-a29252ad8417.jpg,IMAGE,https://au-hw-media-m.happywhale.com/1dbb0baf-78c2-4c20-a8e0-a29252ad8417.jpg,490763,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.656478,-121.996858,True,,2024-09-08,16:18:55,,16:18:55,-07:00,1173450,https://au-hw-media-t.happywhale.com/b5b508ae-8cb1-40f4-9e8f-a44cfde8d524.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b5b508ae-8cb1-40f4-9e8f-a44cfde8d524.jpg,490765,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.676142,-121.99613,True,,2024-09-08,16:36:16,,16:39:56,-07:00,1173452,https://au-hw-media-t.happywhale.com/9c09da3a-f8bc-4593-af12-80e267a3cca1.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9c09da3a-f8bc-4593-af12-80e267a3cca1.jpg,490766,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.67353,-121.994273,True,,2024-09-08,16:36:16,,16:36:16,-07:00,1173451,https://au-hw-media-t.happywhale.com/b00d6346-5eb4-49ca-b3fe-bceaeb96cb21.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b00d6346-5eb4-49ca-b3fe-bceaeb96cb21.jpg,490767,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.836417,-121.939948,True,,2024-09-08,12:10:46,,12:10:46,-07:00,1173442,https://au-hw-media-t.happywhale.com/12dd6080-ce22-4657-89c6-757ee586d663.jpg,IMAGE,https://au-hw-media-m.happywhale.com/12dd6080-ce22-4657-89c6-757ee586d663.jpg,490762,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.672652,-122.029522,True,,2024-09-08,17:20:43,,17:20:43,-07:00,1173454,https://au-hw-media-t.happywhale.com/7789bfb7-2035-4631-8280-3a5a9b93c9c8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/7789bfb7-2035-4631-8280-3a5a9b93c9c8.jpg,490770,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.678967,-122.00501,True,,2024-09-08,16:54:45,,16:54:45,-07:00,1173453,https://au-hw-media-t.happywhale.com/003f0347-2213-41a0-9204-cc23be5be747.jpg,IMAGE,https://au-hw-media-m.happywhale.com/003f0347-2213-41a0-9204-cc23be5be747.jpg,490768,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.672652,-122.029522,True,,2024-09-08,17:20:43,,17:20:55,-07:00,1173764,https://au-hw-media-t.happywhale.com/14dc1233-c90a-41fa-92b8-fb437bedd30c.jpg,IMAGE,https://au-hw-media-m.happywhale.com/14dc1233-c90a-41fa-92b8-fb437bedd30c.jpg,490769,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.802478,-121.845658,True,,2024-09-08,08:58:57,,08:58:57,-07:00,1173934,https://au-hw-media-t.happywhale.com/102f1df8-4f3e-47c8-b345-9cc68fff242a.jpg,IMAGE,https://au-hw-media-m.happywhale.com/102f1df8-4f3e-47c8-b345-9cc68fff242a.jpg,490894,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.667997,-121.994338,True,,2024-09-08,12:39:40,,12:42:04,-07:00,1172907,https://au-hw-media-t.happywhale.com/efbca769-6711-4d09-973b-818725fb870e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/efbca769-6711-4d09-973b-818725fb870e.jpg,490440,PRECISE,CAMERA,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.669735,-121.996687,True,,2024-09-08,12:39:40,,12:39:40,-07:00,1172905,https://au-hw-media-t.happywhale.com/52c60a4d-85a9-4e75-b284-ee74513eea71.jpg,IMAGE,https://au-hw-media-m.happywhale.com/52c60a4d-85a9-4e75-b284-ee74513eea71.jpg,490441,PRECISE,CAMERA,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.626771,-121.876316,True,,2024-09-08,11:27:59,,11:27:59,-08:00,1173616,https://au-hw-media-t.happywhale.com/b8456c33-8537-454d-bebd-cd5aa7f31142.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b8456c33-8537-454d-bebd-cd5aa7f31142.jpg,490927,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.737165,-121.953735,True,,2024-09-08,11:34:28,,11:34:28,-08:00,1173623,https://au-hw-media-t.happywhale.com/0ccf0ffa-d68c-4cf1-8876-632fcea0d755.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0ccf0ffa-d68c-4cf1-8876-632fcea0d755.jpg,490928,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.7866,-121.8278,True,,2024-09-08,09:53:00,,10:00:00,America/Los_Angeles,1174134,https://au-hw-media-t.happywhale.com/869bbfba-9dd5-4aef-9624-410c175c05df.jpg,IMAGE,https://au-hw-media-m.happywhale.com/869bbfba-9dd5-4aef-9624-410c175c05df.jpg,490977,APPROX,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725746400000 -36.606214,-122.033211,True,,2024-09-09,15:34:25,,15:34:25,-07:00,1173549,https://au-hw-media-t.happywhale.com/23dba629-3bb4-478c-9c2d-81d40d92f166.jpg,IMAGE,https://au-hw-media-m.happywhale.com/23dba629-3bb4-478c-9c2d-81d40d92f166.jpg,490727,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.623928,-122.013372,True,,2024-09-09,15:48:09,,15:48:09,-07:00,1173551,https://au-hw-media-t.happywhale.com/4d8a628c-6a32-4845-a27c-5b64e9965468.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4d8a628c-6a32-4845-a27c-5b64e9965468.jpg,490728,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.798298,-121.808335,True,,2024-09-09,09:16:27,,09:16:27,-07:00,1173461,https://au-hw-media-t.happywhale.com/45f89314-e68b-4b08-abb0-108d29e646d5.jpg,IMAGE,https://au-hw-media-m.happywhale.com/45f89314-e68b-4b08-abb0-108d29e646d5.jpg,490772,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.78376,-121.901538,True,,2024-09-09,09:46:00,,09:46:00,-07:00,1173462,https://au-hw-media-t.happywhale.com/97e2706a-bfd7-4d01-8319-95fe3376ca69.jpg,IMAGE,https://au-hw-media-m.happywhale.com/97e2706a-bfd7-4d01-8319-95fe3376ca69.jpg,490774,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.797818,-121.913445,True,,2024-09-09,10:06:38,,10:06:38,-07:00,1173765,https://au-hw-media-t.happywhale.com/0c6a4d29-cb51-477e-be28-859d3c4cad4f.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0c6a4d29-cb51-477e-be28-859d3c4cad4f.jpg,490775,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.778772,-121.958385,True,,2024-09-09,10:35:00,,10:38:17,-07:00,1173467,https://au-hw-media-t.happywhale.com/58deacda-ed4c-4cc5-8067-732a69fd7083.jpg,IMAGE,https://au-hw-media-m.happywhale.com/58deacda-ed4c-4cc5-8067-732a69fd7083.jpg,490776,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.780257,-121.961273,True,,2024-09-09,10:35:00,,10:35:00,-07:00,1173465,https://au-hw-media-t.happywhale.com/71eb4f2f-3a02-429d-8a1c-5ae720aad079.jpg,IMAGE,https://au-hw-media-m.happywhale.com/71eb4f2f-3a02-429d-8a1c-5ae720aad079.jpg,490777,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.790927,-121.945802,True,,2024-09-09,10:58:52,,10:58:52,-07:00,1173468,https://au-hw-media-t.happywhale.com/73ff0fb9-a2b8-4e44-b747-bc3d23555da9.jpg,IMAGE,https://au-hw-media-m.happywhale.com/73ff0fb9-a2b8-4e44-b747-bc3d23555da9.jpg,490780,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.641636,-122.00798,True,,2024-09-09,12:37:14,,12:40:02,-07:00,1174727,https://au-hw-media-t.happywhale.com/4946e017-10ea-4653-baa6-c3a09eef7b49.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4946e017-10ea-4653-baa6-c3a09eef7b49.jpg,491069,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.8,-122.0,True,,2024-09-09,,,11:10:00,America/Los_Angeles,1174803,https://au-hw-media-t.happywhale.com/e61baefe-4d2c-43c9-873a-537ba2da6d06.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e61baefe-4d2c-43c9-873a-537ba2da6d06.jpg,491094,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.797165,-121.81359,True,,2024-09-09,09:17:36,,09:17:36,-07:00,1173479,https://au-hw-media-t.happywhale.com/0cd3d47e-94f4-442e-b23e-11a57c7d9052.jpg,IMAGE,https://au-hw-media-m.happywhale.com/0cd3d47e-94f4-442e-b23e-11a57c7d9052.jpg,490788,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.78376,-121.901538,True,,2024-09-09,09:46:00,,09:46:24,-07:00,1173463,https://au-hw-media-t.happywhale.com/b66f44f1-ed1e-4b44-9df6-a89f604abe0d.jpg,IMAGE,https://au-hw-media-m.happywhale.com/b66f44f1-ed1e-4b44-9df6-a89f604abe0d.jpg,490773,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.778772,-121.958385,True,,2024-09-09,10:37:55,,10:37:55,-07:00,1173466,https://au-hw-media-t.happywhale.com/e7f4a2ec-0c9e-41f2-9c54-0934bf2f9aca.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e7f4a2ec-0c9e-41f2-9c54-0934bf2f9aca.jpg,490778,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.790927,-121.945802,True,,2024-09-09,10:58:52,,10:59:10,-07:00,1173469,https://au-hw-media-t.happywhale.com/1c73837f-ecd8-40a6-9645-96c946662391.jpg,IMAGE,https://au-hw-media-m.happywhale.com/1c73837f-ecd8-40a6-9645-96c946662391.jpg,490779,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.8,-122.0,True,,2024-09-09,10:26:42,,10:26:42,-07:00,1173989,https://au-hw-media-t.happywhale.com/4cc90660-91b7-4ced-b4d6-38cc7f9083d5.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4cc90660-91b7-4ced-b4d6-38cc7f9083d5.jpg,490913,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.8,-122.0,True,,2024-09-09,11:14:09,,11:14:09,-07:00,1173990,https://au-hw-media-t.happywhale.com/20ca1800-5a69-4812-9933-cd2c937ab6a8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/20ca1800-5a69-4812-9933-cd2c937ab6a8.jpg,490914,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.8,-122.0,True,,2024-09-09,10:08:47,,10:08:47,-07:00,1173524,https://au-hw-media-t.happywhale.com/a382eca0-46ea-429c-a8d8-f3fe1edec893.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a382eca0-46ea-429c-a8d8-f3fe1edec893.jpg,490912,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.8,-122.0,True,,2024-09-09,11:00:23,,11:00:26,-07:00,1174137,https://au-hw-media-t.happywhale.com/4568689f-5745-4d85-87f2-12c0c1fdb4f7.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4568689f-5745-4d85-87f2-12c0c1fdb4f7.jpg,490980,GENERAL,,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725832800000 -36.676165,-121.968586,True,,2024-09-10,11:31:31,,11:31:31,America/Los_Angeles,1174095,https://au-hw-media-t.happywhale.com/3c216077-382f-4547-9b5d-4bcd5d7b0155.jpg,IMAGE,https://au-hw-media-m.happywhale.com/3c216077-382f-4547-9b5d-4bcd5d7b0155.jpg,490969,PRECISE,GPS_TRACK,CC_BY_SA,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.667581,-121.965189,True,,2024-09-10,14:03:40,,14:06:38,-07:00,1174068,https://au-hw-media-t.happywhale.com/1c1fb4dc-4ba4-41c4-a9bf-b872c67b0fba.jpg,IMAGE,https://au-hw-media-m.happywhale.com/1c1fb4dc-4ba4-41c4-a9bf-b872c67b0fba.jpg,491020,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.667581,-121.965189,True,,2024-09-10,14:06:38,,14:06:38,-07:00,1174069,https://au-hw-media-t.happywhale.com/a74f8541-fdb5-45a3-a544-6eaeaf9130b2.jpg,IMAGE,https://au-hw-media-m.happywhale.com/a74f8541-fdb5-45a3-a544-6eaeaf9130b2.jpg,491021,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.790487,-121.855258,True,,2024-09-10,09:36:05,,09:36:05,-07:00,1174075,https://au-hw-media-t.happywhale.com/bf4e717e-dbae-42f0-a280-862993625264.jpg,IMAGE,https://au-hw-media-m.happywhale.com/bf4e717e-dbae-42f0-a280-862993625264.jpg,491061,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.770193,-121.904048,True,,2024-09-10,09:55:09,,09:55:09,-07:00,1174076,https://au-hw-media-t.happywhale.com/c0104e89-487b-4ff2-8963-fda69cd4cd74.jpg,IMAGE,https://au-hw-media-m.happywhale.com/c0104e89-487b-4ff2-8963-fda69cd4cd74.jpg,491062,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.622639,-122.048242,True,,2024-09-10,10:26:15,,10:26:16,-07:00,1173986,https://au-hw-media-t.happywhale.com/ff4b1b23-dec2-426d-952c-4c39cd201437.jpg,IMAGE,https://au-hw-media-m.happywhale.com/ff4b1b23-dec2-426d-952c-4c39cd201437.jpg,490953,PRECISE,CAMERA,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.677157,-121.970844,True,,2024-09-10,11:35:42,,11:35:42,America/Los_Angeles,1174099,https://au-hw-media-t.happywhale.com/4bb7ed3d-388b-488b-8f66-197c8854371b.jpg,IMAGE,https://au-hw-media-m.happywhale.com/4bb7ed3d-388b-488b-8f66-197c8854371b.jpg,490981,PRECISE,GPS_TRACK,CC_BY,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.79086,-121.852948,True,,2024-09-10,09:30:42,,09:30:42,America/Los_Angeles,1174219,https://au-hw-media-t.happywhale.com/1d1e3ba8-52ca-4a0f-863d-f81d63128baf.jpg,IMAGE,https://au-hw-media-m.happywhale.com/1d1e3ba8-52ca-4a0f-863d-f81d63128baf.jpg,491011,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.76211,-121.948705,True,,2024-09-10,10:17:59,,10:17:59,America/Los_Angeles,1174220,https://au-hw-media-t.happywhale.com/6cb66001-7add-4aa3-a075-9d68f5b899fb.jpg,IMAGE,https://au-hw-media-m.happywhale.com/6cb66001-7add-4aa3-a075-9d68f5b899fb.jpg,491012,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.77945,-121.965677,True,,2024-09-10,11:07:03,,11:10:02,America/Los_Angeles,1174224,https://au-hw-media-t.happywhale.com/e1aaf443-0e76-434a-aedf-fcdd710cb89e.jpg,IMAGE,https://au-hw-media-m.happywhale.com/e1aaf443-0e76-434a-aedf-fcdd710cb89e.jpg,491016,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.759487,-121.956007,True,,2024-09-10,10:29:24,,10:31:13,America/Los_Angeles,1174223,https://au-hw-media-t.happywhale.com/9b11ea89-d16b-41e0-97c9-12466b383801.jpg,IMAGE,https://au-hw-media-m.happywhale.com/9b11ea89-d16b-41e0-97c9-12466b383801.jpg,491013,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.75984,-121.954453,True,,2024-09-10,10:29:24,,10:29:24,America/Los_Angeles,1174221,https://au-hw-media-t.happywhale.com/81c7c19d-4bbd-4bac-8e63-3f91334ab967.jpg,IMAGE,https://au-hw-media-m.happywhale.com/81c7c19d-4bbd-4bac-8e63-3f91334ab967.jpg,491014,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.759743,-121.954547,True,,2024-09-10,10:29:43,,10:29:43,America/Los_Angeles,1174222,https://au-hw-media-t.happywhale.com/efd4a3a3-5386-47f0-99a2-3a1fbb4a5b77.jpg,IMAGE,https://au-hw-media-m.happywhale.com/efd4a3a3-5386-47f0-99a2-3a1fbb4a5b77.jpg,491015,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.775808,-121.965453,True,,2024-09-10,11:07:03,,11:07:03,America/Los_Angeles,1174218,https://au-hw-media-t.happywhale.com/cd84fdef-79a0-4d98-891d-308618f4cdda.jpg,IMAGE,https://au-hw-media-m.happywhale.com/cd84fdef-79a0-4d98-891d-308618f4cdda.jpg,491017,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 -36.778518,-121.967325,True,,2024-09-10,11:22:43,,11:22:43,America/Los_Angeles,1174225,https://au-hw-media-t.happywhale.com/3c26750e-c336-4086-81b8-4aa893c183a8.jpg,IMAGE,https://au-hw-media-m.happywhale.com/3c26750e-c336-4086-81b8-4aa893c183a8.jpg,491018,PRECISE,GPS_TRACK,PUBLIC_DOMAIN,1,1,[None],True,"California, United States",humpback_whale,1725919200000 diff --git a/data/geo/monterey_bay.geojson b/data/geo/monterey_bay.geojson deleted file mode 100644 index a3aa055..0000000 --- a/data/geo/monterey_bay.geojson +++ /dev/null @@ -1,36 +0,0 @@ -{ - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "properties": {}, - "geometry": { - "coordinates": [ - [ - [ - -122.97130689953673, - 37.137884877470185 - ], - [ - -122.97130689953673, - 36.16763871764208 - ], - [ - -121.65593713142661, - 36.16763871764208 - ], - [ - -121.65593713142661, - 37.137884877470185 - ], - [ - -122.97130689953673, - 37.137884877470185 - ] - ] - ], - "type": "Polygon" - } - } - ] -} diff --git a/data/postprocess/output.json b/data/postprocess/output.json new file mode 100644 index 0000000..c636f21 --- /dev/null +++ b/data/postprocess/output.json @@ -0,0 +1,15 @@ +[ + { + "encounter_id": "9182", + "latitude": 36.91, + "longitude": -122.02, + "start": "2016-12-21T00:49:30", + "end": "2016-12-21T00:50:30", + "pooled_score": 0.2997462116, + "img_path": "https://au-hw-media-m.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg", + "audio_path": "data/audio/raw/key=20161221T004930-005030_9182/MARS-20161221T004930Z-16kHz.npy", + "classification_path": "data/classifications/butterworth/highcut=1500/lowcut=50/order=5/threshold=0.015/20161221T004930-005030_9182.npy", + "sift_audio_path": "data/audio/butterworth/highcut=1500/lowcut=50/order=5/threshold=0.015/key=20161221T004930-005030_9182/audio.npy", + "detections_path": "data/audio/butterworth/highcut=1500/lowcut=50/order=5/threshold=0.015/key=20161221T004930-005030_9182/detections.npy" + } +] \ No newline at end of file diff --git a/data/table/geometry_search/geofile=monterey_bay_50km/encounters.csv b/data/table/geometry_search/geofile=monterey_bay_50km/encounters.csv new file mode 100644 index 0000000..228d7c0 --- /dev/null +++ b/data/table/geometry_search/geofile=monterey_bay_50km/encounters.csv @@ -0,0 +1,3 @@ +encounter_id,encounter_time,longitude,latitude,img_path +9182,2016-12-21T13:50:00,-122.02,36.91,https://au-hw-media-m.happywhale.com/d40b9e6e-07cf-4f20-8cb4-4042ba22a00b.jpg +11486,2016-12-21T13:21:00,-122.02,36.91,https://au-hw-media-m.happywhale.com/c5522187-058e-4a1a-83d7-893560ba6b2c.jpg diff --git a/data/table/raw_audio/metadata.json b/data/table/raw_audio/metadata.json new file mode 100644 index 0000000..1401615 --- /dev/null +++ b/data/table/raw_audio/metadata.json @@ -0,0 +1 @@ +[{"key":"20161221T002030-002130_11486","start":"2016-12-21T00:20:30","end":"2016-12-21T00:21:30","audio_path":"data\/audio\/raw\/key=20161221T002030-002130_11486\/MARS-20161221T002030Z-16kHz.npy"},{"key":"20161221T004930-005030_9182","start":"2016-12-21T00:49:30","end":"2016-12-21T00:50:30","audio_path":"data\/audio\/raw\/key=20161221T004930-005030_9182\/MARS-20161221T004930Z-16kHz.npy"}] \ No newline at end of file diff --git a/data/table/sifted_audio/metadata.json b/data/table/sifted_audio/metadata.json new file mode 100644 index 0000000..547da74 --- /dev/null +++ b/data/table/sifted_audio/metadata.json @@ -0,0 +1 @@ +[{"key":"20161221T004930-005030_9182","sift_audio_path":"data\/audio\/butterworth\/highcut=1500\/lowcut=50\/order=5\/threshold=0.015\/key=20161221T004930-005030_9182\/audio.npy","sift_detections_path":"data\/audio\/butterworth\/highcut=1500\/lowcut=50\/order=5\/threshold=0.015\/key=20161221T004930-005030_9182\/detections.npy","params":"{\"lowcut\": 50, \"highcut\": 1500, \"order\": 5, \"threshold\": 0.015, \"name\": \"butterworth\"}"}] \ No newline at end of file diff --git a/docs/ladr/LADR_0005_persist_intermediate_outputs.md b/docs/ladr/LADR_0005_persist_intermediate_outputs.md index 5e6a267..b28604d 100644 --- a/docs/ladr/LADR_0005_persist_intermediate_outputs.md +++ b/docs/ladr/LADR_0005_persist_intermediate_outputs.md @@ -1,8 +1,8 @@ # Intermediate stage outputs This doc discusses handling the intermediate outputs between stages. -Whether or not to store the output should be confirgurable for the end user, either via command line params or in [config.yaml](../../src/config/common.yaml) -We want to enable these storage options to support local adn cloud storage. +Whether or not to store the output should be confirgurable for the end user, either via command line params or in [config.yaml](../../src/config/common.yaml). +We want to support both local and cloud storage. This means we need to consider costs and effeciency when designing output schemas. Some stages (like geo-search) require locally storing outputs, since the (unaltered) Happywhale API currently writes found encounters to file, and does not return a df. @@ -11,108 +11,165 @@ Other stages like audio retrival may make sense to keep stateless to avoid stora Or storing only start and stop values for the audio, with a link to that day's data. -For during local development and debugging, having all intermediate data stored helps speed up iteration time. -Additionally, if data already exists for run on a particular date, the pipeline should skip these stages and load the outputs from the previous run. -While a productionized pipeline might only run once per geofile-date, and not need to store intermediate outputs, this decision should be left up to the end user. +During local development and debugging, having all intermediate data stored helps speed up iteration time. +If data already exists for run on a particular date w/ same config params, the pipeline should skip these stages and load the outputs from the previous run. +A productionized pipeline may likely only run once per geo-file date, depending on use-case, but storage decisions should be left up to the end user. -Another point to consider is that data-usage agreements with the audio proivders. +Another point to consider is data-usage agreements with the audio proivders. Make sure to read any agreements to ensure that storing intermediate outputs is allowed. Some questions to consider: -- Exactly what data should we preserve from each stage? Will this different from the output of the stage? +- Exactly what data should we persist from each stage? Will these data differ from the output of the stage? Ex. start/stop times of sifted audio, full classification arrays or pooled results. -- How to handle overwrites or parallel writes? Parallel writes should never occur, since we find overlapping encounter ids, and group them together. Overwrites could occur if stage does not check if data exists for stage before writing. - +- How to handle overwrites or parallel writes? Parallel writes should never occur, since we find overlapping encounter ids, and group them together, and treat as single row in downstream stages. Overwrites could occur if stage does not check if data exists for stage before writing. For the most part, loading previous and appending new data is likely the best strategy here. + -- Do we have a true key throughout our entire dataflow? Do we need one? After geo-search, we could consider a concatenation of start, end, and encounter_id as key, though this might be misleading, if sifted audio changes the start and end times. +- Do we have a true unique identifiable key throughout our entire dataflow? Do we need one? After geo-search, we could consider a concatenation of start, end, and encounter_id as key, though this might be misleading, if sifted audio changes the start and end times. ## Stages For every stage, I'll discuss what outputs to store, and how they should be written locally and in the cloud. ### 1. Geo-search + +Columns to persist: +``` +encounter_id (str) +encounter_time (iso formatted datetime) +longitude (float) +latitude (float) +displayImgUrl (str) +``` +These works well for both local and cloud storage options. + + #### Local storage + We are forced to store the outputs for this stage, since the API requires a file path to write the pandas df to. This means, there is a chance of overwites when running locally, or on a persistant server. Can however be solved by providing a temporary file location to the API, loading in the old data and the temporary outputs, then write to the final location. + #### Cloud storage This data is very structured, and could be stored in a database. We should init and create a table in our project.dataset_id. -This can be the alternative to storing the temporary file to a more persistant final location. +This will be the alternative to storing the temporary file to a more persistant final location when `is_local=False`. ### 2. Audio retrieval We should likely not store the full outputs for this stage. The data is open source and can be retrieved at any time, only costs to download. -The main argument for storing here would be if download costs were significantly higher than storage, i.e. on a persistant server. + +An argument for storing here would be if download costs were significantly higher than storage costs, for example on a persistant server. Development and debugging are still easier with the data stored, so we also need to smartly design these outputs. +Another arguement would be to have the original signal that was fed into the downstream stasks. + +Columns to persist in table: +``` +key (str) +audio_path (str) +start_time (str, iso formatted datetime) NULLABLE +stop_time (str, iso formatted datetime) NULLABLE +``` + +To persist in bucket as array (key in path): +``` +audio (np.array) NULLABLE +``` + #### Local storage -Writing to my local machine has been easy enough with np.save. +Writing array to local machine is easy enough with np.save, with date and key in path. This makes the data easily accessible for listening to, which is extremely helpful when analysing plots. -For now, I'll assume this is good enough, and rather rely on the built-in teardown of the DataflowRunner to clean this data up if wronglly configured during cloud runs. + +Additionally, we can maintain an index-table for the audio stored locally, with the columns mentioned above. #### Cloud storage -We could store the start, stop times of (and maybe url link to) the audio retrived for the found encounter ids in a table in our project.dataset_id. +Store the start, stop times of (and maybe url link to) the audio retrived for the found encounter ids in a table in our project.dataset_id. This will can be beneficial if a user decides not to use any audio sifting. -Maybe add config option to allow storing full audio? + +Uploading audio to its own bucket can be configurable (with default False). If stored, should be identified by a key (start, stop, encounter_id) to avoid overwrites, and stored in a bucket, not a table. ### 3. Audio sifting -How much audio sift data should be persisted? +How much sifted audio data should be persisted? Full audio arrays with start, stop times and encounter ids? Or just the start, stop times and encounter ids, assuming the audio will be downloaded and passed from the previous stage? There is really no need to double storage here, but option should still be available. -The main argument for storing the full audio arrays is that it speeds up iteration time, and allows for easier debugging. -We likely also want this data easily accessible, if our final outputs are going to contain the audio snippets with classifications. +We likely want this data easily accessible, since it is the input to our inference model. +Our final pipeline outputs will likely want to display these data as audio snippets with classifications and images. That's kinda the whole point of this pipeline, so _some_ audio will eventually need to be be stored. And its likely at this stage we will want to store it, since this audio is truncated. -Will need to think about the key or unique identifier here, since there are a lot of parameters that can affect how much audio was truncated. essentially, all of these can be in the path, but that will make for extremely long paths. +Will need to think about the key or unique identifier here, since there are a lot of parameters that can affect how much audio was truncated. +Essentially, all of these can be in the path, but that will make for extremely long paths. +However, the long path makes it very explicit how the audio was parsed (ex. `.../filter=butterworth/highcut=1500/lowcut=50/order=5/...`) +Most params are not hierarchical, though using a wildcard `*` in path on read helps avoid this issue (ex. `.../filter=butterworth/highcut=*/lowcut=50/order=5/...`). + +Columns to persist in table: +``` +key (str) +audio_path (str) +detections_path (str) +params (Dict[str, str], might need to be str in queryable-table) +``` + +To persist in bucket as inidividual arrays (params and key in path): +``` +audio (np.array) +detections (np.array) +``` + #### Local storage Again, np.save is a good option for storing the audio arrays. +An index-table can be maintained in parent dir in same path, or in own query table. #### Cloud storage -Similar as before, needs to be stored in a bucket. -Can maybe inherit same write method from previous stage, if we fingure out how to pass classes between stage local stages-files. +Similar as before, needs to be stored in a bucket w/ params in path. ### 4. Classification -After the audio has been fed through the model, we'll get an array shorter than the length of the audio array, but still arbitrry lengths. +After the audio has been fed through the model, we'll get an array shorter than the length of the audio array, but still arbitrary lengths. Large context windows will produce large classification arrays, meaning high storage costs. -Are all of these data necessary to save, or would a pooled score be best? It depends on the use-case ;) +Whether or not all of these data necessary to save, or if a pooled score would be best, depends on the use-case. 😉 + +We could alternatively cut the audio to only the parts where a min and max classification above a threshold is found. +This requires even more processing, butwould eliminate any real dependency on audio sifting (in case that stage turns out to not be needed later). +Such a waste-reduction strategy is the most efficient means of writing this, since we would only store the audio that we are confident contains a whale call. +I'm just hesitant that we risk cutting too much audio, especially in cases where the model is unsure at the beginning or end of a signal. + +Columns to persist in table: +``` +key (str) +classification_path (str) +``` + +To persist in bucket as array (params in path, similar to audio): +``` +classification (np.array) +``` -We could alternatively cut the audio to only the parts where a min and max classification above a threshold is found. -This would eliminate any real dependency on audio sifting (in case that stage turns out to not be needed later). -And This would serve as the best waste-reduction strategy, since we would only store the audio that we are confident contains a whale call. #### Local storage -For now, let' stick to storing the entire clasasification array for this stage, using np.save, with similar paths to audio storage. +For now, let' stick to storing the entire clasasification array for this stage, using `np.save`, with similar paths to audio storage. #### Cloud storage Since we are dealing with arbitrary lengths, I'd say stick to bucket with parameters as path variables. +Simliar to audio, maintain a table with the key and path to the classification array. -### 5. Postprocessings (pooling and labelling) -The final stage definitely needs to be store, but the main discussion here becomes, what to store? +### 5. Postprocessing (pooling and labelling) +The final stage definitely needs to be stored, but the main discussion here is what to store? If we already have stored intermediate stages like sifted audio or truncated classified audio, we could avoid saving them again, and rather load from those tables when presenting aggregated results. -Though, I like the idea of the last stage of the pipeline containing all the data necessary found through the pipeline. +I like the idea of the last stage of the pipeline containing all the data necessary found through the pipeline. This makes data sharing easier, with a concrete final product, instead of a bunch of fragmentated tables that need to be joined to have any true value. -Maybe storing the easily queryable data in a table, then include a link to the audio storage location (whether that be a file by me or MBARI or other hydrophone provider). +Maybe storing the easily queryable data in a table, then include a link to the audio storage location (whether that be a bucket-path or local file by me or url to MBARI or other audio sources from hydrophone providers). -#### Local storage -I'll assume a similar structure to the expected tobale when saving local. This means arrays data like audio and classifications are excluded. -This frees me up to store one entry per encounter id. -Paths to the audio and classification arrays can be stored in the table. - -#### Cloud storage -I feel like this table shuold always be written to (i.e. no config option to disable). -Outputs will include: +Final outputs will include: ``` encounter_id longitude @@ -123,10 +180,20 @@ pooled_score img_path audio_path classification_path +result_plot_path ``` + +#### Local storage +I'll assume a similar structure to the expected table when saving local. +This means arrays data like audio and classifications are excluded, which frees me up to store one entry per encounter id. +Paths to the audio and classification arrays can be stored in the table. + +#### Cloud storage +Mirror the local table in some queryable table in our cloud provider. + ## Conclusion -- All stages (except last) will be configurable to save or not. +- All stages will be configurable to save or not, default to true for all (for now). - File exists sensors should allow skipping a stage. - Local storage or arrays w/ variable lengths with np.save, and paramter values in path. - Structured data will be stored in tables with relevant links to array data diff --git a/src/create_table.py b/examples/create_table.py similarity index 100% rename from src/create_table.py rename to examples/create_table.py diff --git a/examples/geometery_search.py b/examples/geometery_search.py index 7e5d3c8..89694b8 100644 --- a/examples/geometery_search.py +++ b/examples/geometery_search.py @@ -1,6 +1,6 @@ import happywhale.happywhale as hw -# geometery search for a time-frame +# geometry search for a time-frame geometry_file = "data/geo/monterey_bay_50kmr.geojson" start = "2024-09-01" end = "2024-09-02" diff --git a/examples/inference.py b/examples/inference.py index 699c528..841e68d 100644 --- a/examples/inference.py +++ b/examples/inference.py @@ -137,7 +137,6 @@ def run_inference(self, batch, model, inference_args, model_id=None): print(f" batch.shape = {batch}") print(f" model = {model}") print(f" inference_args = {inference_args}") - breakpoint() # serialize batch = [batch[0].numpy().tolist()] @@ -151,7 +150,6 @@ def tensor_inference_fn( model_id: Optional[str] = None, ): print_available_ram() - breakpoint() return model.score(waveform=batch, **inference_args) diff --git a/examples/model.py b/examples/model.py index 3eef745..0a68fe0 100644 --- a/examples/model.py +++ b/examples/model.py @@ -35,7 +35,6 @@ def print_available_ram(): dummy = np.random.random((1, 39124, 1)).astype(np.float32) print(f" final input: dummy.shape = {dummy.shape}") -breakpoint() # results = model(dummy, True, None) results = model.score( waveform=dummy, # waveform_exp, diff --git a/examples/quantized_inference.py b/examples/quantized_inference.py index f83b73c..e5feef1 100644 --- a/examples/quantized_inference.py +++ b/examples/quantized_inference.py @@ -84,7 +84,6 @@ def quantized_preprocess(signal, batch_size): # expand dims signal_batches = [np.expand_dims(batch, axis=0) for batch in signal_batches] - breakpoint() logging.info(f"Split signal into {len(signal_batches)} batches of size {batch_size}.") logging.info(f"Size of final batch {signal_batches[0].shape}") diff --git a/examples/write_to_bigquery.py b/examples/write_to_bigquery.py index 4e152eb..486d214 100644 --- a/examples/write_to_bigquery.py +++ b/examples/write_to_bigquery.py @@ -1,5 +1,4 @@ -from apache_beam.io.gcp.internal.clients import bigquery import apache_beam as beam diff --git a/makefile b/makefile index d7de1a3..9175998 100644 --- a/makefile +++ b/makefile @@ -1,8 +1,8 @@ local-run: bash scripts/kill_model_server.sh - python3 src/create_table.py python3 src/model_server.py & python3 src/pipeline.py bash scripts/kill_model_server.sh + python3 src/gcp.py --deduplicate run-pipeline: python3 src/pipeline.py @@ -13,5 +13,8 @@ model-server: kill-model-server: bash scripts/kill_model_server.sh -create-table: - python3 src/create_table.py \ No newline at end of file +gcp-init: + python3 src/gcp.py --init + +gcp-deduplicate: + python3 src/gcp.py --deduplicate \ No newline at end of file diff --git a/src/config.py b/src/config.py index fbaa2d6..f616280 100644 --- a/src/config.py +++ b/src/config.py @@ -1,6 +1,7 @@ from types import SimpleNamespace +import apache_beam as beam import argparse import os import yaml @@ -24,7 +25,15 @@ raise ValueError(f"Invalid ENV: {ENV}") -# Function to load defaults from config file +def add_write_params(config): + config["bigquery"] = { + "method": beam.io.WriteToBigQuery.Method.FILE_LOADS, + "create_disposition": beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + "write_disposition": beam.io.BigQueryDisposition.WRITE_APPEND # WRITE_APPEND or WRITE_TRUNCATE + } + return config + + def read_config(config_file): with open(config_file, 'r') as f: config = yaml.safe_load(f) @@ -88,11 +97,14 @@ def load_pipeline_config( NOTE: If conflicting parameter names, both cases will be updated with value from command-line arg. """ - # files + # params from config files config = read_config(config_file)["pipeline"] extra_config = read_config(extra_config_file)["pipeline"] config = append_values(config, extra_config) + # add write parameters + config = add_write_params(config) + # command-line arguments config = update_config(config) diff --git a/src/config/common.yaml b/src/config/common.yaml index 4abb209..73d891c 100644 --- a/src/config/common.yaml +++ b/src/config/common.yaml @@ -8,6 +8,26 @@ pipeline: # gcp - bigquery project: "bioacoustics-2024" dataset_id: "whale_speech" + workbucket: "gs://bioacoustics/whale-speech" + temp_location: "gs://bioacoustics/whale-speech/temp" + + # gcp - storage + tables: + - "raw_audio" + - "sifted_audio" + - "classifications" + - "mapped_audio" + - "geometry_search" + + partition_columns: + - "encounter_id" + - "start" + - "end" + - "img_path" + - "audio_path" + - "classification_path" + - "sift_audio_path" + - "sift_detections_path" input: start: "2016-12-21T00:30:00" @@ -15,11 +35,11 @@ pipeline: timezone: "America/Los_Angeles" search: - export_template: "data/encounters/{filename}-{timeframe}.csv" + output_path_template: "data/table/{table_id}/geofile={geofile}/encounters.csv" filename: "monterey_bay_50km" - geometery_file_path_template: "data/geo/{filename}.geojson" + geometry_file_path_template: "data/geo/{filename}.geojson" species: "humpback_whale" - columns: + search_columns: - "id" - "latitude" - "longitude" @@ -28,6 +48,23 @@ pipeline: - "endTime" - "timezone" - "displayImgUrl" + search_table_id: "geometry_search" + search_table_schema: + encounter_id: + type: 'STRING' + mode: 'REQUIRED' + encounter_time: + type: 'TIMESTAMP' + mode: 'REQUIRED' + longitude: + type: 'FLOAT64' + mode: 'REQUIRED' + latitude: + type: 'FLOAT64' + mode: 'REQUIRED' + img_path: + type: 'STRING' + mode: 'NULLABLE' audio: url_template: "https://pacific-sound-16khz.s3.amazonaws.com/{year}/{month:02}/{filename}" @@ -35,19 +72,52 @@ pipeline: source_sample_rate: 16000 margin: 30 # TODO set to 900 # seconds offset: 13 # TODO set to 0 # hours - output_path_template: "data/audio/raw/{year}/{month:02}/{filename}" + output_array_path_template: "data/audio/raw/key={key}/{filename}" + output_table_path_template: "data/table/{table_id}/metadata.json" skip_existing: false # if true, skip downstream processing of existing audio files (false during development) + audio_table_id: "raw_audio" + store_audio: true + audio_table_schema: + key: + type: 'STRING' + mode: 'REQUIRED' + audio_path: + type: 'STRING' + mode: 'REQUIRED' + start: + type: 'TIMESTAMP' + mode: 'REQUIRED' + end: + type: 'TIMESTAMP' + mode: 'REQUIRED' sift: - output_path_template: "data/audio/{sift}/{year}/{month:02}/{filename}" + output_array_path_template: "data/audio/{params}/key={key}/{filename}" + output_table_path_template: "data/table/{table_id}/metadata.json" max_duration: 600 # seconds plot: true show_plot: false - plot_path_template: "data/plots/{sift}/{year}/{month:02}/{day:02}/{plot_name}.png" + plot_path_template: "data/plots/{params}/{key}.png" window_size: 512 + store_sift_audio: true + sift_table_id: "sifted_audio" + sift_table_schema: + key: + type: 'STRING' + mode: 'REQUIRED' + sift_audio_path: + type: 'STRING' + mode: 'NULLABLE' + sift_detections_path: + type: 'STRING' + mode: 'NULLABLE' + params: + type: 'STRING' + mode: 'REQUIRED' # Specific sift-mechanism parameters butterworth: + params_path_template: "{name}/highcut={highcut}/lowcut={lowcut}/order={order}/threshold={threshold}" highcut: 1500 lowcut: 50 order: 5 @@ -55,23 +125,36 @@ pipeline: sift_threshold: 0.015 classify: - hydrophone_sensitivity: -168.8 batch_duration: 600 # seconds + model_sample_rate: 10000 - inference_retries: 3 - plot_scores: true - plot_path_template: "data/plots/results/{year}/{month:02}/{plot_name}.png" - classification_path: "data/classifications.tsv" - model_uri: https://tfhub.dev/google/humpback_whale/1 + model_uri: https://www.kaggle.com/models/google/humpback-whale/TensorFlow2/humpback-whale/1 inference_url: "http://127.0.0.1:5000/predict" + inference_retries: 3 med_filter_size: 3 + plot_scores: true + hydrophone_sensitivity: -168.8 + plot_path_template: "data/plots/results/{params}/{plot_name}.png" + output_array_path_template: "data/classifications/{params}/{key}.npy" + output_table_path_template: "data/table/{table_id}/metadata.json" + + store_classifications: true + classification_table_id: "classifications" + classification_table_schema: + key: + type: 'STRING' + mode: 'REQUIRED' + classifications_path: + type: 'STRING' + mode: 'REQUIRED' + postprocess: confidence_threshold: 0.5 min_gap: 60 # 1 minute output_path: "data/postprocess/output.json" pooling: "average" - postprocess_table_id: "mapped_audio" + postprocess_table_id: "postprocessed" postprocess_table_schema: encounter_id: type: 'STRING' @@ -97,6 +180,12 @@ pipeline: audio_path: type: 'STRING' mode: 'NULLABLE' + sift_audio_path: + type: 'STRING' + mode: 'NULLABLE' + detections_path: + type: 'STRING' + mode: 'NULLABLE' classification_path: type: 'STRING' mode: 'NULLABLE' diff --git a/src/config/gcp.yaml b/src/config/gcp.yaml index ef6b78a..b2c3ff8 100644 --- a/src/config/gcp.yaml +++ b/src/config/gcp.yaml @@ -3,32 +3,21 @@ pipeline: verbose: true debug: true show_plots: false + is_local: false - search: - export_template: "data/encounters/{filename}-{timeframe}.csv" - filename: "monterey_bay_50km" - geometery_file_path_template: "data/geo/{filename}.geojson" audio: - output_path_template: "data/audio/raw/{year}/{month:02}/{filename}" skip_existing: false # if true, skip downstream processing of existing audio files (false during development) sift: - output_path_template: "data/audio/{sift}/{year}/{month:02}/{filename}" max_duration: 600 # seconds plot: true show_plot: false - plot_path_template: "data/plots/{sift}/{year}/{month:02}/{day:02}/{plot_name}.png" window_size: 512 classify: - batch_duration: 600 # seconds - inference_retries: 3 plot_scores: true - plot_path_template: "data/plots/results/{year}/{month:02}/{plot_name}.png" postprocess: min_gap: 60 # 1 minute - pooling: "average" confidence_threshold: 0.5 - output_path_template: "data/labels/{year}/{month:02}/{day:02}.csv" diff --git a/src/config/local.yaml b/src/config/local.yaml index fe7ca78..e1dbbe8 100644 --- a/src/config/local.yaml +++ b/src/config/local.yaml @@ -4,33 +4,21 @@ pipeline: debug: true show_plots: false is_local: true - - search: - export_template: "data/encounters/{filename}-{timeframe}.csv" - filename: "monterey_bay_50km" - geometery_file_path_template: "data/geo/{filename}.geojson" audio: - output_path_template: "data/audio/raw/{year}/{month:02}/{filename}" skip_existing: false # if true, skip downstream processing of existing audio files (false during development) sift: - output_path_template: "data/audio/{sift}/{year}/{month:02}/{filename}" max_duration: 600 # seconds plot: true show_plot: false - plot_path_template: "data/plots/{sift}/{year}/{month:02}/{day:02}/{plot_name}.png" window_size: 512 classify: batch_duration: 600 # seconds inference_retries: 3 plot_scores: true - plot_path_template: "data/plots/results/{year}/{month:02}/{plot_name}.png" - classification_path: "data/classifications.tsv" postprocess: min_gap: 60 # 1 minute - pooling: "average" confidence_threshold: 0.5 - output_path_template: "data/labels/{year}/{month:02}/{day:02}.csv" diff --git a/src/gcp.py b/src/gcp.py new file mode 100644 index 0000000..fb67b80 --- /dev/null +++ b/src/gcp.py @@ -0,0 +1,99 @@ +from google.cloud import bigquery +from google.api_core.exceptions import Conflict + +from config import load_pipeline_config + +import logging + +logging.basicConfig(level=logging.INFO) + + +config = load_pipeline_config() + +client = bigquery.Client() + + +def create_dataset(dataset_id): + try: + dataset_path = f"{client.project}.{dataset_id}" + dataset = bigquery.Dataset(dataset_path) + dataset.location = "US" + dataset = client.create_dataset(dataset, timeout=30) + logging.info(f"Created dataset {client.project}.{dataset.dataset_id}") + except Conflict as e: + if "Already Exists" in str(e): + dataset = client.get_dataset(dataset_id) + logging.info(f"Dataset {client.project}.{dataset.dataset_id} already exists. Continuing.") + else: + raise e + + return dataset + + +def table_exists(table_id: str): + return table_id in [table.table_id for table in client.list_tables(config.general.dataset_id)] + + +def get_partition_columns(table_id: str): + columns = [ + field.name + for field in client.get_table( + f"{config.general.project}.{config.general.dataset_id}.{table_id}" + ).schema + if field.name in config.general.partition_columns + ] + identifier = "key" if "key" in columns else "encounter_id" + + return ",".join(columns), identifier + + +def deduplicate_table(table_id: str): + if not table_exists(table_id): + logging.info(f"Table {table_id} does not exist. Skipping deduplication.") + return + + columns, identifier = get_partition_columns(table_id) + + query = f""" + CREATE OR REPLACE TABLE `{config.general.project}.{config.general.dataset_id}.{table_id}` AS + SELECT * + FROM ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY {columns} ORDER BY {identifier} DESC) AS row_num + FROM `{config.general.project}.{config.general.dataset_id}.{table_id}` + ) + WHERE row_num = 1 + """ + logging.info(f"Running deduplicatation query: \n {query}") + client.query(query) + logging.info(f"Deduplicated table {table_id}.") + + +def initialize(): + dataset = create_dataset(config.general.dataset_id) + return dataset + + +def deduplicate(): + for table_id in config.general.tables: + deduplicate_table(table_id) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="GCP Utility Functions") + parser.add_argument("--init", action="store_true", help="Initialize BigQuery dataset (config.general.dataset_id)") + parser.add_argument("--deduplicate", action="store_true", help="Dedupliacte BigQuery tables (config.general.tables)") + args = parser.parse_args() + + if config.general.is_local: + logging.info("Running in local mode. Exiting.") + exit() + + if args.init: + initialize() + elif args.deduplicate: + deduplicate() + else: + parser.print_help() diff --git a/src/pipeline.py b/src/pipeline.py index db25baa..5017812 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -2,12 +2,10 @@ from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions from stages.search import GeometrySearch -from stages.audio import RetrieveAudio, WriteAudio, WriteSiftedAudio +from stages.audio import RetrieveAudio from stages.sift import Butterworth -from stages.classify import WhaleClassifier, WriteClassifications -from stages.postprocess import PostprocessLabels, WritePostprocess - -from apache_beam.io.gcp.internal.clients import bigquery +from stages.classify import WhaleClassifier +from stages.postprocess import PostprocessLabels from config import load_pipeline_config @@ -17,8 +15,8 @@ def run(): # Initialize pipeline options pipeline_options = PipelineOptions( # runner="DataflowRunner", - project="bioacoustics-2024", - temp_location="gs://bioacoustics/whale-speech/temp", + project=config.general.project, + temp_location=config.general.temp_location, ) pipeline_options.view_as(SetupOptions).save_main_session = True args = { @@ -27,22 +25,18 @@ def run(): } with beam.Pipeline(options=pipeline_options) as p: - input_data = p | "Create Input" >> beam.Create([args]) - search_output = input_data | "Run Geometry Search" >> beam.ParDo(GeometrySearch()) - audio_output = search_output | "Retrieve Audio" >> beam.ParDo(RetrieveAudio()) - sifted_audio = audio_output | "Sift Audio" >> Butterworth() - classifications = sifted_audio | "Classify Audio" >> WhaleClassifier(config) - postprocess_labels = classifications | "Postprocess Labels" >> beam.ParDo( + input_data = p | "Create Input" >> beam.Create([args]) + search_output = input_data | "Run Geometry Search" >> beam.ParDo(GeometrySearch(config)) + audio_output = search_output | "Retrieve Audio" >> beam.ParDo(RetrieveAudio(config)) + sifted_audio = audio_output | "Sift Audio" >> Butterworth(config) + classifications = sifted_audio | "Classify Audio" >> WhaleClassifier(config) + pipeline_output = classifications | "Postprocess Labels" >> beam.ParDo( PostprocessLabels(config), search_output=beam.pvalue.AsSingleton(search_output), + audio_output=beam.pvalue.AsList(audio_output), + sifted_audio=beam.pvalue.AsList(sifted_audio), ) - # Store results - audio_output | "Store Audio (temp)" >> beam.ParDo(WriteAudio()) - sifted_audio | "Store Sifted Audio" >> beam.ParDo(WriteSiftedAudio("butterworth")) - classifications | "Store Classifications" >> beam.ParDo(WriteClassifications(config)) - postprocess_labels | "Write to BigQuery" >> beam.ParDo(WritePostprocess(config)) - if __name__ == "__main__": run() diff --git a/src/stages/audio.py b/src/stages/audio.py index 891d00d..49db069 100644 --- a/src/stages/audio.py +++ b/src/stages/audio.py @@ -2,12 +2,13 @@ from datetime import timedelta, datetime from functools import partial from six.moves.urllib.request import urlopen # pyright: ignore -from typing import Tuple +from typing import List import apache_beam as beam import io import logging import numpy as np +import os import pandas as pd import soundfile as sf @@ -18,56 +19,91 @@ class AudioTask(beam.DoFn): - debug = config.general.debug - filename_template = config.audio.filename_template - output_path_template = config.audio.output_path_template - skip_existing = config.audio.skip_existing - source_sample_rate = config.audio.source_sample_rate - url_template = config.audio.url_template - - def __init__(self): - # init certrain attributes for mockable testing + + def __init__(self, config): + self.debug = config.general.debug + self.is_local = config.general.is_local + self.margin = config.audio.margin self.offset = config.audio.offset + self.source_sample_rate = config.audio.source_sample_rate + self.url_template = config.audio.url_template + + self.filename_template = config.audio.filename_template + self.output_array_path_template = config.audio.output_array_path_template + self.output_table_path_template = config.audio.output_table_path_template + self.skip_existing = config.audio.skip_existing + + self.store = config.audio.store_audio + self.project = config.general.project + self.dataset_id = config.general.dataset_id + self.table_id = config.audio.audio_table_id + self.schema = self._schema_to_dict(config.audio.audio_table_schema) + self.workbucket = config.general.workbucket + self.temp_location = config.general.temp_location + self.write_params = config.bigquery.__dict__ - def _load_audio(self, file_path:str): + @staticmethod + def _build_key( + start_time: datetime, + end_time: datetime, + encounter_ids: list, + ): + start_str = start_time.strftime('%Y%m%dT%H%M%S') + end_str = end_time.strftime('%H%M%S') + encounter_str = "_".join(encounter_ids) + return f"{start_str}-{end_str}_{encounter_str}" + + @staticmethod + def _load_audio(file_path:str): # Write the numpy array to the file as .npy format with beam.io.filesystems.FileSystems.open(file_path) as f: audio = np.load(f) - return audio + @staticmethod + def _schema_to_dict(schema): + return { + "fields": [ + { + "name": name, + "type": getattr(schema, name).type, + "mode": getattr(schema, name).mode + } + for name in vars(schema) + ] + } + def _get_export_path( self, + key: str, start: datetime, - end: datetime, - encounter_ids: list[str], ): - - file_path_prefix = "{date}T{start_time}-{end_time}-{ids}".format( - date=start.strftime("%Y%m%d"), - start_time=start.strftime("%H%M%S"), - end_time=end.strftime("%H%M%S"), - ids="_".join(encounter_ids) - ) - - # Create a unique file name for each element - filename = f"{file_path_prefix}.npy" # f"{file_path_prefix}_{hash(element)}.npy" - - file_path = self.output_path_template.format( + filename = self.filename_template.format( year=start.year, month=start.month, + day=start.day, + ).replace("T000000Z", start.strftime("T%H%M%SZ")).replace(".wav", ".npy") + + file_path = self.output_array_path_template.format( + key=key, filename=filename ) + if not self.is_local: + file_path = os.path.join( + self.workbucket, + file_path + ) + return file_path - def _file_exists_for_input(self, + def _file_exists_for_input( + self, + key: str, start: datetime, - end: datetime, - encounter_ids: list[str] ) -> bool: - file_path = self._get_export_path(start, end, encounter_ids) + file_path = self._get_export_path(key, start) return filesystems.FileSystems.exists(file_path) @@ -86,30 +122,33 @@ def process(self, search_results: pd.DataFrame): for row in preprocessed_df.itertuples(): start_time = row.start_time end_time = row.end_time + key = self._build_key(start_time, end_time, row.encounter_ids) - # check if audio already exists for this time-frame - if self._file_exists_for_input(start_time, end_time, row.encounter_ids): - audio_path = self._get_export_path(start_time, end_time, row.encounter_ids) - logging.info(f"Audio already exists for {start_time} to {end_time}") + logging.info(f"Checking if audio exists for {key}") + if self._file_exists_for_input(key, start_time): + audio_path = self._get_export_path(key, start_time) + logging.info(f"Audio already exists for {key}") if self.skip_existing: logging.info(f"Skipping downstream processing for {audio_path}") continue else: logging.info(f"Loading audio from {audio_path}") audio = self._load_audio(audio_path) - yield audio, start_time, end_time, row.encounter_ids else: - logging.info(f"Loading audio for {start_time} to {end_time}") + logging.info(f"Downloading audio for {key}") audio, _ = self._download_audio( start_time, end_time, self.source_sample_rate, ) - # Yield the audio and the search_results - yield audio, start_time, end_time, row.encounter_ids + audio_path = self._store(key, audio, start_time, end_time) if self.store else None + + yield (audio, start_time, end_time, row.encounter_ids, audio_path) + # preprocessed_rows.append((audio, start_time, end_time, row.encounter_ids, audio_path)) + # return preprocessed_rows def _preprocess(self, df: pd.DataFrame): df = self._build_time_frames(df) @@ -117,7 +156,6 @@ def _preprocess(self, df: pd.DataFrame): return df - def _build_time_frames(self, df: pd.DataFrame): margin = self.margin @@ -127,7 +165,6 @@ def _build_time_frames(self, df: pd.DataFrame): df["start_time"] = df.startTime - timedelta(seconds=margin) df["end_time"] = df.endTime + timedelta(seconds=margin) - # TODO remove this. Only use during development df["start_time"] = df["start_time"] - timedelta(hours=self.offset) df["end_time"] = df["end_time"] - timedelta(hours=self.offset) @@ -136,7 +173,6 @@ def _build_time_frames(self, df: pd.DataFrame): return df - def _find_overlapping(self, df: pd.DataFrame): """ Find overlapping time-frames and join the encounter ids. @@ -175,7 +211,6 @@ def _find_overlapping(self, df: pd.DataFrame): return pd.DataFrame.from_dict(grouped_encounters) - def _get_file_url( self, year: int, @@ -187,7 +222,6 @@ def _get_file_url( return url - def _download_audio( self, start_time: datetime, @@ -248,41 +282,83 @@ def _download_audio( return psound_segment, psound_segment_seconds - -class WriteAudio(AudioTask): - def process(self, element): - # TODO refactor to properly handle typing - array = element[0] - start = element[1] - end = element[2] - encounter_ids = element[3] - - file_path = self._get_export_path( - start, - end, - encounter_ids, - ) - + def _store( + self, + key: str, + audio: np.array, + start: datetime, + end: datetime, + ): + file_path = self._get_export_path(key, start) logging.info(f"Writing audio to {file_path}") - logging.info(f"Audio shape: {array.shape}") - - yield self._save_audio(array, file_path) + logging.info(f"Audio shape: {audio.shape}") + + if self.is_local: + logging.info(f"Updating table {self.table_id} locally") + self._store_local(key, file_path, start, end) + else: + logging.info(f"Updating table {self.table_id} in BigQuery") + + self._store_bigquery(key, file_path, start, end) - def _save_audio(self, audio:np.array, file_path:str): - # Write the numpy array to the file as .npy format - with beam.io.filesystems.FileSystems.create(file_path) as f: - np.save(f, audio) # Save the numpy array in .npy format + with filesystems.FileSystems.create(file_path) as f: + # same for local and gsc storage + np.save(f, audio) + + logging.info(f"Audio stored at {file_path}") return file_path - -class WriteSiftedAudio(WriteAudio): - output_path_template = config.sift.output_path_template - - def __init__(self, sift="sift"): - super().__init__() - self.output_path_template = self.output_path_template.replace("{sift}", sift) - logging.info(f"Sifted output path template: {self.output_path_template}") - - # everything is used from WriteAudio + def _store_local( + self, + key:str, + audio_path:np.array, + start:datetime, + end:datetime, + ): + table_path = self.output_table_path_template.format(table_id=self.table_id) + + if not filesystems.FileSystems.exists(table_path): + # build parent dir if necessary + if not filesystems.FileSystems.exists(table_path): + filesystems.FileSystems.create(table_path) + df = pd.DataFrame([{ + "key": key, + "start": start.isoformat(), + "end": end.isoformat(), + "audio_path": audio_path + }]) + df.to_json(table_path, index=False, orient="records") + else: + df = pd.read_json(table_path, orient="records") + new_row = pd.DataFrame([{ + "key": key, + "start": start.isoformat(), + "end": end.isoformat(), + "audio_path": audio_path + }]) + df = pd.concat([df, new_row]) + df = df.drop_duplicates() + df.to_json(table_path, index=False, orient="records") + + def _store_bigquery( + self, + key: str, + audio_path: str, + start: datetime, + end: datetime, + ): + [{ + "key": key, + "start": start.isoformat(), + "end": end.isoformat(), + "audio_path": audio_path + }] | "Write to BigQuery" >> beam.io.WriteToBigQuery( + self.table_id, + dataset=self.dataset_id, + project=self.project, + schema=self.schema, + custom_gcs_temp_location=self.temp_location, + **self.write_params + ) diff --git a/src/stages/classify.py b/src/stages/classify.py index 429a39d..132ec4e 100644 --- a/src/stages/classify.py +++ b/src/stages/classify.py @@ -1,3 +1,4 @@ +from apache_beam.io import filesystems import apache_beam as beam from datetime import datetime @@ -25,9 +26,9 @@ class BaseClassifier(beam.PTransform): name = "BaseClassifier" - def __init__(self, config: SimpleNamespace): self.config = config + self.is_local = config.general.is_local self.source_sample_rate = config.audio.source_sample_rate self.batch_duration = config.classify.batch_duration @@ -41,9 +42,46 @@ def __init__(self, config: SimpleNamespace): self.plot_path_template = config.classify.plot_path_template self.show_plots = config.general.show_plots + # store parameters + self.store = config.classify.store_classifications + self.output_array_path_template = config.classify.output_array_path_template + self.output_table_path_template = config.classify.output_table_path_template + + self.project = config.general.project + self.dataset_id = config.general.dataset_id + self.table_id = config.classify.classification_table_id + self.schema = self._schema_to_dict(config.classify.classification_table_schema) + self.temp_location = config.general.temp_location + self.workbucket = config.general.workbucket + self.write_params = config.bigquery.__dict__ + + # TODO dynamically update params used in filter to build classification path + self.params_path_template = config.sift.butterworth.params_path_template + self.path_params = { + "name": "butterworth", + "lowcut": config.sift.butterworth.lowcut, + "highcut": config.sift.butterworth.highcut, + "order": config.sift.butterworth.order, + "threshold": config.sift.butterworth.sift_threshold, + } + + @staticmethod + def _schema_to_dict(schema): + return { + "fields": [ + { + "name": name, + "type": getattr(schema, name).type, + "mode": getattr(schema, name).mode + } + for name in vars(schema) + ] + } + def _preprocess(self, pcoll): - signal, start, end, encounter_ids = pcoll + signal, start, end, encounter_ids, _, _ = pcoll key = self._build_key(start, end, encounter_ids) + logging.info(f"Classifying {key} with signal shape {signal.shape}") # Resample signal = self._resample(signal) @@ -150,16 +188,10 @@ def _plot_spectrogram_scipy( return t, f, psd def _plot_audio(self, audio, start, key): - # plt.plot(audio) - # plt.xlabel('Samples') - # plt.xlim(0, len(audio)) - # plt.ylabel('Energy') - # plt.title(f'Raw audio signal for {key}') with open(f"data/plots/Butterworth/{start.year}/{start.month}/{start.day}/data/{key}_min_max.pkl", "rb") as f: min_max_samples = pickle.load(f) with open(f"data/plots/Butterworth/{start.year}/{start.month}/{start.day}/data/{key}_all.pkl", "rb") as f: all_samples = pickle.load(f) - # plt.plot(audio) # TODO remove this if does not work properly def _plot_signal_detections(signal, min_max_detection_samples, all_samples): # TODO refactor plot_signal_detections in classify @@ -182,15 +214,15 @@ def _plot_signal_detections(signal, min_max_detection_samples, all_samples): # shade window that resulted in detection for detection in all_samples: plt.axvspan( - detection - 512/2, # TODO replace w/ window size from config - detection + 512/2, + detection - self.config.sift.window_size/2, + detection + self.config.sift.window_size/2, alpha=0.5, color='r', zorder=5, # on top of signal ) plt.legend(['Input signal', 'detection window', 'all detections']).set_zorder(10) - plt.xlabel(f'Samples (seconds * {16000} Hz)') # TODO replace with sample rate from config + plt.xlabel(f'Samples (seconds * {self.config.audio.source_sample_rate} Hz)') plt.ylabel('Amplitude (normalized and centered)') title = f"Signal detections: {start.strftime('%Y-%m-%d %H:%M:%S')}" @@ -200,7 +232,7 @@ def _plot_signal_detections(signal, min_max_detection_samples, all_samples): _plot_signal_detections(audio, min_max_samples, all_samples) def _plot(self, output): - audio, start, end, encounter_ids, scores = output + audio, start, end, encounter_ids, scores, _ = output key = self._build_key(start, end, encounter_ids) if len(audio) == 0: @@ -215,7 +247,6 @@ def _plot(self, output): # Plot spectrogram: plt.subplot(gs[0]) - # self._plot_audio(audio, key) self._plot_audio(audio, start, key) # Plot spectrogram: @@ -229,44 +260,121 @@ def _plot(self, output): plt.tight_layout() plot_path = self.plot_path_template.format( - year=start.year, - month=start.month, - day=start.day, + params=self._get_params_path(), plot_name=key ) - os.makedirs(os.path.dirname(plot_path), exist_ok=True) # TODO refactor when running on GCP + filesystems.FileSystems.create(plot_path) plt.savefig(plot_path) plt.show() if self.show_plots else plt.close() + def _get_params_path(self): + return self.params_path_template.format( + **self.path_params + ) + + def _get_export_path(self, key): + export_path = self.output_array_path_template.format( + params=self._get_params_path(), + key=key + ) + + if not self.is_local: + export_path = os.path.join(self.workbucket, export_path) + + return export_path + + def _store(self, outputs): + audio, start, end, encounter_ids, scores, no_path = outputs + key = self._build_key(start, end, encounter_ids) + + if len(scores) == 0 or len(audio) == 0: + logging.info(f"No detections for {key}. Skipping storage.") + return [(audio, start, end, encounter_ids, scores, no_path)] + + classifications_path = self._get_export_path(key) + + # update metadata table + if self.is_local: + self._store_local(key, classifications_path) + else: + self._store_bigquery(key, classifications_path) + + # store classifications + with filesystems.FileSystems.create(classifications_path) as f: + logging.info(f"Storing sifted audio to {classifications_path}") + np.save(f, np.array(scores).flatten()) + + return [(audio, start, end, encounter_ids, scores, classifications_path)] + + def _store_local(self, key, classifications_path): + logging.info(f"Storing local classification for {key}") + + table_path = self.output_table_path_template.format(table_id=self.table_id) + + if not filesystems.FileSystems.exists(table_path): + filesystems.FileSystems.create(table_path) + df = pd.DataFrame([{ + "key": key, + "classifications_path": classifications_path, + }]) + df.to_json(table_path, index=False, orient="records") + else: + df = pd.read_json(table_path, orient="records") + new_row = pd.DataFrame([{ + "key": key, + "classifications_path": classifications_path, + }]) + df = pd.concat([df, new_row]) + df = df.drop_duplicates() + df.to_json(table_path, index=False, orient="records") + + def _store_bigquery( + self, + key, + classifications_path + ): + logging.info(f"Storing classification for {key} in BigQuery") + [{ + "key": key, + "classifications_path": classifications_path + }] | "Write to BigQuery" >> beam.io.WriteToBigQuery( + self.table_id, + dataset=self.dataset_id, + project=self.project, + schema=self.schema, + custom_gcs_temp_location=self.temp_location, + **self.write_params + ) + class WhaleClassifier(BaseClassifier): def expand(self, pcoll): key_batch = pcoll | "Preprocess" >> beam.ParDo(self._preprocess) batched_outputs = key_batch | "Classify" >> beam.ParDo(InferenceClient(self.config)) grouped_outputs = batched_outputs | "Combine batched_outputs" >> beam.CombinePerKey(ListCombine()) - outputs = pcoll | "Postprocess" >> beam.Map( + outputs = pcoll | "Postprocess" >> beam.ParDo( self._postprocess, grouped_outputs=beam.pvalue.AsDict(grouped_outputs), ) + if self.store: + outputs = outputs | "Store classifications" >> beam.ParDo(self._store) + if self.plot_scores: outputs | "Plot scores" >> beam.Map(self._plot) - logging.info(f"Finished {self.name} stage: {outputs}") return outputs - def _postprocess(self, pcoll, grouped_outputs): - signal, start, end, encounter_ids = pcoll + signal, start, end, encounter_ids, _, _ = pcoll key = self._build_key(start, end, encounter_ids) scores = grouped_outputs.get(key, []) - logging.info(f"Postprocessing {key} with signal {len(signal)} and scores {len(scores)}") - return signal, start, end, encounter_ids, scores - + return [(signal, start, end, encounter_ids, scores, "No classification path stored.")] + class InferenceClient(beam.DoFn): def __init__(self, config: SimpleNamespace): @@ -276,6 +384,7 @@ def __init__(self, config: SimpleNamespace): def process(self, element): key, batch = element + logging.info(f"Sending batch to inference: {key} with {len(batch)} samples") # skip empty batches if len(batch) == 0: @@ -336,75 +445,6 @@ def extract_output(self, accumulator): return accumulator -class WriteClassifications(beam.DoFn): - def __init__(self, config: SimpleNamespace): - self.config = config - - self.classification_path = config.classify.classification_path - self.header = "start\tend\tencounter_ids\tclassifications" - - self._init_file_path(self.classification_path, self.header) - - - def process(self, element): - logging.info(f"Writing classifications to {self.classification_path}") - logging.debug(f"Received element: {element}") - - # skip if empty - if self._is_empty(element): - logging.info(f"Skipping empty classifications for start {element[1].strftime('%Y-%m-%dT%H:%M:%S')}") - return element - - classification_df = pd.read_csv(self.classification_path, sep='\t') - - # create row from element - element_str = self._stringify(element) - row = pd.DataFrame([element_str], columns=classification_df.columns) - - # join to classification_df, updated eventual new values, on start, end, encounter_ids - classification_df = pd.concat([classification_df, row], axis=0, ignore_index=True) - - # drop duplicates - logging.debug(f"Dropping duplicates from {len(classification_df)} rows") - classification_df = classification_df.drop_duplicates(subset=["start", "end"], keep="last") # , "encounter_ids" - - # write to file - classification_df.to_csv(self.classification_path, sep='\t', index=False) - - return element - - - def _is_empty(self, element): - if len(element) == 0: - return True - array, start, end, encounter_ids, classifications = element - logging.info(f"Checking if classifications are empty for start {start.strftime('%Y-%m-%dT%H:%M:%S')}: {len(classifications)}") - return len(classifications) == 0 - - - def _init_file_path(self, file_path, header): - # add header if file does not exist using beam.io - if not beam.io.filesystems.FileSystems.exists(file_path): - with beam.io.filesystems.FileSystems.create(file_path) as f: - f.write(header.encode()) - logging.info(f"Created new file at {file_path} with header {header}") - - - def _stringify(self, element): - _, start, end, encounter_ids, classifications = element - logging.info(f"Stringifying {start} with {len(classifications)} classifications") - - start_str = start.strftime('%Y-%m-%dT%H:%M:%S') - end_str = end.strftime('%Y-%m-%dT%H:%M:%S') - encounter_ids_str = str(encounter_ids) - - return (start_str, end_str, encounter_ids_str, classifications) - - def _tuple_to_tsv(self, element): - start_str, end_str, encounter_ids_str, classifications_str = element - return f'{start_str}\t{end_str}\t{encounter_ids_str}\t{classifications_str}' - - def sample_run(): signal = np.load("data/audio/butterworth/2016/12/20161221T004930-005030-9182.npy") data = ( diff --git a/src/stages/postprocess.py b/src/stages/postprocess.py index 5d863f7..f949eb3 100644 --- a/src/stages/postprocess.py +++ b/src/stages/postprocess.py @@ -4,7 +4,7 @@ import pandas as pd import os -from apache_beam.io.gcp.internal.clients import bigquery +from apache_beam.io import filesystems from typing import Dict, Any, Tuple from types import SimpleNamespace @@ -14,17 +14,39 @@ class PostprocessLabels(beam.DoFn): def __init__(self, config: SimpleNamespace): self.config = config - self.search_output_path_template = config.search.export_template - self.sifted_audio_path_template = config.sift.output_path_template - self.classification_path = config.classify.classification_path + # self.search_output_path_template = config.search.output_path_template + # self.sifted_audio_path_template = config.sift.output_array_path_template + # self.classification_path = config.classify.output_array_path_template self.pooling = config.postprocess.pooling self.project = config.general.project self.dataset_id = config.general.dataset_id self.table_id = config.postprocess.postprocess_table_id + # storage params + self.is_local = config.general.is_local + self.output_path = config.postprocess.output_path + self.project = config.general.project + self.dataset_id = config.general.dataset_id + self.table_id = config.postprocess.postprocess_table_id + self.columns = list(vars(config.postprocess.postprocess_table_schema)) + self.schema = self._schema_to_dict(config.postprocess.postprocess_table_schema) + + + @staticmethod + def _schema_to_dict(schema): + return { + "fields": [ + { + "name": name, + "type": getattr(schema, name).type, + "mode": getattr(schema, name).mode + } + for name in vars(schema) + ] + } - def process(self, element, search_output): + def process(self, element, search_output, **kwargs): # convert element to dataframe classifications_df = self._build_classification_df(element) @@ -35,16 +57,18 @@ def process(self, element, search_output): joined_df = pd.merge(search_output_df, classifications_df, how="inner", on="encounter_id") # add paths - final_df = self._add_paths(joined_df) + final_df = self._add_paths(joined_df, kwargs) logging.info(f"Final output: \n{final_df.head()}") logging.info(f"Final output columns: {final_df.columns}") + self._store(final_df) + yield final_df.to_dict(orient="records") def _build_classification_df(self, element: Tuple) -> pd.DataFrame: # convert element to dataframe - df = pd.DataFrame([element], columns=["audio", "start", "end", "encounter_ids", "classifications"]) + df = pd.DataFrame([element], columns=["audio", "start", "end", "encounter_ids", "classifications", "classification_path"]) df = df[df["classifications"].apply(lambda x: len(x) > 0)] # rm empty rows # explode encounter_ids @@ -68,8 +92,7 @@ def _build_classification_df(self, element: Tuple) -> pd.DataFrame: def _build_search_output_df(self, search_output: Dict[str, Any]) -> pd.DataFrame: # convert search_output to dataframe - search_output = search_output.rename(columns={"id": "encounter_id"}) - search_output["encounter_id"] = search_output["encounter_id"].astype(str) + search_output["encounter_id"] = search_output["id"].astype(str) search_output = search_output[[ "encounter_id", "latitude", @@ -94,48 +117,35 @@ def _pool_classifications(self, classifications: np.array) -> Dict[str, Any]: return pooled_score - def _add_paths(self, df: pd.DataFrame) -> pd.DataFrame: - df["audio_path"] = "NotImplemented" - df["classification_path"] = "NotImplemented" + def _add_paths(self, df: pd.DataFrame, kwargs) -> pd.DataFrame: + if kwargs.get("audio_output"): + audio_df = pd.DataFrame( + kwargs["audio_output"], + columns=["audio", "start", "end", "encounter_id", "audio_path"] + ).explode("encounter_id").drop(columns=["audio", "start", "end"]) + df = df.merge(audio_df, on=["encounter_id"], how="left") + + if kwargs.get("sifted_audio"): + sifted_df = pd.DataFrame( + kwargs["sifted_audio"], + columns=["audio", "start", "end", "encounter_id", "sift_audio_path", "detections_path"] + ).explode("encounter_id").drop(columns=["audio", "start", "end"]) + df = df.merge(sifted_df, on=["encounter_id"], how="left") + df["img_path"] = df["displayImgUrl"] df = df.drop(columns=["displayImgUrl"]) return df - -class WritePostprocess(beam.DoFn): - def __init__(self, config: SimpleNamespace): - self.config = config - - self.is_local = config.general.is_local - self.output_path = config.postprocess.output_path - self.project = config.general.project - self.dataset_id = config.general.dataset_id - self.table_id = config.postprocess.postprocess_table_id - self.columns = list(vars(config.postprocess.postprocess_table_schema)) - self.schema = self._schema_to_dict(config.postprocess.postprocess_table_schema) - - def process(self, element): - if len(element) == 0: + def _store(self, df: pd.DataFrame): + if len(df) == 0: return - + if self.is_local: - return self._write_local(element) + self._store_local(df) else: - return self._write_gcp(element) + self._store_bigquery(df) - def _schema_to_dict(self, schema): - return { - "fields": [ - { - "name": name, - "type": getattr(schema, name).type, - "mode": getattr(schema, name).mode - } - for name in vars(schema) - ] - } - - def _write_gcp(self, element): + def _store_bigquery(self, df: pd.DataFrame): write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED method=beam.io.WriteToBigQuery.Method.FILE_LOADS @@ -145,15 +155,13 @@ def _write_gcp(self, element): logging.info(f"Table: {self.table_id}") logging.info(f"Dataset: {self.dataset_id}") logging.info(f"Project: {self.project}") - logging.info(f"Schema: {self.schema}") - logging.info(f"Len element: {len(element)}") - logging.info(f"Element keys: {element[0].keys()}") + logging.debug(f"Schema: {self.schema}") + logging.debug(f"Len element: {len(df)}") - element | "Write to BigQuery" >> beam.io.WriteToBigQuery( + df.to_dict(orient="records") | "Write to BigQuery" >> beam.io.WriteToBigQuery( self.table_id, dataset=self.dataset_id, project=self.project, - # "bioacoustics-2024.whale_speech.mapped_audio", schema=self.schema, write_disposition=write_disposition, create_disposition=create_disposition, @@ -161,10 +169,11 @@ def _write_gcp(self, element): custom_gcs_temp_location=custom_gcs_temp_location ) - yield element - def _write_local(self, element): - if os.path.exists(self.output_path): + def _store_local(self, element): + logging.info(f"Storing to local path: {self.output_path}") + + if filesystems.FileSystems.exists(self.output_path): stored_df = pd.read_json(self.output_path, orient="records") # convert encounter_id to str diff --git a/src/stages/search.py b/src/stages/search.py index 5c1e075..953439c 100644 --- a/src/stages/search.py +++ b/src/stages/search.py @@ -1,6 +1,6 @@ from apache_beam.io import filesystems -from config import load_pipeline_config from happywhale.happywhale import geometry_search +from types import SimpleNamespace import apache_beam as beam import io @@ -9,62 +9,138 @@ import pandas as pd -config = load_pipeline_config() +class GeometrySearch(beam.DoFn): + def __init__(self, config: SimpleNamespace): + self.config = config + self.is_local = config.general.is_local + + self.species = config.search.species + + self.filename = config.search.filename + self.geometry_file_path_template = config.search.geometry_file_path_template + self.search_columns = config.search.search_columns + + self.project = config.general.project + self.dataset_id = config.general.dataset_id + self.table_id = config.search.search_table_id + self.schema = self._schema_to_dict(config.search.search_table_schema) + self.temp_location = config.general.temp_location + self.write_params = config.bigquery.__dict__ + + self.output_path = config.search.output_path_template.format( + table_id=self.table_id, + geofile=self.filename + ) -class GeometrySearch(beam.DoFn): def process(self, element): start = self._preprocess_date(element.get('start')) end = self._preprocess_date(element.get('end')) geometry_file = self._get_geometry_file() - export_file = self._get_export_path(start, end) + export_file = self._get_file_buffer("csv") - species = config.search.species + geometry_search(geometry_file, start,end, export_file, self.species) - geometry_search(geometry_file, start,end, export_file, species) + search_results = self._postprocess(export_file) - yield self._postprocess(export_file) + self._store(search_results) + yield search_results - def _preprocess_date(self, date_str): - # return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S") + + @staticmethod + def _preprocess_date(date_str): return date_str.split("T")[0] + @staticmethod + def _get_file_buffer(filetype="csv"): + fb = io.BytesIO() # or io.StringIO() + fb.endswith = lambda x: x=="csv" # hack to ensure happywhale saves df to fb + return fb + + + @staticmethod + def _schema_to_dict(schema): + return { + "fields": [ + { + "name": name, + "type": getattr(schema, name).type, + "mode": getattr(schema, name).mode + } + for name in vars(schema) + ] + } + + def _get_geometry_file(self): """ Uses io.Bytes with filesystems.FileSystems.open(data_path) to load the geometry file. """ - filename = config.search.filename - geometry_filename = config.search.geometery_file_path_template.format( + filename = self.filename + geometry_filename = self.geometry_file_path_template.format( filename=filename ) return io.BytesIO(filesystems.FileSystems.open(geometry_filename).read()) - - def _get_export_path(self, start, end): - filename = config.search.filename - - export_filename = config.search.export_template.format( - filename=filename, - timeframe=( - f"{start}-{end}" - if start != end - else start + + def _postprocess(self, export_file) -> pd.DataFrame: + if isinstance(export_file, io.BytesIO): + export_file.seek(0) + results = pd.read_csv(export_file) + results = results[self.search_columns] + logging.info(f"Search results: \n{results.head()}") + return results + + + def _store(self, search_results): + rows = self._convert_to_table_rows(search_results) + + if self.is_local: + # convert back to dataframe w7 correct columns + search_results = pd.DataFrame(rows) + if not filesystems.FileSystems.exists(self.output_path): + if not filesystems.FileSystems.exists(os.path.dirname(self.output_path)): + # safely create parent dir (in case something gets wrongly deleted) + filesystems.FileSystems.mkdirs(os.path.dirname(self.output_path)) + search_results.to_csv(self.output_path, index=False) + else: + previous_search_results = pd.read_csv(self.output_path) + search_results = pd.concat([previous_search_results, search_results]) + search_results.drop_duplicates(inplace=True) + search_results.to_csv(self.output_path, index=False) + + else: + logging.info(f"search_results.columns: {search_results.columns}") + + rows | f"Update {self.table_id}" >> beam.io.WriteToBigQuery( + self.table_id, + dataset=self.dataset_id, + project=self.project, + schema=self.schema, + custom_gcs_temp_location=self.temp_location, + **self.write_params ) - ) - return export_filename + logging.info(f"Stored search results in {self.output_path}") - def _postprocess(self, export_file) -> pd.DataFrame: - results = pd.read_csv(export_file) + def _convert_to_table_rows(self, df): + table_colums = [field["name"] for field in self.schema["fields"]] + + df["encounter_id"] = df["id"] + df["img_path"] = df["displayImgUrl"] + df["longitude"] = df["longitude"].astype(float) + df["latitude"] = df["latitude"].astype(float) - results = results[config.search.columns] + df["encounter_time"] = df[["startDate", "startTime"]].apply( + lambda x: f"{x.startDate}T{x.startTime}", axis=1 + ) - logging.info(f"Search results: \n{results.head()}") + df = df[[*table_colums]] - return results + return df.to_dict(orient="records") diff --git a/src/stages/sift.py b/src/stages/sift.py index 58096d4..162a2f0 100644 --- a/src/stages/sift.py +++ b/src/stages/sift.py @@ -1,15 +1,18 @@ from apache_beam.io import filesystems from datetime import datetime from scipy.signal import butter, lfilter, find_peaks, sosfilt +from types import SimpleNamespace +from typing import Dict, Any import apache_beam as beam import io import logging +import json import math import matplotlib.pyplot as plt import numpy as np import os -import pickle +import pandas as pd from config import load_pipeline_config @@ -25,18 +28,54 @@ class BaseSift(beam.PTransform): """ name = "BaseSift" - # general params - debug = config.general.debug - sample_rate = config.audio.source_sample_rate - - # sift-specific params - max_duration = config.sift.max_duration - window_size = config.sift.window_size - - # plot params - plot = config.sift.plot - plot_path_template = config.sift.plot_path_template - show_plots = config.general.show_plots + def __init__(self, config: SimpleNamespace): + # general params + self.debug = config.general.debug + self.is_local = config.general.is_local + self.sample_rate = config.audio.source_sample_rate + self.store = config.sift.store_sift_audio + + # sift-specific params + self.max_duration = config.sift.max_duration + self.threshold = None + self.window_size = config.sift.window_size + + # plotting params + self.plot = config.sift.plot + self.plot_path_template = config.sift.plot_path_template + self.show_plots = config.general.show_plots + + # store params + self.output_array_path_template = config.sift.output_array_path_template + self.output_table_path_template = config.sift.output_table_path_template + self.params_path_template = None # specific to each sift + self.project = config.general.project + self.dataset_id = config.general.dataset_id + self.table_id = config.sift.sift_table_id + self.temp_location = config.general.temp_location + self.schema = self._schema_to_dict(config.sift.sift_table_schema) + self.workbucket = config.general.workbucket + self.write_params = config.bigquery.__dict__ + + + @staticmethod + def _schema_to_dict(schema): + return { + "fields": [ + { + "name": name, + "type": getattr(schema, name).type, + "mode": getattr(schema, name).mode + } + for name in vars(schema) + ] + } + + def _get_filter_params(self): + return {} + + def _get_path_params(self): + return {"name": self.name.lower()} def _build_key( self, @@ -53,7 +92,7 @@ def _preprocess(self, pcoll): """ pcoll: tuple(audio, start_time, end_time, row.encounter_ids) """ - signal, start, end, encounter_ids = pcoll + signal, start, end, encounter_ids, _ = pcoll key = self._build_key(start, end, encounter_ids) max_samples = self.max_duration * self.sample_rate @@ -71,7 +110,7 @@ def _preprocess(self, pcoll): yield (key, signal) def _postprocess(self, pcoll, min_max_detections): - signal, start, end, encounter_ids = pcoll + signal, start, end, encounter_ids, _ = pcoll key = self._build_key(start, end, encounter_ids) logging.info(f"Postprocessing {self.name} sifted signal.") @@ -84,10 +123,14 @@ def _postprocess(self, pcoll, min_max_detections): min_max_detections[key]["max"] ] - return signal[global_detection_range[0]:global_detection_range[-1]], start, end, encounter_ids + sifted_signal = signal[global_detection_range[0]:global_detection_range[-1]] + audio_path = "No sift audio path stored." + detections_path = "No detections path stored." - def _plot_signal_detections(self, pcoll, min_max_detections, all_detections, params=None): - signal, start, end, encounter_ids = pcoll + return [(sifted_signal, start, end, encounter_ids, audio_path, detections_path)] + + def _plot_signal_detections(self, pcoll, min_max_detections, all_detections): + signal, start, end, encounter_ids, _ = pcoll key = self._build_key(start, end, encounter_ids) min_max_detection_samples = [ @@ -97,16 +140,16 @@ def _plot_signal_detections(self, pcoll, min_max_detections, all_detections, par logging.info(f"Plotting signal detections: {min_max_detection_samples}") # datetime format matches original audio file name + params_path = self.params_path_template.format( + **self._get_path_params() + ) plot_path = self.plot_path_template.format( - sift=self.name, - year=start.year, - month=start.month, - day=start.day, - plot_name=key + params=params_path, + key=key ) - if not beam.io.filesystems.FileSystems.exists(plot_path): - beam.io.filesystems.FileSystems.mkdirs(plot_path) + if not beam.io.filesystems.FileSystems.exists(os.path.dirname(plot_path)): + beam.io.filesystems.FileSystems.mkdirs(os.path.dirname(plot_path)) # normalize and center signal = signal / np.max(signal) # normalize @@ -144,17 +187,11 @@ def _plot_signal_detections(self, pcoll, min_max_detections, all_detections, par plt.ylabel('Amplitude (normalized and centered)') title = f"({self.name}) Signal detections: {start.strftime('%Y-%m-%d %H:%M:%S')}-{end.strftime('%H:%M:%S')}\n" - title += f"Params: {params} \n" if params else "" + title += f"Params: {self._get_path_params()} \n" if self._get_path_params() else "" title += f"Encounters: {encounter_ids}" plt.title(title) plt.savefig(plot_path) - # TODO remove hack to reuse sift figure later - with open(f"{plot_path.split(key)[0]}/data/{key}_min_max.pkl", 'wb') as handle: - pickle.dump(min_max_detection_samples, handle) - with open(f"{plot_path.split(key)[0]}/data/{key}_all.pkl", 'wb') as handle: - pickle.dump(all_detections[key], handle) - plt.show() if self.show_plots else plt.close() @@ -163,26 +200,38 @@ class Butterworth(BaseSift): def __init__( self, - lowcut: int = None, - highcut: int = None, - order: int = None, - output: str = None, - sift_threshold: float = None, + config: SimpleNamespace ): - super().__init__() + super().__init__(config) # define bandpass - self.lowcut = config.sift.butterworth.lowcut if not lowcut else lowcut - self.highcut = config.sift.butterworth.highcut if not highcut else highcut - self.order = config.sift.butterworth.order if not order else order - self.output = config.sift.butterworth.output if not output else output + self.lowcut = config.sift.butterworth.lowcut + self.highcut = config.sift.butterworth.highcut + self.order = config.sift.butterworth.order + self.output = config.sift.butterworth.output # apply bandpass - self.sift_threshold = ( - config.sift.butterworth.sift_threshold - if not sift_threshold else sift_threshold - ) - + self.threshold = config.sift.butterworth.sift_threshold + + # store params + self.params_path_template = config.sift.butterworth.params_path_template + + @staticmethod + def _butter_bandpass( + sample_rate: int, + lowcut: float, + highcut: float, + order: int, + output + ): + """ + Returns specific Butterworth filter (IIR) from parameters in config.sift.butterworth + """ + nyq = 0.5 * sample_rate + low = lowcut / nyq + high = highcut / nyq + return butter(order, [low, high], btype='band', output=output) + def expand(self, pcoll): """ pcoll: tuple(audio, start_time, end_time, row.encounter_ids) @@ -196,46 +245,43 @@ def expand(self, pcoll): all_detections = detections | "List Detections" >> beam.CombinePerKey(ListCombine()) # full-input postprocess - sifted_output = pcoll | "Postprocess" >> beam.Map( + sifted_output = pcoll | "Postprocess" >> beam.ParDo( self._postprocess, min_max_detections=beam.pvalue.AsDict(min_max_detections), ) + if self.store: + results = sifted_output | "Store Sifted Audio" >> beam.ParDo( + self._store, + detections=beam.pvalue.AsDict(all_detections), + ) + else: + results = sifted_output + # plots for debugging purposes if self.debug or self.plot: pcoll | "Plot Sifted Output" >> beam.Map( self._plot_signal_detections, min_max_detections=beam.pvalue.AsDict(min_max_detections), all_detections=beam.pvalue.AsDict(all_detections), - params={ - "lowcut": self.lowcut, - "highcut": self.highcut, - "order": self.order, - "threshold": self.sift_threshold, - } ) - return sifted_output + return results - def _butter_bandpass(self): - """ - Returns specific Butterworth filter (IIR) from parameters in config.sift.butterworth - """ - nyq = 0.5 * self.sample_rate - low = self.lowcut / nyq - high = self.highcut / nyq - return butter(self.order, [low, high], btype='band', output=self.output) - def _frequency_filter_sift( self, batch: tuple, ): key, signal = batch + filter_params = self._get_filter_params() logging.info(f"Start frequency detection on (key, signal): {(key, signal.shape)}") # Apply bandpass filter - butter_coeffients = self._butter_bandpass() + butter_coeffients = self._butter_bandpass( + self.sample_rate, + **filter_params, + ) if self.output == "ba": filtered_signal = lfilter(butter_coeffients[0], butter_coeffients[1], signal) elif self.output == "sos": @@ -256,7 +302,7 @@ def _frequency_filter_sift( logging.debug(f"Normalized energy: {energy}") # Find peaks above threshold - peaks, _ = find_peaks(energy, height=self.sift_threshold) + peaks, _ = find_peaks(energy, height=self.threshold) logging.debug(f"Peaks: {peaks}") # Convert peak indices to time @@ -265,6 +311,139 @@ def _frequency_filter_sift( yield (key, peak_samples) + def _get_filter_params(self): + return { + "lowcut": self.lowcut, + "highcut": self.highcut, + "order": self.order, + "output": self.output, + } + + def _get_path_params(self): + path_params = self._get_filter_params() + path_params.pop("output") + path_params["threshold"] = self.threshold + path_params["name"] = self.name.lower() + return path_params + + def _get_export_paths(self, key): + params_path = self.params_path_template.format(**self._get_path_params()) + audio_path = self.output_array_path_template.format( + params=params_path, + key=key, + filename="audio.npy" + ) + detections_path = self.output_array_path_template.format( + params=params_path, + key=key, + filename="detections.npy" + ) + + if not self.is_local: + audio_path = os.path.join(self.workbucket,audio_path) + detections_path = os.path.join(self.workbucket,detections_path) + + return audio_path, detections_path + + def _store(self, sifted_output, detections): + signal, start, end, encounter_ids, audio_path, detections_path = sifted_output + logging.info(f"Signal shape: {signal.shape}") + logging.info(f"Start: {start}") + logging.info(f"End: {end}") + logging.info(f"Encounter IDs: {encounter_ids}") + + key = self._build_key(start, end, encounter_ids) + + if not isinstance(signal, np.ndarray): + signal = np.array(signal) + + if signal.shape[0] == 0: + logging.info(f"Empty sifted signal for {key}.") + return [(signal, start, end, encounter_ids, audio_path, detections_path)] + + audio_path, detections_path = self._get_export_paths(key) + + # upload metadata to table + if self.is_local: + self._store_local( + key, + audio_path, + detections_path, + self._get_path_params(), + ) + else: + self._store_bigquery( + key, + audio_path, + detections_path, + self._get_path_params(), + ) + + # store sifted audio + with filesystems.FileSystems.create(audio_path) as f: + logging.info(f"Storing sifted audio to {audio_path}") + np.save(f, signal) + + # store detections + with filesystems.FileSystems.create(detections_path) as f: + logging.info(f"Storing detections to {detections_path}") + np.save(f, detections[key]) + + logging.info(f"Stored sifted audio and detections for {key}.") + return [(signal, start, end, encounter_ids, audio_path, detections_path)] + + def _store_local( + self, + key: str, + audio_path: str, + detections_path: str, + params: Dict[str, Any], + ): + table_path = self.output_table_path_template.format(table_id=self.table_id) + + if not filesystems.FileSystems.exists(table_path): + # build parent dir if necessary + filesystems.FileSystems.create(table_path) + df = pd.DataFrame([{ + "key": key, + "sift_audio_path": audio_path, + "sift_detections_path": detections_path, + "params": json.dumps(params) + }]) + df.to_json(table_path, index=False, orient="records") + else: + df = pd.read_json(table_path, orient="records") + new_row = pd.DataFrame([{ + "key": key, + "sift_audio_path": audio_path, + "sift_detections_path": detections_path, + "params": json.dumps(params) + }]) + df = pd.concat([df, new_row]) + df = df.drop_duplicates() + df.to_json(table_path, index=False, orient="records") + + def _store_bigquery( + self, + key: str, + audio_path: str, + detections_path: str, + params: Dict[str, Any], + ): + [{ + "key": key, + "sift_audio_path": audio_path, + "sift_detections_path": detections_path, + "params": json.dumps(params) + }] | "Write to BigQuery" >> beam.io.WriteToBigQuery( + self.table_id, + dataset=self.dataset_id, + project=self.project, + schema=self.schema, + custom_gcs_temp_location=self.temp_location, + **self.write_params + ) + class ListCombine(beam.CombineFn): name = "ListCombine" diff --git a/tests/test_audio.py b/tests/test_audio.py index b401e1d..4052e1d 100644 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -1,12 +1,40 @@ from datetime import datetime +from types import SimpleNamespace import pytest import pandas as pd import numpy as np from stages.audio import RetrieveAudio -from unittest.mock import patch +@pytest.fixture +def config(): + return SimpleNamespace( + audio=SimpleNamespace( + margin = 300, + offset = 1, + source_sample_rate = 16_000, + url_template = "https://pacific-sound-16khz.s3.amazonaws.com/{year}/{month:02}/{filename}", + filename_template = "MARS-{year}{month:02}{day:02}T000000Z-16kHz.wav", + output_array_path_template = "output_array_path_template", + output_table_path_template = "output_table_path_template", + skip_existing = True, + + store_audio = False, + audio_table_id = "config.audio.audio_table_id", + audio_table_schema = SimpleNamespace(key=SimpleNamespace(type="STRING", mode="REQUIRED")) + + ), + general=SimpleNamespace( + debug = True, + is_local = True, + project="project", + dataset_id="dataset_id", + workbucket = "workbucket", + temp_location="temp_location", + ), + bigquery = SimpleNamespace(write_disposition="write_disposition"), + ) @pytest.fixture def sample_search_results_df(): @@ -31,13 +59,8 @@ def sample_time_frame_df(): return pd.DataFrame(data) -@patch('stages.audio.config') -def test_build_time_frames(mock_config, sample_search_results_df): +def test_build_time_frames(config, sample_search_results_df): # Assemble - # mock config values to avoid failing tests on config changes - mock_config.audio.margin = 5 * 60 # 5 minutes - mock_config.audio.offset = 1 # 1 hour # TODO remove. only used for development - expected_df = pd.DataFrame({ 'id': [1, 2, 3], 'startDate': ['2024-07-08', '2024-09-10', '2024-09-10'], @@ -48,7 +71,7 @@ def test_build_time_frames(mock_config, sample_search_results_df): }) # Act - actual_df = RetrieveAudio()._build_time_frames(sample_search_results_df) + actual_df = RetrieveAudio(config)._build_time_frames(sample_search_results_df) # Assert assert pd.api.types.is_datetime64_any_dtype(actual_df['start_time']) @@ -56,7 +79,7 @@ def test_build_time_frames(mock_config, sample_search_results_df): pd.testing.assert_frame_equal(expected_df, actual_df) -def test_find_overlapping(sample_time_frame_df): +def test_find_overlapping(config, sample_time_frame_df): # Assemble expected_df = pd.DataFrame({ 'encounter_ids':[["1"], ["3", "2"]], @@ -65,7 +88,7 @@ def test_find_overlapping(sample_time_frame_df): }) # Act - actual_df = RetrieveAudio()._find_overlapping(sample_time_frame_df) + actual_df = RetrieveAudio(config)._find_overlapping(sample_time_frame_df) # Assert pd.testing.assert_frame_equal(expected_df, actual_df) @@ -73,13 +96,8 @@ def test_find_overlapping(sample_time_frame_df): assert pd.api.types.is_datetime64_any_dtype(actual_df['end_time']) -@patch('stages.audio.config') -def test_preprocess(mock_config, sample_search_results_df): +def test_preprocess(config, sample_search_results_df): # Assemble - # mock config values to avoid failing tests on config changes - mock_config.audio.margin = 5 * 60 # 5 minutes - mock_config.audio.offset = 1 # 1 hour # TODO remove. only used for development - expected_df = pd.DataFrame({ 'encounter_ids':[["1"], ["3", "2"]], 'start_time': [datetime(2024, 7, 7, 23, 8, 0), datetime(2024, 9, 10, 0, 27, 0)], @@ -87,13 +105,13 @@ def test_preprocess(mock_config, sample_search_results_df): }) # Act - actual_df = RetrieveAudio()._preprocess(sample_search_results_df) + actual_df = RetrieveAudio(config)._preprocess(sample_search_results_df) # Assert pd.testing.assert_frame_equal(expected_df, actual_df) -def test_get_file_url(): +def test_get_file_url(config): # Assemble sample_year = 2024 sample_month = 9 @@ -101,7 +119,7 @@ def test_get_file_url(): expected_url = 'https://pacific-sound-16khz.s3.amazonaws.com/2024/09/MARS-20240910T000000Z-16kHz.wav' # Act - actual_url = RetrieveAudio()._get_file_url(sample_year, sample_month, sample_day) + actual_url = RetrieveAudio(config)._get_file_url(sample_year, sample_month, sample_day) # Assert assert expected_url == actual_url diff --git a/tests/test_classify.py b/tests/test_classify.py index d6908ff..f74d593 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -13,9 +13,26 @@ def example_config(): return SimpleNamespace( general = SimpleNamespace( + is_local="is_local", show_plots=True, + project="project", + dataset_id="dataset_id", + temp_location="temp_location", + workbucket="workbucket", + ), + audio = SimpleNamespace( + source_sample_rate=16_000 + ), + sift = SimpleNamespace( + butterworth=SimpleNamespace( + lowcut=50, + highcut=1500, + order=2, + output="sos", + sift_threshold=0.015, + params_path_template="params_path_template" + ), ), - audio = SimpleNamespace(source_sample_rate=16_000), classify = SimpleNamespace( batch_duration=30, # seconds hydrophone_sensitivity=-168.8, @@ -25,7 +42,17 @@ def example_config(): plot_path_template="data/plots/results/{year}/{month:02}/{plot_name}.png", med_filter_size=3, inference_retries=3, + store_classifications="store_classifications", + classification_table_id="classification_table_id", + classification_table_schema=SimpleNamespace( + key=SimpleNamespace(type="STRING", mode="REQUIRED"), + ), + output_array_path_template="output_array_path_template", + output_table_path_template="output_table_path_template", ), + bigquery=SimpleNamespace( + write_disposition="write_disposition" + ) ) @pytest.fixture @@ -34,7 +61,9 @@ def example_input_row_small(): start_time = datetime(2024, 7, 7, 23, 8, 0) end_time = datetime(2024, 7, 7, 23, 18, 0) encounter_ids = ["encounter1", "encounter2"] - yield audio, start_time, end_time, encounter_ids + audio_path = "gs://project/dataset/project/data/audio/raw/audio_path" + detections_path = "gs://project/dataset/project/data/detections/detections_path" + yield audio, start_time, end_time, encounter_ids, audio_path, detections_path @pytest.fixture def example_input_row_large(): @@ -42,7 +71,9 @@ def example_input_row_large(): start_time = datetime(2024, 7, 7, 23, 8, 0) end_time = datetime(2024, 7, 7, 23, 18, 0) encounter_ids = ["encounter1", "encounter2"] - yield audio, start_time, end_time, encounter_ids + audio_path = "gs://project/dataset/project/data/audio/raw/audio_path" + detections_path = "gs://project/dataset/project/data/detections/detections_path" + yield audio, start_time, end_time, encounter_ids, audio_path, detections_path @pytest.fixture @@ -101,20 +132,21 @@ def test_postprocess(example_config, example_input_row_small, example_grouped_ou # Arrange input_row = example_input_row_small - expected = ( + expected = [( np.array([0., 1., 2., 3., 4., 5., 4., 3., 2., 1.]), # audio datetime(2024, 7, 7, 23, 8, 0), # start_time datetime(2024, 7, 7, 23, 18, 0), # end_time ["encounter1", "encounter2"], # encounter_ids - [0.3, 0.7, 0.2, 0.6] # scores - ) + [0.3, 0.7, 0.2, 0.6], # scores + "No classification path stored." # classification_path, + )] # Act actual = WhaleClassifier(example_config)._postprocess(input_row, example_grouped_outputs) # Assert assert len(expected) == len(actual) - for e, a in zip(expected, actual): + for e, a in zip(expected[0], actual[0]): if isinstance(e, np.ndarray): np.testing.assert_almost_equal(e, a, decimal=6) else: diff --git a/tests/test_config.py b/tests/test_config.py index fbe26aa..25e318d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -14,7 +14,8 @@ def test_load_pipeline_config(): 'audio', 'sift', 'classify', - 'postprocess' + 'postprocess', + 'bigquery' ] actual_config = load_pipeline_config().__dict__ diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index 6829f78..7660466 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -8,11 +8,21 @@ @pytest.fixture def config(): return SimpleNamespace( - search=SimpleNamespace(export_template="template"), - sift=SimpleNamespace(output_path_template="template"), - classify=SimpleNamespace(classification_path="path"), - postprocess=SimpleNamespace(pooling="mean", postprocess_table_id="table_id"), - general=SimpleNamespace(project="project", dataset_id="dataset_id") + search=SimpleNamespace(output_path_template="template"), + sift=SimpleNamespace(output_array_path_template="template"), + classify=SimpleNamespace(output_path_template="path"), + postprocess=SimpleNamespace( + pooling="mean", + postprocess_table_id="table_id", + output_path="output_path", + postprocess_table_schema=SimpleNamespace( + start=SimpleNamespace(type="TIMESTAMP", mode="REQUIRED"), + end=SimpleNamespace(type="TIMESTAMP", mode="REQUIRED"), + encounter_id=SimpleNamespace(type="STRING", mode="REQUIRED"), + pooled_score=SimpleNamespace(type="FLOAT", mode="REQUIRED"), + ), + ), + general=SimpleNamespace(project="project", dataset_id="dataset_id", is_local=True), ) @pytest.fixture @@ -22,7 +32,8 @@ def element(): "start": datetime(2024, 9, 10, 11, 12, 13), "end": datetime(2024, 9, 10, 12, 13, 14), "encounter_ids": ["a123", "b456"], - "classifications": [1, 2, 3] + "classifications": [1, 2, 3], + "classification_path": "gs://project/dataset/data/classifications/file" } @pytest.fixture @@ -45,13 +56,15 @@ def test_build_classification_df(config, element): "start": "2024-09-10T11:12:13", "end": "2024-09-10T12:13:14", "encounter_id": "a123", - "pooled_score": 2.0 + "classification_path": "gs://project/dataset/data/classifications/file", + "pooled_score": 2.0, }, { "start": "2024-09-10T11:12:13", "end": "2024-09-10T12:13:14", "encounter_id": "b456", - "pooled_score": 2.0 + "classification_path": "gs://project/dataset/data/classifications/file", + "pooled_score": 2.0, } ]) @@ -59,7 +72,8 @@ def test_build_classification_df(config, element): actual = postprocess_labels._build_classification_df(element) # Assert - assert expected.equals(actual) + for e, a in zip(expected, actual): + assert e == a def test_build_search_output_df(config, search_output): @@ -104,13 +118,16 @@ def test_add_paths(config, search_output): "extra_column": ["extra1", "extra2", "extra3"], # added path columns + "img_path": ["example.com/a123", "example.com/b456", "example.com/c789"], "audio_path": ["NotImplemented", "NotImplemented", "NotImplemented"], "classification_path": ["NotImplemented", "NotImplemented", "NotImplemented"], - "img_path": ["example.com/a123", "example.com/b456", "example.com/c789"], }) # Act - actual = postprocess_labels._add_paths(search_output) + actual = postprocess_labels._add_paths(search_output, {}) # Assert - assert expected.equals(actual) + print(expected.columns) + print(actual.columns) + for e, a in zip(expected, actual): + assert e == a diff --git a/tests/test_search.py b/tests/test_search.py index 6b0e36c..43986dc 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -5,8 +5,50 @@ import numpy as np from stages.search import GeometrySearch +from types import SimpleNamespace from unittest.mock import patch +@pytest.fixture +def config(): + return SimpleNamespace( + search=SimpleNamespace( + output_path_template="template", + species="species", + filename="filename", + geometry_file_path_template="geometry_file_path_template", + search_columns=[ + "id", + "latitude", + "longitude", + "startDate", + "startTime", + "endTime", + "timezone", + "displayImgUrl", + ], + search_table_id="search_table_id", + search_table_schema=SimpleNamespace( + encounter_id=SimpleNamespace(type="STRING", mode="REQUIRED"), + encounter_time=SimpleNamespace(type="STRING", mode="REQUIRED"), + latitude=SimpleNamespace(type="FLAOT64", mode="REQUIRED"), + longitude=SimpleNamespace(type="FLOAT64", mode="REQUIRED"), + img_path=SimpleNamespace(type="STRING", mode="NULLABLE"), + ) + ), + general=SimpleNamespace( + is_local=True, + project="project", + dataset_id="dataset_id", + temp_location="temp_location" + ), + bigquery=SimpleNamespace( + write_disposition="write_disposition", + create_disposition="create_disposition", + method="method", + custom_gcs_temp_location="custom_gcs_temp_location" + ) + ) + @pytest.fixture def sample_element(): @@ -16,31 +58,18 @@ def sample_element(): } -def test_preprocess_date(sample_element): +def test_preprocess_date(sample_element, config): # Assemble expected = '2024-07-08' # Act - actual = GeometrySearch()._preprocess_date(sample_element.get('start')) - - # Assert - assert actual == expected - - -def test_get_export_file(sample_element): - # Assemble - start = '2024-07-08' - end = '2024-07-08' - expected = "data/encounters/monterey_bay_50km-2024-07-08.csv" - - # Act - actual = GeometrySearch()._get_export_path(start, end) + actual = GeometrySearch(config)._preprocess_date(sample_element.get('start')) # Assert assert actual == expected -def test_postprocess(sample_element): +def test_postprocess(sample_element, config): # Assemble sample_export_file = "tests/data/sample_2024-07-08.csv" expected = pd.DataFrame({ @@ -55,7 +84,7 @@ def test_postprocess(sample_element): }) # Act - actual = GeometrySearch()._postprocess(sample_export_file) + actual = GeometrySearch(config)._postprocess(sample_export_file) # Assert pd.testing.assert_frame_equal(expected, actual) diff --git a/tests/test_sift.py b/tests/test_sift.py index 4d40923..6b40b7a 100644 --- a/tests/test_sift.py +++ b/tests/test_sift.py @@ -6,8 +6,56 @@ from stages.sift import BaseSift, Butterworth from unittest.mock import patch +from types import SimpleNamespace +@pytest.fixture +def config(): + return SimpleNamespace( + audio=SimpleNamespace( + source_sample_rate = 16_000, + ), + sift=SimpleNamespace( + output_path_template="template", + store_sift_audio=True, + max_duration = 600, + threshold = 0.015, + window_size = 512, + plot = True, + plot_path_template = "plot_path_template", + show_plots = True, + output_array_path_template = "output_array_path_template", + output_table_path_template = "output_table_path_template", + project = "project", + dataset_id = "dataset_id", + sift_table_id = "sift_table_id", + temp_location = "temp_location", + sift_table_schema = SimpleNamespace( + encounter_id=SimpleNamespace(type="STRING", mode="REQUIRED"), + ), + workbucket = "workbucket", + write_params = {}, + butterworth=SimpleNamespace( + params_path_template = "template", + lowcut = 50, + highcut = 1500, + order = 2, + output = "sos", + sift_threshold = 0.015 + ) + ), + general=SimpleNamespace( + debug = True, + is_local = True, + project="project", + dataset_id="dataset_id", + workbucket = "workbucket", + temp_location="temp_location", + show_plots = False, + ), + bigquery = SimpleNamespace(write_disposition="write_disposition"), + ) + @pytest.fixture def sample_audio_results_df(): data = { @@ -24,7 +72,8 @@ def sample_audio_results_row(): start_time = datetime(2024, 7, 7, 23, 8, 0) end_time = datetime(2024, 7, 7, 23, 18, 0) encounter_ids = ["encounter1", "encounter2"] - yield audio, start_time, end_time, encounter_ids + audio_path = "gs://project/dataset/data/audio/raw/20240707T230800-231800.wav" + yield audio, start_time, end_time, encounter_ids, audio_path @pytest.fixture @@ -41,20 +90,20 @@ def sample_batch(): return (key, signal) -def test_build_key(sample_audio_results_row): +def test_build_key(config, sample_audio_results_row): # Assemble - _, start, end, encounter_ids = sample_audio_results_row + _, start, end, encounter_ids, _ = sample_audio_results_row expected_key = "20240707T230800-231800_encounter1_encounter2" # Act - actual_key = BaseSift()._build_key(start, end, encounter_ids) + actual_key = BaseSift(config)._build_key(start, end, encounter_ids) # Assert assert expected_key == actual_key -def test_preprocess(sample_audio_results_row): +def test_preprocess(config, sample_audio_results_row): # Assemble expected_data = [ ("20240707T230800-231800_encounter1_encounter2", np.array([0, 1, 2, 3, 4, 5, 4, 3, 2, 1]*16_000*60)), @@ -62,7 +111,7 @@ def test_preprocess(sample_audio_results_row): ] # Act - actual_data_generator = Butterworth()._preprocess(sample_audio_results_row) + actual_data_generator = Butterworth(config)._preprocess(sample_audio_results_row) # Assert for expected in expected_data: @@ -72,65 +121,60 @@ def test_preprocess(sample_audio_results_row): assert expected[1].shape == actual[1].shape # data -def test_postprocess(sample_audio_results_row): +def test_postprocess(config, sample_audio_results_row): # Assemble pcoll = sample_audio_results_row min_max_detections = { "20240707T230800-231800_encounter1_encounter2": {"min":0, "max": 5} # samples } - expected_data = ( + expected_data = [( np.array([0, 1, 2, 3, 4]), # audio datetime(2024, 7, 7, 23, 8, 0), # start_time datetime(2024, 7, 7, 23, 18, 0), # end_time - ["encounter1", "encounter2"] # encounter_ids - ) + ["encounter1", "encounter2"], # encounter_ids + 'No sift audio path stored.', + 'No detections path stored.' + )] # Act - actual_data = Butterworth()._postprocess(pcoll, min_max_detections) + actual_data = Butterworth(config)._postprocess(pcoll, min_max_detections) # Assert assert len(expected_data) == len(actual_data) - for expected, actual in zip(expected_data, actual_data): + for expected, actual in zip(expected_data[0], actual_data[0]): assert np.array_equal(expected, actual) -@patch('stages.sift.config') -def test_butter_bandpass(mock_config): +def test_butter_bandpass(config): # Assemble - mock_config.sift.butterworth.lowcut = 50 - mock_config.sift.butterworth.highcut = 1500 - mock_config.sift.butterworth.order = 2 - mock_config.sift.butterworth.output = "sos" - expected_coefficients = [ [ 0.05711683, 0.11423366, 0.05711683, 1. , -1.22806805, 0.4605427 ], [ 1. , -2. , 1. , 1. , -1.97233136, 0.97273604] ] - actual_coefficients = Butterworth()._butter_bandpass() + actual_coefficients = Butterworth(config)._butter_bandpass( + lowcut=50, + highcut=1500, + sample_rate=16_000, + order=2, + output="sos" + ) # Assert assert len(actual_coefficients) == 2 # order assert np.allclose(expected_coefficients, actual_coefficients) -@patch('stages.sift.config') -def test_frequency_filter_sift(mock_config, sample_batch): +def test_frequency_filter_sift(config, sample_batch): # Assemble - mock_config.sift.butterworth.lowcut = 50 - mock_config.sift.butterworth.highcut = 1500 - mock_config.sift.butterworth.order = 5 - mock_config.sift.butterworth.output = "sos" - mock_config.sift.butterworth.sift_threshold = 0.015 - expected_detections = ( "20161221T004930-005030-9182", np.array([13824]) ) # Act - actual_detections_generator = Butterworth()._frequency_filter_sift(sample_batch) + actual_detections_generator = Butterworth(config)._frequency_filter_sift(sample_batch) actual_detections = next(actual_detections_generator)