Merge pull request #30 from urbangrammarai/feature/create-grid-from-b…

…oundary

Initial create grid of chips

.gitignore

@@ -115,10 +115,12 @@ docs/build
 docs/source/_static
 docs/source/_templates
 
-# Test data
-pyveg/testdata/test_ne_england/
-
 # Virtual Environments
 env/
+
+# Test data
+pyveg/testdata/test_ne_england/
+birmingham-test__2022-*/
+Sentinel2-52.50N-1.90W-birmingham-reproject-test__2022-*/
+pyveg/testdata/Sentinel2/tmp_png/*
 liverpool-bng__*/
-pyveg/testdata/Sentinel2/tmp_png/

environment.yml

@@ -7,3 +7,5 @@ dependencies:
   - earthengine-api
   - rasterio
   - coverage
+  - icecream
+  - gdal

pyveg/configs/config_liverpool_bng_example.py

@@ -0,0 +1,40 @@
+from pyveg.configs.collections import data_collections
+from pyveg.coordinates import coordinate_store
+
+name = "liverpool"
+
+# Define location to save all outputs
+output_location = "liverpool-bng"
+#output_location_type = "azure"
+output_location_type = "local"
+
+# parse selection. Note (long, lat) GEE convention.
+#entry = coordinate_store.loc[coordinate_id]
+# modify this line to set coords based on entries in `coordinates.py`
+#coordinate_id = "00"
+
+# parse selection. Note (long, lat) GEE convention.
+#entry = coordinate_store.loc[coordinate_id]
+#coordinates = (entry.longitude, entry.latitude)
+
+# bounds = [ 327771, 384988, 339052, 395656 ]
+bounds = [297566, 345093, 387814, 430437]
+date_range = ["2016-05-01", "2016-08-01"]
+
+# From the dictionary entries in data_collections.py, which shall we use
+# (these will become "Sequences")
+collections_to_use = ["Sentinel2"]
+
+# Dictionary defining what Modules should run in each Sequence.
+
+modules_to_use = {
+    "Sentinel2": [
+        "VegetationDownloader",
+        "VegetationImageProcessor",
+    ],
+}
+
+# The following demonstrates how parameters can be set for individual Modules or Sequences:
+special_config = {"Sentinel2": {"time_per_point": "3m"}     # this is a whole Sequence
+                  #    # and another Module
+                  }

pyveg/requirements.txt

@@ -12,7 +12,7 @@ pytest
 requests
 dateparser
 python-igraph
-pandas==1.1.1
+pandas
 geopandas
 earthengine-api
 geetools

pyveg/scripts/create_chip_grid.py

@@ -0,0 +1,118 @@
+from datetime import datetime
+
+import fiona
+from icecream import ic
+
+from pyveg.src.grid_creator import (
+    coastline_to_poly,
+    create_grid_of_chips,
+    create_raster_of_chips,
+    get_parent_image_id,
+)
+
+# A collection of config options
+config = {
+    "bounding_box": [0, 0, 700000, 1300000],
+    "target_crs": "EPSG:27700",
+    "path_to_coastline": "zip://../chips/strtgi_essh_gb.zip!strtgi_essh_gb/data/coastline.shp",
+    "output_path": "./data",
+    "pixel_scale": 10,
+}
+
+
+"""
+The `output_options` dict should be specified in the format
+<chip_width_in_pixels> : <base_output_filename>
+
+One output file name will be created for each key/values pair
+"""
+# Production values
+output_options = {
+    32: "chips_32",
+    32 * 16: "images_0512",
+    32 * 32: "images_1024",
+}
+
+# Testing values
+# output_options = {
+#     32 * 16: "images_0512",
+#     32 * 32: "images_1024",
+# }
+
+
+def get_larger_chip_id_func(chip_dimension: int):
+    def apply_larger_chip_id(row):
+        return get_parent_image_id(
+            child_x=row["x_lower_left"],
+            child_y=row["y_lower_left"],
+            parent_chip_dimension=chip_dimension,
+        )
+
+    return apply_larger_chip_id
+
+
+if __name__ == "__main__":
+
+    try:
+        coast = coastline_to_poly(
+            path_to_coastline=config["path_to_coastline"], crs=config["target_crs"]
+        )
+    except fiona.errors.DriverError:
+        ic("warning: coastline data not found")
+        coast = None
+
+    for chip_px_width, layer_name in output_options.items():
+        start_time = datetime.now()
+        ic(start_time)
+
+        # First create the raster
+        create_raster_of_chips(
+            chip_px_width=chip_px_width,
+            pixel_scale=config["pixel_scale"],
+            target_crs=config["target_crs"],
+            bounding_box=config["bounding_box"],
+            base_output_filename=layer_name,
+        )
+
+        # Next create the grid as polygons
+        chips_gdf = create_grid_of_chips(
+            chip_px_width=chip_px_width,
+            pixel_scale=config["pixel_scale"],
+            target_crs=config["target_crs"],
+            bounding_box=config["bounding_box"],
+        )
+
+        stage1_time = datetime.now()
+        ic(stage1_time - start_time)
+
+        # Add `on_land` column if the coast data is available
+        if coast:
+            chips_gdf["on_land"] = chips_gdf["geometry"].intersects(coast)
+
+        stage2_time = datetime.now()
+        ic(stage2_time - stage1_time)
+
+        # # Now add cross-references to image ids (for different size images)
+        # for larger_chip_width, larger_name in output_options.items():
+        #     ic(larger_chip_width, chip_px_width)
+        #     if larger_chip_width > chip_px_width:
+        #         get_larger_chip_id = get_larger_chip_id_func(
+        #             larger_chip_width * config["pixel_scale"]
+        #         )
+        #         chips_gdf[f"{larger_name}_id"] = chips_gdf.apply(
+        #             get_larger_chip_id, axis=1
+        #         )
+
+        # ic(chips_gdf.head(10))
+
+        # Write to geoparquet file
+        chips_gdf.to_parquet(f"{layer_name}.parquet")
+
+        stage3_time = datetime.now()
+        ic(stage3_time - stage2_time)
+
+        # Write to GeoPackage DBN (useful for display in GIS)
+        # chips_gdf.to_file("chips.gpkg", layer=layer_name)
+        stage4_time = datetime.now()
+        ic(stage4_time - stage3_time)
+        ic(stage4_time - start_time)

pyveg/src/download_modules.py

@@ -13,6 +13,9 @@
 from pyveg.src.gee_interface import add_NDVI, apply_mask_cloud
 from pyveg.src.pyveg_pipeline import BaseModule, logger
 
+# from datetime import datetime, timedelta
+
+
 ee.Initialize()
 
 # silence google API WARNING
@@ -146,6 +149,7 @@ def prep_data(self, date_range):
         image_list = self.prep_images(dataset)
         url_list = []
         for image in image_list:
+            ic(image)
             # get a URL from which we can download the resulting data
             try:
                 url = image.getDownloadURL(

pyveg/src/gee_interface.py

@@ -3,15 +3,17 @@
 """
 
 import os
-import shutil
-from datetime import datetime
 
-import cv2 as cv
+# import cv2 as cv
 import ee
 from geetools import cloud_mask
 
 from .file_utils import download_and_unzip
 
+# import shutil
+# from datetime import datetime
+
+
 ee.Initialize()