add overlay, dump to file, tweak cli and categories

harvest_restrictions.py

@@ -176,8 +176,8 @@ def validate_sources(sources, validate_data=True, alias=None):
     return sources
 
 
-def download(source):
-    """download layer from source to a standardized geodataframe"""
+def download_source(source):
+    """download data from source to a standardized geodataframe"""
 
     # download WFS
     if source["source_type"] == "BCGW":
@@ -214,10 +214,11 @@ def download(source):
     df["__index__"] = source["index"]
     df["__description__"] = source["description"]
     df["__alias__"] = source["alias"].lower()
+    df["__primary_key__"] = ""
     if source["primary_key"]:
-        df["__primary_key__"] = source["primary_key"]
-    else:
-        df["__primary_key__"] = ""
+        df["__primary_key__"] = df[source["primary_key"].lower()].astype(
+            "str"
+        )  # handle pks as strings
 
     # rename columns that we want to retain
     for key, value in source["field_mapper"].items():
@@ -247,11 +248,6 @@ def download(source):
     return df
 
 
-# def process(out_path):
-#    """clean and overlay input data, dump to file if specified"""
-#    DB.execute("create table designations_cleaned ")
-
-
 @click.group()
 def cli():
     pass
@@ -307,7 +303,7 @@ def validate(alias, sources_file, verbose, quiet):
 )
 @verbose_opt
 @quiet_opt
-def setup(alias, sources_file, out_path, no_validate, verbose, quiet):
+def download(alias, sources_file, out_path, no_validate, verbose, quiet):
     configure_logging((verbose - quiet))
 
     # load sources file
@@ -325,11 +321,11 @@ def setup(alias, sources_file, out_path, no_validate, verbose, quiet):
 
     # download each data source
     for source in sources:
-        df = download(source)
+        df = download_source(source)
 
         # load to postgres, writing everything to the same initial table
         LOG.info(f"Writing {source['alias']} to postgres")
-        df.to_postgis("restrictions_source", DB, if_exists="append")
+        df.to_postgis("designations_source", DB, if_exists="append")
 
         # dump to file if out_path specified
         if out_path:
@@ -346,15 +342,8 @@ def setup(alias, sources_file, out_path, no_validate, verbose, quiet):
             LOG.info(f"Writing {alias} to {out_file}")
             df.to_parquet(out_file)
 
-    # download additional supporting datasets
-    if not alias:
-        for table in [
-            "WHSE_BASEMAPPING.BCGS_20K_GRID",
-            "WHSE_WILDLIFE_MANAGEMENT.CRIMS_MARINE_ECOSECTION",
-            "WHSE_LEGAL_ADMIN_BOUNDARIES.ABMS_PROVINCE_SP",
-            "WHSE_BASEMAPPING.NTS_250K_GRID",
-        ]:
-            bcdata.bc2pg(table, os.environ["DATABASE_URL"])
+    # download tiles
+    bcdata.bc2pg("WHSE_BASEMAPPING.NTS_250K_GRID", os.environ["DATABASE_URL"])
 
 
 if __name__ == "__main__":

process.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+set -euxo pipefail
+
+# create output table
+psql $DATABASE_URL -c "DROP TABLE IF EXISTS designations;
+  CREATE TABLE designations (
+    designations_id serial primary key,
+    alias text,
+    description text,
+    primary_key text,
+    name text,
+    harvest_restriction integer,
+    mine_restriction integer,
+    og_restriction integer,
+    all_aliases text[],
+    all_descriptions text[],
+    all_primary_keys text[],
+    all_names text[],
+    all_harvest_restrictions integer[],
+    all_mine_restrictions integer[],
+    all_og_restrictions integer[],
+    map_tile text,
+    geom geometry(MULTIPOLYGON, 3005)
+  );"
+
+# run overlay
+psql $DATABASE_URL -tXA \
+-c "SELECT DISTINCT map_tile
+    FROM whse_basemapping.nts_250k_grid
+    ORDER BY map_tile" \
+    | parallel --tag psql $DATABASE_URL -f sql/overlay.sql -v tile={1}
+
+# dump result to file
+ogr2ogr   \
+  -f OpenFileGDB \
+  harvest_restrictions.gdb \
+  "PG:$DATABASE_URL" \
+  --config OPENFILEGDB_DEFAULT_STRING_WIDTH 255 \
+  -nlt MULTIPOLYGON \
+  -nln harvest_restrictions \
+  -lco CREATE_SHAPE_AREA_AND_LENGTH_FIELDS=YES \
+  -mapfieldtype Integer64=Integer \
+  -sql "SELECT designations_id as harvest_restrictions_id,
+    alias as harvest_restriction,
+    description,
+    primary_key,
+    name,
+    harvest_restriction as harvest_restriction_class,
+    case
+      when harvest_restriction = 1 then 'Protected'
+      when harvest_restriction = 2 then 'Prohibited'
+      when harvest_restriction = 3 then 'High Restricted'
+      when harvest_restriction = 4 then 'Medium Restricted'
+      when harvest_restriction = 5 then 'Low Restricted'
+      when harvest_restriction = 6 then 'No Special Restriction'
+    end as harvest_restriction_level,
+    array_to_string(all_aliases, ';') as all_harvest_restrictions,
+    array_to_string(all_descriptions, ';') as all_descriptions,
+    array_to_string(all_primary_keys, ';') as all_primary_keys,
+    array_to_string(all_names, ';') as all_names,
+    array_to_string(all_harvest_restrictions, ';') as all_harvest_restriction_classes,
+    map_tile text,
+    geom
+  from designations
+  where all_harvest_restrictions @> ARRAY[6]" \