Config files and progress on (#45)

* modify table creation, precalculate propagation, add columns for regulatory structures

* build assign table with regulatory data and propagate down

* use config file for inputs

* optimize imports

* remove column name choices from config file and capitalize constants

* move drain, gauge, reg table paths to config file

* set default process count to 1

* more vocab changes and update the sample script

examples/saber_config_example.yml

@@ -0,0 +1,18 @@
+# file paths to input data
+workdir: ''
+
+x_fdc_train: ''
+x_fdc_all: ''
+
+drain_table: ''
+gauge_table: ''
+regulate_table: ''
+
+drain_gis: ''
+gauge_gis: ''
+
+gauge_data: ''
+hindcast_zarr: ''
+
+# options for processing data
+n_processes: 1

examples/saber_script_short.py

@@ -1,13 +1,12 @@
 import logging
 
-import numpy as np
-import pandas as pd
-
 import saber
 
-np.seterr(all="ignore")
+# USER INPUTS - POPULATE THESE PATHS
+config_file = '/Users/rchales/Projects/saber-hbc/examples/config.yml'  # Path to the configuration file
+log_path = ''  # leave blank to write to console
+# END USER INPUTS
 
-log_path = '/Users/rchales/Projects/geoglows_saber/log.log'
 logging.basicConfig(
     level=logging.INFO,
     filename=log_path,
@@ -19,40 +18,33 @@
 if __name__ == "__main__":
     logger = logging.getLogger(__name__)
 
-    # USER INPUTS - POPULATE THESE PATHS
-    workdir = ''
-    x_fdc_train = ''
-    x_fdc_all = ''
-    drain_gis = ''
-    gauge_gis = ''
-    gauge_data = ''
-    hindcast_zarr = ''
-    n_processes = 1
-    # END USER INPUTS
+    # Read the Config File
+    saber.io.read_config(config_file)
+    saber.io.init_workdir(overwrite=False)
 
     # Generate Clusters and Plots
     logger.info('Create Clusters and Plots')
-    saber.cluster.cluster(workdir, x_fdc_train)
+    saber.cluster.cluster()
     # Before continuing, review the clustering results and select the best n_clusters for the next function
-    saber.cluster.predict_labels(workdir, n_clusters=5, x=pd.read_parquet(x_fdc_all))
-
-    # Generate Assignments Table and Make Assignments
-    logger.info('Make Assignments of Ungauged Basins to Gauges')
-    assign_df = saber.assign.generate(workdir)
-    assign_df = saber.assign.mp_assign_all(workdir, assign_df)
-
-    # Recommended Optional - Generate GIS files to visually inspect the assignments
+    saber.cluster.predict_labels(n_clusters=5)
+
+    # Generate Assignments Table and Propagate from Gauges, Dams/Reservoirs
+    logger.info('Generating Assignments Table')
+    assign_df = saber.table.init()
+    assign_df = saber.table.mp_prop_gauges(assign_df)
+    assign_df = saber.table.mp_prop_regulated(assign_df)
+    saber.io.write_table(assign_df, 'assign_table')  # cache results
+
+    # Optional - Compute performance metrics at gauged locations
+    logger.info('Perform Bootstrap Validation')
+    bs_assign_df = saber.bstrap.mp_table(assign_df)
+    bs_metrics_df = saber.bstrap.mp_metrics(bs_assign_df)
+    saber.bstrap.gauge_metric_map(bs_metrics_df)
+
+    # Optional - Make all assignments
+    logger.info('Make Assignments')
+    assign_df = saber.assign.mp_assign(assign_df)
     logger.info('Generating GIS files')
-    saber.gis.create_maps(workdir, assign_df, drain_gis)
-
-    # Recommended Optional - Compute performance metrics
-    logger.info('Compute Performance Metrics')
-    bs_assign_df = saber.bstrap.mp_table(workdir, assign_df, n_processes)
-    bs_metrics_df = saber.bstrap.mp_metrics(workdir, bs_assign_df, gauge_data, hindcast_zarr, n_processes=n_processes)
-    saber.bstrap.merge_metrics_and_gis(workdir, gauge_gis, bs_metrics_df)
-
-    # # Optional - Compute the Corrected Simulation Data
-    # logger.info('Compute Corrected Simulation Data')
-    # saber.calibrate.mp_saber(assign_df, hindcast_zarr, gauge_data)
+    saber.gis.create_maps(assign_df)
 
     logger.info('SABER Completed')

saber/__init__.py

@@ -1,14 +1,15 @@
 import saber.assign
+import saber.bstrap
 import saber.calibrate
 import saber.cluster
 import saber.gis
 import saber.io
-import saber.bstrap
+import saber.table
 
 __all__ = [
-    'io', 'cluster', 'assign', 'gis', 'calibrate', 'bstrap',
+    'io', 'table', 'cluster', 'assign', 'gis', 'calibrate', 'bstrap',
 ]
 
 __author__ = 'Riley C. Hales'
-__version__ = '0.7.0'
+__version__ = '0.8.0'
 __license__ = 'BSD 3 Clause Clear'