Merge pull request #104 from sunt05/dev-test

Dev test

docs/source/version-history/20190829.rst

@@ -29,6 +29,8 @@ Version 20190829
 
   1. ESTM is not supported yet.
   2. BLUEWS, a CBL modules in SUEWS, is not supported yet.
+  3. Performance in parallel mode can be worse than
+     serial mode sometimes due to heavy (de)-serialisation loads.
 
 
 

docs/source/version-history/dev.rst

@@ -0,0 +1,34 @@
+.. _new_latest:
+
+
+Version dev
+======================================================
+
+
+- **New**
+
+  None.
+
+- **Improvement**
+
+  1. Improved performance of the parallel mode.
+
+- **Changes**
+
+  None.
+
+
+- **Fix**
+
+  None.
+
+
+- **Known issue**
+
+  1. ESTM is not supported yet.
+  2. BLUEWS, a CBL modules in SUEWS, is not supported yet.
+  3. Performance in parallel mode is NOT supported on Windows
+     due to system limitation.
+
+
+

src/setup.py

@@ -36,6 +36,7 @@ def readme():
       install_requires=[
           'numpy>=1.15.2',
           'pandas>=0.25.1',
+          'tables', # for dumping in hdf5
           'scipy',
           'dask[complete]',  # needs all dask and its dependencies
           'f90nml',

src/supy/_load.py

@@ -1,5 +1,5 @@
-from scipy.interpolate import interp1d
 import functools
+import logging
 from ast import literal_eval
 from datetime import timedelta
 from multiprocessing import cpu_count
@@ -9,6 +9,7 @@
 import numpy as np
 import pandas as pd
 from dask import dataframe as dd
+from scipy.interpolate import interp1d
 from supy_driver import suews_driver as sd
 
 from ._env import path_supy_module
@@ -226,8 +227,7 @@ def load_SUEWS_nml(path_file):
         df_res = pd.DataFrame(f90nml.read(str_file))
         return df_res
     except FileNotFoundError:
-        print('{path} does not exists!'.format(path=path_file))
-
+        logging.exception(f'{path_file} does not exists!')
 
 # def load_SUEWS_RunControl(path_file):
 #     lib_RunControl = load_SUEWS_nml(path_file)
@@ -242,7 +242,7 @@ def load_SUEWS_table(path_file):
     try:
         path_file = path_file.resolve()
     except FileNotFoundError:
-        print('{path} does not exists!'.format(path=path_file))
+        logging.exception(f'{path_file} does not exists!')
     else:
         # fileX = path_insensitive(fileX)
         str_file = str(path_file)
@@ -802,7 +802,8 @@ def gather_code_set(code, dict_var2SiteSelect):
             elif isinstance(v, dict):
                 set_res.update(v.keys())
             else:
-                print(k, v)
+                logging.info(f'{k},{v}')
+
         elif isinstance(v, dict):
             # print(res,v)
             set_res.update(list(gather_code_set(code, v)))
@@ -843,10 +844,10 @@ def build_code_df(code, path_input, df_base):
     try:
         df_code = df_code0.loc[list_code, list_keys]
     except Exception as e:
-        print('Entries missing from {lib}'.format(lib=lib_code))
-        print('\nlist_code:\n', list_code, '\n', df_code0.index)
-        print('\nlist_keys\n', list_keys, '\n', df_code0.columns)
-
+        logging.exception(f'Entries missing from {lib_code}')
+        logging.exception(f'list_code:\n {list_code} \n {df_code0.index}')
+        logging.exception(f'list_keys:\n {list_keys} \n {df_code0.columns}')
+        logging.exception(f'Entries missing from {lib_code}')
         raise e
 
     df_code.index = df_base.index
@@ -1562,7 +1563,7 @@ def load_InitialCond_grid_df(path_runcontrol, force_reload=True):
         lookup_KeySeq_lib.cache_clear()
         gen_all_code_df.cache_clear()
         build_code_exp_df.cache_clear()
-        print('All cache cleared.')
+        logging.info('All cache cleared.')
 
     # load base df of InitialCond
     df_init = load_SUEWS_InitialCond_df(path_runcontrol)