Merge remote-tracking branch 'upstream/master' into EQhazard

modules/Workflow/WorkflowApplications.json

@@ -82,7 +82,7 @@
           "ExecutablePath": "applications/createAIM/GeoJSON_to_ASSET/GeoJSON_to_ASSET.py",
         "ApplicationSpecificInputs": [
           {
-            "id": "assetFile",
+            "id": "assetSourceFile",
             "type": "path",
             "description": "path to asset database file"
           },

modules/createAIM/GeoJSON_to_ASSET/GeoJSON_to_ASSET.py

@@ -193,9 +193,8 @@ def defineConnectivities(self, AIM_id_prefix = None, edges_file_name = None,\
         self.gdf = edges
         return
 
-def split_and_select_components(input_config):
+def split_and_select_components(input_config, asset_source_file):
     component_dict = dict()
-    asset_source_file = input_config["assetSourceFile"]
     with open(asset_source_file, 'r', encoding="utf-8") as f:
         source_data = json.load(f)
     crs = source_data["crs"]
@@ -251,7 +250,7 @@ def init_workdir(component_dict, outDir):
         component_dir.update({comp:compDir})
     return component_dir
 
-def create_asset_files(output_file, 
+def create_asset_files(output_file, asset_source_file,
     asset_type, input_file, doParallel): 
     # check if running parallel
     numP = 1
@@ -281,7 +280,9 @@ def create_asset_files(output_file,
         ["ApplicationData"]
     # if input_config.get("Roadway", None):
     #     roadSegLength = float(input_config['Roadway'].get('maxRoadLength_m', "100000"))
-    component_dict = split_and_select_components(input_config)
+
+    # assetSourceFile passed through command may be different from input_config when run on designsafe
+    component_dict = split_and_select_components(input_config, asset_source_file)
     component_dir = init_workdir(component_dict, outDir)
     assets_array = []
     for component_type, component_data in component_dict.items():
@@ -341,6 +342,7 @@ def create_asset_files(output_file,
 
     parser = argparse.ArgumentParser()
     parser.add_argument('--assetFile')
+    parser.add_argument('--assetSourceFile')
     parser.add_argument('--assetType')
     parser.add_argument('--inputJsonFile')
     parser.add_argument('--doParallel', default="False")    
@@ -350,7 +352,8 @@ def create_asset_files(output_file,
     args = parser.parse_args()
 
     if args.getRV:
-        sys.exit(create_asset_files(args.assetFile, args.assetType,\
+        sys.exit(create_asset_files(args.assetFile, args.assetSourceFile,\
+                                    args.assetType,\
                                     args.inputJsonFile, args.doParallel))
     else:
         pass # not used

modules/createEVENT/HighRiseTPU/HighRiseTPU.cpp

@@ -528,7 +528,7 @@ int addEvent(json_t *generalInfo, json_t *currentEvent, json_t *outputEvent, boo
       std::string pyArgs = "HighRiseTPU.py tmpSimCenterHighRiseTPU.mat tmpSimCenterHighRiseTPU.json";
       std::string command = "python3 ";
 #ifdef _WIN32
-      command = "python "
+      command = "python ";
 #endif
       command += pyArgs;
       system(command.c_str());

modules/createEVENT/LowRiseTPU/LowRiseTPU.cpp

@@ -534,7 +534,7 @@ int addEvent(json_t *generalInfo, json_t *currentEvent, json_t *outputEvent, boo
       std::string pyArgs = "LowRiseTPU.py tmpSimCenterLowRiseTPU.mat tmpSimCenterLowRiseTPU.json";
       std::string command = "python3 ";
 #ifdef _WIN32
-      command = "python "      
+      command = "python ";      
 #endif
       command += pyArgs;
       system(command.c_str());

modules/performRegionalMapping/SiteSpecifiedEvents/SSE.py

@@ -46,8 +46,9 @@
 import pandas as pd
 from scipy.cluster.vq import vq
 import importlib
+import os
 
-def create_event(asset_file, event_grid_file, doParallel):
+def create_event(asset_file, event_grid_file, multipleEvents, doParallel):
 
 
     # check if running parallel
@@ -172,13 +173,25 @@ def create_event(asset_file, event_grid_file, doParallel):
                 event_df = pd.read_csv(event_dir / event_collection_file, header=0)
 
                 # append the GM record name to the event list
-                event_list.append(event_df.iloc[idx,0])
+                event_list.append(event_df.iloc[0,0])
 
                 # append the scale factor (or 1.0) to the scale list
                 if len(event_df.columns) > 1:
-                    scale_list.append(float(event_df.iloc[idx,1]))
+                    scale_list.append(float(event_df.iloc[0,1]))
                 else:
                     scale_list.append(1.0)
+
+                # If GP_file contains multiple events    
+                if multipleEvents:
+                    # Read the GP_file
+                    if event_df.shape[0] > 1:
+                        for row in range(1,event_df.shape[0]):
+                            event_list.append(event_df.iloc[row,0])
+                            # append the scale factor (or 1.0) to the scale list
+                            if len(event_df.columns) > 1:
+                                scale_list.append(float(event_df.iloc[row,1]))
+                            else:
+                                scale_list.append(1.0)
 
             # if the grid has intensity measures
             elif event_type == 'intensityMeasure':
@@ -189,6 +202,16 @@ def create_event(asset_file, event_grid_file, doParallel):
                 # IM collections are not scaled
                 scale_list.append(1.0)
 
+                # If GP_file contains multiple events  
+                if multipleEvents:
+                    # Read the GP_file
+                    GP_file = os.path.join(event_dir, closestPnt['GP_file'])
+                    GP_file_df = pd.read_csv(GP_file, header=0)
+                    if GP_file_df.shape[0] > 1:
+                        for row in range(1,GP_file_df.shape[0]):
+                            event_list.append(closestPnt['GP_file']+f'x{row}')
+                            scale_list.append(1.0)
+
         # TODO: update the LLNL input data and remove this clause
         else:
             event_list = []
@@ -210,8 +233,7 @@ def create_event(asset_file, event_grid_file, doParallel):
 
 
         # save the event dictionary to the AIM
-        # TODO: we assume there is only one event
-        # handling multiple events will require more sophisticated inputs
+
         # save the event dictionary to the BIM                                          
         asset_data['Events'] = [{}]        
         asset_data['Events'][0] = {
@@ -230,9 +252,10 @@ def create_event(asset_file, event_grid_file, doParallel):
     parser = argparse.ArgumentParser()
     parser.add_argument('--assetFile')
     parser.add_argument('--filenameEVENTgrid')
+    parser.add_argument('--multipleEvents', default="True")
     parser.add_argument('--doParallel', default="False")    
     parser.add_argument("-n", "--numP", default='8')
     parser.add_argument("-m", "--mpiExec", default='mpixece')
     args = parser.parse_args()
 
-    create_event(args.assetFile, args.filenameEVENTgrid, args.doParallel)
+    create_event(args.assetFile, args.filenameEVENTgrid, args.multipleEvents, args.doParallel)

modules/performUQ/SimCenterUQ/PLoM/PLoM.py

@@ -630,7 +630,9 @@ def ISDEGeneration(self, n_mc = 5, tol_PCA2 = 1e-5, tol = 0.02, max_iter = 50, s
             self.C_h_hat_eta = plom.covariance(self.g_c(self.x_mean+(self.phi).dot(np.diag(self.mu)).dot(self.H)))
 
             #scaling beta
-            self.beta_c_normalized = self.beta_c_aux(self.beta_c, self.x_min, self.alpha)
+            #self.beta_c_normalized = self.beta_c_aux(self.beta_c, self.x_min, self.alpha)
+            # KZ, 07/24
+            self.beta_c_normalized = self.beta_c_aux(self.beta_c, self.X)
             self.b_c, self.psi = plom.PCA2(self.C_h_hat_eta, self.beta_c_normalized, tol_PCA2)
             self.nu_c = len(self.b_c)
 

modules/performUQ/SimCenterUQ/runPLoM.py

@@ -451,19 +451,66 @@ def _parse_plom_parameters(self, surrogateInfo):
         try:
             self.n_mc = int(surrogateInfo['newSampleRatio'])
             self.epsilonPCA = surrogateInfo.get("epsilonPCA",1e-6)
-            self.smootherKDE = surrogateInfo.get("smootherKDE",25)
+            # KZ, 07/24: adding customized option for smootherKDE factor
+            self.smootherKDE_Customize = surrogateInfo.get("smootherKDE_Customize",False)
+            if self.smootherKDE_Customize:
+                # KZ, 07/24: taking in user-defined function filepath and directory
+                self.smootherKDE_file = surrogateInfo.get("smootherKDE_path",False)
+                self.smootherKDE_dir = surrogateInfo.get("smootherKDE_pathPath",False)
+                if self.smootherKDE_file and self.smootherKDE_dir:
+                    # KZ, 07/24: both file and file path received
+                    # Note that the file is saved by the frontend to the work_dir -> overwrite self.smootherKDE_file
+                    self.smootherKDE_file = os.path.join(work_dir, "templatedir", self.smootherKDE_file)
+                    if not os.path.isfile(self.smootherKDE_file):
+                        # not found the file
+                        msg = 'Error finding user-defined function file for KDE: {}.'.format(self.smootherKDE_file)
+                        errlog.exit(msg)
+                else:
+                    # KZ, 07/24: missing user-defined file
+                    msg = 'Error loading user-defined function file for KDE.'
+                    errlog.exit(msg)
+            else:
+                # KZ, 07/24: get user defined smootherKDE
+                self.smootherKDE = surrogateInfo.get("smootherKDE",25)
+            #self.smootherKDE = surrogateInfo.get("smootherKDE",25)
             self.randomSeed = surrogateInfo.get("randomSeed",None)
             self.diffMap = surrogateInfo.get("diffusionMaps",True)
             self.logTransform = surrogateInfo.get("logTransform",False)
             self.constraintsFlag = surrogateInfo.get("constraints",False)
-            self.kdeTolerance = surrogateInfo.get("kdeTolerance",0.1)
+            # KZ: 07/24: adding customized option for kdeTolerance
+            self.kdeTolerance_Customize = surrogateInfo.get("tolKDE_Customize",False)
+            if self.kdeTolerance_Customize:
+                # KZ, 07/24: taking in user-defined function filepath and directory
+                self.kdeTolerance_file = surrogateInfo.get("tolKDE_path",False)
+                self.kdeTolerance_dir = surrogateInfo.get("tolKDE_pathPath",False)
+                if self.kdeTolerance_file and self.kdeTolerance_dir:
+                    # KZ, 07/24: both file and file path received
+                    # Note that the file is saved by the frontend to the work_dir -> overwrite self.kdeTolerance_file
+                    self.kdeTolerance_file = os.path.join(work_dir, "templatedir", self.kdeTolerance_file)
+                    if not os.path.isfile(self.kdeTolerance_file):
+                        # not found the file
+                        msg = 'Error finding user-defined function file for KDE: {}.'.format(self.kdeTolerance_file)
+                        errlog.exit(msg)
+                else:
+                    # KZ, 07/24: missing user-defined file
+                    msg = 'Error loading user-defined function file for KDE tolerance.'
+                    errlog.exit(msg)
+            else:
+                self.kdeTolerance = surrogateInfo.get("kdeTolerance",0.1)
+            #self.kdeTolerance = surrogateInfo.get("kdeTolerance",0.1)
             if self.constraintsFlag:
                 self.constraintsFile = os.path.join(work_dir, "templatedir/plomConstraints.py")
             self.numIter = surrogateInfo.get("numIter",50)
             self.tolIter = surrogateInfo.get("tolIter",0.02)
             self.preTrained = surrogateInfo.get("preTrained",False)
             if self.preTrained:
                 self.preTrainedModel = os.path.join(work_dir, "templatedir/surrogatePLoM.h5")
+
+            # KZ, 07/24: loading hyperparameter functions (evaluating self.kdeTolerance and self.smootherKDE from user-defined case)
+            if self._load_hyperparameter():
+                msg = 'runPLoM._parse_plom_parameters: Error in loading hyperparameter functions.'
+                self.errlog.exit(msg)
+
         except:
             run_flag = 1
 
@@ -578,6 +625,39 @@ def _load_variables(self, do_sampling, do_simulation):
 
         # return
         return run_flag
+
+
+    # KZ, 07/24: loading user-defined hyper-parameter files
+    def _load_hyperparameter(self):
+        run_flag = 0
+        try:
+            # load constraints first
+            constr_file = Path(self.constraintsFile).resolve()
+            sys.path.insert(0, str(constr_file.parent) + '/')
+            constr_script = importlib.__import__(constr_file.name[:-3], globals(), locals(), [], 0)
+            self.beta_c = constr_script.beta_c()
+            print("beta_c = ", self.beta_c)
+            # if smootherKDE
+            if self.smootherKDE_Customize:
+                kde_file = Path(self.smootherKDE_file).resolve()
+                sys.path.insert(0, str(kde_file.parent) + '/')
+                kde_script = importlib.__import__(kde_file.name[:-3], globals(), locals(), [], 0)
+                self.get_epsilon_k = kde_script.get_epsilon_k
+                # evaluating the function
+                self.smootherKDE = self.get_epsilon_k(self.beta_c)
+                print('epsilon_k = ',self.smootherKDE)
+            # if tolKDE
+            if self.kdeTolerance_Customize:
+                beta_file = Path(self.kdeTolerance_file).resolve()
+                sys.path.insert(0, str(beta_file.parent) + '/')
+                beta_script = importlib.__import__(beta_file.name[:-3], globals(), locals(), [], 0)
+                self.get_epsilon_db = beta_script.get_epsilon_db
+                # evaluating the function
+                self.kdeTolerance = self.get_epsilon_db(self.beta_c)
+                print('epsilon_db = ',self.kdeTolerance)
+        except:
+            run_flag = 1
+        return run_flag
 
 
     def train_model(self, model_name='SurrogatePLoM'):