From 64f35198f9a3b8764fe37e3735a49aded9098cfb Mon Sep 17 00:00:00 2001 From: Hooman Zabeti Date: Tue, 6 Apr 2021 23:58:22 -0700 Subject: [PATCH] 4th module is renamed. utils is updated. config is updated. --- config.yml | 50 +++++------ group_testing.py | 2 +- ...g_with_pulp.py => group_testing_decoder.py | 0 group_testing_mp.py | 3 +- utils.py | 86 +++++++++---------- 5 files changed, 65 insertions(+), 76 deletions(-) rename model_preprocessing_with_pulp.py => group_testing_decoder.py (100%) diff --git a/config.yml b/config.yml index 4d09b4e..9c4943f 100644 --- a/config.yml +++ b/config.yml @@ -1,58 +1,51 @@ --- -design_param: - 'general': - 'seed': - 'mode': 'range' - 'values': [0,1,1] - 'N': - 'mode': 'scalar' - 'values': [1000] - 'm': - 'mode': 'range' - 'values': [10,30,10] - 'verbose': False - 'groups': - input: 'design_matrix.csv' +design: + groups: + #input: 'design_matrix.csv' params: + seed: 10 + N: 1000 + 'm': + 'mode': 'range' + 'values': [10,30,10] + 'verbose': False 'run_ID': 'debugging' 'plotting': False 'saving': True 'graph_gen_method': 'no_multiple' 'group_size': - 'mode': 'scalar' + 'mode': 'list' 'values': [16] 'max_tests_per_individual': - 'mode': 'scalar' + 'mode': 'list' 'values': [16] 'individual_status': input: 'individual_status.csv' params: 's': - 'mode': 'scalar' + 'mode': 'list' 'values': [50] 'test_results': 'input': 'test_results.csv' 'params': 'test_noise_methods': ['permutation'] 'permutation_noise_prob': - 'mode': 'scalar' + 'mode': 'list' 'values': [0.01] 'theta_l': - 'mode': 'scalar' + 'mode': 'list' 'values': [0.00] 'theta_u': - 'mode': 'scalar' + 'mode': 'list' 'values': [0.0625] 'binary_symmetric_noise_prob': - 'mode': 'scalar' + 'mode': 'list' 'values': [0.26] -decoder_param: - 'decode': +decode: + 'decoder': 'lambda_w': 1 - 'lambda_e': - 'mode': 'scalar' - 'values': [0.125,0.25,0.5,1,2,4,8] + 'lambda_e': 0.25 'defective_num_lower_bound': null 'sensitivity_threshold': null 'specificity_threshold': null @@ -60,13 +53,10 @@ decoder_param: 'lp_relaxation': False 'solver_name': 'CPLEX_PY' 'solver_options': - 'mode': 'exact' 'timeLimit': 1800 - 'lambda_selection': - 'cross_validation': False + 'lambda_selection': True 'number_of_folds': 2 'cv_param': - 'mode': 'exact' 'lambda_e': [0.125,0.25,0.5,1,2,4,8] #'mip':True 'evaluation': diff --git a/group_testing.py b/group_testing.py index 19f3c7f..815b341 100644 --- a/group_testing.py +++ b/group_testing.py @@ -13,7 +13,7 @@ from group_testing_optimizer import GT_optimizer from group_testing_evaluation import decoder_evaluation from group_testing_reporter import decoder_reporter -from model_preprocessing_with_pulp import * +from group_testing_decoder import * import os import decoder import argparse diff --git a/model_preprocessing_with_pulp.py b/group_testing_decoder.py similarity index 100% rename from model_preprocessing_with_pulp.py rename to group_testing_decoder.py diff --git a/group_testing_mp.py b/group_testing_mp.py index ba3eb09..c86d86b 100644 --- a/group_testing_mp.py +++ b/group_testing_mp.py @@ -7,7 +7,7 @@ from generate_groups import gen_measurement_matrix from generate_individual_status import gen_status_vector from group_testing_evaluation import decoder_evaluation -from model_preprocessing_with_pulp import * +from group_testing_decoder import * from multiprocessing import Pool from multiprocessing import cpu_count import numpy as np @@ -157,7 +157,6 @@ def multi_process_group_testing(design_param, decoder_param): # param = config_decoder(config_dict['param']) # lambda_selection = param['lambda_selection'] design_param, decoder_param = config_reader('config.yml') - print(design_param,decoder_param) # output files path path = os.getcwd() currentDate = datetime.datetime.now() diff --git a/utils.py b/utils.py index 7e92f86..2d34a3f 100644 --- a/utils.py +++ b/utils.py @@ -34,13 +34,15 @@ def config_decoder(config_inpt): for keys, vals in config_inpt.items(): if isinstance(vals, dict): #print(vals) - if config_inpt[keys]['mode'] == 'range': - config_inpt[keys] = np.arange(*config_inpt[keys]['values']) - elif config_inpt[keys]['mode'] == 'scalar': - config_inpt[keys] = config_inpt[keys]['values'] + if 'mode' in config_inpt[keys].keys(): + if config_inpt[keys]['mode'] == 'range': + config_inpt[keys] = np.arange(*config_inpt[keys]['values']) + elif config_inpt[keys]['mode'] == 'list': + config_inpt[keys] = config_inpt[keys]['values'] # TODO: remove mode = exact there is no need for it! - elif config_inpt[keys]['mode'] == 'exact': - config_inpt[keys].pop('mode') + #elif config_inpt[keys]['mode'] == 'dict': + else: + #config_inpt[keys].pop('mode') config_inpt[keys] = [config_inpt[keys]] else: config_inpt[keys] = [vals] @@ -66,26 +68,26 @@ def config_reader(config_file_name): e = sys.exc_info()[0] print("config file can not be found!") print("Error:", e) - design_param = {'general': False, 'generate_groups': False, 'generate_individual_status': False, + design_param = {'generate_groups': False, 'generate_individual_status': False, 'generate_test_results': False, 'test_results': False} - decoder_param = {'decode': False, 'lambda_selection': False, 'evaluation': False} + decoder_param = {'decoder': False, 'lambda_selection': False, 'evaluation': False} # Load params - if 'design_param' in config_dict.keys(): - try: - # assert 'general' in config_dict['design_param'].keys(), "You should define general properties of design!" - general_param = config_dict['design_param']['general'] - design_param['general'] = True - design_param.update(general_param) - except KeyError: - print("Warning: 'general' block is not found in the config file! We try to recover essential information " - "from the input files!") - assert 'groups' in config_dict['design_param'].keys(), \ + if 'design' in config_dict.keys(): + # try: + # # assert 'general' in config_dict['design_param'].keys(), "You should define general properties of design!" + # general_param = config_dict['design_param']['general'] + # design_param['general'] = True + # design_param.update(general_param) + # except KeyError: + # print("Warning: 'general' block is not found in the config file! We try to recover essential information " + # "from the input files!") + assert 'groups' in config_dict['design'].keys(), \ "You should define the 'groups' block in the config file!" - generate_groups = config_input_or_params(config_dict['design_param']['groups'], 'groups', 'generate_groups') + generate_groups = config_input_or_params(config_dict['design']['groups'], 'groups', 'generate_groups') design_param.update(generate_groups) try: generate_individual_status = config_input_or_params( - current_dict=config_dict['design_param']['individual_status'], + current_dict=config_dict['design']['individual_status'], block_name='individual_status', generate_label='generate_individual_status') design_param.update(generate_individual_status) @@ -94,46 +96,43 @@ def config_reader(config_file_name): "if the results need to be evaluated!") # if 'test_results' in config_dict['design_param'].keys(): try: - generate_test_results = config_input_or_params(config_dict['design_param']['test_results'], 'test_results', + generate_test_results = config_input_or_params(config_dict['design']['test_results'], 'test_results', 'generate_test_results') design_param.update(generate_test_results) design_param['test_results'] = True except KeyError: print("Warning: 'test_results' block is not found in the config file! Test results is necessary for" " decoding!") - if 'decoder_param' in config_dict.keys(): + if 'decode' in config_dict.keys(): assert design_param['test_results'], "It is not possible to decode without test results! Please define the " \ "'test_results' block in the config file." - if 'decode' in config_dict['decoder_param'].keys(): + if 'decoder' in config_dict['decode'].keys(): try: - decode_param = config_dict['decoder_param']['decode'] - decoder_param['decode'] = True + decode_param = config_dict['decode']['decoder'] + decoder_param['decoder'] = True decoder_param.update(decode_param) except: - # TODO: this should be KeyError - e = sys.exc_info()[0] - print("config file format is not correct!") - print("Error1:", e) - if 'lambda_selection' in config_dict['decoder_param'].keys(): - try: - lambda_selection_param = config_dict['decoder_param']['lambda_selection'] - decoder_param['lambda_selection'] = True - decoder_param.update(lambda_selection_param) - except: - # TODO: this should be KeyError + print("decoder format in the config file is not correct!") e = sys.exc_info()[0] - print("config file format is not correct!") - print("Error2:", e) - if 'evaluation' in config_dict['decoder_param'].keys(): + print("Error:", e) + # if 'lambda_selection' in config_dict['decoder_param'].keys(): + # try: + # lambda_selection_param = config_dict['decoder_param']['lambda_selection'] + # decoder_param['lambda_selection'] = True + # decoder_param.update(lambda_selection_param) + # except: + # e = sys.exc_info()[0] + # print("config file format is not correct!") + # print("Error2:", e) + if 'evaluation' in config_dict['decode'].keys(): try: - evaluation_param = config_dict['decoder_param']['evaluation'] + evaluation_param = config_dict['decode']['evaluation'] decoder_param['evaluation'] = True decoder_param.update(evaluation_param) except: - # TODO: this should be KeyError + print("evaluation format in the config file is not correct!") e = sys.exc_info()[0] - print("config file format is not correct!") - print("Error3:", e) + print("Error:", e) return config_decoder(design_param), config_decoder(decoder_param) @@ -157,3 +156,4 @@ def path_generator(file_path, file_name, file_format): if __name__ == '__main__': design_param, decoder_param = config_reader('config.yml') print(design_param, decoder_param) + print(len(design_param), len(decoder_param))