diff --git a/predict_batch.py b/predict_batch.py index 863b4e0..2c6b758 100644 --- a/predict_batch.py +++ b/predict_batch.py @@ -22,8 +22,8 @@ # MODELS IN USE # Default trained for N1 normalisation -model_source_path_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/' -model_source_path_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/' +model_source_path_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/' +model_source_path_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/' # MODELS TO DEBUG THAT FEATURE N4 NORMALISATION # modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/debug-normalizing/oligo-n4/0_custom/' @@ -32,11 +32,11 @@ # MODELS TO BE VALIDATED # modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/oligo_fieldTest_WObrightness_longer/0_custom/' -source_dir_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_14/' -source_dir_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_14/' +source_dir_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_18/' +source_dir_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_18/' -source_dir_paper_redo_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_6/' -source_dir_paper_redo_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_6/' +source_dir_paper_redo_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_6/' +source_dir_paper_redo_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_6/' # ######### To validate, use these whole well experiments: ######### # source_dir = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/wholeWell/oligo/unannotated/' @@ -254,7 +254,7 @@ def main(args): print('Number of arguments:', len(args), 'arguments.') print('Argument List:', str(args)) - custom_paths = False + custom_paths = False; for arg in args: arg = str(arg).lower() print('Evaluating arg: "'+arg+'".') @@ -273,67 +273,98 @@ def custom_paths_predict(): print('Your input: '+model_source_path); - pass - def prodi_gpu_predict(): print('Running Predictions.') - use_oligo = False + use_oligo = True use_neuron = True + use_glia = False + use_old = False - use_debug = True - use_paper = False + use_debug = False + use_paper = True skip_predicted = True n_jobs: int = 20 # Paper Models trained for N4 - model_source_path_oligo_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/' - model_source_path_neuron_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/' + model_source_path_oligo_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/' + model_source_path_neuron_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/' + model_source_path_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia/smote/' + + # .h5 dirs to be predicted for the paper + # source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/' + # source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/' + + # .h5 dirs to be predicted for efsa or endpoints + source_dir_paper_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints5/' + source_dir_paper_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints5/' + + source_dir_paper_oligo2 = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints6/' + source_dir_paper_neuron2 = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints6/' + + source_dir_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/glia_01/' + + + if use_glia: + predict_batch(model_source_path=model_source_path_glia, source_dir=source_dir_glia, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") - # .h dirs to be predicted for the paper - source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/' - source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/' - # source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_13/' - # source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_13/' + if use_paper: + if use_neuron: + predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") + + predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron2, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") + if use_oligo: + predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") + predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo2, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") + if use_old: + if use_neuron: + predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron, + normalize_enum=1, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="0") + if use_oligo: + predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo, + normalize_enum=1, + n_jobs=n_jobs, + skip_predicted=skip_predicted, + gpu_index_string="1") + if use_debug: + predict_batch(model_source_path=model_source_path_oligo_paper, + source_dir=source_dir_oligo, + normalize_enum=4, + n_jobs=n_jobs, + skip_predicted=False, + gpu_index_string="0") predict_batch(model_source_path=model_source_path_neuron_paper, - source_dir='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_debug/', + source_dir=source_dir_neuron, normalize_enum=4, - n_jobs=1, + n_jobs=n_jobs, skip_predicted=False, gpu_index_string="0") - else: - if use_paper: - if use_neuron: - predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_redo_paper_neuron, - normalize_enum=4, - n_jobs=n_jobs, - skip_predicted=skip_predicted, - gpu_index_string="0") - if use_oligo: - predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_redo_paper_oligo, - normalize_enum=4, - n_jobs=n_jobs, - skip_predicted=skip_predicted, - gpu_index_string="1") - else: - if use_neuron: - predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron, - normalize_enum=1, - n_jobs=n_jobs, - skip_predicted=skip_predicted, - gpu_index_string="0") - - if use_oligo: - predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo, - normalize_enum=1, - n_jobs=n_jobs, - skip_predicted=skip_predicted, - gpu_index_string="1") - - print(gct() + ' All Predictions done. Have a nice day. =)') diff --git a/train_model.py b/train_model.py index 1800107..4d28d02 100644 --- a/train_model.py +++ b/train_model.py @@ -1145,91 +1145,79 @@ def decode_history_key(key: str) -> str: return key +glia_path_train = [ + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_ECM pretreated_trainingData_glia', + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_spheres pretreated_trainingData_glia', + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK274_normal_trainingData_glia', + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK135_trainingData_glia', + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK136_trainingData_glia' +] + +glia_path_test='/home/nilfoe/bioinf/Omnisphero/CNN-glia/test/glia/' + +glia_path_val = [ + '/home/nilfoe/bioinf/Omnisphero/CNN-glia/validation/glia/' +] + def main(): # AUGMENTATION data_gen = get_default_augmenter() + n_jobs = 20 - out_path_base = out_path + 'paper-final_no-datagen' + os.sep - out_path_oligo = out_path_base + 'oligo' + os.sep - out_path_neuron = out_path_base + 'neuron' + os.sep - - out_path_oligo_debug = out_path_base + 'oligo_debug' + os.sep - - oligo_mode = True - neuron_mode = False - debug_mode = False - n_jobs = 40 + out_path = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia-extendedData/' print('Sleeping....') # time.sleep(18000) - if debug_mode: - train_model( - training_path_list=debug_oligos, - validation_path_list=debug_oligos_validation, - test_data_path='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/training/oligo_kontrolliert_test/', - # data_gen=data_gen, - use_SMOTE=False, - out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron_kontrolliert_undersampled' + os.sep, - gpu_index_string="3", - optimizer='SGD', - normalize_enum=4, - under_sample_train_data=True, under_sample_val_data=True, - epochs=5, - n_jobs=25 - ) - return + train_model( + training_path_list=glia_path_train, + validation_path_list=glia_path_val, + test_data_path=glia_path_test, + use_SMOTE=False, + out_path=out_path + 'base' + os.sep, + gpu_index_string="0", + optimizer='SGD', + n_jobs=n_jobs, + epochs=5000 + ) - if oligo_mode: - train_model( - training_path_list=final_oligos_validated, - validation_path_list=final_oligos_validated_validation_set, - test_data_path=test_data_path_oligo, - # training_path_list=oligo_paper_bleedthrough_path_train, - # validation_path_list=oligo_paper_bleedthrough_path_val, - # test_data_path=oligo_paper_bleedthrough_path_test, - # data_gen=data_gen, - use_SMOTE=False, - out_path=out_path + 'paper-final_datagen' + os.sep + 'oligo-normalize4-undersampling' + os.sep, - under_sample_train_data=True, under_sample_val_data=True, - gpu_index_string="1", - optimizer='SGD', - n_jobs=n_jobs, - epochs=5000 - ) + train_model( + training_path_list=glia_path_train, + validation_path_list=glia_path_val, + test_data_path=glia_path_test, + use_SMOTE=False, + out_path=out_path + 'datagen' + os.sep, + gpu_index_string="0", + optimizer='SGD', + n_jobs=n_jobs, + data_gen=data_gen, + epochs=5000 + ) - # paper-final_no-datagen\neuron - if neuron_mode: - train_model( - training_path_list=final_neurons_validated, - validation_path_list=final_neurons_validated_validation_set, - test_data_path=test_data_path_neuron, - # training_path_list=neuron_paper_bleedthrough_path_train, - # validation_path_list=neuron_paper_bleedthrough_path_val, - # test_data_path=neuron_paper_bleedthrough_path_test, - # data_gen=data_gen, - use_SMOTE=False, - out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron-normalize4-undersampling' + os.sep, - under_sample_train_data=True, under_sample_val_data=True, - normalize_enum=4, - gpu_index_string="0", - optimizer='SGD', - n_jobs=n_jobs, - epochs=5000 - ) + train_model( + training_path_list=glia_path_train, + validation_path_list=glia_path_val, + test_data_path=glia_path_test, + use_SMOTE=True, + out_path=out_path + 'smote' + os.sep, + gpu_index_string="0", + optimizer='SGD', + n_jobs=n_jobs, + epochs=5000 + ) - # out_path_oligo = out_path+'oligo'+os.sep - # out_path_neuron = out_path+'neuron'+os.sep - - # - # train_model_scrambling(path_candidate_list=final_oligos_validated, - # test_data_path=test_data_path_oligo, - # out_path=out_path_oligo, - # validation_count=1) - # train_model_scrambling(path_candidate_list=final_neurons_validated, - # test_data_path=test_data_path_neuron, - # out_path=out_path_neuron, - # validation_count=1) + train_model( + training_path_list=glia_path_train, + validation_path_list=glia_path_val, + test_data_path=glia_path_test, + use_SMOTE=True, + out_path=out_path + 'datagen_and_smote' + os.sep, + gpu_index_string="0", + optimizer='SGD', + n_jobs=n_jobs, + data_gen=data_gen, + epochs=5000 + ) print('Finished all trainings. Goodbye.')