Skip to content

Commit

Permalink
Merge branch 'master' into zipped_h5
Browse files Browse the repository at this point in the history 
# Conflicts:
#	predict_batch.py
  • Loading branch information
NilsFo committed Apr 12, 2021
2 parents bd6e8fc + a005619 commit f8f7ef1
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 125 deletions.
133 changes: 82 additions & 51 deletions predict_batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@

# MODELS IN USE
# Default trained for N1 normalisation
model_source_path_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/'
model_source_path_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/'
model_source_path_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/'
model_source_path_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/'

# MODELS TO DEBUG THAT FEATURE N4 NORMALISATION
# modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/debug-normalizing/oligo-n4/0_custom/'
Expand All @@ -32,11 +32,11 @@
# MODELS TO BE VALIDATED
# modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/oligo_fieldTest_WObrightness_longer/0_custom/'

source_dir_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_14/'
source_dir_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_14/'
source_dir_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_18/'
source_dir_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_18/'

source_dir_paper_redo_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_6/'
source_dir_paper_redo_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_6/'
source_dir_paper_redo_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_6/'
source_dir_paper_redo_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_6/'

# ######### To validate, use these whole well experiments: #########
# source_dir = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/wholeWell/oligo/unannotated/'
Expand Down Expand Up @@ -254,7 +254,7 @@ def main(args):
print('Number of arguments:', len(args), 'arguments.')
print('Argument List:', str(args))

custom_paths = False
custom_paths = False;
for arg in args:
arg = str(arg).lower()
print('Evaluating arg: "'+arg+'".')
Expand All @@ -273,67 +273,98 @@ def custom_paths_predict():
print('Your input: '+model_source_path);


pass


def prodi_gpu_predict():
print('Running Predictions.')
use_oligo = False
use_oligo = True
use_neuron = True
use_glia = False
use_old = False

use_debug = True
use_paper = False
use_debug = False
use_paper = True
skip_predicted = True
n_jobs: int = 20

# Paper Models trained for N4
model_source_path_oligo_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/'
model_source_path_neuron_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/'
model_source_path_oligo_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/'
model_source_path_neuron_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/'
model_source_path_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia/smote/'

# .h5 dirs to be predicted for the paper
# source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/'
# source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/'

# .h5 dirs to be predicted for efsa or endpoints
source_dir_paper_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints5/'
source_dir_paper_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints5/'

source_dir_paper_oligo2 = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints6/'
source_dir_paper_neuron2 = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints6/'

source_dir_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/glia_01/'


if use_glia:
predict_batch(model_source_path=model_source_path_glia, source_dir=source_dir_glia,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")

# .h dirs to be predicted for the paper
source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/'
source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/'

# source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_13/'
# source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_13/'
if use_paper:
if use_neuron:
predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")

predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron2,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")
if use_oligo:
predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")
predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo2,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")
if use_old:
if use_neuron:
predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron,
normalize_enum=1,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")
if use_oligo:
predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo,
normalize_enum=1,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="1")


if use_debug:
predict_batch(model_source_path=model_source_path_oligo_paper,
source_dir=source_dir_oligo,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=False,
gpu_index_string="0")
predict_batch(model_source_path=model_source_path_neuron_paper,
source_dir='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_debug/',
source_dir=source_dir_neuron,
normalize_enum=4,
n_jobs=1,
n_jobs=n_jobs,
skip_predicted=False,
gpu_index_string="0")
else:
if use_paper:
if use_neuron:
predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_redo_paper_neuron,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")
if use_oligo:
predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_redo_paper_oligo,
normalize_enum=4,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="1")
else:
if use_neuron:
predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron,
normalize_enum=1,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="0")

if use_oligo:
predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo,
normalize_enum=1,
n_jobs=n_jobs,
skip_predicted=skip_predicted,
gpu_index_string="1")



print(gct() + ' All Predictions done. Have a nice day. =)')

Expand Down
136 changes: 62 additions & 74 deletions train_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1145,91 +1145,79 @@ def decode_history_key(key: str) -> str:
return key


glia_path_train = [
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_ECM pretreated_trainingData_glia',
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_spheres pretreated_trainingData_glia',
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK274_normal_trainingData_glia',
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK135_trainingData_glia',
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK136_trainingData_glia'
]

glia_path_test='/home/nilfoe/bioinf/Omnisphero/CNN-glia/test/glia/'

glia_path_val = [
'/home/nilfoe/bioinf/Omnisphero/CNN-glia/validation/glia/'
]

def main():
# AUGMENTATION
data_gen = get_default_augmenter()
n_jobs = 20

out_path_base = out_path + 'paper-final_no-datagen' + os.sep
out_path_oligo = out_path_base + 'oligo' + os.sep
out_path_neuron = out_path_base + 'neuron' + os.sep

out_path_oligo_debug = out_path_base + 'oligo_debug' + os.sep

oligo_mode = True
neuron_mode = False
debug_mode = False
n_jobs = 40
out_path = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia-extendedData/'

print('Sleeping....')
# time.sleep(18000)

if debug_mode:
train_model(
training_path_list=debug_oligos,
validation_path_list=debug_oligos_validation,
test_data_path='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/training/oligo_kontrolliert_test/',
# data_gen=data_gen,
use_SMOTE=False,
out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron_kontrolliert_undersampled' + os.sep,
gpu_index_string="3",
optimizer='SGD',
normalize_enum=4,
under_sample_train_data=True, under_sample_val_data=True,
epochs=5,
n_jobs=25
)
return
train_model(
training_path_list=glia_path_train,
validation_path_list=glia_path_val,
test_data_path=glia_path_test,
use_SMOTE=False,
out_path=out_path + 'base' + os.sep,
gpu_index_string="0",
optimizer='SGD',
n_jobs=n_jobs,
epochs=5000
)

if oligo_mode:
train_model(
training_path_list=final_oligos_validated,
validation_path_list=final_oligos_validated_validation_set,
test_data_path=test_data_path_oligo,
# training_path_list=oligo_paper_bleedthrough_path_train,
# validation_path_list=oligo_paper_bleedthrough_path_val,
# test_data_path=oligo_paper_bleedthrough_path_test,
# data_gen=data_gen,
use_SMOTE=False,
out_path=out_path + 'paper-final_datagen' + os.sep + 'oligo-normalize4-undersampling' + os.sep,
under_sample_train_data=True, under_sample_val_data=True,
gpu_index_string="1",
optimizer='SGD',
n_jobs=n_jobs,
epochs=5000
)
train_model(
training_path_list=glia_path_train,
validation_path_list=glia_path_val,
test_data_path=glia_path_test,
use_SMOTE=False,
out_path=out_path + 'datagen' + os.sep,
gpu_index_string="0",
optimizer='SGD',
n_jobs=n_jobs,
data_gen=data_gen,
epochs=5000
)

# paper-final_no-datagen\neuron
if neuron_mode:
train_model(
training_path_list=final_neurons_validated,
validation_path_list=final_neurons_validated_validation_set,
test_data_path=test_data_path_neuron,
# training_path_list=neuron_paper_bleedthrough_path_train,
# validation_path_list=neuron_paper_bleedthrough_path_val,
# test_data_path=neuron_paper_bleedthrough_path_test,
# data_gen=data_gen,
use_SMOTE=False,
out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron-normalize4-undersampling' + os.sep,
under_sample_train_data=True, under_sample_val_data=True,
normalize_enum=4,
gpu_index_string="0",
optimizer='SGD',
n_jobs=n_jobs,
epochs=5000
)
train_model(
training_path_list=glia_path_train,
validation_path_list=glia_path_val,
test_data_path=glia_path_test,
use_SMOTE=True,
out_path=out_path + 'smote' + os.sep,
gpu_index_string="0",
optimizer='SGD',
n_jobs=n_jobs,
epochs=5000
)

# out_path_oligo = out_path+'oligo'+os.sep
# out_path_neuron = out_path+'neuron'+os.sep

#
# train_model_scrambling(path_candidate_list=final_oligos_validated,
# test_data_path=test_data_path_oligo,
# out_path=out_path_oligo,
# validation_count=1)
# train_model_scrambling(path_candidate_list=final_neurons_validated,
# test_data_path=test_data_path_neuron,
# out_path=out_path_neuron,
# validation_count=1)
train_model(
training_path_list=glia_path_train,
validation_path_list=glia_path_val,
test_data_path=glia_path_test,
use_SMOTE=True,
out_path=out_path + 'datagen_and_smote' + os.sep,
gpu_index_string="0",
optimizer='SGD',
n_jobs=n_jobs,
data_gen=data_gen,
epochs=5000
)

print('Finished all trainings. Goodbye.')

Expand Down

0 comments on commit f8f7ef1

Please sign in to comment.