diff --git a/src/coreneuron/io/nrn_setup.cpp b/src/coreneuron/io/nrn_setup.cpp index 1f6a955b6c..8cdd33bee2 100644 --- a/src/coreneuron/io/nrn_setup.cpp +++ b/src/coreneuron/io/nrn_setup.cpp @@ -169,14 +169,19 @@ std::vector nrnthreads_netcon_srcgid; /// in order to use the correct neg_gid2out[tid] map std::vector> nrnthreads_netcon_negsrcgid_tid; -/* read files.dat file and distribute cellgroups to all mpi ranks */ -void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { +/// read files.dat file and distribute cellgroups among all mpi ranks +/// +/// Each entry in the files.dat file is the id of a cell group (also its first gid) +/// We do round-robin to associate cell groups to mpi ranks +std::vector nrn_read_filesdat(const char* filesdat) { + std::vector rank_cell_groups; + + // Direct transfer mode patstimtype = nrn_get_mechtype("PatternStim"); if (corenrn_embedded && !corenrn_file_mode) { - ngrp = corenrn_embedded_nthread; - grp = new int[ngrp + 1]; - (*nrn2core_group_ids_)(grp); - return; + rank_cell_groups.resize(corenrn_embedded_nthread); + (*nrn2core_group_ids_)(rank_cell_groups.data()); + return rank_cell_groups; } FILE* fp = fopen(filesdat, "r"); @@ -207,13 +212,12 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { "Info : The number of input datasets are less than ranks, some ranks will be idle!\n"); } - ngrp = 0; - grp = new int[iNumFiles / nrnmpi_numprocs + 1]; + // preallocate for the general case + rank_cell_groups.reserve(iNumFiles / nrnmpi_numprocs + 1); // irerate over gids in files.dat for (int iNum = 0; iNum < iNumFiles; ++iNum) { int iFile; - nrn_assert(fscanf(fp, "%d\n", &iFile) == 1); if ((iNum % nrnmpi_numprocs) == nrnmpi_myid) { // A "-1" entry means that this rank should not be assigned further gid groups. @@ -222,12 +226,12 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { if (iFile == -1) { break; } - grp[ngrp] = iFile; - ngrp++; + rank_cell_groups.push_back(iFile); } } fclose(fp); + return rank_cell_groups; } void netpar_tid_gid2ps(int tid, int gid, PreSyn** ps, InputPreSyn** psi) { @@ -415,22 +419,19 @@ void nrn_setup(const char* filesdat, double* mindelay) { double time = nrn_wtime(); - int ngroup; - int* gidgroups; - nrn_read_filesdat(ngroup, gidgroups, filesdat); - UserParams userParams(ngroup, - gidgroups, + UserParams userParams(nrn_read_filesdat(filesdat), datpath, strlen(restore_path) == 0 ? datpath : restore_path, checkPoints); - // temporary bug work around. If any process has multiple threads, no // process can have a single thread. So, for now, if one thread, make two. // Fortunately, empty threads work fine. // Allocate NrnThread* nrn_threads of size ngroup (minimum 2) // Note that rank with 0 dataset/cellgroup works fine - nrn_threads_create(userParams.ngroup <= 1 ? 2 : userParams.ngroup); + int n_cell_groups = userParams.cell_groups.size(); + int n_threads = n_cell_groups <= 1 ? 2 : n_cell_groups; + nrn_threads_create(n_threads); // from nrn_has_net_event create pnttype2presyn for use in phase2. auto& memb_func = corenrn.get_memb_funcs(); @@ -457,7 +458,7 @@ void nrn_setup(const char* filesdat, /// Reserve vector of maps of size ngroup for negative gid-s /// std::vector< std::map > neg_gid2out; - neg_gid2out.resize(userParams.ngroup); + neg_gid2out.resize(n_cell_groups); // bug fix. gid2out is cumulative over all threads and so do not // know how many there are til after phase1 @@ -509,13 +510,12 @@ void nrn_setup(const char* filesdat, nrn_partrans::setup_info_ = new SetupTransferInfo[nrn_nthread]; coreneuron::phase_wrapper(userParams); } else { - nrn_partrans::setup_info_ = (*nrn2core_get_partrans_setup_info_)(userParams.ngroup, - nrn_nthread, - sizeof(sgid_t)); + nrn_partrans::setup_info_ = + (*nrn2core_get_partrans_setup_info_)(n_cell_groups, nrn_nthread, sizeof(sgid_t)); } nrn_multithread_job(nrn_partrans::gap_data_indices_setup); - nrn_partrans::gap_mpi_setup(userParams.ngroup); + nrn_partrans::gap_mpi_setup(n_cell_groups); // Whether allocated in NEURON or here, delete here. delete[] nrn_partrans::setup_info_; @@ -570,8 +570,6 @@ void nrn_setup(const char* filesdat, printf(" Model size : %.2lf GB\n", model_size_bytes / (1024. * 1024. * 1024.)); } } - - delete[] userParams.gidgroups; } void setup_ThreadData(NrnThread& nt) { diff --git a/src/coreneuron/io/nrn_setup.hpp b/src/coreneuron/io/nrn_setup.hpp index 389b90c15a..167bc6dc1c 100644 --- a/src/coreneuron/io/nrn_setup.hpp +++ b/src/coreneuron/io/nrn_setup.hpp @@ -104,7 +104,7 @@ inline void read_phase_aux(NrnThread& nt, UserParams& userParams) { template inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memory_transfer) { int i = nt->id; - if (i < userParams.ngroup) { + if (i < userParams.cell_groups.size()) { if (!in_memory_transfer) { const char* data_dir = userParams.path; // directory to read could be different for phase 2 if we are restoring @@ -115,8 +115,8 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo } std::string fname = std::string(data_dir) + "/" + - std::to_string(userParams.gidgroups[i]) + "_" + getPhaseName

() + - ".dat"; + std::to_string(userParams.cell_groups[i]) + "_" + + getPhaseName

() + ".dat"; // Avoid trying to open the gid_gap.dat file if it doesn't exist when there are no // gap junctions in this gid. diff --git a/src/coreneuron/io/phase2.cpp b/src/coreneuron/io/phase2.cpp index 3c712f1f5b..b7961b51bb 100644 --- a/src/coreneuron/io/phase2.cpp +++ b/src/coreneuron/io/phase2.cpp @@ -962,7 +962,7 @@ void Phase2::set_vec_play(NrnThread& nt, NrnThreadChkpnt& ntc) { void Phase2::populate(NrnThread& nt, const UserParams& userParams) { NrnThreadChkpnt& ntc = nrnthread_chkpnt[nt.id]; - ntc.file_id = userParams.gidgroups[nt.id]; + ntc.file_id = userParams.cell_groups[nt.id]; nt.ncell = n_real_cell; nt.end = n_node; diff --git a/src/coreneuron/io/user_params.hpp b/src/coreneuron/io/user_params.hpp index f7f52a209c..cdcff0963e 100644 --- a/src/coreneuron/io/user_params.hpp +++ b/src/coreneuron/io/user_params.hpp @@ -16,23 +16,19 @@ class CheckPoints; /// Before it was globals variables, group them to give them as a single argument. /// They have for the most part, nothing related to each other. struct UserParams { - UserParams(int ngroup_, - int* gidgroups_, + UserParams(std::vector&& cell_groups_, const char* path_, const char* restore_path_, CheckPoints& checkPoints_) - : ngroup(ngroup_) - , gidgroups(gidgroups_) + : cell_groups(std::move(cell_groups_)) , path(path_) , restore_path(restore_path_) - , file_reader(ngroup_) + , file_reader(cell_groups.size()) , checkPoints(checkPoints_) {} /// direct memory mode with neuron, do not open files - /// Number of local cell groups - const int ngroup; - /// Array of cell group numbers (indices) - const int* const gidgroups; + /// Vector of cell group numbers (indices) + std::vector cell_groups; /// path to dataset file const char* const path; /// Dataset path from where simulation is being restored