Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nrn_setup: use std::vector for cell_groups (dat_files) #3185

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
45 changes: 20 additions & 25 deletions src/coreneuron/io/nrn_setup.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,13 @@ std::vector<int*> nrnthreads_netcon_srcgid;
std::vector<std::vector<int>> nrnthreads_netcon_negsrcgid_tid;

/* read files.dat file and distribute cellgroups to all mpi ranks */
void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
std::vector<int> nrn_read_filesdat(const char* filesdat) {
std::vector<int> rank_dat_files;
ferdonline marked this conversation as resolved.
Show resolved Hide resolved
patstimtype = nrn_get_mechtype("PatternStim");
if (corenrn_embedded && !corenrn_file_mode) {
ngrp = corenrn_embedded_nthread;
grp = new int[ngrp + 1];
(*nrn2core_group_ids_)(grp);
return;
rank_dat_files.resize(corenrn_embedded_nthread);
ferdonline marked this conversation as resolved.
Show resolved Hide resolved
(*nrn2core_group_ids_)(rank_dat_files.data());
return rank_dat_files;
}

FILE* fp = fopen(filesdat, "r");
Expand Down Expand Up @@ -207,21 +207,18 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
"Info : The number of input datasets are less than ranks, some ranks will be idle!\n");
}

ngrp = 0;
grp = new int[iNumFiles / nrnmpi_numprocs + 1];
ferdonline marked this conversation as resolved.
Show resolved Hide resolved

// irerate over gids in files.dat
for (int iNum = 0; iNum < iNumFiles; ++iNum) {
int iFile;

nrn_assert(fscanf(fp, "%d\n", &iFile) == 1);
if ((iNum % nrnmpi_numprocs) == nrnmpi_myid) {
grp[ngrp] = iFile;
ngrp++;
rank_dat_files.push_back(iFile);
}
}

fclose(fp);
return rank_dat_files;
}

void netpar_tid_gid2ps(int tid, int gid, PreSyn** ps, InputPreSyn** psi) {
Expand Down Expand Up @@ -409,22 +406,22 @@ void nrn_setup(const char* filesdat,
double* mindelay) {
double time = nrn_wtime();

int ngroup;
int* gidgroups;
nrn_read_filesdat(ngroup, gidgroups, filesdat);
UserParams userParams(ngroup,
gidgroups,
datpath,
strlen(restore_path) == 0 ? datpath : restore_path,
checkPoints);

UserParams userParams(
nrn_read_filesdat(filesdat),
datpath,
strlen(restore_path) == 0 ? datpath : restore_path,
checkPoints
);

// temporary bug work around. If any process has multiple threads, no
// process can have a single thread. So, for now, if one thread, make two.
// Fortunately, empty threads work fine.
// Allocate NrnThread* nrn_threads of size ngroup (minimum 2)
// Note that rank with 0 dataset/cellgroup works fine
nrn_threads_create(userParams.ngroup <= 1 ? 2 : userParams.ngroup);
int n_cell_groups = userParams.cell_groups.size();
int n_threads = n_cell_groups <= 1 ? 2 : n_cell_groups;
userParams.cell_groups.reserve(n_threads);
ferdonline marked this conversation as resolved.
Show resolved Hide resolved
nrn_threads_create(n_threads);

// from nrn_has_net_event create pnttype2presyn for use in phase2.
auto& memb_func = corenrn.get_memb_funcs();
Expand All @@ -451,7 +448,7 @@ void nrn_setup(const char* filesdat,

/// Reserve vector of maps of size ngroup for negative gid-s
/// std::vector< std::map<int, PreSyn*> > neg_gid2out;
neg_gid2out.resize(userParams.ngroup);
neg_gid2out.resize(n_cell_groups);

// bug fix. gid2out is cumulative over all threads and so do not
// know how many there are til after phase1
Expand Down Expand Up @@ -503,13 +500,13 @@ void nrn_setup(const char* filesdat,
nrn_partrans::setup_info_ = new SetupTransferInfo[nrn_nthread];
coreneuron::phase_wrapper<coreneuron::gap>(userParams);
} else {
nrn_partrans::setup_info_ = (*nrn2core_get_partrans_setup_info_)(userParams.ngroup,
nrn_partrans::setup_info_ = (*nrn2core_get_partrans_setup_info_)(n_cell_groups,
nrn_nthread,
sizeof(sgid_t));
}

nrn_multithread_job(nrn_partrans::gap_data_indices_setup);
nrn_partrans::gap_mpi_setup(userParams.ngroup);
nrn_partrans::gap_mpi_setup(n_cell_groups);

// Whether allocated in NEURON or here, delete here.
delete[] nrn_partrans::setup_info_;
Expand Down Expand Up @@ -564,8 +561,6 @@ void nrn_setup(const char* filesdat,
printf(" Model size : %.2lf GB\n", model_size_bytes / (1024. * 1024. * 1024.));
}
}

delete[] userParams.gidgroups;
}

void setup_ThreadData(NrnThread& nt) {
Expand Down
4 changes: 2 additions & 2 deletions src/coreneuron/io/nrn_setup.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ inline void read_phase_aux<gap>(NrnThread& nt, UserParams& userParams) {
template <phase P>
inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memory_transfer) {
int i = nt->id;
if (i < userParams.ngroup) {
if (i < userParams.cell_groups.size()) {
if (!in_memory_transfer) {
const char* data_dir = userParams.path;
// directory to read could be different for phase 2 if we are restoring
Expand All @@ -115,7 +115,7 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo
}

std::string fname = std::string(data_dir) + "/" +
std::to_string(userParams.gidgroups[i]) + "_" + getPhaseName<P>() +
std::to_string(userParams.cell_groups[i]) + "_" + getPhaseName<P>() +
".dat";

// Avoid trying to open the gid_gap.dat file if it doesn't exist when there are no
Expand Down
2 changes: 1 addition & 1 deletion src/coreneuron/io/phase2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -962,7 +962,7 @@ void Phase2::set_vec_play(NrnThread& nt, NrnThreadChkpnt& ntc) {

void Phase2::populate(NrnThread& nt, const UserParams& userParams) {
NrnThreadChkpnt& ntc = nrnthread_chkpnt[nt.id];
ntc.file_id = userParams.gidgroups[nt.id];
ntc.file_id = userParams.cell_groups[nt.id];

nt.ncell = n_real_cell;
nt.end = n_node;
Expand Down
14 changes: 5 additions & 9 deletions src/coreneuron/io/user_params.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,19 @@ class CheckPoints;
/// Before it was globals variables, group them to give them as a single argument.
/// They have for the most part, nothing related to each other.
struct UserParams {
UserParams(int ngroup_,
int* gidgroups_,
UserParams(std::vector<int>&& cell_groups_,
ferdonline marked this conversation as resolved.
Show resolved Hide resolved
const char* path_,
const char* restore_path_,
CheckPoints& checkPoints_)
: ngroup(ngroup_)
, gidgroups(gidgroups_)
: cell_groups(std::move(cell_groups_))
, path(path_)
, restore_path(restore_path_)
, file_reader(ngroup_)
, file_reader(cell_groups.size())
, checkPoints(checkPoints_) {}

/// direct memory mode with neuron, do not open files
/// Number of local cell groups
const int ngroup;
/// Array of cell group numbers (indices)
const int* const gidgroups;
/// Vector of cell group numbers (indices)
std::vector<int> cell_groups;
/// path to dataset file
const char* const path;
/// Dataset path from where simulation is being restored
Expand Down
Loading