Skip to content

Commit

Permalink
nrn_setup use array for cell_groups
Browse files Browse the repository at this point in the history
  • Loading branch information
ferdonline committed Nov 7, 2024
1 parent 2abbd25 commit c30f255
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 37 deletions.
45 changes: 20 additions & 25 deletions src/coreneuron/io/nrn_setup.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,13 @@ std::vector<int*> nrnthreads_netcon_srcgid;
std::vector<std::vector<int>> nrnthreads_netcon_negsrcgid_tid;

/* read files.dat file and distribute cellgroups to all mpi ranks */
void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
std::vector<int> nrn_read_filesdat(const char* filesdat) {
std::vector<int> rank_dat_files;
patstimtype = nrn_get_mechtype("PatternStim");
if (corenrn_embedded && !corenrn_file_mode) {
ngrp = corenrn_embedded_nthread;
grp = new int[ngrp + 1];
(*nrn2core_group_ids_)(grp);
return;
rank_dat_files.resize(corenrn_embedded_nthread);
(*nrn2core_group_ids_)(rank_dat_files.data());
return rank_dat_files;
}

FILE* fp = fopen(filesdat, "r");
Expand Down Expand Up @@ -207,21 +207,18 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
"Info : The number of input datasets are less than ranks, some ranks will be idle!\n");
}

ngrp = 0;
grp = new int[iNumFiles / nrnmpi_numprocs + 1];

// irerate over gids in files.dat
for (int iNum = 0; iNum < iNumFiles; ++iNum) {
int iFile;

nrn_assert(fscanf(fp, "%d\n", &iFile) == 1);
if ((iNum % nrnmpi_numprocs) == nrnmpi_myid) {
grp[ngrp] = iFile;
ngrp++;
rank_dat_files.push_back(iFile);
}
}

fclose(fp);
return rank_dat_files;
}

void netpar_tid_gid2ps(int tid, int gid, PreSyn** ps, InputPreSyn** psi) {
Expand Down Expand Up @@ -409,22 +406,22 @@ void nrn_setup(const char* filesdat,
double* mindelay) {
double time = nrn_wtime();

int ngroup;
int* gidgroups;
nrn_read_filesdat(ngroup, gidgroups, filesdat);
UserParams userParams(ngroup,
gidgroups,
datpath,
strlen(restore_path) == 0 ? datpath : restore_path,
checkPoints);

UserParams userParams(
nrn_read_filesdat(filesdat),
datpath,
strlen(restore_path) == 0 ? datpath : restore_path,
checkPoints
);

// temporary bug work around. If any process has multiple threads, no
// process can have a single thread. So, for now, if one thread, make two.
// Fortunately, empty threads work fine.
// Allocate NrnThread* nrn_threads of size ngroup (minimum 2)
// Note that rank with 0 dataset/cellgroup works fine
nrn_threads_create(userParams.ngroup <= 1 ? 2 : userParams.ngroup);
int n_cell_groups = userParams.cell_groups.size();
int n_threads = n_cell_groups <= 1 ? 2 : n_cell_groups;
userParams.cell_groups.reserve(n_threads);
nrn_threads_create(n_threads);

// from nrn_has_net_event create pnttype2presyn for use in phase2.
auto& memb_func = corenrn.get_memb_funcs();
Expand All @@ -451,7 +448,7 @@ void nrn_setup(const char* filesdat,

/// Reserve vector of maps of size ngroup for negative gid-s
/// std::vector< std::map<int, PreSyn*> > neg_gid2out;
neg_gid2out.resize(userParams.ngroup);
neg_gid2out.resize(n_cell_groups);

// bug fix. gid2out is cumulative over all threads and so do not
// know how many there are til after phase1
Expand Down Expand Up @@ -503,13 +500,13 @@ void nrn_setup(const char* filesdat,
nrn_partrans::setup_info_ = new SetupTransferInfo[nrn_nthread];
coreneuron::phase_wrapper<coreneuron::gap>(userParams);
} else {
nrn_partrans::setup_info_ = (*nrn2core_get_partrans_setup_info_)(userParams.ngroup,
nrn_partrans::setup_info_ = (*nrn2core_get_partrans_setup_info_)(n_cell_groups,
nrn_nthread,
sizeof(sgid_t));
}

nrn_multithread_job(nrn_partrans::gap_data_indices_setup);
nrn_partrans::gap_mpi_setup(userParams.ngroup);
nrn_partrans::gap_mpi_setup(n_cell_groups);

// Whether allocated in NEURON or here, delete here.
delete[] nrn_partrans::setup_info_;
Expand Down Expand Up @@ -564,8 +561,6 @@ void nrn_setup(const char* filesdat,
printf(" Model size : %.2lf GB\n", model_size_bytes / (1024. * 1024. * 1024.));
}
}

delete[] userParams.gidgroups;
}

void setup_ThreadData(NrnThread& nt) {
Expand Down
4 changes: 2 additions & 2 deletions src/coreneuron/io/nrn_setup.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ inline void read_phase_aux<gap>(NrnThread& nt, UserParams& userParams) {
template <phase P>
inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memory_transfer) {
int i = nt->id;
if (i < userParams.ngroup) {
if (i < userParams.cell_groups.size()) {
if (!in_memory_transfer) {
const char* data_dir = userParams.path;
// directory to read could be different for phase 2 if we are restoring
Expand All @@ -115,7 +115,7 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo
}

std::string fname = std::string(data_dir) + "/" +
std::to_string(userParams.gidgroups[i]) + "_" + getPhaseName<P>() +
std::to_string(userParams.cell_groups[i]) + "_" + getPhaseName<P>() +
".dat";

// Avoid trying to open the gid_gap.dat file if it doesn't exist when there are no
Expand Down
2 changes: 1 addition & 1 deletion src/coreneuron/io/phase2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -962,7 +962,7 @@ void Phase2::set_vec_play(NrnThread& nt, NrnThreadChkpnt& ntc) {

void Phase2::populate(NrnThread& nt, const UserParams& userParams) {
NrnThreadChkpnt& ntc = nrnthread_chkpnt[nt.id];
ntc.file_id = userParams.gidgroups[nt.id];
ntc.file_id = userParams.cell_groups[nt.id];

nt.ncell = n_real_cell;
nt.end = n_node;
Expand Down
14 changes: 5 additions & 9 deletions src/coreneuron/io/user_params.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,19 @@ class CheckPoints;
/// Before it was globals variables, group them to give them as a single argument.
/// They have for the most part, nothing related to each other.
struct UserParams {
UserParams(int ngroup_,
int* gidgroups_,
UserParams(std::vector<int>&& cell_groups_,
const char* path_,
const char* restore_path_,
CheckPoints& checkPoints_)
: ngroup(ngroup_)
, gidgroups(gidgroups_)
: cell_groups(std::move(cell_groups_))
, path(path_)
, restore_path(restore_path_)
, file_reader(ngroup_)
, file_reader(cell_groups_.size())
, checkPoints(checkPoints_) {}

/// direct memory mode with neuron, do not open files
/// Number of local cell groups
const int ngroup;
/// Array of cell group numbers (indices)
const int* const gidgroups;
/// Vector of cell group numbers (indices)
std::vector<int> cell_groups;
/// path to dataset file
const char* const path;
/// Dataset path from where simulation is being restored
Expand Down

0 comments on commit c30f255

Please sign in to comment.