diff --git a/.gitignore b/.gitignore index 62594fc0bc..7bedc9af56 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ virtualenv docs/_build docs/_generated .vscode +src/nrnoc/hh.mod diff --git a/src/coreneuron/io/nrn_filehandler.cpp b/src/coreneuron/io/nrn_filehandler.cpp index 40171aec7e..9a16c257c2 100644 --- a/src/coreneuron/io/nrn_filehandler.cpp +++ b/src/coreneuron/io/nrn_filehandler.cpp @@ -7,8 +7,11 @@ */ #include +#include #include "coreneuron/io/nrn_filehandler.hpp" #include "coreneuron/nrnconf.h" +#include "coreneuron/mpi/nrnmpi.h" +#include "coreneuron/mpi/core/nrnmpi.hpp" namespace coreneuron { FileHandler::FileHandler(const std::string& filename) @@ -22,10 +25,29 @@ bool FileHandler::file_exist(const std::string& filename) { return (stat(filename.c_str(), &buffer) == 0); } -void FileHandler::open(const std::string& filename, std::ios::openmode mode) { +std::string FileHandler::get_rank_fname(const char* basepath, bool create_folder) { + // TODO: Change this for equivalent MPI functions to get the node ID + std::string nodepath = ""; + if (const char* node_id = std::getenv("SLURM_NODEID")) { + const int factor = 20; + nodepath = std::to_string(std::atoi(node_id) / factor) + "/" + node_id; + } else if (const char* hostname = std::getenv("HOSTNAME")) { + nodepath = hostname; + } + // Create subfolder for the rank, based on the node + std::string path = std::string(basepath) + "/" + nodepath; + if (create_folder && !std::filesystem::exists(path)) { + std::filesystem::create_directories(path); + } + + return (path + "/" + std::to_string(nrnmpi_myid) + ".dat"); +} + +void FileHandler::open(const std::string& filename, size_t offset, std::ios::openmode mode) { nrn_assert((mode & (std::ios::in | std::ios::out))); close(); F.open(filename, mode | std::ios::binary); + F.seekg(offset, std::ios::beg); if (!F.is_open()) { std::cerr << "cannot open file '" << filename << "'" << std::endl; } @@ -47,7 +69,7 @@ bool FileHandler::eof() { return true; } int a = F.get(); - if (F.eof()) { + if (F.eof() || (char) a == '\0') { return true; } F.putback(a); diff --git a/src/coreneuron/io/nrn_filehandler.hpp b/src/coreneuron/io/nrn_filehandler.hpp index 9ac7f048e8..d4342de036 100644 --- a/src/coreneuron/io/nrn_filehandler.hpp +++ b/src/coreneuron/io/nrn_filehandler.hpp @@ -56,7 +56,9 @@ class FileHandler { explicit FileHandler(const std::string& filename); /** Preserving chkpnt state, move to a new file. */ - void open(const std::string& filename, std::ios::openmode mode = std::ios::in); + void open(const std::string& filename, + size_t offset = 0, + std::ios::openmode mode = std::ios::in); /** Is the file not open */ bool fail() const { @@ -65,6 +67,8 @@ class FileHandler { static bool file_exist(const std::string& filename); + static std::string get_rank_fname(const char* basepath, bool create_folder = true); + /** nothing more to read */ bool eof(); diff --git a/src/coreneuron/io/nrn_setup.cpp b/src/coreneuron/io/nrn_setup.cpp index c346f84bf5..ed7ae3a30f 100644 --- a/src/coreneuron/io/nrn_setup.cpp +++ b/src/coreneuron/io/nrn_setup.cpp @@ -168,7 +168,11 @@ std::vector nrnthreads_netcon_srcgid; std::vector> nrnthreads_netcon_negsrcgid_tid; /* read files.dat file and distribute cellgroups to all mpi ranks */ -void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { +void nrn_read_filesdat(int& ngrp, + int*& grp, + int& num_offsets, + size_t*& file_offsets, + const char* filesdat) { patstimtype = nrn_get_mechtype("PatternStim"); if (corenrn_embedded) { ngrp = corenrn_embedded_nthread; @@ -200,6 +204,8 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { } } + nrn_assert(fscanf(fp, "%d\n", &num_offsets) == 1); + if (nrnmpi_numprocs > iNumFiles && nrnmpi_myid == 0) { printf( "Info : The number of input datasets are less than ranks, some ranks will be idle!\n"); @@ -207,12 +213,18 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) { ngrp = 0; grp = new int[iNumFiles / nrnmpi_numprocs + 1]; + file_offsets = new size_t[num_offsets * (iNumFiles / nrnmpi_numprocs + 1)]; // irerate over gids in files.dat + size_t offsets_idx = 0; for (int iNum = 0; iNum < iNumFiles; ++iNum) { int iFile; nrn_assert(fscanf(fp, "%d\n", &iFile) == 1); + for (int i = 0; i < num_offsets; i++, offsets_idx++) { + nrn_assert(fscanf(fp, "%zu\n", &file_offsets[ngrp * num_offsets + i]) == 1); + } + if ((iNum % nrnmpi_numprocs) == nrnmpi_myid) { grp[ngrp] = iFile; ngrp++; @@ -409,9 +421,13 @@ void nrn_setup(const char* filesdat, int ngroup; int* gidgroups; - nrn_read_filesdat(ngroup, gidgroups, filesdat); + int num_offsets; + size_t* file_offsets; + nrn_read_filesdat(ngroup, gidgroups, num_offsets, file_offsets, filesdat); UserParams userParams(ngroup, gidgroups, + num_offsets, + file_offsets, datpath, strlen(restore_path) == 0 ? datpath : restore_path, checkPoints); diff --git a/src/coreneuron/io/nrn_setup.hpp b/src/coreneuron/io/nrn_setup.hpp index 389b90c15a..4fedf4f733 100644 --- a/src/coreneuron/io/nrn_setup.hpp +++ b/src/coreneuron/io/nrn_setup.hpp @@ -8,6 +8,7 @@ #pragma once +#include #include #include "coreneuron/sim/multicore.hpp" #include "coreneuron/io/nrn_filehandler.hpp" @@ -16,6 +17,7 @@ #include "coreneuron/io/mem_layout_util.hpp" #include "coreneuron/io/nrn_checkpoint.hpp" + namespace coreneuron { void read_phase1(NrnThread& nt, UserParams& userParams); void read_phase2(NrnThread& nt, UserParams& userParams); @@ -100,6 +102,7 @@ inline void read_phase_aux(NrnThread& nt, UserParams& userParams) { read_phasegap(nt, userParams); } + /// Reading phase wrapper for each neuron group. template inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memory_transfer) { @@ -114,9 +117,8 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo data_dir = userParams.restore_path; } - std::string fname = std::string(data_dir) + "/" + - std::to_string(userParams.gidgroups[i]) + "_" + getPhaseName

() + - ".dat"; + size_t file_offset = userParams.file_offsets[i * userParams.num_offsets + P - 1]; + const auto& fname = FileHandler::get_rank_fname(data_dir); // Avoid trying to open the gid_gap.dat file if it doesn't exist when there are no // gap junctions in this gid. @@ -128,7 +130,7 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo userParams.file_reader[i].close(); } else { // if no file failed to open or not opened at all - userParams.file_reader[i].open(fname); + userParams.file_reader[i].open(fname, file_offset); } } read_phase_aux

(*nt, userParams); diff --git a/src/coreneuron/io/user_params.hpp b/src/coreneuron/io/user_params.hpp index f7f52a209c..6337001166 100644 --- a/src/coreneuron/io/user_params.hpp +++ b/src/coreneuron/io/user_params.hpp @@ -18,11 +18,15 @@ class CheckPoints; struct UserParams { UserParams(int ngroup_, int* gidgroups_, + int num_offsets_, + size_t* file_offsets_, const char* path_, const char* restore_path_, CheckPoints& checkPoints_) : ngroup(ngroup_) , gidgroups(gidgroups_) + , num_offsets(num_offsets_) + , file_offsets(file_offsets_) , path(path_) , restore_path(restore_path_) , file_reader(ngroup_) @@ -33,6 +37,10 @@ struct UserParams { const int ngroup; /// Array of cell group numbers (indices) const int* const gidgroups; + /// Number of offsets per cell group + const int num_offsets; + /// Array of offsets inside file with cell groups (indices) + const size_t* const file_offsets; /// path to dataset file const char* const path; /// Dataset path from where simulation is being restored diff --git a/src/nrniv/nrncore_write.cpp b/src/nrniv/nrncore_write.cpp index a3b5d8d94c..fa709c7b1d 100644 --- a/src/nrniv/nrncore_write.cpp +++ b/src/nrniv/nrncore_write.cpp @@ -219,16 +219,22 @@ static part1_ret part1() { } static void part2(const char* path) { + std::vector offsets(4); + CellGroup* cgs = cellgroups_; for (int i = 0; i < nrn_nthread; ++i) { chkpnt = 0; - write_nrnthread(path, nrn_threads[i], cgs[i]); + const auto& nrnthread_offsets = write_nrnthread(path, nrn_threads[i], cgs[i]); + if (nrnthread_offsets[1] > 0) { + offsets[0] = nrnthread_offsets[0]; + offsets[1] = nrnthread_offsets[1]; + } } /** write mapping information */ if (mapinfo.size()) { int gid = cgs[0].group_id; - nrn_write_mapping_info(path, gid, mapinfo); + offsets[2] = nrn_write_mapping_info(path, gid, mapinfo); mapinfo.clear(); } @@ -238,7 +244,7 @@ static void part2(const char* path) { for (int i = 0; i < nrn_nthread; ++i) { group_ids[i] = cgs[i].group_id; } - nrnbbcore_gap_write(path, group_ids); + offsets[3] = nrnbbcore_gap_write(path, group_ids); delete[] group_ids; } @@ -262,7 +268,7 @@ static void part2(const char* path) { hoc_execerror("Second arg must be Vector or double.", NULL); } } - write_nrnthread_task(path, cgs, append); + write_nrnthread_task(path, cgs, append, offsets); } part2_clean(); diff --git a/src/nrniv/nrncore_write/io/nrncore_io.cpp b/src/nrniv/nrncore_write/io/nrncore_io.cpp index ef41bc3f48..f0e4ab962e 100644 --- a/src/nrniv/nrncore_write/io/nrncore_io.cpp +++ b/src/nrniv/nrncore_write/io/nrncore_io.cpp @@ -13,6 +13,7 @@ #include "vrecitem.h" // for nrnbbcore_vecplay_write #include #include +#include #include "nrnsection_mapping.h" extern short* nrn_is_artificial_; @@ -54,6 +55,24 @@ std::string get_filename(const std::string& path, std::string file_name) { return fname; } +std::string get_rank_fname(const char* basepath, bool create_folder = true) { + // TODO: Change this for equivalent MPI functions to get the node ID + std::string nodepath = ""; + if (const char* node_id = std::getenv("SLURM_NODEID")) { + const int factor = 20; + nodepath = std::to_string(std::atoi(node_id) / factor) + "/" + node_id; + } else if (const char* hostname = std::getenv("HOSTNAME")) { + nodepath = hostname; + } + // Create subfolder for the rank, based on the node + std::string path = std::string(basepath) + "/" + nodepath; + if (create_folder && !std::filesystem::exists(path)) { + std::filesystem::create_directories(path); + } + + return (path + "/" + std::to_string(nrnmpi_myid) + ".dat"); +} + void write_memb_mech_types(const char* fname) { if (nrnmpi_myid > 0) { @@ -113,18 +132,23 @@ void write_globals(const char* fname) { fclose(f); } +std::array write_nrnthread(const char* path, NrnThread& nt, CellGroup& cg) { + std::array offsets = {0, 0}; -void write_nrnthread(const char* path, NrnThread& nt, CellGroup& cg) { - char fname[1000]; if (cg.n_output <= 0) { - return; + return offsets; } assert(cg.group_id >= 0); - nrn_assert(snprintf(fname, 1000, "%s/%d_1.dat", path, cg.group_id) < 1000); - FILE* f = fopen(fname, "wb"); + + const auto& fname = get_rank_fname(path); + FILE* f = fopen(fname.c_str(), "ab"); if (!f) { - hoc_execerror("nrncore_write write_nrnthread could not open for writing:", fname); + hoc_execerror("nrncore_write write_nrnthread could not open for writing:", fname.c_str()); } + + // Set the first offset inside the file + offsets[0] = ftell(f); + fprintf(f, "%s\n", bbcore_write_version); // nrnthread_dat1(int tid, int& n_presyn, int& n_netcon, int*& output_gid, int*& netcon_srcgid); @@ -138,13 +162,12 @@ void write_nrnthread(const char* path, NrnThread& nt, CellGroup& cg) { delete[] cg.netcon_srcgid; cg.netcon_srcgid = NULL; } - fclose(f); - nrn_assert(snprintf(fname, 1000, "%s/%d_2.dat", path, cg.group_id) < 1000); - f = fopen(fname, "w"); - if (!f) { - hoc_execerror("nrncore_write write_nrnthread could not open for writing:", fname); - } + // Mark the end of the file with '\0' + fputc(0, f); + + // Set the second offset inside the file + offsets[1] = ftell(f); fprintf(f, "%s\n", bbcore_write_version); @@ -285,7 +308,12 @@ void write_nrnthread(const char* path, NrnThread& nt, CellGroup& cg) { nrnbbcore_vecplay_write(f, nt); + // Mark the end of the file with '\0' + fputc(0, f); + fclose(f); + + return offsets; } @@ -351,7 +379,10 @@ static void fgets_no_newline(char* s, int size, FILE* f) { * ... * idN */ -void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { +void write_nrnthread_task(const char* path, + CellGroup* cgs, + bool append, + std::vector& file_offsets) { // ids of datasets that will be created std::vector iSend; @@ -386,6 +417,7 @@ void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { // total number of datasets across all ranks int iSumThread = 0; + int num_offsets = file_offsets.size(); // calculate mpi displacements if (nrnmpi_myid == 0) { @@ -412,6 +444,10 @@ void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { iRecvVec[iInt] = iSend[iInt]; } } + + // Collect the offsets + file_offsets.resize(num_offsets * (nrnmpi_myid == 0 ? nrnmpi_numprocs : 1ULL)); + nrnmpi_sizet_gather(file_offsets.data(), file_offsets.data(), num_offsets, 0); #else for (int iInt = 0; iInt < num_datasets; ++iInt) { iRecvVec[iInt] = iSend[iInt]; @@ -500,15 +536,21 @@ void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { fseek(fp, pos, SEEK_SET); } fprintf(fp, "%10d\n", iSumThread); + fprintf(fp, "%d\n", num_offsets); if (append) { // Start writing the groupids starting at the end of the file. fseek(fp, 0, SEEK_END); } - // write all dataset ids + // write all dataset + offsets + size_t offsets_idx = 0; for (int i = 0; i < iRecvVec.size(); ++i) { fprintf(fp, "%d\n", iRecvVec[i]); + + for (int i = 0; i < num_offsets; i++, offsets_idx++) { + fprintf(fp, "%zu\n", file_offsets[offsets_idx]); + } } fclose(fp); @@ -516,18 +558,17 @@ void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { } /** @brief dump mapping information to gid_3.dat file */ -void nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo) { - /** full path of mapping file */ - std::stringstream ss; - ss << path << "/" << gid << "_3.dat"; - - std::string fname(ss.str()); - FILE* f = fopen(fname.c_str(), "w"); - +size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo) { + size_t offset = 0; + const auto& fname = get_rank_fname(path); + FILE* f = fopen(fname.c_str(), "ab"); if (!f) { hoc_execerror("nrnbbcore_write could not open for writing:", fname.c_str()); } + // Set the offset inside the file + offset = ftell(f); + fprintf(f, "%s\n", bbcore_write_version); /** number of gids in NrnThread */ @@ -562,5 +603,11 @@ void nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo) { } } } + + // Mark the end of the file with '\0' + fputc(0, f); + fclose(f); + + return offset; } diff --git a/src/nrniv/nrncore_write/io/nrncore_io.h b/src/nrniv/nrncore_write/io/nrncore_io.h index ee690b694d..903fc9b7fa 100644 --- a/src/nrniv/nrncore_write/io/nrncore_io.h +++ b/src/nrniv/nrncore_write/io/nrncore_io.h @@ -2,6 +2,7 @@ #define NRN_NRNCORE_IO_H #include "hocdec.h" +#include #include #include @@ -27,7 +28,7 @@ extern int chkpnt; void write_memb_mech_types(const char* fname); void write_globals(const char* fname); -void write_nrnthread(const char* fname, NrnThread& nt, CellGroup& cg); +std::array write_nrnthread(const char* fname, NrnThread& nt, CellGroup& cg); void writeint_(int* p, size_t size, FILE* f); void writedbl_(double* p, size_t size, FILE* f); @@ -38,10 +39,13 @@ struct Memb_list; using bbcore_write_t = void (*)(double*, int*, int*, int*, Memb_list*, std::size_t, Datum*, Datum*, NrnThread*); -void write_nrnthread_task(const char*, CellGroup* cgs, bool append); +void write_nrnthread_task(const char*, + CellGroup* cgs, + bool append, + std::vector& file_offsets); void nrnbbcore_vecplay_write(FILE* f, NrnThread& nt); -void nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo); +size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo); #endif // NRN_NRNCORE_IO_H diff --git a/src/nrniv/nrnsection_mapping.h b/src/nrniv/nrnsection_mapping.h index 5ebeaf2a78..7e12864ddc 100644 --- a/src/nrniv/nrnsection_mapping.h +++ b/src/nrniv/nrnsection_mapping.h @@ -143,6 +143,6 @@ struct NrnMappingInfo { } }; -void nrn_write_mapping_info(const char*, int, NrnMappingInfo&); +size_t nrn_write_mapping_info(const char*, int, NrnMappingInfo&); #endif // NRN_SECTION_MAPPING diff --git a/src/nrniv/partrans.cpp b/src/nrniv/partrans.cpp index c632b2a845..625d4b0bd4 100644 --- a/src/nrniv/partrans.cpp +++ b/src/nrniv/partrans.cpp @@ -119,6 +119,7 @@ extern void (*nrnthread_vi_compute_)(NrnThread*); extern void (*nrnmpi_v_transfer_)(); // before nrnthread_v_transfer and after update. Called by // thread 0. extern void (*nrn_mk_transfer_thread_data_)(); +extern std::string get_rank_fname(const char* basepath, bool create_folder = true); #if NRNMPI extern double nrnmpi_transfer_wait_; #endif @@ -965,9 +966,11 @@ SetupTransferInfo* nrn_get_partrans_setup_info(int ngroup, int cn_nthread, size_ } size_t nrnbbcore_gap_write(const char* path, int* group_ids) { + size_t offset = 0; + auto gi = nrncore_transfer_info(nrn_nthread); // gi stood for gapinfo if (gi == nullptr) { - return 0; + return offset; } // print the files @@ -978,10 +981,16 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) { continue; } - char fname[1000]; - Sprintf(fname, "%s/%d_gap.dat", path, group_ids[tid]); - FILE* f = fopen(fname, "wb"); - assert(f); + const std::string fname = get_rank_fname(path); + + FILE* f = fopen(fname.c_str(), "ab"); + if (!f) { + hoc_execerror("nrnbbcore_write could not open for writing:", fname.c_str()); + } + + // Set the offset inside the file + offset = ftell(f); + fprintf(f, "%s\n", bbcore_write_version); fprintf(f, "%d sizeof_sid_t\n", int(sizeof(sgid_t))); @@ -1005,12 +1014,15 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) { CHKPNT fwrite(g.tar_index.data(), ntar, sizeof(int), f); } + // Mark the end of the file with '\0' + fputc(0, f); + fclose(f); } // cleanup delete[] gi; - return 0; + return offset; } static SetupTransferInfo* nrncore_transfer_info(int cn_nthread) { diff --git a/src/nrnmpi/mpispike.cpp b/src/nrnmpi/mpispike.cpp index 317f27f7b9..a717be2105 100644 --- a/src/nrnmpi/mpispike.cpp +++ b/src/nrnmpi/mpispike.cpp @@ -396,6 +396,10 @@ extern void nrnmpi_int_gatherv(int* s, int scnt, int* r, int* rcnt, int* rdispl, MPI_Gatherv(s, scnt, MPI_INT, r, rcnt, rdispl, MPI_INT, root, nrnmpi_comm); } +extern void nrnmpi_sizet_gather(size_t* s, size_t* r, int cnt, int root) { + MPI_Gather(s, cnt, MPI_UNSIGNED_LONG_LONG, r, cnt, MPI_UNSIGNED_LONG_LONG, root, nrnmpi_comm); +} + extern void nrnmpi_char_gatherv(char* s, int scnt, char* r, int* rcnt, int* rdispl, int root) { MPI_Gatherv(s, scnt, MPI_CHAR, r, rcnt, rdispl, MPI_CHAR, root, nrnmpi_comm); } diff --git a/src/nrnmpi/nrnmpidec.h b/src/nrnmpi/nrnmpidec.h index 34dc84782d..8e732f8dd2 100644 --- a/src/nrnmpi/nrnmpidec.h +++ b/src/nrnmpi/nrnmpidec.h @@ -83,6 +83,7 @@ extern double nrnmpi_mindelay(double maxdel); extern int nrnmpi_int_allmax(int i); extern void nrnmpi_int_gather(int* s, int* r, int cnt, int root); extern void nrnmpi_int_gatherv(int* s, int scnt, int* r, int* rcnt, int* rdispl, int root); +extern void nrnmpi_sizet_gather(size_t* s, size_t* r, int cnt, int root); extern void nrnmpi_char_gatherv(char* s, int scnt, char* r, int* rcnt, int* rdispl, int root); extern void nrnmpi_int_scatter(int* s, int* r, int cnt, int root); extern void nrnmpi_char_scatterv(char* s, int* scnt, int* sdispl, char* r, int rcnt, int root);