From f100cc9e557fe8b3b0f766f55ea00bf36e794e2c Mon Sep 17 00:00:00 2001 From: Sergio Date: Mon, 25 Sep 2023 18:15:39 +0200 Subject: [PATCH] Temp. commit - Added support to gather all of the offsets to write in 'files.dat' --- src/coreneuron/io/nrn_filehandler.cpp | 4 +- src/nrniv/nrncore_write.cpp | 2 +- src/nrniv/nrncore_write/io/nrncore_io.cpp | 148 ++++++++++++---------- src/nrniv/nrncore_write/io/nrncore_io.h | 2 +- src/nrniv/partrans.cpp | 5 +- src/nrnmpi/mpispike.cpp | 4 + 6 files changed, 97 insertions(+), 68 deletions(-) diff --git a/src/coreneuron/io/nrn_filehandler.cpp b/src/coreneuron/io/nrn_filehandler.cpp index 40171aec7e..91b3bdc475 100644 --- a/src/coreneuron/io/nrn_filehandler.cpp +++ b/src/coreneuron/io/nrn_filehandler.cpp @@ -7,6 +7,7 @@ */ #include +#include #include "coreneuron/io/nrn_filehandler.hpp" #include "coreneuron/nrnconf.h" @@ -30,6 +31,7 @@ void FileHandler::open(const std::string& filename, std::ios::openmode mode) { std::cerr << "cannot open file '" << filename << "'" << std::endl; } nrn_assert(F.is_open()); + current_filename = filename; current_mode = mode; char version[256]; if (current_mode & std::ios::in) { @@ -47,7 +49,7 @@ bool FileHandler::eof() { return true; } int a = F.get(); - if (F.eof()) { + if (F.eof() || (char)a == '\0') { return true; } F.putback(a); diff --git a/src/nrniv/nrncore_write.cpp b/src/nrniv/nrncore_write.cpp index 71d006ea1b..8c08d9df6b 100644 --- a/src/nrniv/nrncore_write.cpp +++ b/src/nrniv/nrncore_write.cpp @@ -254,7 +254,7 @@ size_t get_filesize(const char* fname) { } static void part2(const char* path) { - std::array offsets; + std::array offsets; CellGroup* cgs = cellgroups_; for (int i = 0; i < nrn_nthread; ++i) { diff --git a/src/nrniv/nrncore_write/io/nrncore_io.cpp b/src/nrniv/nrncore_write/io/nrncore_io.cpp index 70dfe87247..16242c2141 100644 --- a/src/nrniv/nrncore_write/io/nrncore_io.cpp +++ b/src/nrniv/nrncore_write/io/nrncore_io.cpp @@ -165,6 +165,9 @@ std::array write_nrnthread(const char* path, NrnThread& nt, CellGroup cg.netcon_srcgid = NULL; } + // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + fputc(0, f); + // Set the second offset inside the file offsets[1] = ftell(f); @@ -307,6 +310,9 @@ std::array write_nrnthread(const char* path, NrnThread& nt, CellGroup nrnbbcore_vecplay_write(f, nt); + // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + fputc(0, f); + fclose(f); return offsets; @@ -375,72 +381,82 @@ static void fgets_no_newline(char* s, int size, FILE* f) { * ... * idN */ -void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) { - // ids of datasets that will be created - std::vector iSend; - - // ignore empty nrnthread (has -1 id) - for (int iInt = 0; iInt < nrn_nthread; ++iInt) { - if (cgs[iInt].group_id >= 0) { - iSend.push_back(cgs[iInt].group_id); - } - } +void write_nrnthread_task(const char* path, CellGroup* cgs, bool append, std::vector file_offsets) { +// // ids of datasets that will be created +// std::vector iSend; + +// // ignore empty nrnthread (has -1 id) +// for (int iInt = 0; iInt < nrn_nthread; ++iInt) { +// if (cgs[iInt].group_id >= 0) { +// iSend.push_back(cgs[iInt].group_id); +// } +// } + +// // receive and displacement buffers for mpi +// std::vector iRecv, iDispl; + +// if (nrnmpi_myid == 0) { +// iRecv.resize(nrnmpi_numprocs); +// iDispl.resize(nrnmpi_numprocs); +// } + +// // number of datasets on the current rank +// int num_datasets = iSend.size(); + +// #ifdef NRNMPI +// // gather number of datasets from each task +// if (nrnmpi_numprocs > 1) { +// nrnmpi_int_gather(&num_datasets, begin_ptr(iRecv), 1, 0); +// } else { +// iRecv[0] = num_datasets; +// } +// #else +// iRecv[0] = num_datasets; +// #endif + +// // total number of datasets across all ranks +// int iSumThread = 0; + +// // calculate mpi displacements +// if (nrnmpi_myid == 0) { +// for (int iInt = 0; iInt < nrnmpi_numprocs; ++iInt) { +// iDispl[iInt] = iSumThread; +// iSumThread += iRecv[iInt]; +// } +// } + +// // buffer for receiving all dataset ids +// std::vector iRecvVec(iSumThread); + +// #ifdef NRNMPI +// // gather ids into the array with correspondent offsets +// if (nrnmpi_numprocs > 1) { +// nrnmpi_int_gatherv(begin_ptr(iSend), +// num_datasets, +// begin_ptr(iRecvVec), +// begin_ptr(iRecv), +// begin_ptr(iDispl), +// 0); +// } else { +// for (int iInt = 0; iInt < num_datasets; ++iInt) { +// iRecvVec[iInt] = iSend[iInt]; +// } +// } +// #else +// for (int iInt = 0; iInt < num_datasets; ++iInt) { +// iRecvVec[iInt] = iSend[iInt]; +// } +// #endif + + int num_offsets = file_offsets.size(); + file_offsets.resize(file_offsets.size() * (nrnmpi_myid == 0 ? nrnmpi_numprocs : 1ULL)); + nrnmpi_sizet_gather(file_offsets.data(), file_offsets.data(), num_offsets, 0); + + + + // <<<<<<<<<<<<<<<<<<<<<<<<< The writing part is not done!! Just collecitng the offsets from all of the ranks, and we have to write that to the file, somehow - // receive and displacement buffers for mpi - std::vector iRecv, iDispl; - - if (nrnmpi_myid == 0) { - iRecv.resize(nrnmpi_numprocs); - iDispl.resize(nrnmpi_numprocs); - } - // number of datasets on the current rank - int num_datasets = iSend.size(); - -#ifdef NRNMPI - // gather number of datasets from each task - if (nrnmpi_numprocs > 1) { - nrnmpi_int_gather(&num_datasets, begin_ptr(iRecv), 1, 0); - } else { - iRecv[0] = num_datasets; - } -#else - iRecv[0] = num_datasets; -#endif - - // total number of datasets across all ranks - int iSumThread = 0; - - // calculate mpi displacements - if (nrnmpi_myid == 0) { - for (int iInt = 0; iInt < nrnmpi_numprocs; ++iInt) { - iDispl[iInt] = iSumThread; - iSumThread += iRecv[iInt]; - } - } - - // buffer for receiving all dataset ids - std::vector iRecvVec(iSumThread); - -#ifdef NRNMPI - // gather ids into the array with correspondent offsets - if (nrnmpi_numprocs > 1) { - nrnmpi_int_gatherv(begin_ptr(iSend), - num_datasets, - begin_ptr(iRecvVec), - begin_ptr(iRecv), - begin_ptr(iDispl), - 0); - } else { - for (int iInt = 0; iInt < num_datasets; ++iInt) { - iRecvVec[iInt] = iSend[iInt]; - } - } -#else - for (int iInt = 0; iInt < num_datasets; ++iInt) { - iRecvVec[iInt] = iSend[iInt]; - } -#endif /// Writing the file with task, correspondent number of threads and list of correspondent first /// gids @@ -584,6 +600,10 @@ size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo) } } } + + // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + fputc(0, f); + fclose(f); return offset; diff --git a/src/nrniv/nrncore_write/io/nrncore_io.h b/src/nrniv/nrncore_write/io/nrncore_io.h index 4c247f755e..a4976d214c 100644 --- a/src/nrniv/nrncore_write/io/nrncore_io.h +++ b/src/nrniv/nrncore_write/io/nrncore_io.h @@ -39,7 +39,7 @@ struct Memb_list; using bbcore_write_t = void (*)(double*, int*, int*, int*, Memb_list*, std::size_t, Datum*, Datum*, NrnThread*); -void write_nrnthread_task(const char*, CellGroup* cgs, bool append); +void write_nrnthread_task(const char*, CellGroup* cgs, bool append, std::vector file_offsets); void nrnbbcore_vecplay_write(FILE* f, NrnThread& nt); size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo); diff --git a/src/nrniv/partrans.cpp b/src/nrniv/partrans.cpp index b28c951e03..4dc603b421 100644 --- a/src/nrniv/partrans.cpp +++ b/src/nrniv/partrans.cpp @@ -996,7 +996,7 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) { } // print the files - for (int tid = 0; tid < nrn_nthread; ++tid) { + for (int tid = 0; tid < nrn_nthread; ++tid) { // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< Sometimes more than one has data!! auto& g = gi[tid]; if (g.src_sid.empty() && g.tar_sid.empty()) { // no file @@ -1034,6 +1034,9 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) { CHKPNT fwrite(g.tar_index.data(), ntar, sizeof(int), f); } + // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + fputc(0, f); + fclose(f); } diff --git a/src/nrnmpi/mpispike.cpp b/src/nrnmpi/mpispike.cpp index 317f27f7b9..a717be2105 100644 --- a/src/nrnmpi/mpispike.cpp +++ b/src/nrnmpi/mpispike.cpp @@ -396,6 +396,10 @@ extern void nrnmpi_int_gatherv(int* s, int scnt, int* r, int* rcnt, int* rdispl, MPI_Gatherv(s, scnt, MPI_INT, r, rcnt, rdispl, MPI_INT, root, nrnmpi_comm); } +extern void nrnmpi_sizet_gather(size_t* s, size_t* r, int cnt, int root) { + MPI_Gather(s, cnt, MPI_UNSIGNED_LONG_LONG, r, cnt, MPI_UNSIGNED_LONG_LONG, root, nrnmpi_comm); +} + extern void nrnmpi_char_gatherv(char* s, int scnt, char* r, int* rcnt, int* rdispl, int root) { MPI_Gatherv(s, scnt, MPI_CHAR, r, rcnt, rdispl, MPI_CHAR, root, nrnmpi_comm); }