Temp. commit - Added support to gather all of the offsets to write in…

… 'files.dat'

src/coreneuron/io/nrn_filehandler.cpp

@@ -7,6 +7,7 @@
 */
 
 #include <iostream>
+#include <regex>
 #include "coreneuron/io/nrn_filehandler.hpp"
 #include "coreneuron/nrnconf.h"
 
@@ -30,6 +31,7 @@ void FileHandler::open(const std::string& filename, std::ios::openmode mode) {
         std::cerr << "cannot open file '" << filename << "'" << std::endl;
     }
     nrn_assert(F.is_open());
+    current_filename = filename;
     current_mode = mode;
     char version[256];
     if (current_mode & std::ios::in) {
@@ -47,7 +49,7 @@ bool FileHandler::eof() {
         return true;
     }
     int a = F.get();
-    if (F.eof()) {
+    if (F.eof() || (char)a == '\0') {
         return true;
     }
     F.putback(a);

src/nrniv/nrncore_write.cpp

@@ -254,7 +254,7 @@ size_t get_filesize(const char* fname) {
 }
 
 static void part2(const char* path) {
-    std::array<size_t, 5> offsets;
+    std::array<size_t, 4> offsets;
 
     CellGroup* cgs = cellgroups_;
     for (int i = 0; i < nrn_nthread; ++i) {

src/nrniv/nrncore_write/io/nrncore_io.cpp

@@ -165,6 +165,9 @@ std::array<size_t, 2> write_nrnthread(const char* path, NrnThread& nt, CellGroup
         cg.netcon_srcgid = NULL;
     }
 
+    // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+    fputc(0, f);
+
     // Set the second offset inside the file
     offsets[1] = ftell(f);
 
@@ -307,6 +310,9 @@ std::array<size_t, 2> write_nrnthread(const char* path, NrnThread& nt, CellGroup
 
     nrnbbcore_vecplay_write(f, nt);
 
+    // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+    fputc(0, f);
+
     fclose(f);
 
     return offsets;
@@ -375,72 +381,82 @@ static void fgets_no_newline(char* s, int size, FILE* f) {
  *     ...
  *     idN
  */
-void write_nrnthread_task(const char* path, CellGroup* cgs, bool append) {
-    // ids of datasets that will be created
-    std::vector<int> iSend;
-
-    // ignore empty nrnthread (has -1 id)
-    for (int iInt = 0; iInt < nrn_nthread; ++iInt) {
-        if (cgs[iInt].group_id >= 0) {
-            iSend.push_back(cgs[iInt].group_id);
-        }
-    }
+void write_nrnthread_task(const char* path, CellGroup* cgs, bool append, std::vector<size_t> file_offsets) {
+//     // ids of datasets that will be created
+//     std::vector<int> iSend;
+
+//     // ignore empty nrnthread (has -1 id)
+//     for (int iInt = 0; iInt < nrn_nthread; ++iInt) {
+//         if (cgs[iInt].group_id >= 0) {
+//             iSend.push_back(cgs[iInt].group_id);
+//         }
+//     }
+
+//     // receive and displacement buffers for mpi
+//     std::vector<int> iRecv, iDispl;
+
+//     if (nrnmpi_myid == 0) {
+//         iRecv.resize(nrnmpi_numprocs);
+//         iDispl.resize(nrnmpi_numprocs);
+//     }
+
+//     // number of datasets on the current rank
+//     int num_datasets = iSend.size();
+
+// #ifdef NRNMPI
+//     // gather number of datasets from each task
+//     if (nrnmpi_numprocs > 1) {
+//         nrnmpi_int_gather(&num_datasets, begin_ptr(iRecv), 1, 0);
+//     } else {
+//         iRecv[0] = num_datasets;
+//     }
+// #else
+//     iRecv[0] = num_datasets;
+// #endif
+
+//     // total number of datasets across all ranks
+//     int iSumThread = 0;
+
+//     // calculate mpi displacements
+//     if (nrnmpi_myid == 0) {
+//         for (int iInt = 0; iInt < nrnmpi_numprocs; ++iInt) {
+//             iDispl[iInt] = iSumThread;
+//             iSumThread += iRecv[iInt];
+//         }
+//     }
+
+//     // buffer for receiving all dataset ids
+//     std::vector<int> iRecvVec(iSumThread);
+
+// #ifdef NRNMPI
+//     // gather ids into the array with correspondent offsets
+//     if (nrnmpi_numprocs > 1) {
+//         nrnmpi_int_gatherv(begin_ptr(iSend),
+//                            num_datasets,
+//                            begin_ptr(iRecvVec),
+//                            begin_ptr(iRecv),
+//                            begin_ptr(iDispl),
+//                            0);
+//     } else {
+//         for (int iInt = 0; iInt < num_datasets; ++iInt) {
+//             iRecvVec[iInt] = iSend[iInt];
+//         }
+//     }
+// #else
+//     for (int iInt = 0; iInt < num_datasets; ++iInt) {
+//         iRecvVec[iInt] = iSend[iInt];
+//     }
+// #endif
+
+    int num_offsets = file_offsets.size();
+    file_offsets.resize(file_offsets.size() * (nrnmpi_myid == 0 ? nrnmpi_numprocs : 1ULL));
+    nrnmpi_sizet_gather(file_offsets.data(), file_offsets.data(), num_offsets, 0);
+
+
+
+    // <<<<<<<<<<<<<<<<<<<<<<<<< The writing part is not done!! Just collecitng the offsets from all of the ranks, and we have to write that to the file, somehow
 
-    // receive and displacement buffers for mpi
-    std::vector<int> iRecv, iDispl;
-
-    if (nrnmpi_myid == 0) {
-        iRecv.resize(nrnmpi_numprocs);
-        iDispl.resize(nrnmpi_numprocs);
-    }
 
-    // number of datasets on the current rank
-    int num_datasets = iSend.size();
-
-#ifdef NRNMPI
-    // gather number of datasets from each task
-    if (nrnmpi_numprocs > 1) {
-        nrnmpi_int_gather(&num_datasets, begin_ptr(iRecv), 1, 0);
-    } else {
-        iRecv[0] = num_datasets;
-    }
-#else
-    iRecv[0] = num_datasets;
-#endif
-
-    // total number of datasets across all ranks
-    int iSumThread = 0;
-
-    // calculate mpi displacements
-    if (nrnmpi_myid == 0) {
-        for (int iInt = 0; iInt < nrnmpi_numprocs; ++iInt) {
-            iDispl[iInt] = iSumThread;
-            iSumThread += iRecv[iInt];
-        }
-    }
-
-    // buffer for receiving all dataset ids
-    std::vector<int> iRecvVec(iSumThread);
-
-#ifdef NRNMPI
-    // gather ids into the array with correspondent offsets
-    if (nrnmpi_numprocs > 1) {
-        nrnmpi_int_gatherv(begin_ptr(iSend),
-                           num_datasets,
-                           begin_ptr(iRecvVec),
-                           begin_ptr(iRecv),
-                           begin_ptr(iDispl),
-                           0);
-    } else {
-        for (int iInt = 0; iInt < num_datasets; ++iInt) {
-            iRecvVec[iInt] = iSend[iInt];
-        }
-    }
-#else
-    for (int iInt = 0; iInt < num_datasets; ++iInt) {
-        iRecvVec[iInt] = iSend[iInt];
-    }
-#endif
 
     /// Writing the file with task, correspondent number of threads and list of correspondent first
     /// gids
@@ -584,6 +600,10 @@ size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo)
             }
         }
     }
+
+    // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+    fputc(0, f);
+
     fclose(f);
 
     return offset;

src/nrniv/nrncore_write/io/nrncore_io.h

@@ -39,7 +39,7 @@ struct Memb_list;
 using bbcore_write_t =
     void (*)(double*, int*, int*, int*, Memb_list*, std::size_t, Datum*, Datum*, NrnThread*);
 
-void write_nrnthread_task(const char*, CellGroup* cgs, bool append);
+void write_nrnthread_task(const char*, CellGroup* cgs, bool append, std::vector<size_t> file_offsets);
 void nrnbbcore_vecplay_write(FILE* f, NrnThread& nt);
 
 size_t nrn_write_mapping_info(const char* path, int gid, NrnMappingInfo& minfo);

src/nrniv/partrans.cpp

@@ -996,7 +996,7 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) {
     }
 
     // print the files
-    for (int tid = 0; tid < nrn_nthread; ++tid) {
+    for (int tid = 0; tid < nrn_nthread; ++tid) { // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< Sometimes more than one has data!!
         auto& g = gi[tid];
 
         if (g.src_sid.empty() && g.tar_sid.empty()) {  // no file
@@ -1034,6 +1034,9 @@ size_t nrnbbcore_gap_write(const char* path, int* group_ids) {
             CHKPNT fwrite(g.tar_index.data(), ntar, sizeof(int), f);
         }
 
+        // Mark the end of the file with '\0' <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+        fputc(0, f);
+
         fclose(f);
     }
 

src/nrnmpi/mpispike.cpp

@@ -396,6 +396,10 @@ extern void nrnmpi_int_gatherv(int* s, int scnt, int* r, int* rcnt, int* rdispl,
     MPI_Gatherv(s, scnt, MPI_INT, r, rcnt, rdispl, MPI_INT, root, nrnmpi_comm);
 }
 
+extern void nrnmpi_sizet_gather(size_t* s, size_t* r, int cnt, int root) {
+    MPI_Gather(s, cnt, MPI_UNSIGNED_LONG_LONG, r, cnt, MPI_UNSIGNED_LONG_LONG, root, nrnmpi_comm);
+}
+
 extern void nrnmpi_char_gatherv(char* s, int scnt, char* r, int* rcnt, int* rdispl, int root) {
     MPI_Gatherv(s, scnt, MPI_CHAR, r, rcnt, rdispl, MPI_CHAR, root, nrnmpi_comm);
 }