Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve CoreNEURON file transfer mode #2525

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ virtualenv
docs/_build
docs/_generated
.vscode
src/nrnoc/hh.mod
26 changes: 24 additions & 2 deletions src/coreneuron/io/nrn_filehandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@
*/

#include <iostream>
#include <filesystem>
#include "coreneuron/io/nrn_filehandler.hpp"
#include "coreneuron/nrnconf.h"
#include "coreneuron/mpi/nrnmpi.h"
#include "coreneuron/mpi/core/nrnmpi.hpp"

namespace coreneuron {
FileHandler::FileHandler(const std::string& filename)
Expand All @@ -22,10 +25,29 @@ bool FileHandler::file_exist(const std::string& filename) {
return (stat(filename.c_str(), &buffer) == 0);
}

void FileHandler::open(const std::string& filename, std::ios::openmode mode) {
std::string FileHandler::get_rank_fname(const char* basepath, bool create_folder) {
// TODO: Change this for equivalent MPI functions to get the node ID
std::string nodepath = "";
if (const char* node_id = std::getenv("SLURM_NODEID")) {
const int factor = 20;
nodepath = std::to_string(std::atoi(node_id) / factor) + "/" + node_id;
} else if (const char* hostname = std::getenv("HOSTNAME")) {
nodepath = hostname;
}
// Create subfolder for the rank, based on the node
std::string path = std::string(basepath) + "/" + nodepath;
if (create_folder && !std::filesystem::exists(path)) {
std::filesystem::create_directories(path);
}

return (path + "/" + std::to_string(nrnmpi_myid) + ".dat");
}

void FileHandler::open(const std::string& filename, size_t offset, std::ios::openmode mode) {
nrn_assert((mode & (std::ios::in | std::ios::out)));
close();
F.open(filename, mode | std::ios::binary);
F.seekg(offset, std::ios::beg);
if (!F.is_open()) {
std::cerr << "cannot open file '" << filename << "'" << std::endl;
}
Expand All @@ -47,7 +69,7 @@ bool FileHandler::eof() {
return true;
}
int a = F.get();
if (F.eof()) {
if (F.eof() || (char) a == '\0') {
return true;
}
F.putback(a);
Expand Down
6 changes: 5 additions & 1 deletion src/coreneuron/io/nrn_filehandler.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ class FileHandler {
explicit FileHandler(const std::string& filename);

/** Preserving chkpnt state, move to a new file. */
void open(const std::string& filename, std::ios::openmode mode = std::ios::in);
void open(const std::string& filename,
size_t offset = 0,
std::ios::openmode mode = std::ios::in);

/** Is the file not open */
bool fail() const {
Expand All @@ -65,6 +67,8 @@ class FileHandler {

static bool file_exist(const std::string& filename);

static std::string get_rank_fname(const char* basepath, bool create_folder = true);

/** nothing more to read */
bool eof();

Expand Down
20 changes: 18 additions & 2 deletions src/coreneuron/io/nrn_setup.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,11 @@ std::vector<int*> nrnthreads_netcon_srcgid;
std::vector<std::vector<int>> nrnthreads_netcon_negsrcgid_tid;

/* read files.dat file and distribute cellgroups to all mpi ranks */
void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
void nrn_read_filesdat(int& ngrp,
int*& grp,
int& num_offsets,
size_t*& file_offsets,
const char* filesdat) {
patstimtype = nrn_get_mechtype("PatternStim");
if (corenrn_embedded) {
ngrp = corenrn_embedded_nthread;
Expand Down Expand Up @@ -200,19 +204,27 @@ void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
}
}

nrn_assert(fscanf(fp, "%d\n", &num_offsets) == 1);

if (nrnmpi_numprocs > iNumFiles && nrnmpi_myid == 0) {
printf(
"Info : The number of input datasets are less than ranks, some ranks will be idle!\n");
}

ngrp = 0;
grp = new int[iNumFiles / nrnmpi_numprocs + 1];
file_offsets = new size_t[num_offsets * (iNumFiles / nrnmpi_numprocs + 1)];

// irerate over gids in files.dat
size_t offsets_idx = 0;
for (int iNum = 0; iNum < iNumFiles; ++iNum) {
int iFile;

nrn_assert(fscanf(fp, "%d\n", &iFile) == 1);
for (int i = 0; i < num_offsets; i++, offsets_idx++) {
nrn_assert(fscanf(fp, "%zu\n", &file_offsets[ngrp * num_offsets + i]) == 1);
}

if ((iNum % nrnmpi_numprocs) == nrnmpi_myid) {
grp[ngrp] = iFile;
ngrp++;
Expand Down Expand Up @@ -409,9 +421,13 @@ void nrn_setup(const char* filesdat,

int ngroup;
int* gidgroups;
nrn_read_filesdat(ngroup, gidgroups, filesdat);
int num_offsets;
size_t* file_offsets;
nrn_read_filesdat(ngroup, gidgroups, num_offsets, file_offsets, filesdat);
UserParams userParams(ngroup,
gidgroups,
num_offsets,
file_offsets,
datpath,
strlen(restore_path) == 0 ? datpath : restore_path,
checkPoints);
Expand Down
10 changes: 6 additions & 4 deletions src/coreneuron/io/nrn_setup.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

#pragma once

#include <filesystem>
#include <string>
#include "coreneuron/sim/multicore.hpp"
#include "coreneuron/io/nrn_filehandler.hpp"
Expand All @@ -16,6 +17,7 @@
#include "coreneuron/io/mem_layout_util.hpp"
#include "coreneuron/io/nrn_checkpoint.hpp"


namespace coreneuron {
void read_phase1(NrnThread& nt, UserParams& userParams);
void read_phase2(NrnThread& nt, UserParams& userParams);
Expand Down Expand Up @@ -100,6 +102,7 @@ inline void read_phase_aux<gap>(NrnThread& nt, UserParams& userParams) {
read_phasegap(nt, userParams);
}


/// Reading phase wrapper for each neuron group.
template <phase P>
inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memory_transfer) {
Expand All @@ -114,9 +117,8 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo
data_dir = userParams.restore_path;
}

std::string fname = std::string(data_dir) + "/" +
std::to_string(userParams.gidgroups[i]) + "_" + getPhaseName<P>() +
".dat";
size_t file_offset = userParams.file_offsets[i * userParams.num_offsets + P - 1];
const auto& fname = FileHandler::get_rank_fname(data_dir);

// Avoid trying to open the gid_gap.dat file if it doesn't exist when there are no
// gap junctions in this gid.
Expand All @@ -128,7 +130,7 @@ inline void* phase_wrapper_w(NrnThread* nt, UserParams& userParams, bool in_memo
userParams.file_reader[i].close();
} else {
// if no file failed to open or not opened at all
userParams.file_reader[i].open(fname);
userParams.file_reader[i].open(fname, file_offset);
}
}
read_phase_aux<P>(*nt, userParams);
Expand Down
8 changes: 8 additions & 0 deletions src/coreneuron/io/user_params.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@ class CheckPoints;
struct UserParams {
UserParams(int ngroup_,
int* gidgroups_,
int num_offsets_,
size_t* file_offsets_,
const char* path_,
const char* restore_path_,
CheckPoints& checkPoints_)
: ngroup(ngroup_)
, gidgroups(gidgroups_)
, num_offsets(num_offsets_)
, file_offsets(file_offsets_)
, path(path_)
, restore_path(restore_path_)
, file_reader(ngroup_)
Expand All @@ -33,6 +37,10 @@ struct UserParams {
const int ngroup;
/// Array of cell group numbers (indices)
const int* const gidgroups;
/// Number of offsets per cell group
const int num_offsets;
/// Array of offsets inside file with cell groups (indices)
const size_t* const file_offsets;
/// path to dataset file
const char* const path;
/// Dataset path from where simulation is being restored
Expand Down
14 changes: 10 additions & 4 deletions src/nrniv/nrncore_write.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -219,16 +219,22 @@ static part1_ret part1() {
}

static void part2(const char* path) {
std::vector<size_t> offsets(4);

CellGroup* cgs = cellgroups_;
for (int i = 0; i < nrn_nthread; ++i) {
chkpnt = 0;
write_nrnthread(path, nrn_threads[i], cgs[i]);
const auto& nrnthread_offsets = write_nrnthread(path, nrn_threads[i], cgs[i]);
if (nrnthread_offsets[1] > 0) {
offsets[0] = nrnthread_offsets[0];
offsets[1] = nrnthread_offsets[1];
}
}

/** write mapping information */
if (mapinfo.size()) {
int gid = cgs[0].group_id;
nrn_write_mapping_info(path, gid, mapinfo);
offsets[2] = nrn_write_mapping_info(path, gid, mapinfo);
mapinfo.clear();
}

Expand All @@ -238,7 +244,7 @@ static void part2(const char* path) {
for (int i = 0; i < nrn_nthread; ++i) {
group_ids[i] = cgs[i].group_id;
}
nrnbbcore_gap_write(path, group_ids);
offsets[3] = nrnbbcore_gap_write(path, group_ids);
delete[] group_ids;
}

Expand All @@ -262,7 +268,7 @@ static void part2(const char* path) {
hoc_execerror("Second arg must be Vector or double.", NULL);
}
}
write_nrnthread_task(path, cgs, append);
write_nrnthread_task(path, cgs, append, offsets);
}

part2_clean();
Expand Down
Loading
Loading