Skip to content

Commit

Permalink
Reworked FileLocation class (#2)
Browse files Browse the repository at this point in the history
  • Loading branch information
@henricasanova
henricasanova committed Sep 25, 2019
1 parent a43ef19 commit 12a3d9f
Show file tree
Hide file tree
Showing 11 changed files with 297 additions and 224 deletions.
3 changes: 1 addition & 2 deletions include/wrench/managers/DataMovementManager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,7 @@ namespace wrench {
bool operator==(const CopyRequestSpecs &rhs) const {
return (file == rhs.file) &&
(dst->getStorageService() == rhs.dst->getStorageService()) &&
(dst->getMountPoint() == rhs.dst->getMountPoint()) &&
(dst->getDirectory() == rhs.dst->getDirectory());
(dst->getAbsolutePathAtMountPoint() == rhs.dst->getAbsolutePathAtMountPoint());
}
};

Expand Down
25 changes: 12 additions & 13 deletions include/wrench/services/storage/storage_helpers/FileLocation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,23 +29,18 @@ namespace wrench {
static std::shared_ptr<FileLocation> LOCATION(std::shared_ptr<StorageService> ss);

static std::shared_ptr<FileLocation> LOCATION(std::shared_ptr<StorageService> ss,
std::string mp);

static std::shared_ptr<FileLocation> LOCATION(std::shared_ptr<StorageService> ss,
std::string mp,
std::string dir);
std::string absolute_path);

std::shared_ptr<StorageService> getStorageService();
std::string getMountPoint();
std::string getDirectory();
std::string getAbsolutePathAtMountPoint();

std::string toString();


bool operator==(const std::shared_ptr<FileLocation> &rhs) {
return ((this->getStorageService() == rhs->getStorageService()) and
(this->getMountPoint() == rhs->getMountPoint()) and
(this->getDirectory() == rhs->getDirectory()));
(this->getAbsolutePathAtMountPoint() == rhs->getAbsolutePathAtMountPoint()));
}

/**
Expand All @@ -57,20 +52,24 @@ namespace wrench {
return not FileLocation::operator==(rhs);
}


private:

friend class LogicalFileSystem;

/**
* @brief Constructor
* @param ss: the storage service
* @param mp: the mount point
* @param dir: the directory
* @param ap: the absolute path
*/
FileLocation(std::shared_ptr<StorageService> ss, std::string mp, std::string dir) :
storage_service(ss), mount_point(mp), directory(dir) { }
FileLocation(std::shared_ptr<StorageService> ss, std::string mp, std::string apamp) :
storage_service(ss), mount_point(mp), absolute_path_at_mount_point(apamp) { }

std::shared_ptr<StorageService> storage_service;
std::string mount_point;
std::string directory;
std::string absolute_path_at_mount_point;

static std::string sanitizePath(std::string path);

};

Expand Down
38 changes: 20 additions & 18 deletions include/wrench/simulation/Simulation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,8 @@ namespace wrench {
static double getHostMemoryCapacity(std::string hostname);
static unsigned long getHostNumCores(std::string hostname);
static double getHostFlopRate(std::string hostname);
static bool isHostOn(std::string hostname);
static void turnOnHost(std::string hostname);
static void turnOffHost(std::string hostname);
static bool isLinkOn(std::string linkname);
static void turnOnLink(std::string linkname);
static void turnOffLink(std::string linkname);

void launch();
bool isRunning();


/**
* @brief Method to add a service to the simulation
Expand All @@ -82,14 +74,8 @@ namespace wrench {
return s;
}

void addService(std::shared_ptr<ComputeService> service);
void addService(std::shared_ptr<StorageService> service);
void addService(std::shared_ptr<NetworkProximityService> service);
void addService(std::shared_ptr<WMS> service);
void addService(std::shared_ptr<FileRegistryService> service);

void stageFile(WorkflowFile *file, std::shared_ptr<FileLocation> location);
void stageFiles(std::map<WorkflowFile *, std::shared_ptr<FileLocation>> file_locations);
void stageFile(WorkflowFile *file, std::shared_ptr<StorageService> storage_service, std::string absolute_path);

SimulationOutput &getOutput();

Expand All @@ -99,19 +85,29 @@ namespace wrench {
// double getEnergyTimestamp(const std::string &hostname, bool can_record = false);

// pstate related calls
void setPstate(const std::string &hostname, int pstate);
static int getNumberofPstates(const std::string &hostname);
static int getCurrentPstate(const std::string &hostname);
static double getMinPowerConsumption(const std::string &hostname);
static double getMaxPowerConsumption(const std::string &hostname);
static std::vector<int> getListOfPstates(const std::string &hostname);

void stageFile(WorkflowFile *file, std::shared_ptr<FileLocation> location);


/***********************/
/** \cond DEVELOPER */
/***********************/

static bool isHostOn(std::string hostname);
static void turnOnHost(std::string hostname);
static void turnOffHost(std::string hostname);
static bool isLinkOn(std::string linkname);
static void turnOnLink(std::string linkname);
static void turnOffLink(std::string linkname);

// pstate related calls
void setPstate(const std::string &hostname, int pstate);
static int getCurrentPstate(const std::string &hostname);


std::shared_ptr<ComputeService> startNewService(ComputeService *service);
std::shared_ptr<StorageService> startNewService(StorageService *service);
Expand Down Expand Up @@ -151,8 +147,14 @@ namespace wrench {
std::set<std::shared_ptr<StorageService>> storage_services;

void checkSimulationSetup();
bool isRunning();

void startAllProcesses();
void addService(std::shared_ptr<ComputeService> service);
void addService(std::shared_ptr<StorageService> service);
void addService(std::shared_ptr<NetworkProximityService> service);
void addService(std::shared_ptr<WMS> service);
void addService(std::shared_ptr<FileRegistryService> service);

std::string getWRENCHVersionString() { return WRENCH_VERSION_STRING; }

Expand All @@ -161,7 +163,7 @@ namespace wrench {
unsigned int on_state_change_callback_id;


};
};

};

Expand Down
13 changes: 4 additions & 9 deletions src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -977,8 +977,8 @@ namespace wrench {
try {
StorageService::deleteFile(f,
FileLocation::LOCATION(this->getScratch(),
this->getScratch()->getMountPoint(),
job->getName()),
this->getScratch()->getMountPoint() +
"/" + job->getName()),
nullptr);
} catch (WorkflowExecutionException &e) {
// ignore (perhaps it was never written)
Expand Down Expand Up @@ -1166,8 +1166,7 @@ namespace wrench {
for (auto const &f : this->files_in_scratch[job]) {
StorageService::deleteFile(f,
FileLocation::LOCATION(this->getScratch(),
this->getScratch()->getMountPoint(),
job->getName()),
this->getScratch()->getMountPoint() + job->getName()),
nullptr);
}
this->files_in_scratch[job].clear();
Expand Down Expand Up @@ -1523,11 +1522,7 @@ namespace wrench {
dict,
this->getMessagePayloadValue(
ComputeServiceMessagePayload::RESOURCE_DESCRIPTION_ANSWER_MESSAGE_PAYLOAD));
// try {
S4U_Mailbox::dputMessage(answer_mailbox, answer_message);
// } catch (std::shared_ptr<NetworkError> &cause) {
// return;
// }
}

/**
Expand All @@ -1541,7 +1536,7 @@ namespace wrench {
try {
StorageService::deleteFile(f,
FileLocation::LOCATION(this->getScratch(),
this->getScratch()->getMountPoint(),
this->getScratch()->getMountPoint() +
j.first->getName()));
} catch (WorkflowExecutionException &e) {
throw;
Expand Down
2 changes: 1 addition & 1 deletion src/wrench/services/file_registry/FileRegistryService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ namespace wrench {
for (auto const &l : this->entries[file]) {
if ((l->getStorageService() == location->getStorageService()) &&
(l->getMountPoint() == location->getMountPoint()) &&
(l->getDirectory() == location->getDirectory())) {
(l->getAbsolutePathAtMountPoint() == location->getAbsolutePathAtMountPoint())) {
this->entries[file].erase(l);
return true;
}
Expand Down
4 changes: 2 additions & 2 deletions src/wrench/services/storage/StorageService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ namespace wrench {
*/
void StorageService::stageFile(WorkflowFile *file, std::shared_ptr<FileLocation> location) {

location->getStorageService()->stageFile(file, location->getMountPoint(), location->getDirectory());
location->getStorageService()->stageFile(file, location->getMountPoint(), location->getAbsolutePathAtMountPoint());
}

/**
Expand Down Expand Up @@ -630,7 +630,7 @@ namespace wrench {
*/
std::string StorageService::getMountPoint() {
if (this->hasMultipleMountPoints()) {
throw std::invalid_argument("StorageService::getMountPoint(): The storage service has more than one mount point");
throw std::invalid_argument("StorageService::getAbsolutePath(): The storage service has more than one mount point");
}
return this->file_systems.begin()->first;
}
Expand Down
32 changes: 16 additions & 16 deletions src/wrench/services/storage/simple/SimpleStorageService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ namespace wrench {
} else if (auto msg = std::dynamic_pointer_cast<StorageServiceFileLookupRequestMessage>(message)) {

auto fs = this->file_systems[msg->location->getMountPoint()].get();
bool file_found = fs->isFileInDirectory(msg->file, msg->location->getDirectory());
bool file_found = fs->isFileInDirectory(msg->file, msg->location->getAbsolutePathAtMountPoint());

S4U_Mailbox::dputMessage(msg->answer_mailbox,
new StorageServiceFileLookupAnswerMessage(msg->file, file_found,
Expand Down Expand Up @@ -230,15 +230,15 @@ namespace wrench {
bool SimpleStorageService::processFileWriteRequest(WorkflowFile *file, std::shared_ptr<FileLocation> location,
std::string answer_mailbox, unsigned long buffer_size) {

auto fs = this->file_systems[location->getMountPoint()].get();
auto fs = this->file_systems[location->getAbsolutePathAtMountPoint()].get();

// If the file is not already there, do a capacity check/update
// (If the file is already there, then there will just be an overwrite. Note that
// if the overwrite fails, then the file will disappear, which is expected)


if ((not fs->doesDirectoryExist(location->getDirectory())) or
(not fs->isFileInDirectory(file, location->getDirectory()))) {
if ((not fs->doesDirectoryExist(location->getAbsolutePathAtMountPoint())) or
(not fs->isFileInDirectory(file, location->getAbsolutePathAtMountPoint()))) {

if (not fs->hasEnoughFreeSpace(file->getSize())) {
try {
Expand Down Expand Up @@ -315,10 +315,10 @@ namespace wrench {
// Figure out whether this succeeds or not
std::shared_ptr<FailureCause> failure_cause = nullptr;

auto fs = this->file_systems[location->getMountPoint()].get();
auto fs = this->file_systems[location->getAbsolutePathAtMountPoint()].get();

if ((not fs->doesDirectoryExist(location->getDirectory())) or
(not fs->isFileInDirectory(file, location->getDirectory()))) {
if ((not fs->doesDirectoryExist(location->getAbsolutePathAtMountPoint())) or
(not fs->isFileInDirectory(file, location->getAbsolutePathAtMountPoint()))) {
WRENCH_INFO("Received a a read request for a file I don't have (%s)", location->toString().c_str());
failure_cause = std::shared_ptr<FailureCause>(
new FileNotFound(file, location));
Expand Down Expand Up @@ -378,11 +378,11 @@ namespace wrench {
// if the overwrite fails, then the file will disappear, which is expected)

// File System at the destination
auto fs = this->file_systems[dst_location->getMountPoint()].get();
auto fs = this->file_systems[dst_location->getAbsolutePathAtMountPoint()].get();


if ((not fs->doesDirectoryExist(dst_location->getDirectory())) or
(not fs->isFileInDirectory(file, dst_location->getDirectory()))) {
if ((not fs->doesDirectoryExist(dst_location->getAbsolutePathAtMountPoint())) or
(not fs->isFileInDirectory(file, dst_location->getAbsolutePathAtMountPoint()))) {

if (not fs->hasEnoughFreeSpace(file->getSize())) {

Expand Down Expand Up @@ -484,15 +484,15 @@ namespace wrench {
if (dst_location and (dst_location->getStorageService().get() == this)) {
if (success) {
this->file_systems[dst_location->getMountPoint()]->storeFileInDirectory(
file, dst_location->getDirectory());
file, dst_location->getAbsolutePathAtMountPoint());
// Deal with time stamps
if (start_timestamp != nullptr) {
this->simulation->getOutput().addTimestamp<SimulationTimestampFileCopyCompletion>(
new SimulationTimestampFileCopyCompletion(start_timestamp));
}
} else {
// Process the failure, meaning, just un-decrease the free space
this->file_systems[dst_location->getMountPoint()]->increaseFreeSpace(file->getSize());
this->file_systems[dst_location->getAbsolutePathAtMountPoint()]->increaseFreeSpace(file->getSize());
}
}

Expand Down Expand Up @@ -532,14 +532,14 @@ namespace wrench {
std::string answer_mailbox) {
std::shared_ptr<FailureCause> failure_cause = nullptr;

auto fs = this->file_systems[location->getMountPoint()].get();
auto fs = this->file_systems[location->getAbsolutePathAtMountPoint()].get();

if ((not fs->doesDirectoryExist(location->getDirectory())) or
(not fs->isFileInDirectory(file, location->getDirectory()))) {
if ((not fs->doesDirectoryExist(location->getAbsolutePathAtMountPoint())) or
(not fs->isFileInDirectory(file, location->getAbsolutePathAtMountPoint()))) {
failure_cause = std::shared_ptr<FailureCause>(
new FileNotFound(file, location));
} else {
fs->removeFileFromDirectory(file, location->getDirectory());
fs->removeFileFromDirectory(file, location->getAbsolutePathAtMountPoint());
}

S4U_Mailbox::dputMessage(answer_mailbox,
Expand Down
Loading

0 comments on commit 12a3d9f

Please sign in to comment.