From 531b6eb5c6914ea91e578465139eb82e7c344729 Mon Sep 17 00:00:00 2001 From: henricasanova Date: Tue, 1 Oct 2019 20:59:02 -1000 Subject: [PATCH] (#2) tmp commit --- .../wrench/services/compute/ComputeService.h | 13 ++-- .../storage/simple/SimpleStorageService.h | 2 + .../storage/storage_helpers/FileLocation.h | 2 + .../storage_helpers/FileTransferThread.h | 8 +++ .../wrench/simgrid_S4U_util/S4U_Simulation.h | 2 +- .../bare_metal/BareMetalComputeService.cpp | 1 - .../compute/htcondor/HTCondorService.cpp | 2 +- .../work_unit_executor/WorkunitExecutor.cpp | 3 + .../services/storage/StorageService.cpp | 25 +++++++ .../storage/StorageServiceMessage.cpp | 4 +- .../services/storage/StorageServiceMessage.h | 12 ++++ .../storage/simple/SimpleStorageService.cpp | 39 ++++++++--- .../storage/storage_helpers/FileLocation.cpp | 51 ++++++++++---- .../storage_helpers/FileTransferThread.cpp | 34 +++++++++- .../FileTransferThreadMessage.h | 13 +++- .../storage_helpers/LocicalFileSystem.cpp | 66 +++++++++++-------- .../storage_helpers/LogicalFileSystem.h | 5 +- .../simgrid_S4U_util/S4U_Simulation.cpp | 2 +- .../BareMetalComputeServiceOneTaskTest.cpp | 42 ++++++------ ...areMetalComputeServiceTestStandardJobs.cpp | 18 ++--- test/compute_services/HTCondorServiceTest.cpp | 4 +- test/compute_services/ScratchSpaceTest.cpp | 18 ++--- .../VirtualizedClusterServiceTest.cpp | 14 ++-- test/wms/MultipleWMSTest.cpp | 2 +- 24 files changed, 273 insertions(+), 109 deletions(-) diff --git a/include/wrench/services/compute/ComputeService.h b/include/wrench/services/compute/ComputeService.h index bf7583acd2..841d3c1fc4 100644 --- a/include/wrench/services/compute/ComputeService.h +++ b/include/wrench/services/compute/ComputeService.h @@ -39,12 +39,12 @@ namespace wrench { /***********************/ friend class StandardJobExecutorTest; + friend class Simulation; /***********************/ /** \endcond **/ /***********************/ - friend class Simulation; public: @@ -98,7 +98,6 @@ namespace wrench { double getFreeScratchSpaceSize(); - std::shared_ptr getScratch(); /***********************/ /** \endcond **/ @@ -109,18 +108,18 @@ namespace wrench { /***********************/ - /** + /** * @brief Method to submit a standard job to the service - * + * * @param job: The job being submitted * @param service_specific_arguments: the set of service-specific arguments */ virtual void submitStandardJob(StandardJob *job, std::map &service_specific_arguments) = 0; - /** + /** * @brief Method to submit a pilot job to the service - * + * * @param job: The job being submitted * @param service_specific_arguments: the set of service-specific arguments */ @@ -146,6 +145,8 @@ namespace wrench { protected: + std::shared_ptr getScratch(); + ComputeService(const std::string &hostname, std::string service_name, std::string mailbox_name_prefix, diff --git a/include/wrench/services/storage/simple/SimpleStorageService.h b/include/wrench/services/storage/simple/SimpleStorageService.h index f41247f859..62c465dd07 100644 --- a/include/wrench/services/storage/simple/SimpleStorageService.h +++ b/include/wrench/services/storage/simple/SimpleStorageService.h @@ -111,6 +111,8 @@ namespace wrench { std::shared_ptr dst_location, bool success, std::shared_ptr failure_cause, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, SimulationTimestampFileCopyStart *start_timestamp); diff --git a/include/wrench/services/storage/storage_helpers/FileLocation.h b/include/wrench/services/storage/storage_helpers/FileLocation.h index bf94bc7bd0..706f8b80b4 100644 --- a/include/wrench/services/storage/storage_helpers/FileLocation.h +++ b/include/wrench/services/storage/storage_helpers/FileLocation.h @@ -25,6 +25,7 @@ namespace wrench { public: + static std::shared_ptr SCRATCH; static std::shared_ptr LOCATION(std::shared_ptr ss); static std::shared_ptr LOCATION(std::shared_ptr ss, @@ -33,6 +34,7 @@ namespace wrench { std::shared_ptr getStorageService(); std::string getMountPoint(); std::string getAbsolutePathAtMountPoint(); + std::string getFullAbsolutePath(); std::string toString(); diff --git a/include/wrench/services/storage/storage_helpers/FileTransferThread.h b/include/wrench/services/storage/storage_helpers/FileTransferThread.h index befd76d56f..2b5ff09478 100644 --- a/include/wrench/services/storage/storage_helpers/FileTransferThread.h +++ b/include/wrench/services/storage/storage_helpers/FileTransferThread.h @@ -34,6 +34,8 @@ namespace wrench { WorkflowFile *file, std::string src_mailbox, std::shared_ptr dst_location, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp = nullptr); @@ -43,6 +45,8 @@ namespace wrench { WorkflowFile *file, std::shared_ptr src_location, std::string dst_mailbox, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp = nullptr); @@ -52,6 +56,8 @@ namespace wrench { WorkflowFile *file, std::shared_ptr src_location, std::shared_ptr dsg_location, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp = nullptr); @@ -73,6 +79,8 @@ namespace wrench { std::string dst_mailbox; std::shared_ptr dst_location; + std::string answer_mailbox_if_read; + std::string answer_mailbox_if_write; std::string answer_mailbox_if_copy; unsigned long buffer_size; SimulationTimestampFileCopyStart *start_timestamp; diff --git a/include/wrench/simgrid_S4U_util/S4U_Simulation.h b/include/wrench/simgrid_S4U_util/S4U_Simulation.h index 8106c1d019..30de931a7c 100644 --- a/include/wrench/simgrid_S4U_util/S4U_Simulation.h +++ b/include/wrench/simgrid_S4U_util/S4U_Simulation.h @@ -56,7 +56,7 @@ namespace wrench { static void readFromDisk(double num_bytes, std::string hostname, std::string absolute_path); static double getDiskCapacity(std::string hostname, std::string mount_point); std::set getDisks(std::string hostname); - static bool hostHasDisk(std::string hostname, std::string mount_point); + static bool hostHasMountPoint(std::string hostname, std::string mount_point); static void yield(); static std::string getHostProperty(std::string hostname, std::string property_name); diff --git a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp index 74ecd610ae..4bb8afe021 100644 --- a/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp +++ b/src/wrench/services/compute/bare_metal/BareMetalComputeService.cpp @@ -43,7 +43,6 @@ namespace wrench { * @brief Destructor */ BareMetalComputeService::~BareMetalComputeService() { - WRENCH_INFO("IN DESTRUCTOR"); this->default_property_values.clear(); } diff --git a/src/wrench/services/compute/htcondor/HTCondorService.cpp b/src/wrench/services/compute/htcondor/HTCondorService.cpp index 013b4db958..38c506052a 100644 --- a/src/wrench/services/compute/htcondor/HTCondorService.cpp +++ b/src/wrench/services/compute/htcondor/HTCondorService.cpp @@ -82,7 +82,7 @@ namespace wrench { this->mailbox_name, new ComputeServiceSubmitStandardJobRequestMessage( answer_mailbox, job, service_specific_args, - this->getMessagePayloadValue( + this->SgetMessagePayloadValue( HTCondorServiceMessagePayload::SUBMIT_STANDARD_JOB_REQUEST_MESSAGE_PAYLOAD))); } catch (std::shared_ptr &cause) { throw WorkflowExecutionException(cause); diff --git a/src/wrench/services/compute/work_unit_executor/WorkunitExecutor.cpp b/src/wrench/services/compute/work_unit_executor/WorkunitExecutor.cpp index c8e0293aaf..afafcc3dcc 100644 --- a/src/wrench/services/compute/work_unit_executor/WorkunitExecutor.cpp +++ b/src/wrench/services/compute/work_unit_executor/WorkunitExecutor.cpp @@ -327,6 +327,9 @@ namespace wrench { if (work->file_locations.find(f) != work->file_locations.end()) { files_to_read[f] = work->file_locations[f]; } else { + if (this->scratch_space == nullptr) { + WRENCH_INFO("DUCK!!"); + } files_to_read[f] = FileLocation::LOCATION(this->scratch_space, job->getName()); this->files_stored_in_scratch.insert(f); } diff --git a/src/wrench/services/storage/StorageService.cpp b/src/wrench/services/storage/StorageService.cpp index 68e57e2a25..594266a386 100644 --- a/src/wrench/services/storage/StorageService.cpp +++ b/src/wrench/services/storage/StorageService.cpp @@ -287,6 +287,18 @@ namespace wrench { file_content_message->getName() + "] message!"); } } + + //Waiting for the final ack + try { + message = S4U_Mailbox::getMessage(answer_mailbox, storage_service->network_timeout); + } catch (std::shared_ptr &cause) { + throw WorkflowExecutionException(cause); + } + if (not std::dynamic_pointer_cast(message)) { + throw std::runtime_error("StorageService::writeFile(): Received an unexpected [" + + message->getName() + "] message!"); + } + } } else { @@ -360,8 +372,21 @@ namespace wrench { } catch (std::shared_ptr &cause) { throw WorkflowExecutionException(cause); } + + //Waiting for the final ack + + try { + message = S4U_Mailbox::getMessage(answer_mailbox, storage_service->network_timeout); + } catch (std::shared_ptr &cause) { + throw WorkflowExecutionException(cause); + } + if (not std::dynamic_pointer_cast(message)) { + throw std::runtime_error("StorageService::writeFile(): Received an unexpected [" + + message->getName() + "] message!"); + } } + } else { throw std::runtime_error("StorageService::writeFile(): Received an unexpected [" + message->getName() + "] message!"); diff --git a/src/wrench/services/storage/StorageServiceMessage.cpp b/src/wrench/services/storage/StorageServiceMessage.cpp index d061e1d812..6a14bf26e7 100644 --- a/src/wrench/services/storage/StorageServiceMessage.cpp +++ b/src/wrench/services/storage/StorageServiceMessage.cpp @@ -289,7 +289,9 @@ namespace wrench { std::string data_write_mailbox_name, double payload) : StorageServiceMessage( "FILE_WRITE_ANSWER", payload) { - if ((file == nullptr) || (location == nullptr) || (data_write_mailbox_name == "") || + if ((file == nullptr) || (location == nullptr) || + (success && (data_write_mailbox_name.empty())) || + (!success && (!data_write_mailbox_name.empty())) || (success && (failure_cause != nullptr)) || (!success && (failure_cause == nullptr))) { throw std::invalid_argument( "StorageServiceFileWriteAnswerMessage::StorageServiceFileWriteAnswerMessage(): Invalid arguments"); diff --git a/src/wrench/services/storage/StorageServiceMessage.h b/src/wrench/services/storage/StorageServiceMessage.h index 492dbe42e4..e81a314998 100644 --- a/src/wrench/services/storage/StorageServiceMessage.h +++ b/src/wrench/services/storage/StorageServiceMessage.h @@ -182,6 +182,8 @@ namespace wrench { }; + + /** * @brief A message sent to a StorageService to write a file */ @@ -286,6 +288,16 @@ namespace wrench { bool last_chunk; }; + + /** + * @brief A message sent by a StorageService as an ack + */ + class StorageServiceAckMessage : public StorageServiceMessage { + public: + StorageServiceAckMessage() : StorageServiceMessage("ACK",0) {} + }; + + /***********************/ /** \endcond */ /***********************/ diff --git a/src/wrench/services/storage/simple/SimpleStorageService.cpp b/src/wrench/services/storage/simple/SimpleStorageService.cpp index 052687a6af..f50f11691b 100644 --- a/src/wrench/services/storage/simple/SimpleStorageService.cpp +++ b/src/wrench/services/storage/simple/SimpleStorageService.cpp @@ -114,7 +114,6 @@ namespace wrench { for (auto const &fs : this->file_systems) { message += "\n - " + fs.first + ":" + std::to_string(fs.second->getTotalCapacity()/(1000*1000*1000)) + "GB"; } - WRENCH_INFO("%s", message.c_str()); /** Main loop **/ @@ -205,6 +204,7 @@ namespace wrench { msg->answer_mailbox, msg->start_timestamp); } else if (auto msg = std::dynamic_pointer_cast(message)) { + return processFileTransferThreadNotification( msg->file_transfer_thread, msg->file, @@ -214,6 +214,8 @@ namespace wrench { msg->dst_location, msg->success, msg->failure_cause, + msg->answer_mailbox_if_read, + msg->answer_mailbox_if_write, msg->answer_mailbox_if_copy, msg->start_time_stamp); } else { @@ -266,6 +268,7 @@ namespace wrench { } if (failure_cause == nullptr) { + auto fs = this->file_systems[location->getMountPoint()].get(); if (not fs->doesDirectoryExist(location->getAbsolutePathAtMountPoint())) { @@ -273,7 +276,7 @@ namespace wrench { } // Update occupied space, in advance (will have to be decreased later in case of failure) - fs->decreaseFreeSpace(file->getSize()); + fs->reserveSpace(file, location->getAbsolutePathAtMountPoint()); // Generate a mailbox_name name on which to receive the file std::string file_reception_mailbox = S4U_Mailbox::generateUniqueMailboxName("file_reception"); @@ -297,6 +300,8 @@ namespace wrench { file_reception_mailbox, location, "", + answer_mailbox, + "", buffer_size)); ftt->simulation = this->simulation; @@ -380,6 +385,8 @@ namespace wrench { file, location, mailbox_to_receive_the_file_content, + answer_mailbox, + "", "", buffer_size)); ftt->simulation = this->simulation; @@ -469,7 +476,7 @@ namespace wrench { } return true; } - fs->decreaseFreeSpace(file->getSize()); + fs->reserveSpace(file, dst_location->getAbsolutePathAtMountPoint()); } @@ -484,6 +491,8 @@ namespace wrench { file, src_location, dst_location, + "", + "", answer_mailbox, this->buffer_size, start_timestamp)); ftt->simulation = this->simulation; @@ -517,6 +526,8 @@ namespace wrench { * @param dst_location: the transfer's destination location (or nullptr if destination was not a location) * @param success: whether the transfer succeeded or not * @param failure_cause: the failure cause (nullptr if success) + * @param answer_mailbox_if_read: the mailbox to send a read notification ("" if not a copy) + * @param answer_mailbox_if_write: the mailbox to send a write notification ("" if not a copy) * @param answer_mailbox_if_copy: the mailbox to send a copy notification ("" if not a copy) * @param start_timestamp: a start file copy time stamp * @return false if the daemon should terminate @@ -529,6 +540,8 @@ namespace wrench { std::shared_ptr dst_location, bool success, std::shared_ptr failure_cause, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, SimulationTimestampFileCopyStart *start_timestamp) { @@ -539,7 +552,6 @@ namespace wrench { this->running_file_transfer_threads.erase(ftt); } - // Was the destination me? if (dst_location and (dst_location->getStorageService().get() == this)) { if (success) { @@ -552,11 +564,23 @@ namespace wrench { } } else { // Process the failure, meaning, just un-decrease the free space - this->file_systems[dst_location->getAbsolutePathAtMountPoint()]->increaseFreeSpace(file->getSize()); + this->file_systems[dst_location->getMountPoint()]->unreserveSpace(file, dst_location->getAbsolutePathAtMountPoint()); } } - // Send back the corresponding ack if this was a copy + // Send back the relevant ack if this was a read + if (not answer_mailbox_if_read.empty()) { + WRENCH_INFO("Sending back an ack since this was a file read and some client is waiting for me to say something"); + S4U_Mailbox::dputMessage(answer_mailbox_if_read, new StorageServiceAckMessage()); + } + + // Send back the relevant ack if this was a write + if (not answer_mailbox_if_write.empty()) { + WRENCH_INFO("Sending back an ack since this was a file write and some client is waiting for me to say something"); + S4U_Mailbox::dputMessage(answer_mailbox_if_write, new StorageServiceAckMessage()); + } + + // Send back the relevant ack if this was a copy if (not answer_mailbox_if_copy.empty()) { WRENCH_INFO("Sending back an ack since this was a file copy and some client is waiting for me to say something"); if ((src_location == nullptr) or (dst_location == nullptr)) { @@ -575,7 +599,6 @@ namespace wrench { SimpleStorageServiceMessagePayload::FILE_COPY_ANSWER_MESSAGE_PAYLOAD))); } - return true; } @@ -592,7 +615,7 @@ namespace wrench { std::string answer_mailbox) { std::shared_ptr failure_cause = nullptr; - auto fs = this->file_systems[location->getAbsolutePathAtMountPoint()].get(); + auto fs = this->file_systems[location->getMountPoint()].get(); if ((not fs->doesDirectoryExist(location->getAbsolutePathAtMountPoint())) or (not fs->isFileInDirectory(file, location->getAbsolutePathAtMountPoint()))) { diff --git a/src/wrench/services/storage/storage_helpers/FileLocation.cpp b/src/wrench/services/storage/storage_helpers/FileLocation.cpp index 671faa4d1c..45e54c6104 100644 --- a/src/wrench/services/storage/storage_helpers/FileLocation.cpp +++ b/src/wrench/services/storage/storage_helpers/FileLocation.cpp @@ -16,6 +16,8 @@ namespace wrench { + std::shared_ptr FileLocation::SCRATCH = std::shared_ptr(new FileLocation(nullptr, "", "")); + /** * @brief File location specifier for a storage service's (single) mount point root @@ -29,6 +31,7 @@ namespace wrench { if (ss == nullptr) { throw std::invalid_argument("FileLocation::LOCATION(): Cannot pass nullptr storage service"); } + if (ss->hasMultipleMountPoints()) { throw std::invalid_argument("FileLocation::LOCATION(): Storage Service has multiple mount points. " "Call the version of this method that takes a mount point argument"); @@ -39,7 +42,7 @@ namespace wrench { /** * @brief File location specifier given an absolute path at a storage service * - * @param ss: a storage service + * @param ss: a storage service or ComputeService::SCRATCH * @param absolute_path: an absolute path at the storage service to a directory (that may contain files) * @return a file location specification * @@ -47,16 +50,14 @@ namespace wrench { */ std::shared_ptr FileLocation::LOCATION(std::shared_ptr ss, std::string absolute_path) { - if ((ss == nullptr) or (absolute_path.empty())) { - throw std::invalid_argument("FileLocation::LOCATION(): Invalid arguments"); + if (ss == nullptr) { + throw std::invalid_argument("FileLocation::LOCATION(): Invalid storage service argument"); } - - try { - absolute_path = FileLocation::sanitizePath(absolute_path); - } catch (std::invalid_argument &e) { - throw; + if (absolute_path.empty()) { + throw std::invalid_argument("FileLocation::LOCATION(): must specify a non-empty path"); } + absolute_path = FileLocation::sanitizePath(absolute_path); std::string mount_point = ""; for (auto const &mp : ss->getMountPoints()) { @@ -71,20 +72,26 @@ namespace wrench { mount_point = "/"; } else { throw std::invalid_argument("FileLocation::LOCATION(): Invalid path '" + - absolute_path +"' at storage service '" + ss->getName() + "'"); + absolute_path + "' at storage service '" + ss->getName() + "'"); } } absolute_path.replace(0, mount_point.length(), "/"); + absolute_path = sanitizePath(absolute_path); - return std::shared_ptr(new FileLocation(ss, mount_point, sanitizePath(absolute_path))); + return std::shared_ptr(new FileLocation(ss, mount_point, absolute_path)); } /** * @brief Give a ::" string for the location */ std::string FileLocation::toString() { - return this->storage_service->getName() + ":" + sanitizePath(this->mount_point + this->absolute_path_at_mount_point); + if (this == FileLocation::SCRATCH.get()) { + return "scratch"; + } else { + return this->storage_service->getName() + ":" + + sanitizePath(this->mount_point + this->absolute_path_at_mount_point); + } } /** @@ -92,6 +99,9 @@ namespace wrench { * @return a storage service */ std::shared_ptr FileLocation::getStorageService() { + if (this == FileLocation::SCRATCH.get()) { + throw std::invalid_argument("FileLocation::getStorageService(): Method cannot be called on FileLocation::SCRATCH"); + } return this->storage_service; } @@ -100,17 +110,34 @@ namespace wrench { * @return a mount point */ std::string FileLocation::getMountPoint() { + if (this == FileLocation::SCRATCH.get()) { + throw std::invalid_argument("FileLocation::getMountPoint(): Method cannot be called on FileLocation::SCRATCH"); + } return this->mount_point; } /** - * @brief Get the location's storage service's mount point + * @brief Get the location's path at mount point * @return */ std::string FileLocation::getAbsolutePathAtMountPoint() { + if (this == FileLocation::SCRATCH.get()) { + throw std::invalid_argument("FileLocation::getAbsolutePathAtMountPoint(): Method cannot be called on FileLocation::SCRATCH"); + } return this->absolute_path_at_mount_point; } + /** + * @brief Get the location's full absolute path + * @return + */ + std::string FileLocation::getFullAbsolutePath() { + if (this == FileLocation::SCRATCH.get()) { + throw std::invalid_argument("FileLocation::getFullAbsolutePath(): Method cannot be called on FileLocation::SCRATCH"); + } + return FileLocation::sanitizePath(this->mount_point + "/" + this->absolute_path_at_mount_point); + } + /** * @brief Method to sanitize an absolute path diff --git a/src/wrench/services/storage/storage_helpers/FileTransferThread.cpp b/src/wrench/services/storage/storage_helpers/FileTransferThread.cpp index caf0c293dc..5e8eeba15b 100644 --- a/src/wrench/services/storage/storage_helpers/FileTransferThread.cpp +++ b/src/wrench/services/storage/storage_helpers/FileTransferThread.cpp @@ -30,6 +30,10 @@ namespace wrench { * @param file: the file corresponding to the connection * @param src_mailbox: the a source mailbox to receive data from * @param dst_location: a location to write data to + * @param answer_mailbox_if_read: the mailbox to send an answer to in case this was a file read ("" if none). This + * will simply be reported to the parent service, who may use it as needed + * @param answer_mailbox_if_write: the mailbox to send an answer to in case this was a file write ("" if none). This + * will simply be reported to the parent service, who may use it as needed * @param answer_mailbox_if_copy: the mailbox to send an answer to in case this was a file copy ("" if none). This * will simply be reported to the parent service, who may use it as needed * @param buffer_size: the buffer size to use @@ -40,12 +44,16 @@ namespace wrench { WorkflowFile *file, std::string src_mailbox, std::shared_ptr dst_location, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp) : Service(hostname, "file_transfer_thread", "file_transfer_thread"), parent(parent), file(file), + answer_mailbox_if_read(answer_mailbox_if_read), + answer_mailbox_if_write(answer_mailbox_if_write), answer_mailbox_if_copy(answer_mailbox_if_copy), buffer_size(buffer_size), start_timestamp(start_timestamp) @@ -63,6 +71,10 @@ namespace wrench { * @param file: the file corresponding to the connection * @param src_location: a location to read data from * @param dst_mailbox: a mailbox to send data to + * @param answer_mailbox_if_read: the mailbox to send an answer to in case this was a file read ("" if none). This + * will simply be reported to the parent service, who may use it as needed + * @param answer_mailbox_if_write: the mailbox to send an answer to in case this was a file write ("" if none). This + * will simply be reported to the parent service, who may use it as needed * @param answer_mailbox_if_copy: the mailbox to send an answer to in case this was a file copy ("" if none). This * will simply be reported to the parent service, who may use it as needed * @param buffer_size: the buffer size to use @@ -73,12 +85,16 @@ namespace wrench { WorkflowFile *file, std::shared_ptr src_location, std::string dst_mailbox, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp) : Service(hostname, "file_transfer_thread", "file_transfer_thread"), parent(parent), file(file), + answer_mailbox_if_read(answer_mailbox_if_read), + answer_mailbox_if_write(answer_mailbox_if_write), answer_mailbox_if_copy(answer_mailbox_if_copy), buffer_size(buffer_size), start_timestamp(start_timestamp) @@ -96,6 +112,10 @@ namespace wrench { * @param file: the file corresponding to the connection * @param src_location: a location to read data from * @param dst_location: a location to send data to + * @param answer_mailbox_if_read: the mailbox to send an answer to in case this was a file read ("" if none). This + * will simply be reported to the parent service, who may use it as needed + * @param answer_mailbox_if_write: the mailbox to send an answer to in case this was a file write ("" if none). This + * will simply be reported to the parent service, who may use it as needed * @param answer_mailbox_if_copy: the mailbox to send an answer to in case this was a file copy ("" if none). This * will simply be reported to the parent service, who may use it as needed * @param buffer_size: the buffer size to use @@ -106,12 +126,16 @@ namespace wrench { WorkflowFile *file, std::shared_ptr src_location, std::shared_ptr dst_location, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, unsigned long buffer_size, SimulationTimestampFileCopyStart *start_timestamp) : Service(hostname, "file_transfer_thread", "file_transfer_thread"), parent(parent), file(file), + answer_mailbox_if_read(answer_mailbox_if_read), + answer_mailbox_if_write(answer_mailbox_if_write), answer_mailbox_if_copy(answer_mailbox_if_copy), buffer_size(buffer_size), start_timestamp(start_timestamp) @@ -141,13 +165,17 @@ namespace wrench { FileTransferThreadNotificationMessage *msg_to_send_back = nullptr; std::shared_ptr failure_cause = nullptr; - WRENCH_INFO("New FileTransferThread (file=%s, src_mailbox=%s; src_location=%s; dst_mailbox=%s; dst_location=%s; answer_mailbox_if_copy=%s", + WRENCH_INFO("New FileTransferThread (file=%s, src_mailbox=%s; src_location=%s; dst_mailbox=%s; dst_location=%s; " + "answer_mailbox_if_copy=%s; answer_mailbox_if_copy=%s; answer_mailbox_if_copy=%s", file->getID().c_str(), (src_mailbox.empty() ? "none" : src_mailbox.c_str()), (src_location == nullptr ? "none" : src_location->toString().c_str()), (dst_mailbox.empty() ? "none" : dst_mailbox.c_str()), (dst_location == nullptr ? "none" : dst_location->toString().c_str()), - (answer_mailbox_if_copy.empty() ? "none" : answer_mailbox_if_copy.c_str())); + (answer_mailbox_if_read.empty() ? "none" : answer_mailbox_if_read.c_str()), + (answer_mailbox_if_write.empty() ? "none" : answer_mailbox_if_write.c_str()), + (answer_mailbox_if_copy.empty() ? "none" : answer_mailbox_if_copy.c_str()) + ); // Create a message to send back (some field of which may be overwritten below) msg_to_send_back = new FileTransferThreadNotificationMessage( @@ -157,6 +185,8 @@ namespace wrench { this->src_location, this->dst_mailbox, this->dst_location, + this->answer_mailbox_if_read, + this->answer_mailbox_if_write, this->answer_mailbox_if_copy, true, nullptr, this->start_timestamp); diff --git a/src/wrench/services/storage/storage_helpers/FileTransferThreadMessage.h b/src/wrench/services/storage/storage_helpers/FileTransferThreadMessage.h index 23aeae5121..5bbaa0bcf0 100644 --- a/src/wrench/services/storage/storage_helpers/FileTransferThreadMessage.h +++ b/src/wrench/services/storage/storage_helpers/FileTransferThreadMessage.h @@ -57,6 +57,8 @@ namespace wrench { * @param src_location: the source location of the transfer (or nullptr if source wasn't a location) * @param dst_mailbox: the destination mailbox of the transfer (or "" if source wasn't a mailbox) * @param dst_location: the destination location of the transfer (or nullptr if source wasn't a location) + * @param answer_mailbox_if_read: the mailbox that a "read is done" may be sent to if necessary + * @param answer_mailbox_if_write: the mailbox that a "write is done" may be sent to if necessary * @param answer_mailbox_if_copy: the mailbox that a "copy is done/failed" may be sent if necessary * @param success: whether the transfer succeeded * @param failure_cause: the failure cause (nullptr if success) @@ -68,6 +70,8 @@ namespace wrench { std::shared_ptr src_location, std::string dst_mailbox, std::shared_ptr dst_location, + std::string answer_mailbox_if_read, + std::string answer_mailbox_if_write, std::string answer_mailbox_if_copy, bool success, std::shared_ptr failure_cause, SimulationTimestampFileCopyStart *start_time_stamp) : @@ -76,7 +80,10 @@ namespace wrench { file(file), src_mailbox(src_mailbox), src_location(src_location), dst_mailbox(dst_mailbox), dst_location(dst_location), - answer_mailbox_if_copy(answer_mailbox_if_copy), success(success), + answer_mailbox_if_read(answer_mailbox_if_read), + answer_mailbox_if_write(answer_mailbox_if_write), + answer_mailbox_if_copy(answer_mailbox_if_copy), + success(success), failure_cause(failure_cause), start_time_stamp(start_time_stamp) {} /** @brief File transfer thread that sent this message */ @@ -94,6 +101,10 @@ namespace wrench { /** @brief Destination location (or nullptr if source wasn't a location) */ std::shared_ptr dst_location; + /** @brief If this was a file read, the mailbox to which an answer should be send */ + std::string answer_mailbox_if_read; + /** @brief If this was a file write, the mailbox to which an answer should be send */ + std::string answer_mailbox_if_write; /** @brief If this was a file copy, the mailbox to which an answer should be send */ std::string answer_mailbox_if_copy; /** @brief Whether the transfer succeeded or not */ diff --git a/src/wrench/services/storage/storage_helpers/LocicalFileSystem.cpp b/src/wrench/services/storage/storage_helpers/LocicalFileSystem.cpp index 43e906a980..27f35eecb9 100644 --- a/src/wrench/services/storage/storage_helpers/LocicalFileSystem.cpp +++ b/src/wrench/services/storage/storage_helpers/LocicalFileSystem.cpp @@ -23,15 +23,7 @@ namespace wrench { */ LogicalFileSystem::LogicalFileSystem(std::string hostname, std::string mount_point) { - // Sanitize the mount_point - if (mount_point.at(0) != '/') { - mount_point = "/" + mount_point; - } - if (mount_point.at(mount_point.length()-1) != '/') { - mount_point += "/"; - } - - mount_point = FileLocation::sanitizePath(mount_point); + mount_point = FileLocation::sanitizePath("/" + mount_point + "/"); // WRENCH_INFO("NEW %s", mount_point.c_str()); // Check uniqueness @@ -40,7 +32,7 @@ namespace wrench { mount_point + " at host " + hostname + " already exists"); } - if (mount_point != "/") { // "/" is obviously a prefix, but it's okstr(), + if (mount_point != "/") { // "/" is obviously a prefix, but it's ok // Check non-proper-prefixness for (auto const &mp : LogicalFileSystem::mount_points) { @@ -60,8 +52,7 @@ namespace wrench { LogicalFileSystem::mount_points.insert(hostname+":"+mount_point); this->mount_point = mount_point; - this->content.insert(std::make_pair("/", (std::set){})); -// this->content["/"] = {}; + this->content["/"] = {}; this->total_capacity = S4U_Simulation::getDiskCapacity(hostname, mount_point); this->occupied_space = 0; } @@ -115,14 +106,19 @@ namespace wrench { /** * @brief Store file in directory - * @param absolute_path: the directory's absolute path + * @param absolute_path: the directory's absolute path (at the mount point) * * @throw std::invalid_argument */ void LogicalFileSystem::storeFileInDirectory(WorkflowFile *file, std::string absolute_path) { assertDirectoryExist(absolute_path); this->content[absolute_path].insert(file); - this->occupied_space += file->getSize(); + std::string key = FileLocation::sanitizePath(absolute_path) + file->getID(); + if (this->reserved_space.find(key) == this->reserved_space.end()) { + this->occupied_space += file->getSize(); + } else { + this->reserved_space.erase(key); + } } /** @@ -202,27 +198,45 @@ namespace wrench { } /** - * @brief Decrease the amount of free space the service "thinks" it has - * @param num_bytes: the number of bytes by which to decrease + * @brief Reserve space for a file that will be stored + * @param file: the file + * @param absolute_path: the path where it will be written * @throw std::invalid_argument */ - void LogicalFileSystem::decreaseFreeSpace(double num_bytes) { - if (this->total_capacity - this->occupied_space < num_bytes) { - throw std::invalid_argument("LogicalFileSystem::decrementFreeSpace(): Not enough free space"); + void LogicalFileSystem::reserveSpace(WorkflowFile *file, std::string absolute_path) { + std::string key = FileLocation::sanitizePath(absolute_path) + file->getID(); + + if (this->total_capacity - this->occupied_space < file->getSize()) { + throw std::invalid_argument("LogicalFileSystem::reserveSpace(): Not enough free space"); } - this->occupied_space += num_bytes; + if (this->reserved_space.find(key) != this->reserved_space.end()) { + throw std::runtime_error("LogicalFileSystem::reserveSpace(): Space was already being reserved for storing file " + + file->getID() + "at path " + absolute_path); + } + this->reserved_space[key] = file->getSize(); + this->occupied_space += file->getSize(); } /** - * @brief Increase the amount of free space the service "thinks" it has - * @param num_bytes: the number of bytes by which to increase + * @brief Unreserve space that was saved for a file (likely a failed transfer) + * @param file: the file + * @param absolute_path: the path where it would have been written * @throw std::invalid_argument */ - void LogicalFileSystem::increaseFreeSpace(double num_bytes) { - if (this->occupied_space + num_bytes > this->total_capacity) { - throw std::invalid_argument("LogicalFileSystem::decrementFreeSpace(): No enough capacity"); + void LogicalFileSystem::unreserveSpace(WorkflowFile *file, std::string absolute_path) { + std::string key = FileLocation::sanitizePath(absolute_path) + file->getID(); + + if (this->occupied_space + file->getSize() > this->total_capacity) { + throw std::invalid_argument("LogicalFileSystem::unreserveSpace(): No enough capacity"); } - this->occupied_space -= num_bytes; + + if (this->reserved_space.find(key) == this->reserved_space.end()) { + throw std::runtime_error("LogicalFileSystem::unreserveSpace(): Space was not being reserved for storing file " + + file->getID() + "at path " + absolute_path); + } + + this->reserved_space.erase(key); + this->occupied_space -= file->getSize(); } } diff --git a/src/wrench/services/storage/storage_helpers/LogicalFileSystem.h b/src/wrench/services/storage/storage_helpers/LogicalFileSystem.h index 957799387d..c62694c4ae 100644 --- a/src/wrench/services/storage/storage_helpers/LogicalFileSystem.h +++ b/src/wrench/services/storage/storage_helpers/LogicalFileSystem.h @@ -32,8 +32,8 @@ namespace wrench { double getTotalCapacity(); bool hasEnoughFreeSpace(double bytes); double getFreeSpace(); - void decreaseFreeSpace(double num_bytes); - void increaseFreeSpace(double num_bytes); + void reserveSpace(WorkflowFile *file, std::string absolute_path); + void unreserveSpace(WorkflowFile *file, std::string absolute_path); void createDirectory(std::string absolute_path); bool doesDirectoryExist(std::string absolute_path); @@ -55,6 +55,7 @@ namespace wrench { std::string mount_point; double total_capacity; double occupied_space; + std::map reserved_space; void assertDirectoryExist(std::string absolute_path) { if (not this->doesDirectoryExist(absolute_path)) { diff --git a/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp b/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp index 84b39f3646..633a28c720 100644 --- a/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp +++ b/src/wrench/simgrid_S4U_util/S4U_Simulation.cpp @@ -647,7 +647,7 @@ namespace wrench { * @param mount_point: the mount point * @return true if the host has a disk attached to the specified mount point, false otherwise */ - bool S4U_Simulation::hostHasDisk(std::string hostname, std::string mount_point) { + bool S4U_Simulation::hostHasMountPoint(std::string hostname, std::string mount_point) { simgrid::s4u::Host *host; try { diff --git a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceOneTaskTest.cpp b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceOneTaskTest.cpp index 2dc1266935..687672d4d9 100644 --- a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceOneTaskTest.cpp +++ b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceOneTaskTest.cpp @@ -90,7 +90,7 @@ class BareMetalComputeServiceOneTaskTest : public ::testing::Test { " " " " " " - " " + " " " " " " " " @@ -108,7 +108,7 @@ class BareMetalComputeServiceOneTaskTest : public ::testing::Test { " " " " " " - " " + " " " " " " " " @@ -126,7 +126,7 @@ class BareMetalComputeServiceOneTaskTest : public ::testing::Test { " " " " " " - " " + " " " " " " " " @@ -1088,12 +1088,14 @@ class ExecutionWithPrePostCopiesAndCleanupTestWMS : public wrench::WMS { auto job_manager = this->createJobManager(); // Create a job - wrench::StandardJob *job = job_manager->createStandardJob({test->task}, - {{test->input_file,wrench::FileLocation::LOCATION((test->storage_service1))},{test->output_file,wrench::FileLocation::LOCATION(test->storage_service2)}}, //changed this since we don't have default storage now - {std::make_tuple(test->input_file, wrench::FileLocation::LOCATION((test->storage_service1)), wrench::FileLocation::LOCATION(test->storage_service2))}, - {std::make_tuple(test->output_file, wrench::FileLocation::LOCATION(test->storage_service2), wrench::FileLocation::LOCATION((test->storage_service1)))}, - {std::make_tuple(test->input_file, wrench::FileLocation::LOCATION(test->storage_service2)), - std::make_tuple(test->output_file, wrench::FileLocation::LOCATION(test->storage_service2))}); + wrench::StandardJob *job = job_manager->createStandardJob( + {test->task}, + {{test->input_file,wrench::FileLocation::LOCATION((test->storage_service1))}, + {test->output_file,wrench::FileLocation::LOCATION(test->storage_service2)}}, //changed this since we don't have default storage now + {std::make_tuple(test->input_file, wrench::FileLocation::LOCATION((test->storage_service1)), wrench::FileLocation::LOCATION(test->storage_service2))}, + {std::make_tuple(test->output_file, wrench::FileLocation::LOCATION(test->storage_service2), wrench::FileLocation::LOCATION((test->storage_service1)))}, + {std::make_tuple(test->input_file, wrench::FileLocation::LOCATION(test->storage_service2)), + std::make_tuple(test->output_file, wrench::FileLocation::LOCATION(test->storage_service2))}); // Submit the job job_manager->submitJob(job, test->compute_service); @@ -1109,22 +1111,24 @@ class ExecutionWithPrePostCopiesAndCleanupTestWMS : public wrench::WMS { } // Test file locations - if (!this->test->storage_service1->lookupFile( - this->test->input_file, nullptr)) { + if (not wrench::StorageService::lookupFile( + this->test->input_file, + wrench::FileLocation::LOCATION(this->test->storage_service1))) { throw std::runtime_error("Input file should be on Storage Service #1"); } - if (!this->test->storage_service1->lookupFile( - this->test->output_file, nullptr)) { + if (not wrench::StorageService::lookupFile( + this->test->output_file, + wrench::FileLocation::LOCATION(this->test->storage_service1))) { throw std::runtime_error("Output file should be on Storage Service #1"); } - if (this->test->storage_service2->lookupFile( + if (wrench::StorageService::lookupFile( this->test->input_file, - wrench::FileLocation::LOCATION(this->test->storage_service2, "/" + job->getName()))) { + wrench::FileLocation::LOCATION(this->test->storage_service2))) { throw std::runtime_error("Input file should not be on Storage Service #2"); } - if (this->test->storage_service2->lookupFile( + if (wrench::StorageService::lookupFile( this->test->output_file, - wrench::FileLocation::LOCATION(this->test->storage_service2, "/" + job->getName()))) { + wrench::FileLocation::LOCATION(this->test->storage_service2))) { throw std::runtime_error("Output file should not be on Storage Service #2"); } @@ -1169,7 +1173,7 @@ void BareMetalComputeServiceOneTaskTest::do_ExecutionWithPrePostCopiesTaskCleanu ASSERT_NO_THROW(compute_service = simulation->add( new wrench::BareMetalComputeService(hostname, {std::make_pair(hostname, std::make_tuple(wrench::ComputeService::ALL_CORES, wrench::ComputeService::ALL_RAM))}, - {}))); + "", {}))); // Create a WMS std::shared_ptr wms = nullptr;; @@ -1487,7 +1491,7 @@ class ExecutionWithMissingFileTestWMS : public wrench::WMS { // Remove the staged file! wrench::StorageService::deleteFile(test->input_file, - wrench::FileLocation::LOCATION(test->storage_service1)); + wrench::FileLocation::LOCATION(test->storage_service1)); // Create a job wrench::StandardJob *job = job_manager->createStandardJob({test->task}, diff --git a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceTestStandardJobs.cpp b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceTestStandardJobs.cpp index 36a2643d7a..0061870095 100644 --- a/test/compute_services/BareMetalComputeService/BareMetalComputeServiceTestStandardJobs.cpp +++ b/test/compute_services/BareMetalComputeService/BareMetalComputeServiceTestStandardJobs.cpp @@ -293,7 +293,7 @@ class MulticoreComputeServiceBogusNumCoresTestWMS : public wrench::WMS { auto two_core_task_job = job_manager->createStandardJob({this->test->task3}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 1-task job for execution with too few cores @@ -406,7 +406,7 @@ class MulticoreComputeServiceTwoSingleCoreTasksTestWMS : public wrench::WMS { {this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -530,7 +530,7 @@ class MulticoreComputeServiceTwoDualCoreTasksCase1TestWMS : public wrench::WMS { wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task3, this->test->task4}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -654,7 +654,7 @@ class MulticoreComputeServiceTwoDualCoreTasksCase2TestWMS : public wrench::WMS { wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task5, this->test->task6}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -797,7 +797,7 @@ class BareMetalComputeServiceTwoDualCoreTasksCase3TestWMS : public wrench::WMS { {std::make_tuple( this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution (WRONG CS-specific arguments) @@ -888,7 +888,7 @@ class BareMetalComputeServiceTwoDualCoreTasksCase3TestWMS : public wrench::WMS { wrench::StandardJob *two_task_job_2 = job_manager->createStandardJob({this->test->task7, this->test->task8}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); job_manager->submitJob(two_task_job_2, this->test->compute_service, @@ -1016,7 +1016,7 @@ class BareMetalComputeServiceJobTerminationTestWMS : public wrench::WMS { wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -1256,7 +1256,7 @@ class BareMetalComputeServiceCompletedJobTerminationTestWMS : public wrench::WMS wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -1380,7 +1380,7 @@ class BareMetalComputeServiceShutdownComputeServiceWhileJobIsRunningTestWMS : pu wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution diff --git a/test/compute_services/HTCondorServiceTest.cpp b/test/compute_services/HTCondorServiceTest.cpp index 9c1f9a395a..3fac0c361a 100644 --- a/test/compute_services/HTCondorServiceTest.cpp +++ b/test/compute_services/HTCondorServiceTest.cpp @@ -140,7 +140,7 @@ class HTCondorStandardJobTestWMS : public wrench::WMS { {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -261,7 +261,7 @@ class HTCondorSimpleServiceTestWMS : public wrench::WMS { {this->test->task1}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(htcondor_cs->getLocalStorageService()), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution diff --git a/test/compute_services/ScratchSpaceTest.cpp b/test/compute_services/ScratchSpaceTest.cpp index 32c84db78d..8d03ff2d71 100644 --- a/test/compute_services/ScratchSpaceTest.cpp +++ b/test/compute_services/ScratchSpaceTest.cpp @@ -167,7 +167,7 @@ class SimpleScratchSpaceTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -304,7 +304,7 @@ class SimpleScratchSpaceFailureTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file1"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -315,7 +315,7 @@ class SimpleScratchSpaceFailureTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file2"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -557,7 +557,7 @@ class PilotJobScratchSpaceTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file1"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -567,7 +567,7 @@ class PilotJobScratchSpaceTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file2"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -577,7 +577,7 @@ class PilotJobScratchSpaceTestWMS : public wrench::WMS { {}, {std::make_tuple(this->getWorkflow()->getFileByID("input_file3"), wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -753,7 +753,7 @@ class ScratchSpaceRaceConditionTestWMS : public wrench::WMS { {task1, task2}, {}, {std::make_tuple(file, wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Create a second job that: @@ -763,7 +763,7 @@ class ScratchSpaceRaceConditionTestWMS : public wrench::WMS { {task3}, {}, {std::make_tuple(file, wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit both jobs @@ -905,7 +905,7 @@ class ScratchNonScratchPartitionsTestWMS : public wrench::WMS { {task1}, {}, {std::make_tuple(file1, wrench::FileLocation::LOCATION(this->test->storage_service1), - wrench::FileLocation::LOCATION(this->test->compute_service->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit job1 diff --git a/test/compute_services/VirtualizedClusterService/VirtualizedClusterServiceTest.cpp b/test/compute_services/VirtualizedClusterService/VirtualizedClusterServiceTest.cpp index 5259eeb90e..57078a8759 100644 --- a/test/compute_services/VirtualizedClusterService/VirtualizedClusterServiceTest.cpp +++ b/test/compute_services/VirtualizedClusterService/VirtualizedClusterServiceTest.cpp @@ -287,7 +287,7 @@ class CloudStandardJobTestWMS : public wrench::WMS { auto two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(cs->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Try to submit the job directly to the CloudComputeService (which fails) @@ -495,7 +495,7 @@ class CloudStandardJobWithCustomVMNameTestWMS : public wrench::WMS { wrench::StandardJob *two_task_job = job_manager->createStandardJob({this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(vm_cs->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -606,7 +606,7 @@ class VirtualizedClusterVMMigrationTestWMS : public wrench::WMS { {this->test->task1, this->test->task2}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(cs->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the 2-task job for execution @@ -1389,7 +1389,7 @@ class SubmitToVMTestWMS : public wrench::WMS { wrench::StandardJob *job1 = job_manager->createStandardJob({this->test->task1}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(std::get<1>(vm_list[0])->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Trying to submit to a VM that has been shutdown job_manager->submitJob(job1, std::get<1>(vm_list[0])); @@ -1412,7 +1412,7 @@ class SubmitToVMTestWMS : public wrench::WMS { wrench::StandardJob *job1 = job_manager->createStandardJob({this->test->task1}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(std::get<1>(vm_list[1])->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); try { @@ -1629,7 +1629,7 @@ class CloudServiceVMShutdownWhileJobIsRunningTestWMS : public wrench::WMS { {std::make_tuple( this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(vm_cs->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); @@ -1762,7 +1762,7 @@ class CloudServiceVMComputeServiceStopWhileJobIsRunningTestWMS : public wrench:: {this->test->task1}, {}, {std::make_tuple(this->test->input_file, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(vm_cs->getScratch()))}, + wrench::FileLocation::SCRATCH)}, {}, {}); // Submit the job to the VM diff --git a/test/wms/MultipleWMSTest.cpp b/test/wms/MultipleWMSTest.cpp index e4fd43889e..5fd89f4083 100644 --- a/test/wms/MultipleWMSTest.cpp +++ b/test/wms/MultipleWMSTest.cpp @@ -136,7 +136,7 @@ class DeferredWMSStartTestWMS : public wrench::WMS { std::tuple, std::shared_ptr> each_copy = std::make_tuple(it.second, wrench::FileLocation::LOCATION(this->test->storage_service), - wrench::FileLocation::LOCATION(cs->getScratch())); + wrench::FileLocation::SCRATCH); pre_copies.insert(each_copy); }