From 24f72b65363f18239677c2baaeca4504fb6c2014 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 20 Apr 2020 15:01:58 -1000 Subject: [PATCH] (#157) Added a virtual cluster example --- conf/cmake/Examples.cmake | 1 + .../BareMetalBagOfTasks.cpp | 3 +- .../BareMetalChainScratch.cpp | 3 +- .../bare-metal-chain/BareMetalChain.cpp | 3 +- .../bare-metal-complex-job/ComplexJob.cpp | 3 +- .../batch-bag-of-tasks/BatchBagOfTasks.cpp | 3 +- .../TwoTasksAtATimeBatchWMS.cpp | 2 +- .../batch-pilot-job/BatchPilotJob.cpp | 3 +- .../cloud-bag-of-tasks/CloudBagOfTasks.cpp | 7 +- .../CMakeLists.txt | 13 ++ .../TwoTasksAtATimeVirtualizedClusterWMS.cpp | 192 ++++++++++++++++++ .../TwoTasksAtATimeVirtualizedClusterWMS.h | 45 ++++ .../VirtualizedClusterBagOfTasks.cpp | 162 +++++++++++++++ .../four_hosts.xml | 36 ++++ 14 files changed, 459 insertions(+), 17 deletions(-) create mode 100644 examples/basic-examples/virtualized-cluster-bag-of-tasks/CMakeLists.txt create mode 100644 examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.cpp create mode 100644 examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.h create mode 100644 examples/basic-examples/virtualized-cluster-bag-of-tasks/VirtualizedClusterBagOfTasks.cpp create mode 100644 examples/basic-examples/virtualized-cluster-bag-of-tasks/four_hosts.xml diff --git a/conf/cmake/Examples.cmake b/conf/cmake/Examples.cmake index b364afe180..069f364ed8 100644 --- a/conf/cmake/Examples.cmake +++ b/conf/cmake/Examples.cmake @@ -6,6 +6,7 @@ set(EXAMPLES_CMAKEFILES_TXT examples/basic-examples/bare-metal-bag-of-tasks/CMakeLists.txt examples/basic-examples/bare-metal-complex-job/CMakeLists.txt examples/basic-examples/cloud-bag-of-tasks/CMakeLists.txt + examples/basic-examples/virtualized-cluster-bag-of-tasks/CMakeLists.txt examples/basic-examples/batch-bag-of-tasks/CMakeLists.txt examples/basic-examples/batch-pilot-job/CMakeLists.txt ) diff --git a/examples/basic-examples/bare-metal-bag-of-tasks/BareMetalBagOfTasks.cpp b/examples/basic-examples/bare-metal-bag-of-tasks/BareMetalBagOfTasks.cpp index 67394cae93..1b002a3c8a 100644 --- a/examples/basic-examples/bare-metal-bag-of-tasks/BareMetalBagOfTasks.cpp +++ b/examples/basic-examples/bare-metal-bag-of-tasks/BareMetalBagOfTasks.cpp @@ -116,8 +116,7 @@ int main(int argc, char **argv) { "ComputeHost", {"ComputeHost"}, "", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in TwoTasksAtATimeWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::TwoTasksAtATimeWMS({baremetal_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/bare-metal-chain-scratch/BareMetalChainScratch.cpp b/examples/basic-examples/bare-metal-chain-scratch/BareMetalChainScratch.cpp index bcd255f2a7..64a2e1e002 100644 --- a/examples/basic-examples/bare-metal-chain-scratch/BareMetalChainScratch.cpp +++ b/examples/basic-examples/bare-metal-chain-scratch/BareMetalChainScratch.cpp @@ -119,8 +119,7 @@ int main(int argc, char **argv) { "ComputeHost", {"ComputeHost"}, "/scratch/", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in OneTaskAtATimeWMS.cpp - * for more details */ + * for executing the workflow */ auto wms = simulation.add( new wrench::WorkflowAsAsingleJobWMS({baremetal_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/bare-metal-chain/BareMetalChain.cpp b/examples/basic-examples/bare-metal-chain/BareMetalChain.cpp index ca7044c50e..892421ec14 100644 --- a/examples/basic-examples/bare-metal-chain/BareMetalChain.cpp +++ b/examples/basic-examples/bare-metal-chain/BareMetalChain.cpp @@ -118,8 +118,7 @@ int main(int argc, char **argv) { "ComputeHost", {"ComputeHost"}, "", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in OneTaskAtATimeWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::OneTaskAtATimeWMS({baremetal_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/bare-metal-complex-job/ComplexJob.cpp b/examples/basic-examples/bare-metal-complex-job/ComplexJob.cpp index 1a396d6ce6..0dee90b820 100644 --- a/examples/basic-examples/bare-metal-complex-job/ComplexJob.cpp +++ b/examples/basic-examples/bare-metal-complex-job/ComplexJob.cpp @@ -101,8 +101,7 @@ int main(int argc, char **argv) { "ComputeHost", {"ComputeHost"}, "", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in OneTaskAtATimeWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::ComplexJobWMS({baremetal_service}, {storage_service1, storage_service2}, "WMSHost")); diff --git a/examples/basic-examples/batch-bag-of-tasks/BatchBagOfTasks.cpp b/examples/basic-examples/batch-bag-of-tasks/BatchBagOfTasks.cpp index a9c98e7d8a..ddfa996807 100644 --- a/examples/basic-examples/batch-bag-of-tasks/BatchBagOfTasks.cpp +++ b/examples/basic-examples/batch-bag-of-tasks/BatchBagOfTasks.cpp @@ -112,8 +112,7 @@ int main(int argc, char **argv) { "BatchHeadNode", batch_nodes, "", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in TwoTasksAtATimeWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::TwoTasksAtATimeBatchWMS({batch_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/batch-bag-of-tasks/TwoTasksAtATimeBatchWMS.cpp b/examples/basic-examples/batch-bag-of-tasks/TwoTasksAtATimeBatchWMS.cpp index 65ff8bce2a..6c90d5dc90 100644 --- a/examples/basic-examples/batch-bag-of-tasks/TwoTasksAtATimeBatchWMS.cpp +++ b/examples/basic-examples/batch-bag-of-tasks/TwoTasksAtATimeBatchWMS.cpp @@ -78,7 +78,7 @@ namespace wrench { } /* Initialize and seed a RNG */ - std::uniform_int_distribution dist(0, 1000000000); + std::uniform_int_distribution dist(0, 1000000000); std::mt19937 rng(42); /* While the workflow isn't done, repeat the main loop */ diff --git a/examples/basic-examples/batch-pilot-job/BatchPilotJob.cpp b/examples/basic-examples/batch-pilot-job/BatchPilotJob.cpp index c5e5131834..cddc712986 100644 --- a/examples/basic-examples/batch-pilot-job/BatchPilotJob.cpp +++ b/examples/basic-examples/batch-pilot-job/BatchPilotJob.cpp @@ -101,8 +101,7 @@ int main(int argc, char **argv) { "BatchHeadNode", batch_nodes, "/scratch/", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in PilotJobWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::PilotJobWMS({batch_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/cloud-bag-of-tasks/CloudBagOfTasks.cpp b/examples/basic-examples/cloud-bag-of-tasks/CloudBagOfTasks.cpp index 16b36c84ef..5ca11a8871 100644 --- a/examples/basic-examples/cloud-bag-of-tasks/CloudBagOfTasks.cpp +++ b/examples/basic-examples/cloud-bag-of-tasks/CloudBagOfTasks.cpp @@ -55,7 +55,7 @@ int main(int argc, char **argv) { /* Parsing of the command-line arguments for this WRENCH simulation */ if (argc != 3) { - std::cerr << "Usage: " << argv[0] << " [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl; + std::cerr << "Usage: " << argv[0] << " [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl; exit(1); } @@ -80,7 +80,7 @@ int main(int argc, char **argv) { wrench::Workflow workflow; /* Initialize and seed a RNG */ - std::uniform_int_distribution dist(100000000,10000000000); + std::uniform_int_distribution dist(100000000,10000000000); std::mt19937 rng(42); /* Add workflow tasks and files */ @@ -117,8 +117,7 @@ int main(int argc, char **argv) { "CloudProviderHost", cloud_hosts, "", {}, {})); /* Instantiate a WMS, to be stated on WMSHost, which is responsible - * for executing the workflow. See comments in TwoTasksAtATimeWMS.cpp - * for more details */ + * for executing the workflow. */ auto wms = simulation.add( new wrench::TwoTasksAtATimeCloudWMS({cloud_service}, {storage_service}, "WMSHost")); diff --git a/examples/basic-examples/virtualized-cluster-bag-of-tasks/CMakeLists.txt b/examples/basic-examples/virtualized-cluster-bag-of-tasks/CMakeLists.txt new file mode 100644 index 0000000000..5902fe046c --- /dev/null +++ b/examples/basic-examples/virtualized-cluster-bag-of-tasks/CMakeLists.txt @@ -0,0 +1,13 @@ + +set(SOURCE_FILES + TwoTasksAtATimeVirtualizedClusterWMS.h + TwoTasksAtATimeVirtualizedClusterWMS.cpp + VirtualizedClusterBagOfTasks.cpp + ) + +add_executable(wrench-virtualized-cluster-bag-of-tasks-simulator ${SOURCE_FILES}) + +target_link_libraries(wrench-virtualized-cluster-bag-of-tasks-simulator wrench ${SimGrid_LIBRARY} ${PUGIXML_LIBRARY} ${LEMON_LIBRARY}) + +install(TARGETS wrench-virtualized-cluster-bag-of-tasks-simulator DESTINATION bin) + diff --git a/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.cpp b/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.cpp new file mode 100644 index 0000000000..7c7d2fd263 --- /dev/null +++ b/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.cpp @@ -0,0 +1,192 @@ +/** + * Copyright (c) 2017-2018. The WRENCH Team. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +/** + ** A Workflow Management System (WMS) implementation that operates as follows: + ** - Create two VMs, each one running on its own host + ** - While the workflow is not done, repeat: + ** - Pick up to two ready tasks + ** - Submit both of them as two different jobs to the VMs + ** - The most expensive task on the more powerful VM + ** - The least expensive task on the less powerful VM + ** - Each task reads the input file from the StorageService + ** - Each task reads the output file from the StorageService + ** - After a while, migrate one VM + **/ + +#include + +#include "TwoTasksAtATimeVirtualizedClusterWMS.h" + +#define GB 1000000000 + +XBT_LOG_NEW_DEFAULT_CATEGORY(custom_wms, "Log category for TwoTasksAtATimeVirtualizedClusterWMS"); + +namespace wrench { + + /** + * @brief Constructor, which calls the super constructor + * + * @param compute_services: a set of compute services available to run tasks + * @param storage_services: a set of storage services available to store files + * @param hostname: the name of the host on which to start the WMS + */ + TwoTasksAtATimeVirtualizedClusterWMS::TwoTasksAtATimeVirtualizedClusterWMS(const std::set> &compute_services, + const std::set> &storage_services, + const std::string &hostname) : WMS( + nullptr, nullptr, + compute_services, + storage_services, + {}, nullptr, + hostname, + "two-tasks-at-a-time-virtualized-cluster") {} + + /** + * @brief main method of the TwoTasksAtATimeVirtualizedClusterWMS daemon + * + * @return 0 on completion + * + * @throw std::runtime_error + */ + int TwoTasksAtATimeVirtualizedClusterWMS::main() { + + /* Set the logging output to GREEN */ + TerminalOutput::setThisProcessLoggingColor(TerminalOutput::COLOR_GREEN); + + WRENCH_INFO("WMS starting on host %s", Simulation::getHostName().c_str()); + WRENCH_INFO("About to execute a workflow with %lu tasks", this->getWorkflow()->getNumberOfTasks()); + + /* Create a job manager so that we can create/submit jobs */ + auto job_manager = this->createJobManager(); + + /* Get the first available bare-metal compute service and storage service */ + auto virtualized_cluster_service = *(this->getAvailableComputeServices().begin()); + auto storage_service = *(this->getAvailableStorageServices().begin()); + + /* Create a VM instance with 5 cores and one with 2 cores (and 500M of RAM) */ + WRENCH_INFO("Creating a 'large' VM with 5 cores and a 'small' VM with 2 cores, both of them with 5GB RAM"); + auto large_vm = virtualized_cluster_service->createVM(5, 5 * GB); + auto small_vm = virtualized_cluster_service->createVM(2, 5 * GB); + + /* Start the VMs */ + WRENCH_INFO("Start the large VM on host VirtualizedClusterHost1"); + auto large_vm_compute_service = virtualized_cluster_service->startVM(large_vm, "VirtualizedClusterHost1"); + WRENCH_INFO("Start the small VM on host VirtualizedClusterHost2"); + auto small_vm_compute_service = virtualized_cluster_service->startVM(small_vm, "VirtualizedClusterHost2"); + + /* While the workflow isn't done, repeat the main loop */ + while (not this->getWorkflow()->isDone()) { + + /* Get the ready tasks */ + auto ready_tasks = this->getWorkflow()->getReadyTasks(); + + /* Sort them by flops */ + std::sort(ready_tasks.begin(), ready_tasks.end(), + [](const WorkflowTask *t1, const WorkflowTask *t2) -> bool { + + if (t1->getFlops() == t2->getFlops()) { + return ((uintptr_t) t1 > (uintptr_t) t2); + } else { + return (t1->getFlops() < t2->getFlops()); + } + }); + + /* Pick the least and most expensive task */ + auto cheap_ready_task = ready_tasks.at(0); + auto expensive_ready_task = ready_tasks.at(ready_tasks.size() - 1); + + /* Submit the cheap task to the small VM */ + /* First, we need to create a map of file locations, stating for each file + * where is should be read/written */ + std::map> file_locations1; + file_locations1[cheap_ready_task->getInputFiles().at(0)] = FileLocation::LOCATION(storage_service); + file_locations1[cheap_ready_task->getOutputFiles().at(0)] = FileLocation::LOCATION(storage_service); + + /* Create the job */ + WRENCH_INFO("Creating a job to run task %s (%.2lf)", + cheap_ready_task->getID().c_str(), cheap_ready_task->getFlops()); + + auto standard_job1 = job_manager->createStandardJob(cheap_ready_task, file_locations1); + + /* Submit the job to the small VM */ + WRENCH_INFO("Submit this job to the small VM"); + job_manager->submitJob(standard_job1, small_vm_compute_service); + + /* Submit the expensive task to the large VM */ + /* First, we need to create a map of file locations, stating for each file + * where is should be read/written */ + std::map> file_locations2; + file_locations2[expensive_ready_task->getInputFiles().at(0)] = FileLocation::LOCATION(storage_service); + file_locations2[expensive_ready_task->getOutputFiles().at(0)] = FileLocation::LOCATION(storage_service); + + /* Create the job */ + WRENCH_INFO("Creating a job to run task %s (%.2lf)", + expensive_ready_task->getID().c_str(), expensive_ready_task->getFlops()); + + auto standard_job2 = job_manager->createStandardJob(expensive_ready_task, file_locations2); + + /* Submit the job to the large VM */ + WRENCH_INFO("Submit this job to the large VM"); + job_manager->submitJob(standard_job2, large_vm_compute_service); + + /* Sleeping for 10 seconds */ + Simulation::sleep(10); + + /* Migrate the VM */ + WRENCH_INFO("Migrating the small VM from VirtualizedClusterHost1 to VirtualizedClusterHost2"); + virtualized_cluster_service->migrateVM(small_vm, "VirtualizedClusterHost2"); + WRENCH_INFO("VM Migrated!"); + + /* Wait for workflow execution event and process it. In this case we know that + * the event will be a StandardJobCompletionEvent, which is processed by the method + * processEventStandardJobCompletion() that this class overrides. */ + WRENCH_INFO("Wait for next event"); + this->waitForAndProcessNextEvent(); + + /* And again! */ + WRENCH_INFO("Wait for next event again"); + this->waitForAndProcessNextEvent(); + } + + WRENCH_INFO("Workflow execution complete"); + return 0; + } + + /** + * @brief Process a standard job completion event + * + * @param event: the event + */ + void TwoTasksAtATimeVirtualizedClusterWMS::processEventStandardJobCompletion(std::shared_ptr event) { + /* Retrieve the job that this event is for */ + auto job = event->standard_job; + /* Retrieve the job's first (and in our case only) task */ + auto task = job->getTasks().at(0); + WRENCH_INFO("Notified that a standard job has completed task %s", task->getID().c_str()); + } + + /** + * @brief Process a standard job failure event + * + * @param event: the event + */ + void TwoTasksAtATimeVirtualizedClusterWMS::processEventStandardJobFailure(std::shared_ptr event) { + /* Retrieve the job that this event is for */ + auto job = event->standard_job; + /* Retrieve the job's first (and in our case only) task */ + auto task = job->getTasks().at(0); + /* Print some error message */ + WRENCH_INFO("Notified that a standard job has failed for task %s with error %s", + task->getID().c_str(), + event->failure_cause->toString().c_str()); + throw std::runtime_error("ABORTING DUE TO JOB FAILURE"); + } + + +} diff --git a/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.h b/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.h new file mode 100644 index 0000000000..b0c0ee9922 --- /dev/null +++ b/examples/basic-examples/virtualized-cluster-bag-of-tasks/TwoTasksAtATimeVirtualizedClusterWMS.h @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2017-2018. The WRENCH Team. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + + +#ifndef WRENCH_TWO_TASKS_AT_A_TIME_VIRTUALIZED_CLUSTER_H +#define WRENCH_TWO_TASKS_AT_A_TIME_VIRTUALIZED_CLUSTER_H + +#include + + +namespace wrench { + + class Simulation; + + /** + * @brief A Workflow Management System (WMS) implementation (inherits from WMS) + */ + class TwoTasksAtATimeVirtualizedClusterWMS : public WMS { + + public: + // Constructor + TwoTasksAtATimeVirtualizedClusterWMS( + const std::set> &compute_services, + const std::set> &storage_services, + const std::string &hostname); + + protected: + + // Overriden method + void processEventStandardJobCompletion(std::shared_ptr) override; + void processEventStandardJobFailure(std::shared_ptr) override; + + private: + // main() method of the WMS + int main() override; + + }; +} +#endif //WRENCH_TWO_TASKS_AT_A_TIME_VIRTUALIZED_CLUSTER_H diff --git a/examples/basic-examples/virtualized-cluster-bag-of-tasks/VirtualizedClusterBagOfTasks.cpp b/examples/basic-examples/virtualized-cluster-bag-of-tasks/VirtualizedClusterBagOfTasks.cpp new file mode 100644 index 0000000000..fc685caa50 --- /dev/null +++ b/examples/basic-examples/virtualized-cluster-bag-of-tasks/VirtualizedClusterBagOfTasks.cpp @@ -0,0 +1,162 @@ +/** + * Copyright (c) 2017-2018. The WRENCH Team. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + */ + +/** + ** This simulator simulates the execution of a bag-of-tasks workflow, that is, of a workflow + ** in which each task has its own input file and its own output file, and tasks can be + ** executed completely independently + ** + ** InputFile #0 -> Task #0 -> OutputFile #1 + ** ... + ** InputFile #n -> Task #n -> OutputFile #n + ** + ** The compute platform comprises four hosts, WMSHost, VirtualizedClusterProviderHost, VirtualizedClusterHost1, and + ** VirtualizedClusterHost2. On WMSHost runs a simple storage + ** service and a WMS (defined in class TwoTasksAtATimeVirtualizedClusterWMS). On VirtualizedClusterProviderHost runs a virtualized_cluster + ** service, that has access to two hosts: VirtualizedClusterHost1 and VirtualizedClusterHost2. Once the simulation is done, + ** the completion time of each workflow task is printed. + ** + ** Example invocation of the simulator for a 10-task workflow, with only WMS logging: + ** ./virtualized_cluster-bag-of-tasks-simulator 10 ./four_hosts.xml --wrench-no-logs --log=custom_wms.threshold=info + ** + ** Example invocation of the simulator for a 6-task workflow with full logging: + ** ./virtualized_cluster-bag-of-tasks-simulator 6 ./four_hosts.xml + **/ + + +#include +#include + +#include "TwoTasksAtATimeVirtualizedClusterWMS.h" // WMS implementation + +/** + * @brief The Simulator's main function + * + * @param argc: argument count + * @param argv: argument array + * @return 0 on success, non-zero otherwise + */ +int main(int argc, char **argv) { + + /* Declare a WRENCH simulation object */ + wrench::Simulation simulation; + + /* Initialize the simulation, which may entail extracting WRENCH-specific and + * Simgrid-specific command-line arguments that can modify general simulation behavior. + * Two special command-line arguments are --help-wrench and --help-simgrid, which print + * details about available command-line arguments. */ + simulation.init(&argc, argv); + + /* Parsing of the command-line arguments for this WRENCH simulation */ + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " [--wrench-no-logs --log=custom_wms.threshold=info]" << std::endl; + exit(1); + } + + /* Reading and parsing the platform description file, written in XML following the SimGrid-defined DTD, + * to instantiate the simulated platform */ + std::cerr << "Instantiating simulated platform..." << std::endl; + simulation.instantiatePlatform(argv[2]); + + /* Parse the first command-line argument (number of tasks) */ + int num_tasks = 0; + try { + num_tasks = std::atoi(argv[1]); + if (num_tasks % 2) { + throw std::invalid_argument("Number of tasks should be even"); + } + } catch (std::invalid_argument &e) { + std::cerr << "Invalid number of tasks (" << e.what() << ")\n"; + exit(1); + } + + /* Declare a workflow */ + wrench::Workflow workflow; + + /* Initialize and seed a RNG */ + std::uniform_int_distribution dist(100000000,10000000000); + std::mt19937 rng(42); + + /* Add workflow tasks and files */ + for (int i=0; i < num_tasks; i++) { + /* Create a task: 10GFlop, 1 to 10 cores, 0.90 parallel efficiency, 10MB memory footprint */ + auto task = workflow.addTask("task_" + std::to_string(i), dist(rng), 1, 10, 0.90, 1000); + task->addInputFile(workflow.addFile("input_" + std::to_string(i), 10000000)); + task->addOutputFile(workflow.addFile("output_" + std::to_string(i), 10000000)); + } + + /* Instantiate a storage service, and add it to the simulation. + * A wrench::StorageService is an abstraction of a service on + * which files can be written and read. This particular storage service, which is an instance + * of wrench::SimpleStorageService, is started on WMSHost in the + * platform , which has an attached disk mounted at "/". The SimpleStorageService + * is a basic storage service implementation provided by WRENCH. + * Throughout the simulation execution, input/output files of workflow tasks will be located + * in this storage service, and accessed remotely by the compute service. Note that the + * storage service is configured to use a buffer size of 50M when transferring data over + * the network (i.e., to pipeline disk reads/writes and network revs/sends). */ + std::cerr << "Instantiating a SimpleStorageService on WMSHost..." << std::endl; + auto storage_service = simulation.add(new wrench::SimpleStorageService( + "WMSHost", {"/"}, {{wrench::SimpleStorageServiceProperty::BUFFER_SIZE, "50000000"}}, {})); + + /* Instantiate a virtualized cluster compute service, and add it to the simulation. + * A wrench::VirtualizedClusterComputeService is an abstraction of a compute service that corresponds + * to a virtualized_cluster that responds to VM creating requests, and each VM exposes a "bare-metal" compute service. + * This particular service is started on VirtualizedClusterProviderHost, uses VirtualizedClusterHost1 and VirtualizedClusterHost2 + * as hardware resources, and has no scratch storage space (mount point argument = ""). + * This means that tasks running on this service will access data only from remote storage services. */ + std::cerr << "Instantiating a VirtualizedClusterComputeService on VirtualizedClusterProviderHost..." << std::endl; + std::vector virtualized_cluster_hosts = {"VirtualizedClusterHost1", "VirtualizedClusterHost2"}; + auto virtualized_cluster_service = simulation.add(new wrench::VirtualizedClusterComputeService( + "VirtualizedClusterProviderHost", virtualized_cluster_hosts, "", {}, {})); + + /* Instantiate a WMS, to be stated on WMSHost, which is responsible + * for executing the workflow. */ + + auto wms = simulation.add( + new wrench::TwoTasksAtATimeVirtualizedClusterWMS({virtualized_cluster_service}, {storage_service}, "WMSHost")); + + /* Associate the workflow to the WMS */ + wms->addWorkflow(&workflow); + + /* Instantiate a file registry service to be started on WMSHost. This service is + * essentially a replica catalog that stores pairs so that + * any service, in particular a WMS, can discover where workflow files are stored. */ + std::cerr << "Instantiating a FileRegistryService on WMSHost ..." << std::endl; + auto file_registry_service = new wrench::FileRegistryService("WMSHost"); + simulation.add(file_registry_service); + + /* It is necessary to store, or "stage", input files that only input. The getInputFiles() + * method of the Workflow class returns the set of all workflow files that are not generated + * by workflow tasks, and thus are only input files. These files are then staged on the storage service. */ + std::cerr << "Staging task input files..." << std::endl; + for (auto const &f : workflow.getInputFiles()) { + simulation.stageFile(f, storage_service); + } + + /* Launch the simulation. This call only returns when the simulation is complete. */ + std::cerr << "Launching the Simulation..." << std::endl; + try { + simulation.launch(); + } catch (std::runtime_error &e) { + std::cerr << "Exception: " << e.what() << std::endl; + return 1; + } + std::cerr << "Simulation done!" << std::endl; + + /* Simulation results can be examined via simulation.output, which provides access to traces + * of events. In the code below, we print the retrieve the trace of all task completion events, print how + * many such events there are, and print some information for the first such event. */ + auto trace = simulation.getOutput().getTrace(); + for (auto const &item : trace) { + std::cerr << "Task " << item->getContent()->getTask()->getID() << " completed at time " << item->getDate() << std::endl; + } + + return 0; +} diff --git a/examples/basic-examples/virtualized-cluster-bag-of-tasks/four_hosts.xml b/examples/basic-examples/virtualized-cluster-bag-of-tasks/four_hosts.xml new file mode 100644 index 0000000000..31d4f58e98 --- /dev/null +++ b/examples/basic-examples/virtualized-cluster-bag-of-tasks/four_hosts.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +