From 37566be02c1309f31c475b4dfc8defb084e6a159 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 13 Mar 2024 01:29:39 -0500 Subject: [PATCH] Fix openmpi exec --- None | 157 ++++++++++++++++++---------------- jarvis_util/shell/mpi_exec.py | 2 - 2 files changed, 85 insertions(+), 74 deletions(-) diff --git a/None b/None index 5b89413..f1db61d 100644 --- a/None +++ b/None @@ -1,72 +1,85 @@ -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/api/manager.h:78 45907 LoadServerConfig Loading server configuration: /home/llogan/.local/jarvis/jv-shared/mm_kmeans_mega/hermes_run/hermes_server.yaml -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:35 45907 ServerInit Initializing shared memory -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:37 45907 ServerInit Initializing RPC -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:39 45907 ServerInit Initializing thallium -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:42 45907 ServerInit Initializing RPC server -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:44 45907 ServerInit Attempting to start server on: sockets://localhost:8080 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:52 45907 ServerInit Serving ofi+sockets://127.0.0.1:8080 (i.e., sockets://localhost:8080) with 32 RPC threads as node id 1 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:67 45907 ClientInit This client is on node 1 (i.e., localhost, proto: sockets) -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:41 45907 ServerInit Initializing queues + workers -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: hermes_mdm -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: hermes_blob_mdm -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: hermes_bucket_mdm -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: hermes_data_op -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: data_stager -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: posix_bdev -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: ram_bdev -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/api/manager.h:78 45907 LoadServerConfig Loading server configuration: /home/llogan/.local/jarvis/jv-shared/mm_kmeans_mega/hermes_run/hermes_server.yaml -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45950 SetCpuAffinity Affining worker 0 (pid=45950) to 0 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45951 SetCpuAffinity Affining worker 1 (pid=45951) to 1 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45952 SetCpuAffinity Affining worker 2 (pid=45952) to 2 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45953 SetCpuAffinity Affining worker 3 (pid=45953) to 2 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45954 SetCpuAffinity Affining worker 4 (pid=45954) to 2 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45955 SetCpuAffinity Affining worker 5 (pid=45955) to 2 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 45956 SetCpuAffinity Affining worker 6 (pid=45956) to 2 -/home/llogan/Documents/Projects/hermes/hrun/src/work_orchestrator.cc:84 45907 ServerInit Started 7 workers -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: hrun_admin -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 45907 CreateTaskState (node 1) Created an instance of hrun_admin with name hrun_admin and ID 1.0 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: proc_queue -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 45907 CreateTaskState (node 1) Created an instance of proc_queue with name proc_queue and ID 1.1 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: worch_queue_round_robin -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 45907 CreateTaskState (node 1) Created an instance of worch_queue_round_robin with name worch_queue_round_robin and ID 1.2 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: worch_proc_round_robin -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 45907 CreateTaskState (node 1) Created an instance of worch_proc_round_robin with name worch_proc_round_robin and ID 1.3 -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 1, lane 0, worker 0) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 0, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 1, lane 0, worker 2) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 0, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 1, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 2, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 45950 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 3, worker 1) -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 45907 RegisterTaskLib Finished loading the lib: remote_queue -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 45950 CreateTaskState (node 1) Creating task state remote_queue with id 1.6 (task_node=1.7/0) -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/remote_queue/src/remote_queue.cc:83 45950 Construct (node 1) Constructing remote queue (task_node=1.7/0, task_state=1.0, method=0) -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 45950 CreateTaskState (node 1) Created an instance of remote_queue with name remote_queue and ID 1.6 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:73 45907 RunDaemon Starting the daemon on node: 1 -/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:127 45950 StopRuntime Stopping (server mode) -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:91 45950 StopDaemon Sending stop signal to: 1 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:76 45913 operator() Finalizing RPCs on node: 1 -/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:80 45907 RunDaemon Daemon has stopped on node: 1 -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:150 45907 RunDaemon Daemon is running -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:155 45907 RunDaemon Finishing up last requests -/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:157 45907 RunDaemon Daemon is exiting +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/api/manager.h:78 48399 LoadServerConfig Loading server configuration: /home/llogan/.local/jarvis/jv-shared/mm_kmeans_mega/hermes_run/hermes_server.yaml +/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:35 48399 ServerInit Initializing shared memory +/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:37 48399 ServerInit Initializing RPC +/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:39 48399 ServerInit Initializing thallium +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:42 48399 ServerInit Initializing RPC server +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:44 48399 ServerInit Attempting to start server on: sockets://localhost:8080 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:52 48399 ServerInit Serving ofi+sockets://127.0.0.1:8080 (i.e., sockets://localhost:8080) with 32 RPC threads as node id 1 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:67 48399 ClientInit This client is on node 1 (i.e., localhost, proto: sockets) +/home/llogan/Documents/Projects/hermes/hrun/src/hrun_runtime.cc:41 48399 ServerInit Initializing queues + workers +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: hermes_mdm +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: hermes_blob_mdm +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: hermes_bucket_mdm +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: hermes_data_op +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: data_stager +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: posix_bdev +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: ram_bdev +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/api/manager.h:78 48399 LoadServerConfig Loading server configuration: /home/llogan/.local/jarvis/jv-shared/mm_kmeans_mega/hermes_run/hermes_server.yaml +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48442 SetCpuAffinity Affining worker 0 (pid=48442) to 0 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48443 SetCpuAffinity Affining worker 1 (pid=48443) to 1 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48444 SetCpuAffinity Affining worker 2 (pid=48444) to 2 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48445 SetCpuAffinity Affining worker 3 (pid=48445) to 2 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48446 SetCpuAffinity Affining worker 4 (pid=48446) to 2 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48447 SetCpuAffinity Affining worker 5 (pid=48447) to 2 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/work_orchestrator/worker.h:281 48448 SetCpuAffinity Affining worker 6 (pid=48448) to 2 +/home/llogan/Documents/Projects/hermes/hrun/src/work_orchestrator.cc:84 48399 ServerInit Started 7 workers +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: hrun_admin +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48399 CreateTaskState (node 1) Created an instance of hrun_admin with name hrun_admin and ID 1.0 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: proc_queue +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48399 CreateTaskState (node 1) Created an instance of proc_queue with name proc_queue and ID 1.1 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: worch_queue_round_robin +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48399 CreateTaskState (node 1) Created an instance of worch_queue_round_robin with name worch_queue_round_robin and ID 1.2 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: worch_proc_round_robin +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48399 CreateTaskState (node 1) Created an instance of worch_proc_round_robin with name worch_proc_round_robin and ID 1.3 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 1, lane 0, worker 0) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 2, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 3, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.0 (prio 4, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 0, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:77 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 1, lane 0, worker 2) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 2, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:62 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 3, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 0, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 1, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 2, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/worch_queue_round_robin/src/worch_queue_round_robin.cc:70 48442 Schedule (node 1) Scheduling the queue 1.1 (prio 4, lane 3, worker 1) +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:181 48399 RegisterTaskLib Finished loading the lib: remote_queue +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state remote_queue with id 1.6 (task_node=1.7/0) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/remote_queue/src/remote_queue.cc:83 48442 Construct (node 1) Constructing remote queue (task_node=1.7/0, task_state=1.0, method=0) +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of remote_queue with name remote_queue and ID 1.6 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/network/rpc_thallium.h:73 48399 RunDaemon Starting the daemon on node: 1 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_mdm with id 1.9 (task_node=1.8/0) +/home/llogan/Documents/Projects/hermes/include/hermes/config_manager.h:85 48442 LoadServerConfig Loading server configuration: /home/llogan/.local/jarvis/jv-shared/mm_kmeans_mega/hermes_run/hermes_server.yaml +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of hermes_mdm with name hermes_mdm and ID 1.9 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_blob_mdm with id 1.11 (task_node=1.10/0) +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_ram/1 with id 1.12 (task_node=1.10/1) +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of ram_bdev with name hermes_ram/1 and ID 1.12 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_ssd_0/1 with id 1.13 (task_node=1.10/1) +/home/llogan/Documents/Projects/hermes/tasks/posix_bdev/src/posix_bdev.cc:41 48442 Construct Created ssd_0 at /home/llogan/jarvis_data/hermes_data/slab_ssd_0 of size 483183820800 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of posix_bdev with name hermes_ssd_0/1 and ID 1.13 +/home/llogan/Documents/Projects/hermes/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc:106 48442 Construct (node 1) Target 1.12 has bw 4.29497e+10 and score 1 +/home/llogan/Documents/Projects/hermes/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc:106 48442 Construct (node 1) Target 1.13 has bw 5.24288e+08 and score 0 +/home/llogan/Documents/Projects/hermes/tasks/hermes_blob_mdm/src/hermes_blob_mdm.cc:111 48442 Construct (node 1) Created Blob MDM +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of hermes_blob_mdm with name hermes_blob_mdm and ID 1.11 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_bkt_mdm with id 1.15 (task_node=1.14/0) +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of hermes_bucket_mdm with name hermes_bkt_mdm and ID 1.15 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_op_mdm with id 1.17 (task_node=1.16/0) +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of hermes_data_op with name hermes_op_mdm and ID 1.17 +/home/llogan/Documents/Projects/hermes/hrun/tasks_required/hrun_admin/src/hrun_admin.cc:75 48442 CreateTaskState (node 1) Creating task state hermes_stager_mdm with id 1.20 (task_node=1.19/0) +/home/llogan/Documents/Projects/hermes/tasks/data_stager/src/data_stager.cc:31 48442 Construct (node 1) BLOB MDM: 1.11 +/home/llogan/Documents/Projects/hermes/hrun/include/hrun/task_registry/task_registry.h:262 48442 CreateTaskState (node 1) Created an instance of data_stager with name hermes_stager_mdm and ID 1.20 diff --git a/jarvis_util/shell/mpi_exec.py b/jarvis_util/shell/mpi_exec.py index b47def3..03473ee 100644 --- a/jarvis_util/shell/mpi_exec.py +++ b/jarvis_util/shell/mpi_exec.py @@ -81,8 +81,6 @@ def mpicmd(self): else: params.append(f'-n {self.nprocs}') params.append(self.cmd) - - params.append(self.cmd) cmd = ' '.join(params) jutil = JutilManager.get_instance() if jutil.debug_mpi_exec: