From fff2aa31078074fab98732a39fde2f9727c40b66 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 5 Jul 2021 04:06:06 +0100
Subject: [PATCH 01/30] Modify composer to emit device structs in their own
 header and fix names

---
 Source/OrchBase/Composer.cpp | 325 ++++++++++++++++++++++-------------
 Source/OrchBase/Composer.h   |   5 +-
 Source/Softswitch/Makefile   |   1 +
 3 files changed, 211 insertions(+), 120 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index dca6b510..6f26d67d 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -626,9 +626,36 @@ int Composer::generate(GraphI_t* graphI)
         return -2;
     }
     builderGraphI->cores = &(giToCoresFinder->second);
+    
+    
+    // Cache the file provenance info
+    formFileProvenance(builderGraphI);
+    
+    
+    //==========================================================================
+    // Write device structures header header
+    //==========================================================================
+    std::ofstream devst_h;
 
+    std::stringstream devst_hFName;
+    devst_hFName << builderGraphI->outputDir;
+    devst_hFName << "/" << GENERATED_PATH;
+    devst_hFName << "/DeviceStructs.h";
+    
+    std::string devst_hFNameStr = devst_hFName.str();
 
-    //Form Device Type strings for all DevTs in the GraphT
+    devst_h.open(devst_hFNameStr.c_str());
+    if(devst_h.fail()) // Check that the file opened
+    {                 // if it didn't, tell logserver and exit
+        //TODO: Barf
+        //par->Post(816, vars_hFName.str(), OSFixes::getSysErrorString(errno));
+        return -1;
+    }
+    writeFileProvenance(devst_hFNameStr, builderGraphI, devst_h);
+    writeDeviceStructTypesPreamble(devst_h);
+    
+    //Form Device Type strings for all DevTs in the GraphT and write the device
+    //structs
     builderGraphI->clearDevTStrsMap();      // sanity clear
     WALKVECTOR(DevT_t*,graphI->pT->DevT_v,devT)
     {
@@ -636,14 +663,16 @@ int Composer::generate(GraphI_t* graphI)
         {
             formDevTStrings(builderGraphI, (*devT));
         }
+        writeDeviceStructTypes((*devT), devst_h);
     }
-
-    // Cache the file provenance info
-    formFileProvenance(builderGraphI);
-
-
-    //Write global properties and message format headers
-    std::ofstream props_h, pkt_h;
+    writeDeviceStructTypesPostamble(devst_h);
+    devst_h.close();
+    
+    
+    //==========================================================================
+    // Write global properties header
+    //==========================================================================
+    std::ofstream props_h;
 
     std::stringstream props_hFName;
     props_hFName << builderGraphI->outputDir;
@@ -662,8 +691,12 @@ int Composer::generate(GraphI_t* graphI)
     writeFileProvenance(props_hFNameStr, builderGraphI, props_h);
     writeGlobalPropsD(graphI, props_h);
     props_h.close();
-
-
+    
+    
+    //==========================================================================
+    // Write message format header
+    //==========================================================================
+    std::ofstream pkt_h;
     std::stringstream pkt_hFName;
     pkt_hFName << builderGraphI->outputDir;
     pkt_hFName << "/" << GENERATED_PATH;
@@ -683,8 +716,9 @@ int Composer::generate(GraphI_t* graphI)
     pkt_h.close();
 
 
-    //Generate Supervisor, inc Dev> Super map
-
+    //==========================================================================
+    // Generate Supervisor, inc Dev> Super map
+    //==========================================================================
     fprintf(fd,"\tGenerating Supervisor...");
     generateSupervisor(builderGraphI);
     fprintf(fd,"\tDone!\n");
@@ -1985,6 +2019,81 @@ void Composer::writeMessageTypes(GraphI_t* graphI, std::ofstream& pkt_h)
 }
 
 
+/******************************************************************************
+ * Write the Device structs preamble to a common header
+ *****************************************************************************/
+void Composer::writeDeviceStructTypesPreamble(std::ofstream& types_h)
+{
+    types_h << "#ifndef _DEVICESTRUCTS_H_\n";
+    types_h << "#define _DEVICESTRUCTS_H_\n\n";
+
+    types_h << "#include <cstdint>\n";
+    
+    //types_h <<  "#pragma pack(push,1)\n";
+}
+
+
+/******************************************************************************
+ * Write the Device structs to a common header
+ *****************************************************************************/
+void Composer::writeDeviceStructTypes(DevT_t* devT, std::ofstream& types_h)
+{
+    GraphT_t* graphT = devT->par;
+    
+    std::string devTName = devT->Name();  // grab a local copy of the name
+    std::string graphTName = graphT->Name();    // grab copy of the name
+    
+    // Write Properties struct declaration
+    if (devT->pPropsD)
+    {
+        types_h << "typedef struct " << graphTName << "_" << devTName;
+        types_h << "_properties_t \n{\n" << devT->pPropsD->C_src() << "\n} ";
+        types_h << graphTName << "_" << devTName << "_properties_t;\n\n";
+    }
+
+    // Write State struct declaration
+    if (devT->pStateD)
+    {
+        types_h << "typedef struct " << graphTName << "_" << devTName;
+        types_h << "_state_t \n{\n" << devT->pStateD->C_src() << "\n} ";
+        types_h << graphTName << "_" << devTName << "_state_t;\n\n";
+    }
+    
+    // Walk Input pin types
+    WALKVECTOR(PinT_t*,devT->PinTI_v,pinI)
+    {
+        std::string pinIName = (*pinI)->Name();
+        
+        if ((*pinI)->pPropsD)
+        {   // Write the pin's properties struct
+            types_h << "typedef struct " << devTName;
+            types_h << "_InPin_" << pinIName << "_edgeproperties_t \n{\n";
+            types_h << (*pinI)->pPropsD->C_src();
+            types_h << "\n} " << graphTName << "_" << devTName << "_";
+            types_h << pinIName << "_properties_t;\n\n";
+        }
+        
+        if ((*pinI)->pStateD)
+        {   // Write the pin's state struct
+            types_h << "typedef struct " << devTName;
+            types_h << "_InPin_" << pinIName << "_edgestate_t \n{\n";
+            types_h << (*pinI)->pStateD->C_src();
+            types_h << "\n} " << graphTName << "_" << devTName << "_";
+            types_h << pinIName << "_state_t;\n\n";
+        }
+    }
+}
+
+/******************************************************************************
+ * Write the Device structs postamble to a common header
+ *****************************************************************************/
+void Composer::writeDeviceStructTypesPostamble(std::ofstream& types_h)
+{
+    //types_h <<  "#pragma pack(pop)\n";
+    
+    types_h << "#endif /*_DEVICESTRUCTS_H_*/\n\n";
+}
+
 /******************************************************************************
  * Form the common handler strings for a device type if they do not exist
  *****************************************************************************/
@@ -2003,7 +2112,6 @@ void Composer::formDevTStrings(ComposerGraphI_t* builderGraphI, DevT_t* devT)
 
         formHandlerPreamble(dTypStrs);
         formDevTHandlers(dTypStrs);
-        formDevTPropsDStateD(dTypStrs);
         formDevTInputPinHandlers(dTypStrs);
         formDevTOutputPinHandlers(dTypStrs);
 
@@ -2018,6 +2126,9 @@ void Composer::formHandlerPreamble(devTypStrings_t* dTypStrs)
 {
     DevT_t* devT = dTypStrs->devT;  // grab a local copy of the devtype
     std::string devTName = devT->Name();  // grab a local copy of the name
+    
+    GraphT_t* graphT = dTypStrs->graphI->pT;    // grab a local copy of the graphtype
+    std::string graphTName = graphT->Name();    // grab a local copy of the name
 
     std::stringstream handlerPreamble_SS("");
     std::stringstream handlerPreambleS_SS("");        // "normal" state
@@ -2041,12 +2152,12 @@ void Composer::formHandlerPreamble(devTypStrings_t* dTypStrs)
     // deviceProperties (with unused variable handling)
     if (devT->pPropsD)
     {
-        handlerPreamble_SS << "    const devtyp_" << devTName;
-        handlerPreamble_SS << "_props_t* deviceProperties ";
+        handlerPreamble_SS << "    const " << graphTName << "_" << devTName;
+        handlerPreamble_SS << "_properties_t* deviceProperties ";
         handlerPreamble_SS << "OS_ATTRIBUTE_UNUSED= ";
-        handlerPreamble_SS << "static_cast<const devtyp_";
+        handlerPreamble_SS << "static_cast<const " << graphTName << "_";
         handlerPreamble_SS << devTName;
-        handlerPreamble_SS << "_props_t*>(deviceInstance->properties);\n";
+        handlerPreamble_SS << "_properties_t*>(deviceInstance->properties);\n";
         handlerPreamble_SS << "    OS_PRAGMA_UNUSED(deviceProperties)\n";
     }
 
@@ -2054,20 +2165,18 @@ void Composer::formHandlerPreamble(devTypStrings_t* dTypStrs)
     if (devT->pStateD)
     {
         // Const-protected state
-        handlerPreambleCS_SS << "    const devtyp_" << devTName;
+        handlerPreambleCS_SS << "    const " << graphTName << "_" << devTName;
         handlerPreambleCS_SS << "_state_t* deviceState ";
         handlerPreambleCS_SS << "OS_ATTRIBUTE_UNUSED= ";
-        handlerPreambleCS_SS << "static_cast<devtyp_";
-        handlerPreambleCS_SS << devTName;
+        handlerPreambleCS_SS << "static_cast<" << graphTName << "_" << devTName;
         handlerPreambleCS_SS << "_state_t*>(deviceInstance->state);\n";
         handlerPreambleCS_SS << "    OS_PRAGMA_UNUSED(deviceState)\n";
 
         // "normal" state
-        handlerPreambleS_SS << "    devtyp_" << devTName;
+        handlerPreambleS_SS << "    " << graphTName << "_" << devTName;
         handlerPreambleS_SS << "_state_t* deviceState ";
         handlerPreambleS_SS << "OS_ATTRIBUTE_UNUSED= ";
-        handlerPreambleS_SS << "static_cast<devtyp_";
-        handlerPreambleS_SS << devTName;
+        handlerPreambleS_SS << "static_cast<" << graphTName << "_" << devTName;
         handlerPreambleS_SS << "_state_t*>(deviceInstance->state);\n";
         handlerPreambleS_SS << "    OS_PRAGMA_UNUSED(deviceState)\n";
     }
@@ -2117,7 +2226,8 @@ void Composer::formDevTHandlers(devTypStrings_t* dTypStrs)
     handlers_cpp << dTypStrs->handlerPreamble;
     handlers_cpp << dTypStrs->handlerPreambleCS;
     
-    handlers_cpp << "    bool* requestIdle = &deviceInstance->requestIdle;\n";
+    handlers_cpp << "    bool* requestIdle OS_ATTRIBUTE_UNUSED= ";
+    handlers_cpp << "&deviceInstance->requestIdle;\n";
     handlers_cpp << "    OS_PRAGMA_UNUSED(requestIdle)\n";
     
     if (devT->pOnRTS != 0) handlers_cpp << devT->pOnRTS->C_src() << "\n";
@@ -2246,37 +2356,6 @@ void Composer::formDevTHandlers(devTypStrings_t* dTypStrs)
 }
 
 
-/******************************************************************************
- * Form Device Type Properties and State declarations
- *****************************************************************************/
-void Composer::formDevTPropsDStateD(devTypStrings_t* dTypStrs)
-{
-    DevT_t* devT = dTypStrs->devT;  // grab a local copy of the devtype
-    std::string devTName = devT->Name();  // grab a local copy of the name
-
-    std::stringstream vars_h("");
-
-    // Write Properties declaration
-    if (devT->pPropsD)
-    {
-        vars_h << "typedef struct " << devTName << "_properties_t \n{\n";
-        vars_h << devT->pPropsD->C_src();
-        vars_h << "\n} devtyp_" << devTName << "_props_t;\n\n";
-    }
-
-    // Write State declaration
-    if (devT->pStateD)
-    {
-        vars_h << "typedef struct " << devTName << "_state_t \n{\n";
-        vars_h << devT->pStateD->C_src();
-        vars_h << "\n} devtyp_" << devTName << "_state_t;\n\n";
-    }
-
-    // Append to the strings
-    dTypStrs->varsHCommon += vars_h.str();
-}
-
-
 /******************************************************************************
  * Form Device Type Input Pin handler strings
  *****************************************************************************/
@@ -2284,10 +2363,13 @@ void Composer::formDevTInputPinHandlers(devTypStrings_t* dTypStrs)
 {
     DevT_t* devT = dTypStrs->devT;  // grab a local copy of the devtype
     std::string devTName = devT->Name();  // grab a local copy of the name
+    
+    GraphT_t* graphT = dTypStrs->graphI->pT;    // grab the graphtype
+    std::string graphTName = graphT->Name();    // grab a local copy of the name
 
     std::stringstream handlers_h("");
     std::stringstream handlers_cpp("");
-    std::stringstream vars_h("");
+    std::stringstream types_h("");
 
 
 
@@ -2312,43 +2394,28 @@ void Composer::formDevTInputPinHandlers(devTypStrings_t* dTypStrs)
         handlers_cpp << "OS_PRAGMA_UNUSED(edgeInstance)\n";
 
         if ((*pinI)->pPropsD)
-        {
-            vars_h << "typedef struct " << devTName;
-            vars_h << "_InPin_" << pinIName << "_edgeproperties_t \n{\n";
-            vars_h << (*pinI)->pPropsD->C_src();
-            vars_h << "\n} devtyp_" << devTName;
-            vars_h << "_InPin_" << pinIName << "_props_t;\n\n";
-
-            handlers_cpp << "   const devtyp_" << devTName;
-            handlers_cpp << "_InPin_" << pinIName;
-            handlers_cpp << "_props_t* edgeProperties ";
+        {   // If the pin type has properties, 
+            handlers_cpp << "   const " << graphTName << "_" << devTName << "_";
+            handlers_cpp << pinIName << "_properties_t* edgeProperties ";
             handlers_cpp << "OS_ATTRIBUTE_UNUSED= ";
-            handlers_cpp << "static_cast<const devtyp_" << devTName;
-            handlers_cpp << "_InPin_" << pinIName;
-            handlers_cpp << "_props_t*>(edgeInstance->properties);\n";
+            handlers_cpp << "static_cast<const " << graphTName << "_";
+            handlers_cpp << devTName << "_" << pinIName;
+            handlers_cpp << "_properties_t*>(edgeInstance->properties);\n";
             handlers_cpp << "OS_PRAGMA_UNUSED(edgeProperties)\n";
         }
 
         if ((*pinI)->pStateD)
-        {
-            vars_h << "typedef struct " << devTName;
-            vars_h << "_InPin_" << pinIName << "_edgestate_t \n{\n";
-            vars_h << (*pinI)->pStateD->C_src();
-            vars_h << "\n} devtyp_" << devTName;
-            vars_h << "_InPin_" << pinIName << "_state_t;\n\n";
-
-            handlers_cpp << "   devtyp_" << devTName;
-            handlers_cpp << "_InPin_" << pinIName;
-            handlers_cpp << "_state_t* edgeState ";
-            handlers_cpp << "OS_ATTRIBUTE_UNUSED= ";
-            handlers_cpp << "static_cast<devtyp_" << devTName;
-            handlers_cpp << "_InPin_"<< pinIName;
+        {   // If the pin type has state, 
+            handlers_cpp << graphTName << "_" << devTName << "_" << pinIName;
+            handlers_cpp << "_state_t* edgeState OS_ATTRIBUTE_UNUSED= ";
+            handlers_cpp << "static_cast<" << graphTName << "_";
+            handlers_cpp << devTName << "_" << pinIName;
             handlers_cpp << "_state_t*>(edgeInstance->state);\n";
             handlers_cpp << "OS_PRAGMA_UNUSED(edgeState)\n";
         }
 
         if ((*pinI)->pMsg->pPropsD)
-        {
+        {   
             handlers_cpp << "   const pkt_" << (*pinI)->pMsg->Name();
             handlers_cpp << "_pyld_t* message";
             handlers_cpp << " OS_ATTRIBUTE_UNUSED= ";
@@ -2368,7 +2435,7 @@ void Composer::formDevTInputPinHandlers(devTypStrings_t* dTypStrs)
     // Append to the strings
     dTypStrs->handlersH += handlers_h.str();
     dTypStrs->handlersC += handlers_cpp.str();
-    dTypStrs->varsHCommon += vars_h.str();
+    dTypStrs->typesH += types_h.str();
 }
 
 
@@ -2613,6 +2680,7 @@ void Composer::writeCoreVarsHead(AddressComponent coreAddr,
     vars_h << "#include \"softswitch_common.h\"\n";
     vars_h << "#include \"MessageFormats.h\"\n";
     vars_h << "#include \"GlobalProperties.h\"\n\n";
+    vars_h << "#include \"DeviceStructs.h\"\n\n";
 
     vars_cpp << "#include \"vars_" << coreAddr << ".h\"\n";
 }
@@ -2917,6 +2985,8 @@ void Composer::writeThreadContextInitialiser(ComposerGraphI_t* builderGraphI,
 void Composer::writeDevTDeclInit(AddressComponent threadAddr, DevT_t* devT,
                                 std::ofstream& vars_h, std::ofstream& vars_cpp)
 {
+    GraphT_t* graphT = devT->par;
+    
     size_t inTypCnt = devT->PinTI_v.size();       // Number of Input pins
     size_t outTypCnt = devT->PinTO_v.size();      // Number of output pins
 
@@ -2944,10 +3014,16 @@ void Composer::writeDevTDeclInit(AddressComponent threadAddr, DevT_t* devT,
     //TODO: Add v4 handlers
 
     if(devT->pPropsD)
-        vars_cpp << "sizeof(devtyp_" << devT->Name() << "_props_t),";           // sz_props
+    {
+        vars_cpp << "sizeof(" << graphT->Name() << "_";
+        vars_cpp << devT->Name() << "_properties_t),";           // sz_props
+    }
     else vars_cpp << "0,";
     if(devT->pStateD)
-        vars_cpp << "sizeof(devtyp_" << devT->Name() << "_state_t),";           // sz_state
+    {
+        vars_cpp << "sizeof(" << graphT->Name() << "_";
+        vars_cpp << devT->Name() << "_state_t),";           // sz_state
+    }
     else vars_cpp << "0,";
 
     vars_cpp << inTypCnt << ",";                                                // numInputTypes
@@ -2969,6 +3045,8 @@ void Composer::writeInputPinInit(AddressComponent threadAddr, DevT_t* devT,
 {
     unsigned int inTypCnt = devT->PinTI_v.size();       // Number of Input pins
     std::stringstream initialiser;
+    
+    GraphT_t* graphT = devT->par;    // grab a local copy of the graphtype
 
     vars_h << "//------------------------------ Pin Type Tables ";
     vars_h << "-------------------------------\n";
@@ -2978,10 +3056,10 @@ void Composer::writeInputPinInit(AddressComponent threadAddr, DevT_t* devT,
         // Add declaration for the input pins array to relevant vars header
         vars_h << "extern in_pintyp_t Thread_" << threadAddr;
         vars_h << "_DevTyp_0_InputPins[" << inTypCnt << "];\n";
-
+        
         // Build the dev type input pin name string
-        std::string dTypInPin = std::string("devtyp_");
-        dTypInPin += devT->Name() + std::string("_InPin_");
+        std::string dTypInPin = graphT->Name();
+        dTypInPin += std::string("_") + devT->Name() + std::string("_");
 
         initialiser.str("");    // Clear the initialiser
 
@@ -2989,7 +3067,7 @@ void Composer::writeInputPinInit(AddressComponent threadAddr, DevT_t* devT,
         WALKVECTOR(PinT_t*, devT->PinTI_v, ipin) // Build pin initialiser
         {
             initialiser << "{";
-            initialiser << "&" << dTypInPin;
+            initialiser << "&" << "devtyp_" << devT->Name() << "_InPin_";
             initialiser << (*ipin)->Name() << "_Recv_handler,"; // Recv_handler
 
             if ((*ipin)->pMsg->pPropsD)
@@ -3003,7 +3081,7 @@ void Composer::writeInputPinInit(AddressComponent threadAddr, DevT_t* devT,
             if ((*ipin)->pPropsD)
             {
                 initialiser << "sizeof(" << dTypInPin;               // sz_props
-                initialiser << (*ipin)->Name() << "_props_t),";
+                initialiser << (*ipin)->Name() << "_properties_t),";
             }
             else initialiser << "0,";
 
@@ -3116,8 +3194,12 @@ void Composer::writeDevIDecl(AddressComponent threadAddr,
 void Composer::writePinPropsDecl(PinI_t* pinI, std::string& thrDevName,
                                     size_t edgeCount, std::ofstream& vars_h)
 {
-    vars_h << "extern devtyp_" << pinI->pT->par->Name();
-    vars_h << "_InPin_" << pinI->pT->Name() << "_props_t ";
+    std::string devTName = pinI->pT->par->Name();           // grab device name
+    std::string graphTName = pinI->pT->par->par->Name();    // grab Graph name
+    
+    // Format "{graphTypeId}_{deviceTypeId}_{pinName}_properties_t".  
+    vars_h << "extern " << graphTName << "_" << devTName << "_";
+    vars_h << pinI->pT->Name() << "_properties_t ";
     vars_h << thrDevName << "_Pin_" << pinI->pT->Name();
     vars_h << "_InEdgeProps[" << edgeCount << "];\n\n";
 }
@@ -3129,8 +3211,12 @@ void Composer::writePinPropsDecl(PinI_t* pinI, std::string& thrDevName,
 void Composer::writePinStateDecl(PinI_t* pinI, std::string& thrDevName,
                                     size_t edgeCount, std::ofstream& vars_h)
 {
-    vars_h << "extern devtyp_" << pinI->pT->par->Name();
-    vars_h << "_InPin_" << pinI->pT->Name() << "_state_t ";
+    std::string devTName = pinI->pT->par->Name();           // grab device name
+    std::string graphTName = pinI->pT->par->par->Name();    // grab Graph name
+    
+    // Format "{graphTypeId}_{deviceTypeId}_{pinName}_state_t".  
+    vars_h << "extern " << graphTName << "_" << devTName << "_";
+    vars_h << pinI->pT->Name() << "_state_t ";
     vars_h << thrDevName << "_Pin_" << pinI->pT->Name();
     vars_h << "_InEdgeStates[" << edgeCount << "];\n\n";
 }
@@ -3172,6 +3258,8 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
 
     GraphI_t* graphI = builderGraphI->graphI;
     FILE * fd = graphI->par->par->fd;              // Detail output file
+    
+    GraphT_t* graphT = graphI->pT;
 
     // devInst_t initialiser
     std::vector<std::string> devIIStrs(numberOfDevices, "{},");
@@ -3323,9 +3411,6 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
     }
 
 
-
-
-
     // Process the individual initialisers into a coherent string
 
     // Start the devInst_t initialiser
@@ -3334,15 +3419,15 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
     devII << numberOfDevices << "] = {";
 
 
-    // Start the devtyp_X_props_t initialiser
+    // Start the {graphTypeId}_{deviceTypeId}_properties_t initialiser
     devPI.str("");
-    devPI << "devtyp_" << devT->Name() << "_props_t Thread_" << threadAddr;
-    devPI << "_DeviceProperties[" << numberOfDevices << "] = {";
+    devPI << graphT->Name() << "_" << devT->Name() << "_properties_t Thread_";
+    devPI << threadAddr << "_DeviceProperties[" << numberOfDevices << "] = {";
 
-    // Start the devtyp_X_state_t initialiser
+    // Start the {graphTypeId}_{deviceTypeId}_state_t initialiser
     devSI.str("");
-    devSI << "devtyp_" << devT->Name() << "_state_t Thread_" << threadAddr;
-    devSI << "_DeviceState[" << numberOfDevices << "] = {";
+    devSI << graphT->Name() << "_" << devT->Name() << "_state_t Thread_";
+    devSI << threadAddr << "_DeviceState[" << numberOfDevices << "] = {";
 
 
     // Fill in the initialiser strings
@@ -3357,11 +3442,11 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
     devII.seekp(-1, ios_base::cur);   // Remove the stray ,
     devII << "};\n\n";
 
-    // Finish off the devtyp_X_props_t initialiser
+    // Finish off the {graphTypeId}_{deviceTypeId}_properties_t initialiser
     devPI.seekp(-1, ios_base::cur);   // Remove the stray ,
     devPI << "};\n\n";
 
-    // Finish off the devtyp_X_state_t initialiser
+    // Finish off the {graphTypeId}_{deviceTypeId}_state_t initialiser
     devSI.seekp(-1, ios_base::cur);   // Remove the stray ,
     devSI << "};\n\n";
 
@@ -3375,9 +3460,9 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
         vars_cpp << devPI.rdbuf();
 
         // Make sure the properties decl is there.
-        vars_h << "extern devtyp_" << devT->Name() << "_props_t Thread_";
-        vars_h << threadAddr << "_DeviceProperties[";
-        vars_h << numberOfDevices << "];\n";
+        vars_h << "extern " << graphT->Name() << "_" << devT->Name();
+        vars_h << "_properties_t Thread_" << threadAddr ;
+        vars_h << "_DeviceProperties[" << numberOfDevices << "];\n";
     }
 
     if(devT->pStateD)
@@ -3385,8 +3470,9 @@ void Composer::writeThreadDevIDefs(ComposerGraphI_t* builderGraphI,
         vars_cpp << devSI.rdbuf();
 
         // Make sure the state decl is there.
-        vars_h << "extern devtyp_" << devT->Name() << "_state_t Thread_";
-        vars_h << threadAddr << "_DeviceState[" << numberOfDevices << "];\n\n";
+        vars_h << "extern " << graphT->Name() << "_" << devT->Name();
+        vars_h << "_state_t Thread_" << threadAddr ;
+        vars_h << "_DeviceState[" << numberOfDevices << "];\n\n";
     }
 
 }
@@ -3584,19 +3670,20 @@ void Composer::writeDevIInputPinEdgeDefs(GraphI_t* graphI, PinI_t* pinI,
         inEdgeTI << "inEdge_t " << thrDevName << "_Pin_" << pinI->pT->Name();
         inEdgeTI << "_InEdges[" << edgeCount << "] = {";
 
-        // Start the devtyp_X_InPin_Y_state_t initialiser
+        std::string devTName = pinI->pT->par->Name();        // device name
+        std::string graphTName = pinI->pT->par->par->Name(); // Graph name
+    
+        // Start {graphTypeId}_{deviceTypeId}_{pinName}_state_t initialiser".  
         inEdgeStatesI.str("");
-        inEdgeStatesI << "devtyp_" << pinI->pT->par->Name();
-        inEdgeStatesI << "_InPin_" << pinI->pT->Name();
-        inEdgeStatesI << "_state_t ";
+        inEdgeStatesI << graphTName << "_" << devTName << "_";
+        inEdgeStatesI << pinI->pT->Name() << "_state_t ";
         inEdgeStatesI << thrDevName << "_Pin_" << pinI->pT->Name();
         inEdgeStatesI << "_InEdgeStates[" << edgeCount << "] = {";
 
-        // Start the devtyp_X_InPin_Y_props_t initialiser
+        // Start {graphTypeId}_{deviceTypeId}_{pinName}_properties_t initialiser
         inEdgePropsI.str("");
-        inEdgePropsI << "devtyp_" << pinI->pT->par->Name();
-        inEdgePropsI << "_InPin_" << pinI->pT->Name();
-        inEdgePropsI << "_props_t ";
+        inEdgePropsI << graphTName << "_" << devTName << "_";
+        inEdgePropsI << pinI->pT->Name() << "_properties_t ";
         inEdgePropsI << thrDevName << "_Pin_" << pinI->pT->Name();
         inEdgePropsI << "_InEdgeProps[" << edgeCount << "] = {";
 
diff --git a/Source/OrchBase/Composer.h b/Source/OrchBase/Composer.h
index b606591f..33b9809e 100644
--- a/Source/OrchBase/Composer.h
+++ b/Source/OrchBase/Composer.h
@@ -47,6 +47,7 @@ typedef struct devTypStrings_t
     std::string handlersH;
     std::string handlersC;
     std::string varsHCommon;
+    std::string typesH;
 } devTypStrings_t;
 
 typedef enum softswitchLogHandler_t
@@ -163,13 +164,15 @@ void writeGlobalPropsD(GraphI_t*, std::ofstream&);
 void writeGlobalPropsI(GraphI_t*, std::ofstream&);
 void writeMessageTypes(GraphI_t*, std::ofstream&);
 
+void writeDeviceStructTypesPreamble(std::ofstream&);
+void writeDeviceStructTypes(DevT_t*, std::ofstream&);
+void writeDeviceStructTypesPostamble(std::ofstream&);
 
 // Device Type writers
 void formDevTStrings(ComposerGraphI_t*, DevT_t*);
 void populatePinTIdxMap(DevT_t*);
 void formHandlerPreamble(devTypStrings_t*);
 void formDevTHandlers(devTypStrings_t*);
-void formDevTPropsDStateD(devTypStrings_t*);
 void formDevTInputPinHandlers(devTypStrings_t* dTypStrs);
 void formDevTOutputPinHandlers(devTypStrings_t* dTypStrs);
 
diff --git a/Source/Softswitch/Makefile b/Source/Softswitch/Makefile
index c9d0a2a9..169a3a3d 100644
--- a/Source/Softswitch/Makefile
+++ b/Source/Softswitch/Makefile
@@ -113,6 +113,7 @@ vpath PMsg_p.hpp $(PCOMMONINC)
 vpath poets_pkt.h $(PCOMMONINC)
 vpath MessageFormats.h $(GENDIR)
 vpath GlobalProperties.h $(GENDIR)
+vpath DeviceStructs.h $(GENDIR)
 vpath OSFixes.hpp $(GENERICSINC)
 vpath Supervisor.% $(realpath ../$(DEFAULT_GEN_DIR))
 vpath SupervisorApi.% $(realpath ../$(DEFAULT_GEN_DIR))

From 9bc3b38ab9b991816fc7c16a8ba3d8754a458c1b Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 5 Jul 2021 13:13:15 +0100
Subject: [PATCH 02/30] Make it so the Supervisor can access defice struct
 layouts

---
 Source/OrchBase/Composer.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 6f26d67d..e647bd98 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -1668,6 +1668,7 @@ int Composer::generateSupervisor(ComposerGraphI_t* builderGraphI)
     // Include the generated global props and messages for all supervisors. 
     supervisor_h << "#include \"GlobalProperties.h\"\n";
     supervisor_h << "#include \"MessageFormats.h\"\n\n";
+    supervisor_h << "#include \"DeviceStructs.h\"\n\n";
     
     if(graphI->pT->pSup)    // If we have a non-default Supervisor, build it.
     {

From cd4f490f48537901912eb7773c4f7b7ea1b40b32 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 5 Jul 2021 13:35:59 +0100
Subject: [PATCH 03/30] Add packing for message types to partially address #240

---
 Source/OrchBase/Composer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index e647bd98..7257c52b 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -2004,7 +2004,7 @@ void Composer::writeMessageTypes(GraphI_t* graphI, std::ofstream& pkt_h)
 
     pkt_h << "#include <cstdint>\n";
     
-    //pkt_h <<  "#pragma pack(push,1)\n";
+    pkt_h <<  "#pragma pack(push,1)\n";
     
     WALKVECTOR(MsgT_t*,graphT->MsgT_v,msg)
     {
@@ -2014,7 +2014,7 @@ void Composer::writeMessageTypes(GraphI_t* graphI, std::ofstream& pkt_h)
         pkt_h << "\n} pkt_" << (*msg)->Name() << "_pyld_t;\n\n";
     }
     
-    //pkt_h <<  "#pragma pack(pop)\n";
+    pkt_h <<  "#pragma pack(pop)\n";
     
     pkt_h << "#endif /*_MESSAGETYPES_H_*/\n\n";
 }

From 9c70eceea66e238641a233c942067cf16eea0ed3 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Thu, 8 Jul 2021 13:51:39 +0100
Subject: [PATCH 04/30] Fix apsp example so that it compiles

---
 .../ReferenceXML/v4/PEP20/apps/apsp_vec_barrier_150_10.xml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Tests/ReferenceXML/v4/PEP20/apps/apsp_vec_barrier_150_10.xml b/Tests/ReferenceXML/v4/PEP20/apps/apsp_vec_barrier_150_10.xml
index b7911d7d..c27e7819 100644
--- a/Tests/ReferenceXML/v4/PEP20/apps/apsp_vec_barrier_150_10.xml
+++ b/Tests/ReferenceXML/v4/PEP20/apps/apsp_vec_barrier_150_10.xml
@@ -8,6 +8,7 @@ uint32_t gen_k;
 		#include <cmath>
 		#include <utility>
 		#include <cstring>
+		#include <climits>
 
 		const unsigned K_MSG=sizeof(std::declval<apsp_vec_barrier_share_message_t>().distances) / sizeof(std::declval<apsp_vec_barrier_share_message_t>().distances[0]);
 		const unsigned K_STATE=sizeof(std::declval<apsp_vec_barrier_vertex_state_t>().distances) / sizeof(std::declval<apsp_vec_barrier_vertex_state_t>().distances[0]);
@@ -78,7 +79,7 @@ uint32_t distances[8];
 					// Sometimes we need to be precise about exactly how many vertices are in this batch,
 					// but most of the time it is _slightly_ more efficient to use K (instruction count + memory accesses + reg pressure).
 					// There is only one round where curr_k < K, versus floor(n/K) where curr_k==K. Ignored for now.
-					deviceState->curr_k=std::min(K, graphProperties->total_vertices-deviceState->round_base);
+					deviceState->curr_k=std::min(K, (const unsigned)(graphProperties->total_vertices-deviceState->round_base));
 					assert(deviceState->curr_k <= K);
 
 					deviceState->round_base += K;
@@ -222,7 +223,7 @@ uint32_t round_base;
           <OnReceive><![CDATA[
 					deviceState->sum_sum_distances += message->sum_sum_distance;
 					assert(deviceState->round_base < graphProperties->total_vertices);
-					unsigned curr_k = std::min(K, graphProperties->total_vertices - deviceState->round_base);
+					unsigned curr_k = std::min(K, (const unsigned)(graphProperties->total_vertices - deviceState->round_base));
 					assert(curr_k <= K);
 					for(unsigned i=0; i<curr_k; i++){
 						deviceState->max_distances[i] = std::max(deviceState->max_distances[i], message->max_distances[i]);
@@ -248,7 +249,7 @@ uint32_t round_base;
 					assert(deviceState->vertex_count == graphProperties->total_vertices);
 					assert(deviceState->round_base < graphProperties->total_vertices);
 
-					unsigned curr_k = std::min(K, graphProperties->total_vertices - deviceState->round_base);
+					unsigned curr_k = std::min(K, (const unsigned)(graphProperties->total_vertices - deviceState->round_base));
 					assert(curr_k <= K);
 					for(unsigned i=0; i<curr_k; i++){
 						deviceState->sum_max_distances += deviceState->max_distances[i];

From 2e8def337b094488e7e6105c597062cb618dd95a Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Wed, 4 Aug 2021 15:17:10 +0100
Subject: [PATCH 05/30] Enable tests for fixed features

---
 Tests/ReferenceXML/test_compile.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Tests/ReferenceXML/test_compile.sh b/Tests/ReferenceXML/test_compile.sh
index 5ef35dca..6b5a7587 100755
--- a/Tests/ReferenceXML/test_compile.sh
+++ b/Tests/ReferenceXML/test_compile.sh
@@ -99,12 +99,8 @@ function test_compile_failure {
 
 RC=0
 for i in $ORCHROOT/Tests/ReferenceXML/v4/PEP20/apps/*.xml ; do
-    if [[ "$i" == */apsp_vec_barrier_150_10.xml ]] ; then
-        TODO_compile $i "Bug #232 needs to be fixed."
-    elif [[ "$i" == */betweeness_centrality_16_16_20_20_v4.xml ]] ; then
+    if [[ "$i" == */betweeness_centrality_16_16_20_20_v4.xml ]] ; then
         TODO_compile $i "orchestrator needs indexed sends."
-    elif [[ "$i" == */example_device_idle.xml ]] ; then
-        TODO_compile $i "orchestrator needs requestIdle."
     else
         test_compile_success $i
     fi

From c62c205dddedec10f89ab24775562331d0bbc4db Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Tue, 10 Aug 2021 12:39:27 +0100
Subject: [PATCH 06/30] Implement Hardware Idle detection

---
 Source/OrchBase/Composer.cpp                | 11 ++++--
 Source/Softswitch/inc/softswitch_common.h   |  1 +
 Source/Softswitch/src/softswitch_common.cpp | 41 +++++++++++++++++----
 3 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 7257c52b..db75e0ea 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -3007,13 +3007,16 @@ void Composer::writeDevTDeclInit(AddressComponent threadAddr, DevT_t* devT,
     vars_cpp << "= {";
     vars_cpp << "&devtyp_" << devT->Name() << "_RTS_handler,";                  // RTS_Handler
     vars_cpp << "&devtyp_" << devT->Name() << "_OnInit_handler,";               // OnInit_Handler
-    vars_cpp << "&devtyp_" << devT->Name() << "_OnIdle_handler,";               // OnIdle_Handler
-    vars_cpp << "&devtyp_" << devT->Name() << "_OnHWIdle_handler,";             // OnHWIdle_Handler
+    
+    if(devT->pOnDeId) vars_cpp <<"&devtyp_"<<devT->Name()<<"_OnIdle_handler,";  // OnIdle_Handler
+    else vars_cpp << "0,";
+    
+    if(devT->pOnHWId) vars_cpp <<"&devtyp_"<<devT->Name()<<"_OnHWIdle_handler,";// OnHWIdle_Handler
+    else vars_cpp << "0,";
+    
     vars_cpp << "&devtyp_" << devT->Name() << "_OnImpl_handler,";               // OnImpl_Handler
     vars_cpp << "&devtyp_" << devT->Name() << "_OnCtl_handler,";                // OnCtl_Handler
 
-    //TODO: Add v4 handlers
-
     if(devT->pPropsD)
     {
         vars_cpp << "sizeof(" << graphT->Name() << "_";
diff --git a/Source/Softswitch/inc/softswitch_common.h b/Source/Softswitch/inc/softswitch_common.h
index fa805089..2aca8980 100644
--- a/Source/Softswitch/inc/softswitch_common.h
+++ b/Source/Softswitch/inc/softswitch_common.h
@@ -198,6 +198,7 @@ void softswitch_instrumentation(ThreadCtxt_t* thr_ctxt, volatile void* send_buf)
 uint32_t softswitch_onSend(ThreadCtxt_t* thr_ctxt, volatile void* send_buf);
 void softswitch_onReceive(ThreadCtxt_t* ThreadContext, volatile void* recv_buf);
 bool softswitch_onIdle(ThreadCtxt_t* thr_ctxt);
+void softswitch_onHWIdle(ThreadCtxt_t* thr_ctxt);
 uint32_t softswitch_onRTS(ThreadCtxt_t* thr_ctxt, devInst_t* device);
 
 
diff --git a/Source/Softswitch/src/softswitch_common.cpp b/Source/Softswitch/src/softswitch_common.cpp
index 4be28c45..f814ba7c 100644
--- a/Source/Softswitch/src/softswitch_common.cpp
+++ b/Source/Softswitch/src/softswitch_common.cpp
@@ -152,6 +152,9 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
  #endif
 #endif
 
+    // We abuse the fact that we have a single device type for idle handling
+    devInst_t* device = &ThreadContext->devInsts[idleIdx];
+
     while (!ThreadContext->ctlEnd)
     {
     #ifndef DISABLE_SOFTSWITCH_INSTRUMENTATION
@@ -204,13 +207,22 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
         }
 #endif
 
-
-        // Nothing to RX, nothing to TX: iterate through all devices until
-        // something happens or all OnComputes have returned 0.
-        //softswitch_onIdle(ThreadContext);
-        else if(!softswitch_onIdle(ThreadContext))
-        {
-            tinselWaitUntil(TINSEL_CAN_RECV);
+        // Idle handling
+        else if(device->devType->OnIdle_handler 
+                && !softswitch_onIdle(ThreadContext))
+        {   // We have an idle handler and nothing interesting happened in it
+            
+            if(device->devType->OnHWIdle_handler)
+            {   // We have hardware idle
+                if(tinselIdle(true))
+                {   // returned from a synchronisation point
+                    softswitch_onHWIdle(ThreadContext)
+                }
+            }
+            else
+            {   // No hardware idle, sleep untill we can receive
+                tinselWaitUntil(TINSEL_CAN_RECV);
+            }
         }
     }
 }
@@ -234,7 +246,7 @@ void softswitch_finalise(ThreadCtxt_t* ThreadContext)
 
         ThreadContext->rtsStart++;
         if(ThreadContext->rtsStart == maxIdx)
-        {
+        { 
             ThreadContext->rtsStart = 0;
         }
     }
@@ -648,6 +660,19 @@ inline bool softswitch_onIdle(ThreadCtxt_t* ThreadContext)
 }
 
 
+/* Routine called if tinselIdle returns true. This loops over all hosted devices
+ * and calls the device type's OnHWIdle handler followed by ReadyToSend.
+ */
+inline void softswitch_onHWIdle(ThreadCtxt_t* thr_ctxt)
+{
+    for(uint32_t i = 0; i < ThreadContext->numDevInsts; i++)
+    {
+        devInst_t* device = &ThreadContext->devInsts[i];
+        device->devType->OnHWIdle_Handler(ThreadContext->properties, device);
+        softswitch_onRTS(ThreadContext, device);  // Call RTS for device
+    }
+}
+
 inline uint32_t softswitch_onRTS(ThreadCtxt_t* ThreadContext, devInst_t* device)
 {
     // Essentially:

From 7d43872e22187e65dc7d96dd868c9eb91dce9fd8 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Tue, 10 Aug 2021 16:44:36 +0100
Subject: [PATCH 07/30] Fix some typos

---
 Source/Softswitch/src/softswitch_common.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Source/Softswitch/src/softswitch_common.cpp b/Source/Softswitch/src/softswitch_common.cpp
index f814ba7c..d128c176 100644
--- a/Source/Softswitch/src/softswitch_common.cpp
+++ b/Source/Softswitch/src/softswitch_common.cpp
@@ -153,7 +153,7 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
 #endif
 
     // We abuse the fact that we have a single device type for idle handling
-    devInst_t* device = &ThreadContext->devInsts[idleIdx];
+    devInst_t* device = &ThreadContext->devInsts[0];
 
     while (!ThreadContext->ctlEnd)
     {
@@ -208,15 +208,15 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
 #endif
 
         // Idle handling
-        else if(device->devType->OnIdle_handler 
+        else if(device->devType->OnIdle_Handler 
                 && !softswitch_onIdle(ThreadContext))
         {   // We have an idle handler and nothing interesting happened in it
             
-            if(device->devType->OnHWIdle_handler)
+            if(device->devType->OnHWIdle_Handler)
             {   // We have hardware idle
                 if(tinselIdle(true))
                 {   // returned from a synchronisation point
-                    softswitch_onHWIdle(ThreadContext)
+                    softswitch_onHWIdle(ThreadContext);
                 }
             }
             else
@@ -663,7 +663,7 @@ inline bool softswitch_onIdle(ThreadCtxt_t* ThreadContext)
 /* Routine called if tinselIdle returns true. This loops over all hosted devices
  * and calls the device type's OnHWIdle handler followed by ReadyToSend.
  */
-inline void softswitch_onHWIdle(ThreadCtxt_t* thr_ctxt)
+inline void softswitch_onHWIdle(ThreadCtxt_t* ThreadContext)
 {
     for(uint32_t i = 0; i < ThreadContext->numDevInsts; i++)
     {

From f5a58721679a1949dfb11aaf1a0ffdb9653059cb Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 14:22:16 +0100
Subject: [PATCH 08/30] Add new configuration section 'mode', and field
 'single_app_mode', to discriminate between when we are supposed to support
 hardware idle, and when we are not.

---
 .gitmodules                |  4 ----
 Config/Orchestrator.ocfg   | 13 +++++++++----
 Source/Root/OrchConfig.cpp | 18 ++++++++++++++++--
 Source/Root/OrchConfig.h   |  5 +++++
 4 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index ab22aa56..e69de29b 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +0,0 @@
-[submodule "Tinsel"]
-	path = Tinsel
-	url = https://github.com/poetsii/tinsel
-	branch = master
diff --git a/Config/Orchestrator.ocfg b/Config/Orchestrator.ocfg
index f878e144..1a8d08f3 100644
--- a/Config/Orchestrator.ocfg
+++ b/Config/Orchestrator.ocfg
@@ -8,10 +8,10 @@
 
 [Orchestrator_header]
 // All pretty arbitrary; just copied in and stored
-name        = OrchestratorConfiguration
-author      = "MLV and GMB"
-date        = "2021-01-17"
-version     = "0.0.6"
+name        = "OrchestratorConfiguration"
+author      = "MLV"
+date        = "2021-08-27"
+version     = "0.1.0"
 
 // All these may be overridden by the console "path" command
 [default_paths]
@@ -58,6 +58,11 @@ hardware = "../Config/POETSHardwareOneBox.ocfg"
 // Default flags for the cross-compiler
 // build = "\oink -plop ++wheeee  !. <?> "
 
+// Alter the behaviour of the Orchestrator
+[modes]
+single_app_mode = "true"  // Either 'true' or 'false'. Must be true to support
+                          // hardware idle, for now.
+
 // Elaboration messages passed out to the author for errors in processing THIS
 // file They are all classed as "Unrecoverable", not because they are, but
 // because: If you're not a grown-up you shouldn't be mucking about with it
diff --git a/Source/Root/OrchConfig.cpp b/Source/Root/OrchConfig.cpp
index 0b93f499..05c41203 100644
--- a/Source/Root/OrchConfig.cpp
+++ b/Source/Root/OrchConfig.cpp
@@ -27,8 +27,8 @@ ecnt = 0;                              // So far, so good, then. No errors
 JNJ P(WhereAmI);                      // Parse defining file
 ecnt = P.ErrCnt();                     // Syntax cockups?
 if (ecnt!=0) {                         // If so, bail
-  if (P.Td.t==Lex::S_0) IncErr(0,6);
-  else IncErr(0,7);
+  if (P.Td.t==Lex::S_0) IncErr(0,7);
+  else IncErr(0,8);
   return;
 }
 vH sects;
@@ -99,6 +99,20 @@ WALKVECTOR(UIF::Node *,sects,i) {      // Walk the sections
       if ((*k)->str=="build"   ) flags.build    = s;
     }
   }
+  if (sn=="modes") {
+    P.GetVari(*i,varis);
+    WALKVECTOR(UIF::Node *,varis,k) {
+      P.LocValu(*k,valus);
+      if (valus.size()>1) IncErr(P.FndRecd(*k),6);
+      if (valus.empty()) s.clear();
+      else s = valus[0]->str;
+      if ((*k)->str=="single_app_mode") {
+        if (s=="false") modes.single_app_mode = false;
+        else if (s=="true") modes.single_app_mode = true;
+        else IncErr(P.FndRecd(*k),6);
+      }
+    }
+  }
 }
 if (ecnt!=0) Init();              // Any errors, kill the lot
 Force2Linux();                    // Force all the separators to linux-land
diff --git a/Source/Root/OrchConfig.h b/Source/Root/OrchConfig.h
index ebbac36b..7a8344cd 100644
--- a/Source/Root/OrchConfig.h
+++ b/Source/Root/OrchConfig.h
@@ -41,6 +41,7 @@ string                   Placement()   { return setup_files.placement;         }
 string                   RemoteMshp()  { return default_paths.remote_mship;    }
 string                   RemoteOut()   { return default_paths.remote_outdir;   }
 string                   Trace()       { return default_paths.trace;           }
+bool                     SingleApp()   { return modes.single_app_mode;         }
 string                   Stage()       { return default_paths.stage;           }
 string                   Supervisors() { return default_paths.supervisors;     }
 string                   Ulog()        { return default_paths.ulog;            }
@@ -86,6 +87,10 @@ struct flags_t {
   string build;                        // Default x-compiler flags
 } flags;
 
+struct modes_t {
+  bool single_app_mode;                // Hardware-idle friendly?
+} modes;
+
 };
 
 //==============================================================================

From 28c8c099582c8df626a6be5e3229782ebe9c1c97 Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 14:58:03 +0100
Subject: [PATCH 09/30] Deployment logic reads from the aforementioned config
 field, and lets the Mothership know. The Mothership will use global backend
 'start' and 'go' methods instead.

---
 Config/OrchestratorMessages.ocfg     |  3 +
 Source/Common/Pglobals.h             |  2 +
 Source/Mothership/AppDB.cpp          | 14 +++--
 Source/Mothership/AppDB.h            |  2 +-
 Source/Mothership/AppInfo.cpp        |  5 +-
 Source/Mothership/AppInfo.h          |  3 +-
 Source/Mothership/AppTransitions.cpp | 84 +++++++++++++++++-----------
 Source/Mothership/MPIHandlers.cpp    | 10 ++--
 Source/Mothership/MessageUtils.cpp   | 22 +++++++-
 Source/Mothership/Mothership.h       |  4 +-
 Source/OrchBase/Handlers/CmDepl.cpp  |  9 ++-
 11 files changed, 106 insertions(+), 52 deletions(-)

diff --git a/Config/OrchestratorMessages.ocfg b/Config/OrchestratorMessages.ocfg
index 93973a3b..fbca9cf0 100644
--- a/Config/OrchestratorMessages.ocfg
+++ b/Config/OrchestratorMessages.ocfg
@@ -234,7 +234,10 @@
 534(I) : "Mothership (rank %s): All devices on this Mothership for application '%s' have stopped."
 535(I) : "Mothership (rank %s): Recalling application '%s'."
 536(I) : "Mothership (rank %s): Application '%s' recalled. This Mothership has forgotten everything about this application."
+537(E) : "Mothership: Error decoding MPI message with key '0x%s': Expected bool in field %s. Ignoring message."
 
+578(I) : "Mothership: Calling backend->go."
+579(I) : "Mothership: Calling backend->startAll."
 580(E) : "Mothership: Received a log packet with an invalid device index 0x%s."
 581(I) : "Mothership: Consuming log packet from device address 0x%s with name %s."
 582(I) : "Mothership: Received a message containing packets for a supervisor device for application '%s' that is not running (it may be in the process of stopping). Ignoring these packets."
diff --git a/Source/Common/Pglobals.h b/Source/Common/Pglobals.h
index d26e57b5..0651ccd5 100644
--- a/Source/Common/Pglobals.h
+++ b/Source/Common/Pglobals.h
@@ -82,6 +82,8 @@ EXIT |-    |-    |-    | (None)
 SYST |KILL |-    |-    | (None)
 APP  |SPEC |-    |-    | (0:string)Application name
                          (1:uint32_t)Number of expected distribution messages
+                         (2:uint8_t)Application number
+                         (3:bool)Hardware-idle application?
 APP  |DIST |-    |-    | (0:string)Application name
                          (1:string)Code path for this core
                          (2:string)Data path for this core
diff --git a/Source/Mothership/AppDB.cpp b/Source/Mothership/AppDB.cpp
index c09d72dd..c3947072 100644
--- a/Source/Mothership/AppDB.cpp
+++ b/Source/Mothership/AppDB.cpp
@@ -1,10 +1,12 @@
 #include "AppDB.h"
 
 /* Checks appInfos for an application of a given name. If it doesn't exist,
- * AppDB creates it and returns a pointer to it (passing distCountExpected to
- * it as an argument). If it already exists, returns a pointer to the existing
- * application, and does not use distCountExpected. */
-AppInfo* AppDB::check_create_app(std::string name, uint32_t distCountExpected)
+ * AppDB creates it and returns a pointer to it (passing distCountExpected and
+ * soloApp to it as an argument). If it already exists, returns a pointer to
+ * the existing application, does not use distCountExpected, but sets
+ * soloApp. */
+AppInfo* AppDB::check_create_app(std::string name, uint32_t distCountExpected,
+                                 bool soloApp)
 {
     AppInfoIt appFinder = appInfos.find(name);
 
@@ -21,7 +23,7 @@ AppInfo* AppDB::check_create_app(std::string name, uint32_t distCountExpected)
         else
         {
             appInfos.insert(std::pair<std::string, AppInfo>
-                            (name, AppInfo(name, distCountExpected)));
+                            (name, AppInfo(name, distCountExpected, soloApp)));
         }
 
         return &(appInfos.find(name)->second);
@@ -37,7 +39,7 @@ AppInfo* AppDB::check_create_app(std::string name, uint32_t distCountExpected)
 /* Sub-synonym. */
 AppInfo* AppDB::check_create_app(std::string name)
 {
-    return check_create_app(name, 0);
+    return check_create_app(name, 0, false);  /* soloApp argument not used. */
 }
 
 /* Checks appInfos for an application of a given name, returning true if such
diff --git a/Source/Mothership/AppDB.h b/Source/Mothership/AppDB.h
index 32667778..4a067f26 100644
--- a/Source/Mothership/AppDB.h
+++ b/Source/Mothership/AppDB.h
@@ -22,7 +22,7 @@ class AppDB
     std::map<uint32_t, uint32_t> threadToCoreAddr;
     std::map<uint8_t, std::string> numberToApp;
 
-    AppInfo* check_create_app(std::string, uint32_t);
+    AppInfo* check_create_app(std::string, uint32_t, bool);
     AppInfo* check_create_app(std::string);
     bool check_defined_app(std::string);
     void recall_app(AppInfo*);
diff --git a/Source/Mothership/AppInfo.cpp b/Source/Mothership/AppInfo.cpp
index 9b27395e..832d1545 100644
--- a/Source/Mothership/AppInfo.cpp
+++ b/Source/Mothership/AppInfo.cpp
@@ -1,9 +1,10 @@
 #include "AppInfo.h"
 
 /* This constructor is used by SPEC messages. */
-AppInfo::AppInfo(std::string name, uint32_t distCountExpected):
+AppInfo::AppInfo(std::string name, uint32_t distCountExpected, bool soloApp):
     name(name),
-    distCountExpected(distCountExpected)
+    distCountExpected(distCountExpected),
+    soloApp(soloApp)
 {
     pendingCommands = 0;
     distCountCurrent = 0;
diff --git a/Source/Mothership/AppInfo.h b/Source/Mothership/AppInfo.h
index 408462fa..98073c88 100644
--- a/Source/Mothership/AppInfo.h
+++ b/Source/Mothership/AppInfo.h
@@ -39,11 +39,12 @@ enum AppState{UNDERDEFINED,  /* We're still receiving DIST messages. */
 class AppInfo
 {
 public:
-    AppInfo(std::string nameArg, uint32_t distCountExpected);
+    AppInfo(std::string nameArg, uint32_t distCountExpected, bool soloApp);
     AppInfo(std::string nameArg);
 
     std::string name;
     uint32_t distCountExpected;
+    bool soloApp;
     AppState state;
     std::map<uint32_t, CoreInfo> coreInfos;
     std::set<uint32_t> coresLoaded;
diff --git a/Source/Mothership/AppTransitions.cpp b/Source/Mothership/AppTransitions.cpp
index 81320820..22ef2158 100644
--- a/Source/Mothership/AppTransitions.cpp
+++ b/Source/Mothership/AppTransitions.cpp
@@ -58,44 +58,62 @@ void Mothership::initialise_application(AppInfo* app)
                                  meshX, meshY, coreId);
     }
 
-    /* 2: For each core, kick off the threads (mode=false).
-     * 3: For each core, start execution (mode=true). */
-    mode = false;
-    do
+    /* 2: For each core, kick off the threads.
+     * 3: For each core, start execution. */
+
+    /* In solo-app mode, use backend's available global start and go
+     * methods. */
+    if (app->soloApp)
     {
-        for (coreIt = app->coreInfos.begin(); coreIt != app->coreInfos.end();
-             coreIt++)
-        {
-            backend->fromAddr(coreIt->first, &meshX, &meshY, &coreId,
-                              &threadId);
+        debug_post(579, 0);
+        backend->startAll();  /* 2 */
+        debug_post(578, 0);
+        backend->go();  /* 3 */
+    }
 
-            if (!mode)  /* 2 */
-            {
-                debug_post(
-                    587, 4, hex2str(meshX).c_str(), hex2str(meshY).c_str(),
-                    hex2str(coreId).c_str(),
-                    uint2str(coreIt->second.threadsExpected.size()).c_str());
-                /* Note that startOne can hang for the Tinsel backend if the
-                 * number of threads expected is greater than the number of
-                 * threads that the core will start - this is because startOne
-                 * waits for an acknowledgement message from the core that
-                 * varies as a function of the number of threads. If you find
-                 * the above 587 being the last message you see from the MPI
-                 * CNC Resolver thread (for example), this is most likely your
-                 * issue. */
-                backend->startOne(meshX, meshY, coreId,
-                                  coreIt->second.threadsExpected.size());
-            }
-            else  /* 3 */
+    /* Otherwise, things get more complicated... */
+    else
+    {
+        mode = false;  /* 2 when false, 3 when true. */
+        do  /* Simple loop to reduce code duplication (see comment accompanying
+             * this function definition). */
+        {
+            for (coreIt = app->coreInfos.begin();
+                 coreIt != app->coreInfos.end(); coreIt++)
             {
-                debug_post(586, 3, hex2str(meshX).c_str(),
-                           hex2str(meshY).c_str(), hex2str(coreId).c_str());
-                backend->goOne(meshX, meshY, coreId);
+                backend->fromAddr(coreIt->first, &meshX, &meshY, &coreId,
+                                  &threadId);
+
+                if (!mode)  /* 2 */
+                {
+                    debug_post(
+                        587, 4, hex2str(meshX).c_str(), hex2str(meshY).c_str(),
+                        hex2str(coreId).c_str(),
+                        uint2str(coreIt->second.threadsExpected.size())
+                        .c_str());
+                    /* Note that startOne can hang for the Tinsel backend if
+                     * the number of threads expected is greater than the
+                     * number of threads that the core will start - this is
+                     * because startOne waits for an acknowledgement message
+                     * from the core that varies as a function of the number of
+                     * threads. If you find the above 587 being the last
+                     * message you see from the MPI CNC Resolver thread (for
+                     * example), this is most likely your issue. */
+                    backend->startOne(meshX, meshY, coreId,
+                                      coreIt->second.threadsExpected.size());
+                }
+                else  /* 3 */
+                {
+                    debug_post(586, 3, hex2str(meshX).c_str(),
+                               hex2str(meshY).c_str(),
+                               hex2str(coreId).c_str());
+                    backend->goOne(meshX, meshY, coreId);
+                }
             }
-        }
 
-        mode = !mode;
-    } while (mode);
+            mode = !mode;
+        } while (mode);
+    }
 
     /* Good stuff. Now the cores will spin up and send BARRIER messages to the
      * Mothership. */
diff --git a/Source/Mothership/MPIHandlers.cpp b/Source/Mothership/MPIHandlers.cpp
index fec38f8b..f25a22aa 100644
--- a/Source/Mothership/MPIHandlers.cpp
+++ b/Source/Mothership/MPIHandlers.cpp
@@ -87,8 +87,9 @@ unsigned Mothership::handle_msg_app_spec(PMsg_p* message)
     std::string appName;
     uint32_t distCount;
     uint8_t appNumber;
+    bool soloApp;
     if (!decode_app_spec_message(message, &appName, &distCount,
-                                 &appNumber))
+                                 &appNumber, &soloApp))
     {
         debug_post(597, 3, "Q::APP,Q::SPEC", hex2str(message->Key()).c_str(),
                    "Failed to decode.");
@@ -96,12 +97,13 @@ unsigned Mothership::handle_msg_app_spec(PMsg_p* message)
     }
 
     debug_post(597, 3, "Q::APP,Q::SPEC", hex2str(message->Key()).c_str(),
-               dformat("appName=%s, distCount=%u, appNumber=%u",
-                       appName.c_str(), distCount, appNumber).c_str());
+               dformat("appName=%s, distCount=%u, appNumber=%u, soloApp=%s",
+                       appName.c_str(), distCount, appNumber,
+                       soloApp ? "true" : "false").c_str());
 
     /* Ensure application existence idempotently (it might have been created by
      * an AppDist message). */
-    appInfo = appdb.check_create_app(appName, distCount);
+    appInfo = appdb.check_create_app(appName, distCount, soloApp);
 
     /* If the application is not in the UNDERDEFINED state, post bossily and do
      * nothing else. */
diff --git a/Source/Mothership/MessageUtils.cpp b/Source/Mothership/MessageUtils.cpp
index 814a3c43..2c85f141 100644
--- a/Source/Mothership/MessageUtils.cpp
+++ b/Source/Mothership/MessageUtils.cpp
@@ -38,12 +38,14 @@ bool Mothership::decode_app_supd_message(PMsg_p* message, std::string* appName,
 
 bool Mothership::decode_app_spec_message(PMsg_p* message, std::string* appName,
                                          uint32_t* distCount,
-                                         uint8_t* appNumber)
+                                         uint8_t* appNumber,
+                                         bool* soloApp)
 {
     *distCount = 0;
     if(!decode_string_message(message, appName)) return false;
     if(!decode_unsigned_message(message, distCount, 1)) return false;
     if(!decode_char_message(message, appNumber, 2)) return false;
+    if(!decode_bool_message(message, soloApp, 3)) return false;
     return true;
 }
 
@@ -89,6 +91,24 @@ bool Mothership::decode_addressed_packets_message(PMsg_p* message,
     return true;
 }
 
+bool Mothership::decode_bool_message(PMsg_p* message, bool* result,
+                                     unsigned index)
+{
+    int countBuffer;
+    bool* resultBuffer;
+
+    /* Get and check for errors. */
+    resultBuffer = message->Get<bool>(index, countBuffer);
+    if (resultBuffer == PNULL)
+    {
+        *result = 0;
+        Post(537, hex2str(message->Key()), uint2str(index));
+        return false;
+    }
+    *result = *resultBuffer;
+    return true;
+}
+
 bool Mothership::decode_char_message(PMsg_p* message, unsigned char* result,
                                      unsigned index)
 {
diff --git a/Source/Mothership/Mothership.h b/Source/Mothership/Mothership.h
index b3fc99ea..3fafa457 100644
--- a/Source/Mothership/Mothership.h
+++ b/Source/Mothership/Mothership.h
@@ -102,12 +102,14 @@ class Mothership: public CommonBase
     bool decode_app_supd_message(PMsg_p* message, std::string* appName,
                                  std::string* soPath);
     bool decode_app_spec_message(PMsg_p* message, std::string* appName,
-                                 uint32_t* distCount, uint8_t* appNumber);
+                                 uint32_t* distCount, uint8_t* appNumber,
+                                 bool* soloApp);
     bool decode_addresses_message(PMsg_p* message,
                                   std::vector<uint32_t>* addresses,
                                   unsigned index=0);
     bool decode_addressed_packets_message(PMsg_p* message,
         std::vector<P_Addr_Pkt_t>* packets, unsigned index=0);
+    bool decode_bool_message(PMsg_p* message, bool* result, unsigned index=0);
     bool decode_char_message(PMsg_p* message, unsigned char* result,
                              unsigned index=0);
     bool decode_packets_message(PMsg_p* message,
diff --git a/Source/OrchBase/Handlers/CmDepl.cpp b/Source/OrchBase/Handlers/CmDepl.cpp
index a4ab09fe..5c2c1758 100644
--- a/Source/OrchBase/Handlers/CmDepl.cpp
+++ b/Source/OrchBase/Handlers/CmDepl.cpp
@@ -1,8 +1,8 @@
 //------------------------------------------------------------------------------
 
 #include "CmDepl.h"
-#include "OrchBase.h"
 #include "Pglobals.h"
+#include "Root.h"  /* Grabs Orchestrator config, and OrchBase. */
 #include "SupervisorModes.h"
 
 //==============================================================================
@@ -376,9 +376,12 @@ int CmDepl::DeployGraph(GraphI_t* gi)
         specMessage.Put<unsigned>(1, &distCount);
         specMessage.Put<unsigned char>(2,
             static_cast<unsigned char*>(&appNumber));
+        bool soloApp = dynamic_cast<Root *>(par)->pOC->SingleApp();
+        specMessage.Put<bool>(3,&soloApp);
         fprintf(par->fd, "Sending SPEC message to Mothership rank %d, with "
-                "appNumber=%u and distCount=%u...",
-                mothershipPayloadsIt->first, appNumber, distCount);
+                "appNumber=%u, distCount=%u, and soloApp=%s...",
+                mothershipPayloadsIt->first, appNumber, distCount,
+                soloApp ? "true" : "false");
         specMessage.Send();
         fprintf(par->fd, " message sent.\n");
 

From 026550e9e79542d865ae0870aab92fd633f4515b Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 14:58:17 +0100
Subject: [PATCH 10/30] Whitespace fixes.

---
 Source/Mothership/MessageUtils.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Source/Mothership/MessageUtils.cpp b/Source/Mothership/MessageUtils.cpp
index 2c85f141..8c58fa2b 100644
--- a/Source/Mothership/MessageUtils.cpp
+++ b/Source/Mothership/MessageUtils.cpp
@@ -79,9 +79,9 @@ bool Mothership::decode_addressed_packets_message(PMsg_p* message,
     message->Put<P_Addr_Pkt_t>();  // Tell the message its type
 
     packets->clear();
-    
+
     message->Get<P_Addr_Pkt_t>(index, *packets);
-    
+
     if (packets->empty())
     {
         Post(516, hex2str(message->Key()), uint2str(index));
@@ -140,7 +140,7 @@ bool Mothership::decode_packets_message(PMsg_p* message,
     packets->clear();
 
     message->Get<P_Pkt_t>(index, *packets);
-    
+
     // If the packet vector has come back empty, there is an error.
     if (packets->empty())
     {

From 10850e9e297adaedccc2a2cf3c1132281325ef8e Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 17:46:49 +0100
Subject: [PATCH 11/30] Define an `APP,EMPT` message, which is sent by root in
 single-app mode at deploy time. It causes a Mothership to call backend's
 loadAll method.

---
 Source/Common/Pglobals.cpp          |  3 ++-
 Source/Common/Pglobals.h            |  3 +++
 Source/Mothership/MPIHandlers.cpp   | 23 ++++++++++++++++++++++-
 Source/Mothership/MessageUtils.cpp  | 10 ++++++++++
 Source/Mothership/Mothership.h      |  3 +++
 Source/Mothership/ThreadLogic.cpp   |  4 +++-
 Source/OrchBase/Handlers/CmDepl.cpp | 28 +++++++++++++++++++++++++++-
 7 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/Source/Common/Pglobals.cpp b/Source/Common/Pglobals.cpp
index e986fcf7..9738b7e5 100644
--- a/Source/Common/Pglobals.cpp
+++ b/Source/Common/Pglobals.cpp
@@ -40,8 +40,9 @@ const byte Q::SPEC    = 0x50;
 const byte Q::SUPD    = 0x51;
 const byte Q::INIT    = 0x52;
 const byte Q::CNC     = 0x53;
-const byte Q::ACKt     = 0x54;
+const byte Q::ACKt    = 0x54;
 const byte Q::SUPR    = 0x55;
+const byte Q::EMPT    = 0x56;
 // temporary use: for MPI testing ------------------------------------------
 const byte Q::M0      = 0x60;
 const byte Q::M1      = 0x61;
diff --git a/Source/Common/Pglobals.h b/Source/Common/Pglobals.h
index 0651ccd5..f4b36525 100644
--- a/Source/Common/Pglobals.h
+++ b/Source/Common/Pglobals.h
@@ -80,6 +80,8 @@ Mothership
 ----------
 EXIT |-    |-    |-    | (None)
 SYST |KILL |-    |-    | (None)
+APP  |EMPT |-    |-    | (0:string)Code path to broadcast
+                       | (1:string)Data path to broadcast
 APP  |SPEC |-    |-    | (0:string)Application name
                          (1:uint32_t)Number of expected distribution messages
                          (2:uint8_t)Application number
@@ -154,6 +156,7 @@ static const byte INIT;
 static const byte CNC;
 static const byte ACK;
 static const byte SUPR;
+static const byte EMPT;
 // temporary use: for MPI testing ------------------------------------------
 static const byte M0;
 static const byte M1;
diff --git a/Source/Mothership/MPIHandlers.cpp b/Source/Mothership/MPIHandlers.cpp
index f25a22aa..61ab28cd 100644
--- a/Source/Mothership/MPIHandlers.cpp
+++ b/Source/Mothership/MPIHandlers.cpp
@@ -55,7 +55,7 @@ unsigned Mothership::handle_msg_cnc(PMsg_p* message)
     #if ORCHESTRATOR_DEBUG
     std::string key = "Unknown";
     if (message->Key() == PMsg_p::KEY(Q::APP,Q::SPEC))
-        key = "Q::APP,Q::SPEC";
+        key = "Q::APP,Q::EMPT";
     else if (message->Key() == PMsg_p::KEY(Q::APP,Q::SPEC))
         key = "Q::APP,Q::SPEC";
     else if (message->Key() == PMsg_p::KEY(Q::APP,Q::DIST))
@@ -79,6 +79,27 @@ unsigned Mothership::handle_msg_cnc(PMsg_p* message)
     return 0;
 }
 
+unsigned Mothership::handle_msg_app_empt(PMsg_p* message)
+{
+    /* Pull message contents. */
+    std::string codePath;
+    std::string dataPath;
+    if (!decode_app_empt_message(message, &codePath, &dataPath))
+    {
+        debug_post(597, 3, "Q::APP,Q::EMPT", hex2str(message->Key()).c_str(),
+                   "Failed to decode.");
+        return 0;
+    }
+
+    debug_post(597, 3, "Q::APP,Q::EMPT", hex2str(message->Key()).c_str(),
+               dformat("codePath=%s, dataPath=%s",
+                       codePath.c_str(), dataPath.c_str()));
+
+    /* gogogo */
+    backend->loadAll(codePath.c_str(), dataPath.c_str());
+    return 0;
+}
+
 unsigned Mothership::handle_msg_app_spec(PMsg_p* message)
 {
     AppInfo* appInfo;
diff --git a/Source/Mothership/MessageUtils.cpp b/Source/Mothership/MessageUtils.cpp
index 8c58fa2b..75909343 100644
--- a/Source/Mothership/MessageUtils.cpp
+++ b/Source/Mothership/MessageUtils.cpp
@@ -10,6 +10,16 @@
 
 #include "Mothership.h"
 
+bool Mothership::decode_app_empt_message(
+    PMsg_p* message, std::string* codePath, std::string* dataPath)
+{
+    codePath->clear();
+    dataPath->clear();
+    if(!decode_string_message(message, codePath)) return false;
+    if(!decode_string_message(message, dataPath, 1)) return false;
+    return true;
+}
+
 bool Mothership::decode_app_dist_message(
     PMsg_p* message, std::string* appName, std::string* codePath,
     std::string* dataPath, uint32_t* coreAddr,
diff --git a/Source/Mothership/Mothership.h b/Source/Mothership/Mothership.h
index 3fafa457..bc72ca59 100644
--- a/Source/Mothership/Mothership.h
+++ b/Source/Mothership/Mothership.h
@@ -51,6 +51,7 @@ class Mothership: public CommonBase
     void recall_application(AppInfo*);
 
     /* Methods for handling MPI messages (called by consumer threads). */
+    unsigned handle_msg_app_empt(PMsg_p* message);
     unsigned handle_msg_app_spec(PMsg_p* message);
     unsigned handle_msg_app_dist(PMsg_p* message);
     unsigned handle_msg_app_supd(PMsg_p* message);
@@ -95,6 +96,8 @@ class Mothership: public CommonBase
 
     /* Methods for safely decoding MPI messages with certain field
      * configurations. */
+    bool decode_app_empt_message(PMsg_p* message, std::string* codePath,
+                                 std::string* dataPath);
     bool decode_app_dist_message(PMsg_p* message, std::string* appName,
                                  std::string* codePath, std::string* dataPath,
                                  uint32_t* coreAddr,
diff --git a/Source/Mothership/ThreadLogic.cpp b/Source/Mothership/ThreadLogic.cpp
index f69d910b..bc9ee9ab 100644
--- a/Source/Mothership/ThreadLogic.cpp
+++ b/Source/Mothership/ThreadLogic.cpp
@@ -55,7 +55,9 @@ void* ThreadComms::mpi_cnc_resolver(void* mothershipArg)
              messageIt++)
         {
             key = messageIt->Key();
-            if (key == PMsg_p::KEY(Q::APP, Q::SPEC))
+            if (key == PMsg_p::KEY(Q::APP, Q::EMPT))
+                mothership->handle_msg_app_empt(&*messageIt);
+            else if (key == PMsg_p::KEY(Q::APP, Q::SPEC))
                 mothership->handle_msg_app_spec(&*messageIt);
             else if (key == PMsg_p::KEY(Q::APP, Q::DIST))
                 mothership->handle_msg_app_dist(&*messageIt);
diff --git a/Source/OrchBase/Handlers/CmDepl.cpp b/Source/OrchBase/Handlers/CmDepl.cpp
index 5c2c1758..2dd0a68a 100644
--- a/Source/OrchBase/Handlers/CmDepl.cpp
+++ b/Source/OrchBase/Handlers/CmDepl.cpp
@@ -145,6 +145,7 @@ int CmDepl::DeployGraph(GraphI_t* gi)
     PMsg_p specMessage;
     PMsg_p distMessage;
     PMsg_p supdMessage;
+    PMsg_p emptMessage;
     std::vector<PMsg_p*> messages;
     std::vector<PMsg_p*>::iterator messageIt;
 
@@ -347,6 +348,7 @@ int CmDepl::DeployGraph(GraphI_t* gi)
     messages.push_back(&specMessage);
     messages.push_back(&distMessage);
     messages.push_back(&supdMessage);
+    messages.push_back(&emptMessage);
     for (messageIt = messages.begin(); messageIt != messages.end();
          messageIt++)
     {
@@ -356,6 +358,7 @@ int CmDepl::DeployGraph(GraphI_t* gi)
     specMessage.Key(Q::APP, Q::SPEC);
     distMessage.Key(Q::APP, Q::DIST);
     supdMessage.Key(Q::APP, Q::SUPD);
+    emptMessage.Key(Q::APP, Q::EMPT);
 
     /* Iterate through participating Motherships. */
     for (mothershipPayloadsIt = mothershipPayloads.begin();
@@ -369,6 +372,30 @@ int CmDepl::DeployGraph(GraphI_t* gi)
             (*messageIt)->Tgt(mothershipPayloadsIt->first);
         }
 
+        /* Customise and send the EMPT message, if configured to use them. */
+        bool soloApp = dynamic_cast<Root *>(par)->pOC->SingleApp();
+        if (soloApp)
+        {
+            /* TODO: GMB to plonk binary paths here (or at least some way to
+             * grab them). <!> */
+            std::string codePath;
+            std::string dataPath;
+
+            emptMessage.Put(0, &codePath);
+            emptMessage.Put(1, &dataPath);
+            fprintf(par->fd, "Sending EMPT message to Mothership rank %d, "
+                    "with codepath=%s and dataPath=%s...",
+                    mothershipPayloadsIt->first, codePath.c_str(),
+                    dataPath.c_str());
+            emptMessage.Send();
+            fprintf(par->fd, " message sent.\n");
+        }
+        else
+        {
+            fprintf(par->fd, "Not sending EMPT message, as Orchestrator is "
+                    "not configured to operate in single-application mode.\n");
+        }
+
         /* Customise and send the SPEC message. */
         appNumber = 0;  /* This is terrible - only one graph instance can be
                          * loaded at a time! <!> TODO */
@@ -376,7 +403,6 @@ int CmDepl::DeployGraph(GraphI_t* gi)
         specMessage.Put<unsigned>(1, &distCount);
         specMessage.Put<unsigned char>(2,
             static_cast<unsigned char*>(&appNumber));
-        bool soloApp = dynamic_cast<Root *>(par)->pOC->SingleApp();
         specMessage.Put<bool>(3,&soloApp);
         fprintf(par->fd, "Sending SPEC message to Mothership rank %d, with "
                 "appNumber=%u, distCount=%u, and soloApp=%s...",

From 8ef318269c9b88fb80f8e7ff39b2ce057224db67 Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 17:50:55 +0100
Subject: [PATCH 12/30] Add a preprocessor warning to make sure GMB defines
 these paths in future.

---
 Source/OrchBase/Handlers/CmDepl.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Source/OrchBase/Handlers/CmDepl.cpp b/Source/OrchBase/Handlers/CmDepl.cpp
index 2dd0a68a..9f0cb1eb 100644
--- a/Source/OrchBase/Handlers/CmDepl.cpp
+++ b/Source/OrchBase/Handlers/CmDepl.cpp
@@ -378,6 +378,7 @@ int CmDepl::DeployGraph(GraphI_t* gi)
         {
             /* TODO: GMB to plonk binary paths here (or at least some way to
              * grab them). <!> */
+            #warning GMB to define these paths somewhere.
             std::string codePath;
             std::string dataPath;
 

From 02c286df7d2c916ccec1f97bd709d6bcdd925bc4 Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Fri, 27 Aug 2021 18:50:08 +0100
Subject: [PATCH 13/30] Put the Tinsel submodule back (whoops).

---
 .gitmodules | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitmodules b/.gitmodules
index e69de29b..ab22aa56 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "Tinsel"]
+	path = Tinsel
+	url = https://github.com/poetsii/tinsel
+	branch = master

From 61cd47be3f8269e87caf185f102bf0ab8baf8478 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 18 Oct 2021 13:44:29 +0100
Subject: [PATCH 14/30] Softswitch tweak for idle

---
 Source/Softswitch/src/softswitch_main.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Source/Softswitch/src/softswitch_main.cpp b/Source/Softswitch/src/softswitch_main.cpp
index dfbb5d63..ca46574f 100644
--- a/Source/Softswitch/src/softswitch_main.cpp
+++ b/Source/Softswitch/src/softswitch_main.cpp
@@ -14,6 +14,11 @@ void softswitch_main()
 {
     // The thread context will reside at the base of the DRAM memory area.
     PThreadContext* ThreadContext = static_cast<PThreadContext*>(tinselHeapBase());
+    
+    if(ThreadContext->numDevInsts == 0)
+    {   // We have no devices loaded, so should do nothing.
+        while(true) tinselIdle(1);
+    }
 
     // Configure rtsBuf
     outPin_t* rtsBuf[ThreadContext->rtsBuffSize+1];

From 0c36c6077f750c4ede42d0d56cf8bf8d65f77d31 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 25 Oct 2021 12:26:44 +0100
Subject: [PATCH 15/30] Fix makefile to generate dummy binaries

---
 Source/Softswitch/Makefile | 35 ++++++++++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/Source/Softswitch/Makefile b/Source/Softswitch/Makefile
index 169a3a3d..fc737d10 100644
--- a/Source/Softswitch/Makefile
+++ b/Source/Softswitch/Makefile
@@ -142,8 +142,8 @@ VDTARGETS := $(patsubst $(GENINC)/vars_%.h,$(BINDIR)/softswitch_data_%.v,$(wildc
 # targets for thread-local variables
 THREADVARS := $(patsubst $(GENSRC)/%.cpp,%.o,$(wildcard $(GENSRC)/vars_*_*.cpp))
 
-.PHONY: all clean printvars supervisor
-all : $(TARGETS) $(VCTARGETS) $(VDTARGETS) supervisor
+.PHONY: all clean printvars supervisor dummy
+all : $(TARGETS) $(VCTARGETS) $(VDTARGETS) supervisor dummy
 
 # This line can be given a target/prerequisite (tgt : $(TARGETS)) if needed
 #	$(foreach TARGET,$<,cp $(TARGET) $(BINDIR)/$(TARGET))
@@ -155,9 +155,11 @@ SupervisorData.o : supervisor.bin
 	ld -r -b binary -o $@ supervisor.bin
 	rm supervisor.bin
 
+# Supervisor shared object.
 $(BINDIR)/libSupervisor.so: Supervisor.cpp Supervisor.h supervisor_generated.cpp supervisor_generated.h SupervisorApi.h SupervisorApiEntrypoints.h SupervisorData.o
-	$(MPICXX) $(MPICFLAGS) $(MPILDFLAGS) -O3 -Wl,-soname,libSupervisor.so -o $@ $< SupervisorData.o
+	$(MPICXX) $(MPICFLAGS) -pipe $(MPILDFLAGS) -O3 -Wl,-soname,libSupervisor.so -o $@ $< SupervisorData.o
 
+# Split out the code and data from the elf.
 $(BINDIR)/softswitch_code_%.v: $(BINDIR)/softswitch_%.elf
 	$(TINSELBIN)/checkelf.sh $<
 	$(RV_OBJCOPY) -O verilog --only-section=.text $< $@
@@ -176,7 +178,7 @@ io.o : io.c io.h tinsel.h $(TINSELINC)/config.h
 
 ## the pattern substitution should find all vars_x_y.o created from vars_x_y.cpp
 .SECONDEXPANSION:
-$(BINDIR)/softswitch_%.elf : softswitch_main.o softswitch_common.o softswitch.o vars_%.o $$(patsubst $$(SECEXPCPPS),$$(SECEXPOS),$$(wildcard $(GENSRC)/vars_$$*_*.cpp)) handlers_%.o io.o link_%.ld entry.o libc.a libgcc.a
+$(BINDIR)/softswitch_%.elf : softswitch_main.o softswitch_common.o softswitch.o softswitch_vars.o vars_%.o $$(patsubst $$(SECEXPCPPS),$$(SECEXPOS),$$(wildcard $(GENSRC)/vars_$$*_*.cpp)) handlers_%.o io.o link_%.ld entry.o libc.a libgcc.a
 	$(RV_LD) $(LDFLAGS) -T link_$(notdir $*).ld -o $@ $(filter-out link_$(notdir $*).ld, $^)
 
 # this convoluted rule (we hope) will establish as a prerequisite for any file
@@ -205,5 +207,28 @@ link_%.ld : genld.sh
 $(HL)/%.o :
 	make -C $(TINSELHL)
 
+
+# Rules to build the dummy binaries for Hardware Idle detection.
+dummy: $(BINDIR)/threadCtxInit_data.v $(BINDIR)/dummy_code.v
+
+threadCtxInitGenerator : threadCtxInitGenerator.cpp
+	$(MPICXX) -I$(TINSELINC) -std=c++14 -o $@ $<
+
+dummy.o : dummy.cpp
+	$(RV_CPPC) $(CFLAGS) -Wall -c -o $@ $<
+
+$(BINDIR)/dummy.elf : dummy.o link_0.ld entry.o libc.a libgcc.a
+	$(RV_LD) $(LDFLAGS) -T link_0.ld -o $@ $(filter-out link_0.ld, $^)
+
+$(BINDIR)/dummy_code.v : $(BINDIR)/dummy.elf
+	$(TINSELBIN)/checkelf.sh $<
+	$(RV_OBJCOPY) -O verilog --only-section=.text $< $@
+
+$(BINDIR)/threadCtxInit_data.v : threadCtxInitGenerator
+	./$<
+	mv threadCtxInit_data.v $@
+
+
+# Cleanup on aisle 3
 clean :
-	rm -f *.o *.elf link.ld $(BINDIR)/*.elf $(BINDIR)/*.v $(BINDIR)/libSupervisor.so
+	rm -f *.o *.elf link.ld link_*.ld $(BINDIR)/*.elf $(BINDIR)/*.v $(BINDIR)/libSupervisor.so threadCtxInitGenerator

From 0dbe44eb2f1911a0e8a2e862c336b009ef4e0363 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 25 Oct 2021 16:47:13 +0100
Subject: [PATCH 16/30] Make composer store dummy binaries

---
 Source/OrchBase/Composer.cpp | 36 ++++++++++++++++++++++++++++++++++++
 Source/OrchBase/Composer.h   |  3 +++
 2 files changed, 39 insertions(+)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index db75e0ea..665c0958 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -126,6 +126,11 @@ void ComposerGraphI_t::Dump(unsigned off,FILE* file)
         default:        fprintf(file, "**INVALID**\n");
     }
     
+    
+    fprintf(file, "  Hardware Idle Instructions: %s\n",
+                    idleInstructionBinary.c_str());
+    fprintf(file, "  Hardware Idle Data:         %s\n",idleDataBinary.c_str());
+    
     fprintf(file, "\nNitty gritty details:\n");
     fprintf(file, "  Device type strs map size:  %lu \n",
                         static_cast<unsigned long>(devTStrsMap.size()));
@@ -1211,6 +1216,37 @@ int Composer::checkBinaries(ComposerGraphI_t* builderGraphI)
     std::string taskDir(builderGraphI->outputDir);
     std::string elfPath(taskDir + "/bin");
     
+    
+    // Check that the "dummy" binaries for HW idle were generated.
+    FILE* dummyBinary;
+    
+    // Check Dummy Instruction binary and add to GraphI
+    std::string dummyPath = elfPath + "/dummy_code.v";
+    dummyBinary = fopen(dummyPath.c_str(), "r");
+    if(dummyBinary == PNULL)
+    { // Failed to open binary
+        fprintf(fd,"\tFailed to open dummy instruction binary %s after compilation\n",
+                    dummyPath.c_str());
+        return -1;
+    }
+    fclose(dummyBinary);
+    builderGraphI->idleInstructionBinary = dummyPath;
+    
+    
+    // Check Dummy Data binary and add to GraphI
+    dummyPath = elfPath + "/threadCtxInit_data.v";
+    dummyBinary = fopen(dummyPath.c_str(), "r");
+    if(dummyBinary == PNULL)
+    { // Failed to open binary
+        fprintf(fd,"\tFailed to open dummy data binary %s after compilation\n",
+                    dummyPath.c_str());
+        return -1;
+    }
+    fclose(dummyBinary);
+    builderGraphI->idleDataBinary = dummyPath;
+    
+    
+    
     // Check that the core binaries were made and link to each core.
     WALKSET(P_core*,(*(builderGraphI->cores)),coreNode)
     {
diff --git a/Source/OrchBase/Composer.h b/Source/OrchBase/Composer.h
index 33b9809e..f7293064 100644
--- a/Source/OrchBase/Composer.h
+++ b/Source/OrchBase/Composer.h
@@ -98,6 +98,9 @@ typedef struct ComposerGraphI_t
     ssLoopMode_t softswitchLoopMode;
     bool softswitchRequestIdle;
     
+    std::string idleInstructionBinary;   // Dummy instruction binary for HW Idle
+    std::string idleDataBinary;          // Dummy data binary for HW Idle
+    
     // Constructors/Destructors
     ComposerGraphI_t();
     ComposerGraphI_t(GraphI_t*, std::string&);

From e9c983a6bdba2d661bbdd0ca593d7f7bc68ea497 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 25 Oct 2021 18:17:10 +0100
Subject: [PATCH 17/30] add logic to get dummy binaries

---
 Source/OrchBase/Composer.cpp        | 34 +++++++++++++++++++++++++++++
 Source/OrchBase/Composer.h          |  2 ++
 Source/OrchBase/Handlers/CmDepl.cpp |  6 ++---
 3 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 665c0958..901a0f84 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -38,6 +38,9 @@ ComposerGraphI_t::ComposerGraphI_t()
     
     compilationFlags = "";
     provenanceCache = "";
+    
+    idleInstructionBinary = "";
+    idleDataBinary = "";
 }
 
 ComposerGraphI_t::ComposerGraphI_t(GraphI_t* graphIIn, std::string& outputPath)
@@ -59,6 +62,9 @@ ComposerGraphI_t::ComposerGraphI_t(GraphI_t* graphIIn, std::string& outputPath)
     
     compilationFlags = "";
     provenanceCache = "";
+    
+    idleInstructionBinary = "";
+    idleDataBinary = "";
 }
 
 ComposerGraphI_t::~ComposerGraphI_t()
@@ -1017,6 +1023,34 @@ bool Composer::isCompiled(GraphI_t* graphI)
     return builderGraphI->compiled;
 }
 
+/******************************************************************************
+ * Public method to get the paths for the HW Idle binaries
+ *****************************************************************************/
+bool Composer::getDummyPaths(GraphI_t* graphI, std::string& instrBin, 
+                                std::string& dataBin)
+{
+    ComposerGraphI_t* builderGraphI;
+
+    ComposerGraphIMap_t::iterator srch = graphIMap.find(graphI);
+    if (srch == graphIMap.end())
+    {   // The Graph Instance has not been seen before, so not compiled.
+        return false;
+
+    } else {
+        builderGraphI = srch->second;
+    }
+    
+    if(builderGraphI->idleInstructionBinary == "" || 
+        builderGraphI->idleDataBinary == "")
+    {   // We are missing a binary path, return false.
+        return false;
+    }
+    
+    instrBin = builderGraphI->idleInstructionBinary;
+    dataBin = builderGraphI->idleDataBinary;
+    return true;
+}
+
 /******************************************************************************
  * Invoke a clean and then a degenerate
  *****************************************************************************/
diff --git a/Source/OrchBase/Composer.h b/Source/OrchBase/Composer.h
index f7293064..69c62aba 100644
--- a/Source/OrchBase/Composer.h
+++ b/Source/OrchBase/Composer.h
@@ -143,6 +143,8 @@ int         addFlags(GraphI_t*, std::string&);
 bool        isGenerated(GraphI_t*);
 bool        isCompiled(GraphI_t*);
 
+bool        getDummyPaths(GraphI_t*, std::string&, std::string&);   // Get the hardware idle binary names 
+
 void        Show(FILE * = stdout);
 void        Dump(unsigned = 0,FILE * = stdout);
 
diff --git a/Source/OrchBase/Handlers/CmDepl.cpp b/Source/OrchBase/Handlers/CmDepl.cpp
index 9f0cb1eb..688d8742 100644
--- a/Source/OrchBase/Handlers/CmDepl.cpp
+++ b/Source/OrchBase/Handlers/CmDepl.cpp
@@ -376,11 +376,11 @@ int CmDepl::DeployGraph(GraphI_t* gi)
         bool soloApp = dynamic_cast<Root *>(par)->pOC->SingleApp();
         if (soloApp)
         {
-            /* TODO: GMB to plonk binary paths here (or at least some way to
-             * grab them). <!> */
-            #warning GMB to define these paths somewhere.
             std::string codePath;
             std::string dataPath;
+            
+            // Get the dummy binary paths
+            par->pComposer->getDummyPaths(gi, codePath, dataPath);
 
             emptMessage.Put(0, &codePath);
             emptMessage.Put(1, &dataPath);

From 9668721d53b3e54035e7bacd571d48f53cec840e Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 25 Oct 2021 18:19:36 +0100
Subject: [PATCH 18/30] add missing Softswitch files

---
 Source/Softswitch/inc/softswitch_vars.h       | 23 ++++++
 Source/Softswitch/src/dummy.cpp               |  9 +++
 Source/Softswitch/src/softswitch_vars.cpp     | 20 ++++++
 .../Softswitch/src/threadCtxInitGenerator.cpp | 70 +++++++++++++++++++
 4 files changed, 122 insertions(+)
 create mode 100644 Source/Softswitch/inc/softswitch_vars.h
 create mode 100644 Source/Softswitch/src/dummy.cpp
 create mode 100644 Source/Softswitch/src/softswitch_vars.cpp
 create mode 100644 Source/Softswitch/src/threadCtxInitGenerator.cpp

diff --git a/Source/Softswitch/inc/softswitch_vars.h b/Source/Softswitch/inc/softswitch_vars.h
new file mode 100644
index 00000000..80653985
--- /dev/null
+++ b/Source/Softswitch/inc/softswitch_vars.h
@@ -0,0 +1,23 @@
+#ifndef _VARS_H_
+#define _VARS_H_
+#include <cstdint>
+#include "softswitch_common.h"
+
+extern ThreadCtxt_t Thread_0_Context;
+extern ThreadCtxt_t Thread_1_Context;
+extern ThreadCtxt_t Thread_2_Context;
+extern ThreadCtxt_t Thread_3_Context;
+extern ThreadCtxt_t Thread_4_Context;
+extern ThreadCtxt_t Thread_5_Context;
+extern ThreadCtxt_t Thread_6_Context;
+extern ThreadCtxt_t Thread_7_Context;
+extern ThreadCtxt_t Thread_8_Context;
+extern ThreadCtxt_t Thread_9_Context;
+extern ThreadCtxt_t Thread_10_Context;
+extern ThreadCtxt_t Thread_11_Context;
+extern ThreadCtxt_t Thread_12_Context;
+extern ThreadCtxt_t Thread_13_Context;
+extern ThreadCtxt_t Thread_14_Context;
+extern ThreadCtxt_t Thread_15_Context;
+
+#endif /*_VARS_H_*/
\ No newline at end of file
diff --git a/Source/Softswitch/src/dummy.cpp b/Source/Softswitch/src/dummy.cpp
new file mode 100644
index 00000000..47bd6fbe
--- /dev/null
+++ b/Source/Softswitch/src/dummy.cpp
@@ -0,0 +1,9 @@
+#include "tinsel.h"
+
+int main(int argc, char** argv)
+{
+    // Block on tinselIdle to facilitate hardware idle.
+    while(true) tinselIdle(1);
+    
+    return 0;
+}
diff --git a/Source/Softswitch/src/softswitch_vars.cpp b/Source/Softswitch/src/softswitch_vars.cpp
new file mode 100644
index 00000000..347b5f65
--- /dev/null
+++ b/Source/Softswitch/src/softswitch_vars.cpp
@@ -0,0 +1,20 @@
+
+#include "softswitch_common.h"
+#include "softswitch_vars.h"
+
+ThreadCtxt_t Thread_0_Context __attribute__((weak, section (".thr0_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_1_Context __attribute__((weak, section (".thr1_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_2_Context __attribute__((weak, section (".thr2_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_3_Context __attribute__((weak, section (".thr3_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_4_Context __attribute__((weak, section (".thr4_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_5_Context __attribute__((weak, section (".thr5_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_6_Context __attribute__((weak, section (".thr6_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_7_Context __attribute__((weak, section (".thr7_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_8_Context __attribute__((weak, section (".thr8_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_9_Context __attribute__((weak, section (".thr9_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_10_Context __attribute__((weak, section (".thr10_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_11_Context __attribute__((weak, section (".thr11_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_12_Context __attribute__((weak, section (".thr12_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_13_Context __attribute__((weak, section (".thr13_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_14_Context __attribute__((weak, section (".thr14_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+ThreadCtxt_t Thread_15_Context __attribute__((weak, section (".thr15_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
\ No newline at end of file
diff --git a/Source/Softswitch/src/threadCtxInitGenerator.cpp b/Source/Softswitch/src/threadCtxInitGenerator.cpp
new file mode 100644
index 00000000..77e34685
--- /dev/null
+++ b/Source/Softswitch/src/threadCtxInitGenerator.cpp
@@ -0,0 +1,70 @@
+#include "config.h"
+
+#include <string>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <stdio.h>
+#include <math.h>
+
+
+// Number of bytes per thread partition                         2^21 = 2,097,152
+uint32_t BytesPerDRAMPartition=1<<TinselLogBytesPerDRAMPartition;
+
+// The number of cores that share a DRAM                            2^(2+3) = 32
+uint32_t CoresPerDRAM=1<<(TinselLogCoresPerDCache+TinselLogDCachesPerDRAM);
+
+// The number of threads per DRAM                                    32*16 = 512
+uint32_t ThreadsPerDRAM=CoresPerDRAM*TinselThreadsPerCore;
+
+uint32_t InterleavedPartitionOffset=0x80000000;
+
+int main(void)
+{
+    // The output binary
+    std::ofstream dataBin;
+
+    dataBin.open("threadCtxInit_data.v");
+    if(dataBin.fail()) // Check that the file opened
+    {                 // if it didn't, barf
+        std::cout << "***ERROR*** FAILED TO OPEN DATA BINARY\n" << std::endl;
+        return -1;
+    }
+
+    // Populate the bin
+    uint32_t i = 0;
+    for(int coreNum = 0; coreNum<CoresPerDRAM; coreNum++)
+    {
+        for(int threadNum = 1; threadNum<=TinselThreadsPerCore; threadNum++)
+        {
+            uint32_t threadAddr, threadOffset;
+            
+            threadOffset = ((coreNum << TinselLogThreadsPerCore) + threadNum);
+            threadOffset *= BytesPerDRAMPartition;
+            
+            threadAddr = TinselBytesPerDRAM - threadOffset;
+            
+            threadAddr |= InterleavedPartitionOffset;
+            
+            // Write the address
+            dataBin << std::hex << std::uppercase << "@" << threadAddr << "\n";
+            
+            // Write out the initialiser
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n";  
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n";  
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n"; 
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n"; 
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n"; 
+            dataBin << "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \n"; 
+            dataBin << "00 00 00 00 00 00 00 00 " << std::endl;
+            
+            i++;
+        }
+    }
+
+    // And we are done
+    dataBin.close();
+    
+    return 0;
+}
+

From c4450feca850fc03cc249d7fcbceaa5d8c310140 Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Wed, 27 Oct 2021 13:47:45 +0100
Subject: [PATCH 19/30] Add missing entry for APP,EMPT messages in the
 Mothership's MPI message map.

---
 Source/Mothership/MPIHandlers.cpp | 2 +-
 Source/Mothership/Mothership.cpp  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Source/Mothership/MPIHandlers.cpp b/Source/Mothership/MPIHandlers.cpp
index 61ab28cd..2f2d784b 100644
--- a/Source/Mothership/MPIHandlers.cpp
+++ b/Source/Mothership/MPIHandlers.cpp
@@ -54,7 +54,7 @@ unsigned Mothership::handle_msg_cnc(PMsg_p* message)
 {
     #if ORCHESTRATOR_DEBUG
     std::string key = "Unknown";
-    if (message->Key() == PMsg_p::KEY(Q::APP,Q::SPEC))
+    if (message->Key() == PMsg_p::KEY(Q::APP,Q::EMPT))
         key = "Q::APP,Q::EMPT";
     else if (message->Key() == PMsg_p::KEY(Q::APP,Q::SPEC))
         key = "Q::APP,Q::SPEC";
diff --git a/Source/Mothership/Mothership.cpp b/Source/Mothership/Mothership.cpp
index 35f7a471..7052ee70 100644
--- a/Source/Mothership/Mothership.cpp
+++ b/Source/Mothership/Mothership.cpp
@@ -196,6 +196,7 @@ void Mothership::setup_mpi_hooks()
     DebugPrint("[MOTHERSHIP] Setting up MPI hooks.\n");
     FnMap[PMsg_p::KEY(Q::EXIT)] = &Mothership::handle_msg_exit;
     FnMap[PMsg_p::KEY(Q::SYST,Q::KILL)] = &Mothership::handle_msg_syst_kill;
+    FnMap[PMsg_p::KEY(Q::APP,Q::EMPT)] = &Mothership::handle_msg_cnc;
     FnMap[PMsg_p::KEY(Q::APP,Q::SPEC)] = &Mothership::handle_msg_cnc;
     FnMap[PMsg_p::KEY(Q::APP,Q::DIST)] = &Mothership::handle_msg_cnc;
     FnMap[PMsg_p::KEY(Q::APP,Q::SUPD)] = &Mothership::handle_msg_cnc;

From 7fb7f9ecdabf272f91a33336604faca7a9b5436c Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Wed, 27 Oct 2021 13:47:59 +0100
Subject: [PATCH 20/30] Fix bad decode message format.

---
 Config/OrchestratorMessages.ocfg | 2 +-
 Source/Common/Decode.cpp         | 7 ++-----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/Config/OrchestratorMessages.ocfg b/Config/OrchestratorMessages.ocfg
index 9d5de844..27fb0530 100644
--- a/Config/OrchestratorMessages.ocfg
+++ b/Config/OrchestratorMessages.ocfg
@@ -133,7 +133,7 @@
 // Loading, typelinking
 
 100(U) : "MLV: Allocated but as yet unused"
-101(S) : "Decoder in %s (PID %s) has dropped a packet from %s(rank %s) to %s(rank %s) with key 0x%s"
+101(S) : "Decoder in %s (rank %s) has dropped a packet from %s (rank %s) with key 0x%s."
 102(I) : "Task graph default file path is ||%s||"
 103(I) : "New path is ||%s||"
 104(W) : "Filename %s does not parse correctly"
diff --git a/Source/Common/Decode.cpp b/Source/Common/Decode.cpp
index 262d1058..f738e7c5 100644
--- a/Source/Common/Decode.cpp
+++ b/Source/Common/Decode.cpp
@@ -32,12 +32,9 @@ if (CommonBase::FnMap.find(pPkt->Key())!=CommonBase::FnMap.end()) {
                                        // Nope. Kick.
                                        // Pull out the unknown key and post what
                                        // little we know to the LogServer
-Post(101,Sderived,int2str(pPkt->Src()),pPmap->vPmap[pPkt->Src()].P_class,int2str(pPkt->Tgt()),
-     pPmap->vPmap[pPkt->Tgt()].P_class,hex2str(pPkt->Key()));
+Post(101,Sderived,int2str(pPkt->Tgt()),pPmap->vPmap[pPkt->Src()].P_class,
+     int2str(pPkt->Src()),hex2str(pPkt->Key()));
 return 0;                              // Return "keep going" value
 }
 
 //==============================================================================
-
-
-

From f8d517058831c9c84750e2f0fc12d9f743e66961 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <g.bragg@ecs.soton.ac.uk>
Date: Wed, 27 Oct 2021 14:02:47 +0100
Subject: [PATCH 21/30] Add missing library def for linkler

---
 Source/Softswitch/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Source/Softswitch/Makefile b/Source/Softswitch/Makefile
index fc737d10..8a77a4ba 100644
--- a/Source/Softswitch/Makefile
+++ b/Source/Softswitch/Makefile
@@ -212,7 +212,7 @@ $(HL)/%.o :
 dummy: $(BINDIR)/threadCtxInit_data.v $(BINDIR)/dummy_code.v
 
 threadCtxInitGenerator : threadCtxInitGenerator.cpp
-	$(MPICXX) -I$(TINSELINC) -std=c++14 -o $@ $<
+	$(MPICXX) -I$(TINSELINC) -L$(MPICHLIB) -std=c++14 -o $@ $<
 
 dummy.o : dummy.cpp
 	$(RV_CPPC) $(CFLAGS) -Wall -c -o $@ $<

From 09e91f32e8ace2693d2cfc24126c92154c8f9b94 Mon Sep 17 00:00:00 2001
From: Mark Vousden <m.vousden@soton.ac.uk>
Date: Wed, 27 Oct 2021 13:30:56 +0000
Subject: [PATCH 22/30] Fix some misleading reporting in debug mode.

---
 Config/OrchestratorMessages.ocfg  | 2 +-
 Source/Mothership/MPIHandlers.cpp | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Config/OrchestratorMessages.ocfg b/Config/OrchestratorMessages.ocfg
index 27fb0530..73058130 100644
--- a/Config/OrchestratorMessages.ocfg
+++ b/Config/OrchestratorMessages.ocfg
@@ -490,7 +490,7 @@
 574(U) : "MLV: Allocated but as yet unused"
 575(U) : "MLV: Allocated but as yet unused"
 576(U) : "MLV: Allocated but as yet unused"
-577(U) : "MLV: Allocated but as yet unused"
+577(I) : "Mothership: Calling backend->loadAll."
 578(I) : "Mothership: Calling backend->go."
 579(I) : "Mothership: Calling backend->startAll."
 580(E) : "Mothership: Received a log packet with an invalid device index 0x%s."
diff --git a/Source/Mothership/MPIHandlers.cpp b/Source/Mothership/MPIHandlers.cpp
index 2f2d784b..58f77b1b 100644
--- a/Source/Mothership/MPIHandlers.cpp
+++ b/Source/Mothership/MPIHandlers.cpp
@@ -93,9 +93,10 @@ unsigned Mothership::handle_msg_app_empt(PMsg_p* message)
 
     debug_post(597, 3, "Q::APP,Q::EMPT", hex2str(message->Key()).c_str(),
                dformat("codePath=%s, dataPath=%s",
-                       codePath.c_str(), dataPath.c_str()));
+                       codePath.c_str(), dataPath.c_str()).c_str());
 
     /* gogogo */
+    debug_post(577, 0);
     backend->loadAll(codePath.c_str(), dataPath.c_str());
     return 0;
 }

From d30d0f8753c68b3987e5ca25b9b8051f46ad03ad Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Wed, 27 Oct 2021 16:44:26 +0100
Subject: [PATCH 23/30] Fix a warning

---
 Source/Softswitch/src/threadCtxInitGenerator.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Source/Softswitch/src/threadCtxInitGenerator.cpp b/Source/Softswitch/src/threadCtxInitGenerator.cpp
index 77e34685..76b7070f 100644
--- a/Source/Softswitch/src/threadCtxInitGenerator.cpp
+++ b/Source/Softswitch/src/threadCtxInitGenerator.cpp
@@ -33,7 +33,7 @@ int main(void)
 
     // Populate the bin
     uint32_t i = 0;
-    for(int coreNum = 0; coreNum<CoresPerDRAM; coreNum++)
+    for(uint32_t coreNum = 0; coreNum<CoresPerDRAM; coreNum++)
     {
         for(int threadNum = 1; threadNum<=TinselThreadsPerCore; threadNum++)
         {

From f8cc3e9a21beca6578c7233f66add6fa95a861b3 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Wed, 27 Oct 2021 16:45:44 +0100
Subject: [PATCH 24/30] get rid of the weak thread context initialisers as we
 seem to be running into a GCC bug where weak symbols are not replaced by
 strong.

---
 Source/Softswitch/Makefile                |  4 ++--
 Source/Softswitch/inc/softswitch_vars.h   | 23 -----------------------
 Source/Softswitch/src/softswitch_vars.cpp | 20 --------------------
 3 files changed, 2 insertions(+), 45 deletions(-)
 delete mode 100644 Source/Softswitch/inc/softswitch_vars.h
 delete mode 100644 Source/Softswitch/src/softswitch_vars.cpp

diff --git a/Source/Softswitch/Makefile b/Source/Softswitch/Makefile
index fc737d10..4bf9334b 100644
--- a/Source/Softswitch/Makefile
+++ b/Source/Softswitch/Makefile
@@ -178,7 +178,7 @@ io.o : io.c io.h tinsel.h $(TINSELINC)/config.h
 
 ## the pattern substitution should find all vars_x_y.o created from vars_x_y.cpp
 .SECONDEXPANSION:
-$(BINDIR)/softswitch_%.elf : softswitch_main.o softswitch_common.o softswitch.o softswitch_vars.o vars_%.o $$(patsubst $$(SECEXPCPPS),$$(SECEXPOS),$$(wildcard $(GENSRC)/vars_$$*_*.cpp)) handlers_%.o io.o link_%.ld entry.o libc.a libgcc.a
+$(BINDIR)/softswitch_%.elf : softswitch_main.o softswitch_common.o softswitch.o vars_%.o $$(patsubst $$(SECEXPCPPS),$$(SECEXPOS),$$(wildcard $(GENSRC)/vars_$$*_*.cpp)) handlers_%.o io.o link_%.ld entry.o libc.a libgcc.a
 	$(RV_LD) $(LDFLAGS) -T link_$(notdir $*).ld -o $@ $(filter-out link_$(notdir $*).ld, $^)
 
 # this convoluted rule (we hope) will establish as a prerequisite for any file
@@ -212,7 +212,7 @@ $(HL)/%.o :
 dummy: $(BINDIR)/threadCtxInit_data.v $(BINDIR)/dummy_code.v
 
 threadCtxInitGenerator : threadCtxInitGenerator.cpp
-	$(MPICXX) -I$(TINSELINC) -std=c++14 -o $@ $<
+	$(MPICXX) -I$(TINSELINC) -L$(MPICHLIB) -std=c++14 -o $@ $<
 
 dummy.o : dummy.cpp
 	$(RV_CPPC) $(CFLAGS) -Wall -c -o $@ $<
diff --git a/Source/Softswitch/inc/softswitch_vars.h b/Source/Softswitch/inc/softswitch_vars.h
deleted file mode 100644
index 80653985..00000000
--- a/Source/Softswitch/inc/softswitch_vars.h
+++ /dev/null
@@ -1,23 +0,0 @@
-#ifndef _VARS_H_
-#define _VARS_H_
-#include <cstdint>
-#include "softswitch_common.h"
-
-extern ThreadCtxt_t Thread_0_Context;
-extern ThreadCtxt_t Thread_1_Context;
-extern ThreadCtxt_t Thread_2_Context;
-extern ThreadCtxt_t Thread_3_Context;
-extern ThreadCtxt_t Thread_4_Context;
-extern ThreadCtxt_t Thread_5_Context;
-extern ThreadCtxt_t Thread_6_Context;
-extern ThreadCtxt_t Thread_7_Context;
-extern ThreadCtxt_t Thread_8_Context;
-extern ThreadCtxt_t Thread_9_Context;
-extern ThreadCtxt_t Thread_10_Context;
-extern ThreadCtxt_t Thread_11_Context;
-extern ThreadCtxt_t Thread_12_Context;
-extern ThreadCtxt_t Thread_13_Context;
-extern ThreadCtxt_t Thread_14_Context;
-extern ThreadCtxt_t Thread_15_Context;
-
-#endif /*_VARS_H_*/
\ No newline at end of file
diff --git a/Source/Softswitch/src/softswitch_vars.cpp b/Source/Softswitch/src/softswitch_vars.cpp
deleted file mode 100644
index 347b5f65..00000000
--- a/Source/Softswitch/src/softswitch_vars.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-
-#include "softswitch_common.h"
-#include "softswitch_vars.h"
-
-ThreadCtxt_t Thread_0_Context __attribute__((weak, section (".thr0_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_1_Context __attribute__((weak, section (".thr1_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_2_Context __attribute__((weak, section (".thr2_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_3_Context __attribute__((weak, section (".thr3_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_4_Context __attribute__((weak, section (".thr4_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_5_Context __attribute__((weak, section (".thr5_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_6_Context __attribute__((weak, section (".thr6_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_7_Context __attribute__((weak, section (".thr7_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_8_Context __attribute__((weak, section (".thr8_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_9_Context __attribute__((weak, section (".thr9_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_10_Context __attribute__((weak, section (".thr10_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_11_Context __attribute__((weak, section (".thr11_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_12_Context __attribute__((weak, section (".thr12_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_13_Context __attribute__((weak, section (".thr13_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_14_Context __attribute__((weak, section (".thr14_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-ThreadCtxt_t Thread_15_Context __attribute__((weak, section (".thr15_base"))) = {0,PNULL,0,PNULL,PNULL,0,PNULL,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
\ No newline at end of file

From a167578caae0725269e07cf5ec1be368c72bbb81 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <gmb@ecs.soton.ac.uk>
Date: Thu, 28 Oct 2021 14:09:09 +0100
Subject: [PATCH 25/30] first bash at HWIdle barrier release

---
 Source/OrchBase/Composer.cpp                |  3 +++
 Source/Softswitch/src/softswitch_common.cpp | 23 ++++++++++++++-------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 665c0958..516a8d46 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -907,6 +907,9 @@ int Composer::compile(GraphI_t* graphI)
     // Form the Softswitch compilation control string
     //TODO: Make these const strings at the top
     std::string makeArgs = "";
+	
+	//TODO: make this configurable
+	makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
 
     if(builderGraphI->bufferingSoftswitch)
     {   // Softswitch needs to be built in buffering mode
diff --git a/Source/Softswitch/src/softswitch_common.cpp b/Source/Softswitch/src/softswitch_common.cpp
index d128c176..6fc78e4b 100644
--- a/Source/Softswitch/src/softswitch_common.cpp
+++ b/Source/Softswitch/src/softswitch_common.cpp
@@ -43,12 +43,21 @@ void softswitch_init(ThreadCtxt_t* ThreadContext)
     }
 }
 
-// <!> You know it's a hack when the "pragma GCC" comes out.
-#pragma GCC push_options
-#pragma GCC optimize ("O0")
-void softswitch_delay(){for (uint32_t i=0; i<500; i++);}
-#pragma GCC pop_options
-
+/*------------------------------------------------------------------------------
+ * softswitch_delay: Wait for other threads to start so that we don't break sync
+ *----------------------------------------------------------------------------*/
+#ifdef SOFTSWITCH_HWIDLE_BARRIER
+void softswitch_delay(){
+    // Block on a tinselIdle call untill all threads have started
+    tinselIdle(true);        
+}
+#else
+  // <!> You know it's a hack when the "pragma GCC" comes out.
+  #pragma GCC push_options
+  #pragma GCC optimize ("O0")
+void softswitch_delay(){for (uint32_t i=0; i<500; i++);}    // Delay for an arbitary period
+  #pragma GCC pop_options
+#endif
 
 /*------------------------------------------------------------------------------
  * softswitch_barrier: Block until told to continue by the mothership
@@ -141,7 +150,7 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
     volatile void *superBuffer = tinselSendSlotExtra(); // Supervisor send slot
 
 #ifndef DISABLE_SOFTSWITCH_INSTRUMENTATION
-    uint32_t cycles = tinselCycleCount();		// cycle counter is per-core
+    uint32_t cycles = tinselCycleCount();        // cycle counter is per-core
     ThreadContext->lastCycles = cycles;         // save initial cycle count
  #if TinselEnablePerfCount == true
     // Save initial extended instrumentation counts.

From fe5a32ad228c857b07cced80a2cf159fc22f36cc Mon Sep 17 00:00:00 2001
From: Graeme Bragg <gmb@ecs.soton.ac.uk>
Date: Thu, 28 Oct 2021 18:04:42 +0100
Subject: [PATCH 26/30] make it actually use the barrier

---
 Source/OrchBase/Composer.cpp | 4 ++--
 Source/Softswitch/Makefile   | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 516a8d46..f6791ba7 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -908,8 +908,8 @@ int Composer::compile(GraphI_t* graphI)
     //TODO: Make these const strings at the top
     std::string makeArgs = "";
 	
-	//TODO: make this configurable
-	makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
+    //TODO: make this configurable
+    makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
 
     if(builderGraphI->bufferingSoftswitch)
     {   // Softswitch needs to be built in buffering mode
diff --git a/Source/Softswitch/Makefile b/Source/Softswitch/Makefile
index 4bf9334b..7555f688 100644
--- a/Source/Softswitch/Makefile
+++ b/Source/Softswitch/Makefile
@@ -42,6 +42,11 @@ ifndef BINDIR
 BINDIR := ../bin
 endif
 
+
+ifdef SOFTSWITCH_HWIDLE_BARRIER
+LH_CFLAGS := $(LH_CFLAGS) -DSOFTSWITCH_HWIDLE_BARRIER
+endif
+
 ifdef SOFTSWITCH_BUFFERING
 LH_CFLAGS := $(LH_CFLAGS) -DBUFFERING_SOFTSWITCH
 endif

From ee9552483bb9e5da4d33399d5302560f46843085 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Mon, 6 Dec 2021 03:55:21 +0000
Subject: [PATCH 27/30] Make the barrier configurable through Orchestrator
 commands

---
 Source/OrchBase/Composer.cpp        | 54 +++++++++++++++++++++++++++--
 Source/OrchBase/Composer.h          |  2 ++
 Source/OrchBase/Handlers/CmComp.cpp | 25 +++++++++++++
 Source/OrchBase/Handlers/CmComp.h   |  1 +
 4 files changed, 79 insertions(+), 3 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 96d0163e..c74981bb 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -30,6 +30,7 @@ ComposerGraphI_t::ComposerGraphI_t()
     
     // Softswitch control.
     rtsBuffSizeMax = MAX_RTSBUFFSIZE;
+    softswitchHWIdleBarrier = true;     // Default to using Hardware Idle for barrier
     bufferingSoftswitch = false;        // Default to a non-buffering softswitch
     softswitchInstrumentation = true;   // Default to enable instrumentation
     softswitchLogHandler = trivial;     // Default to the trivial log handler
@@ -54,6 +55,7 @@ ComposerGraphI_t::ComposerGraphI_t(GraphI_t* graphIIn, std::string& outputPath)
     
     // Softswitch control.
     rtsBuffSizeMax = MAX_RTSBUFFSIZE;
+    softswitchHWIdleBarrier = true;     // Default to using Hardware Idle for barrier
     bufferingSoftswitch = false;        // Default to a non-buffering softswitch
     softswitchInstrumentation = true;   // Default to enable instrumentation
     softswitchLogHandler = trivial;     // Default to the trivial log handler 
@@ -108,11 +110,13 @@ void ComposerGraphI_t::Dump(unsigned off,FILE* file)
     
     fprintf(file, "\nSoftswitch generation/compilation control:\n");
     fprintf(file, "  Maximum RTS buffer size:    %lu \n",rtsBuffSizeMax);
+    fprintf(file, "  Hardware Idle Barrier:      %s \n",
+                        softswitchHWIdleBarrier ? "true" : "false");
     fprintf(file, "  Buffering Softswitch:       %s \n",
                         bufferingSoftswitch ? "true" : "false");
     fprintf(file, "  Softswitch Instrumentation: %s \n",
                         softswitchInstrumentation ? "true" : "false");
-    fprintf(file, "  Softswitch requestIdle: %s \n",
+    fprintf(file, "  Softswitch requestIdle:     %s \n",
                         softswitchRequestIdle ? "true" : "false");
                         
     fprintf(file, "  Softswitch log handler:     ");
@@ -146,8 +150,12 @@ void ComposerGraphI_t::Dump(unsigned off,FILE* file)
     fprintf(file, "  Provenance string cache:\n%s \n\n",provenanceCache.c_str());
     
     
-    // Form the Softswitch compilation control string
     std::string makeArgs = "";
+    // Form the Softswitch compilation control string
+    if(softswitchHWIdleBarrier)
+    {   // Softswitch needs to be built with a hardware idle barrier
+        makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
+    }
     if(bufferingSoftswitch)
     {   // Softswitch needs to be built in buffering mode
         makeArgs += "SOFTSWITCH_BUFFERING=1 ";
@@ -297,6 +305,40 @@ void Composer::setPlacer(Placer* plc)
     graphIMap.clear();
 }
 
+/******************************************************************************
+ * Set the barrier mode for the softswitch. true = HWIdle, false = Packet based
+ *
+ * Changing this requires a compiled app to be recompiled.
+ *****************************************************************************/
+int Composer::setBarrierMode(GraphI_t* graphI, bool barrierMode)
+{
+    ComposerGraphI_t* builderGraphI;
+    //FILE * fd = graphI->par->par->fd;              // Detail output file
+
+    ComposerGraphIMap_t::iterator srch = graphIMap.find(graphI);
+    if (srch == graphIMap.end())
+    {   // The Graph Instance has not been seen before, map it.
+        builderGraphI = new ComposerGraphI_t(graphI, outputPath);
+
+        // Insert the GraphI
+        std::pair<ComposerGraphIMap_t::iterator, bool> insertedGraphI;
+        insertedGraphI = graphIMap.insert(ComposerGraphIMap_t::value_type
+                                                (graphI, builderGraphI));
+
+    } else {
+        builderGraphI = srch->second;
+    }
+    
+    if(builderGraphI->compiled)
+    {   // Already compiled, need to decompile
+        clean(graphI);
+    }
+    
+    builderGraphI->softswitchHWIdleBarrier = barrierMode;
+    
+    return 0;
+}
+
 
 /******************************************************************************
  * Set the buffering mode for the softswitch
@@ -916,7 +958,10 @@ int Composer::compile(GraphI_t* graphI)
     std::string makeArgs = "";
 	
     //TODO: make this configurable
-    makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
+    if(builderGraphI->softswitchHWIdleBarrier)
+    {   // Softswitch needs to be built with a hardware idle barrier
+        makeArgs += "SOFTSWITCH_HWIDLE_BARRIER=1 ";
+    }
 
     if(builderGraphI->bufferingSoftswitch)
     {   // Softswitch needs to be built in buffering mode
@@ -1396,6 +1441,9 @@ void Composer::formFileProvenance(ComposerGraphI_t* builderGraphI)
     
     
     provStr << " * Softswitch control:\n";
+    provStr << " *   Hardware Idle Barrier:\t";
+    provStr << (builderGraphI->softswitchHWIdleBarrier?"true":"false") << "\n";
+    
     provStr << " *   Buffering mode:\t\t";
     provStr << (builderGraphI->bufferingSoftswitch ? "true" : "false") << "\n";
     
diff --git a/Source/OrchBase/Composer.h b/Source/OrchBase/Composer.h
index 69c62aba..e6b78c9d 100644
--- a/Source/OrchBase/Composer.h
+++ b/Source/OrchBase/Composer.h
@@ -90,6 +90,7 @@ typedef struct ComposerGraphI_t
     bool generated;
     bool compiled;
     
+    bool softswitchHWIdleBarrier;
     bool bufferingSoftswitch;
     unsigned long rtsBuffSizeMax;
     bool softswitchInstrumentation;
@@ -131,6 +132,7 @@ int         clean(GraphI_t*);      // Get rid of built files (e.g. a make clean)
 void        setOutputPath(std::string);
 void        setPlacer(Placer*);
 
+int         setBarrierMode(GraphI_t*, bool);
 int         setBuffMode(GraphI_t*, bool);
 int         setReqIdleMode(GraphI_t*, bool);
 int         setRTSSize(GraphI_t*, unsigned long);
diff --git a/Source/OrchBase/Handlers/CmComp.cpp b/Source/OrchBase/Handlers/CmComp.cpp
index 924b9fcb..492550f8 100644
--- a/Source/OrchBase/Handlers/CmComp.cpp
+++ b/Source/OrchBase/Handlers/CmComp.cpp
@@ -297,6 +297,29 @@ void CmComp::Cm_Reset(Cli::Cl_t clause)
 
 //------------------------------------------------------------------------------
 
+void CmComp::Cm_SoftswitchBarrierMode(Cli::Cl_t clause, bool mode = true)
+{
+    /* Shout if no hardware model is loaded (i.e. there is no placer) */
+    if (par->pPlacer == PNULL)
+    {
+        par->Post(805, "HWIdleBarrier");
+        return;
+    }
+
+    /* Grab the graph instances of interest. */
+    std::set<GraphI_t*> graphs;
+    if (par->GetGraphIs(clause, graphs) == 1) return;
+    
+    /* Set the softswitch buffering mode for each app in sequence. */
+    std::set<GraphI_t*>::iterator graphIt;
+    for (graphIt = graphs.begin(); graphIt != graphs.end(); graphIt++)
+    {
+        par->pComposer->setBarrierMode(*graphIt, mode);
+    }
+}
+
+//------------------------------------------------------------------------------
+
 void CmComp::Cm_SoftswitchBufferMode(Cli::Cl_t clause, bool mode = false)
 {
     /* Shout if no hardware model is loaded (i.e. there is no placer) */
@@ -544,6 +567,8 @@ WALKVECTOR(Cli::Cl_t,pC->Cl_v,i) {     // Walk the clause list
   if (sCl=="rese" ) { Cm_Reset(*i);         continue; }
   
   // Softswitch control commands
+  if (sCl=="hwba" ) { Cm_SoftswitchBarrierMode(*i, true);   continue; }
+  if (sCl=="swba" ) { Cm_SoftswitchBarrierMode(*i, false);   continue; }
   if (sCl=="buff" ) { Cm_SoftswitchBufferMode(*i, true);   continue; }
   if (sCl=="nobu" ) { Cm_SoftswitchBufferMode(*i, false);  continue; }
   if (sCl=="logh" ) { Cm_SoftswitchLogHandler(*i);         continue; }
diff --git a/Source/OrchBase/Handlers/CmComp.h b/Source/OrchBase/Handlers/CmComp.h
index f676f64c..f231d1cd 100644
--- a/Source/OrchBase/Handlers/CmComp.h
+++ b/Source/OrchBase/Handlers/CmComp.h
@@ -25,6 +25,7 @@ void          Cm_Decompose(Cli::Cl_t clause);
 void          Cm_Degenerate(Cli::Cl_t clause);
 void          Cm_Clean(Cli::Cl_t clause);
 void          Cm_Reset(Cli::Cl_t clause);
+void          Cm_SoftswitchBarrierMode(Cli::Cl_t clause, bool mode);
 void          Cm_SoftswitchBufferMode(Cli::Cl_t clause, bool mode);
 void          Cm_SoftswitchReqIdleMode(Cli::Cl_t clause, bool mode);
 void          Cm_SoftswitchInstrMode(Cli::Cl_t clause, bool mode);

From e74e6ee1e353935e074e4f9a522020aa029f9351 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Fri, 10 Dec 2021 10:47:29 +0000
Subject: [PATCH 28/30] Fix idle logic to account for edge cases

---
 Source/Softswitch/src/softswitch_common.cpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/Source/Softswitch/src/softswitch_common.cpp b/Source/Softswitch/src/softswitch_common.cpp
index 6fc78e4b..42d6ad3b 100644
--- a/Source/Softswitch/src/softswitch_common.cpp
+++ b/Source/Softswitch/src/softswitch_common.cpp
@@ -233,6 +233,21 @@ void softswitch_loop(ThreadCtxt_t* ThreadContext)
                 tinselWaitUntil(TINSEL_CAN_RECV);
             }
         }
+        
+        // We don't have an idle handler, but we do have a Hardware Idle handler
+        else if(device->devType->OnHWIdle_Handler)
+        {   // We have hardware idle
+            if(tinselIdle(true))
+            {   // returned from a synchronisation point
+                softswitch_onHWIdle(ThreadContext);
+            }
+        }
+        
+        // We don't have an idle handler or a Hardware Idle handler
+        else
+        {   // No hardware idle, sleep untill we can receive
+            tinselWaitUntil(TINSEL_CAN_RECV);
+        }
     }
 }
 //------------------------------------------------------------------------------

From b9a69d1d7e84751b59bfd9ac33f330f6d0894ca0 Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Thu, 20 Jan 2022 23:19:21 +0000
Subject: [PATCH 29/30] Implement doSend

---
 Source/OrchBase/Composer.cpp                |  8 ++--
 Source/Softswitch/inc/softswitch_common.h   |  3 +-
 Source/Softswitch/src/softswitch_common.cpp | 43 +++++++++++++++------
 3 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index c74981bb..5cf5085a 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -2588,12 +2588,12 @@ void Composer::formDevTOutputPinHandlers(devTypStrings_t* dTypStrs)
         handlers_h << "uint32_t devtyp_" << devTName;
         handlers_h << "_Supervisor_Implicit_OutPin";
         handlers_h << "_Send_handler (const void* __GraphProps, ";
-        handlers_h << "void* __Device, void* pkt);\n";
+        handlers_h << "void* __Device, void* pkt, bool* doSend);\n";
 
         handlers_cpp << "uint32_t devtyp_" << devTName;
         handlers_cpp << "_Supervisor_Implicit_OutPin";
         handlers_cpp << "_Send_handler (const void* __GraphProps, ";
-        handlers_cpp << "void* __Device, void* pkt)\n";
+        handlers_cpp << "void* __Device, void* pkt, bool* doSend)\n";
 
         handlers_cpp << dTypStrs->handlerPreamble;
         handlers_cpp << dTypStrs->handlerPreambleS;
@@ -2629,12 +2629,12 @@ void Composer::formDevTOutputPinHandlers(devTypStrings_t* dTypStrs)
         handlers_h << "uint32_t devtyp_" << devTName;
         handlers_h << "_OutPin_" << pinOName;
         handlers_h << "_Send_handler (const void* __GraphProps, ";
-        handlers_h << "void* __Device, void* pkt);\n";
+        handlers_h << "void* __Device, void* pkt, bool* doSend);\n";
 
         handlers_cpp << "uint32_t devtyp_" << devTName;
         handlers_cpp << "_OutPin_" << pinOName;
         handlers_cpp << "_Send_handler (const void* __GraphProps, ";
-        handlers_cpp << "void* __Device, void* pkt)\n";
+        handlers_cpp << "void* __Device, void* pkt, bool* doSend)\n";
 
         handlers_cpp << dTypStrs->handlerPreamble;
         handlers_cpp << dTypStrs->handlerPreambleS;
diff --git a/Source/Softswitch/inc/softswitch_common.h b/Source/Softswitch/inc/softswitch_common.h
index 2aca8980..e54367e3 100644
--- a/Source/Softswitch/inc/softswitch_common.h
+++ b/Source/Softswitch/inc/softswitch_common.h
@@ -53,7 +53,8 @@ typedef uint32_t (*Recv_handler_t)
 typedef uint32_t (*Send_handler_t)
 (   const void* __GraphProps,
     void*       __Device,
-    void*       pkt
+    void*       pkt,
+    bool*       doSend
 );
 
 typedef struct PInputType
diff --git a/Source/Softswitch/src/softswitch_common.cpp b/Source/Softswitch/src/softswitch_common.cpp
index 42d6ad3b..fe808d8d 100644
--- a/Source/Softswitch/src/softswitch_common.cpp
+++ b/Source/Softswitch/src/softswitch_common.cpp
@@ -450,8 +450,26 @@ inline uint32_t softswitch_onSend(ThreadCtxt_t* ThreadContext, volatile void* se
                     pin->pinType->sz_pkt);
 #else
         // Build the packet in the send slot
-        pin->pinType->Send_Handler(ThreadContext->properties, device, pkt);
+        bool doSnd = true;  // doSend bool
+        
         ThreadContext->txHandlerCount++;         // Increment SendHandler count
+        pin->pinType->Send_Handler(ThreadContext->properties,device,pkt,&doSnd);
+        
+        if(!doSnd)
+        { // If DoSend is false, skip the actual sending and cleanup
+            pin->idxEdges = 0;       // Reset index,
+            pin->sendPending = 0;   // Reset pending
+
+            // Move the circular RTS buffer index
+            ThreadContext->rtsStart++;
+            if(ThreadContext->rtsStart == ThreadContext->rtsBuffSize)
+            {
+                ThreadContext->rtsStart = 0;
+            }
+
+            softswitch_onRTS(ThreadContext, device); // Run the Device's RTS handler.
+            return 0;
+        }
 #endif
     }
     //--------------------------------------------------------------------------
@@ -732,6 +750,17 @@ inline uint32_t softswitch_onRTS(ThreadCtxt_t* ThreadContext, devInst_t* device)
                 if(buffRem > 1)                     //(output_pin->numEdges && )
                 {
                     //output_pin->sendPending++;
+                    // Build the packet in the Packet buffer
+                    bool doSnd = true;  // doSend bool
+                    output_pin->pinType->Send_Handler(ThreadContext->properties,
+                          device, ThreadContext->pktBuf[ThreadContext->rtsEnd],
+                          &doSnd);
+                    ThreadContext->txHandlerCount++;    // ++ SendHandler count
+                    
+                    if(!doSnd)
+                    { // If DoSend is false, skip adding to the packet buffer
+                        continue;
+                    }
 #else
                 //If the pin is not already pending,
                 if(output_pin->sendPending == 0)    // output_pin->numEdges && 
@@ -743,22 +772,14 @@ inline uint32_t softswitch_onRTS(ThreadCtxt_t* ThreadContext, devInst_t* device)
                     // Add pin to RTS list and update end
                     ThreadContext->rtsBuf[ThreadContext->rtsEnd] = output_pin;
 
-#ifdef BUFFERING_SOFTSWITCH
-                    // Build the packet in the Packet buffer
-                    output_pin->pinType->Send_Handler(ThreadContext->properties,
-                          device, ThreadContext->pktBuf[ThreadContext->rtsEnd]);
-                    ThreadContext->txHandlerCount++;    // ++ SendHandler count
-#endif
-
                     ThreadContext->rtsEnd++;
                     if(ThreadContext->rtsEnd == ThreadContext->rtsBuffSize)
                     {
                         ThreadContext->rtsEnd = 0;
                     }
-#ifdef BUFFERING_SOFTSWITCH
-                }
-#else
+
                 }
+#ifndef BUFFERING_SOFTSWITCH
                 else if((output_pin->sendPending == 1) && 
                         (ThreadContext->rtsStart == ThreadContext->rtsEnd))
                 {   // Sanity check to catch buffer overflow

From 1c7e2b9b31fd4062d34463dbdddf71bfb9c3e80a Mon Sep 17 00:00:00 2001
From: Graeme Bragg <helios@m0nsa.com>
Date: Fri, 21 Jan 2022 00:36:02 +0000
Subject: [PATCH 30/30] Modify composer to output global RTSflags and indicies

---
 Source/OrchBase/Composer.cpp | 38 ++++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/Source/OrchBase/Composer.cpp b/Source/OrchBase/Composer.cpp
index 5cf5085a..0112927e 100644
--- a/Source/OrchBase/Composer.cpp
+++ b/Source/OrchBase/Composer.cpp
@@ -2115,7 +2115,7 @@ void Composer::writeGlobalPropsI(GraphI_t* graphI, std::ofstream& props_cpp)
 
 
 /******************************************************************************
- * Write the Message structs to a common header
+ * Write the Message structs and "global" RTS consts to a common header
  *****************************************************************************/
 void Composer::writeMessageTypes(GraphI_t* graphI, std::ofstream& pkt_h)
 {
@@ -2124,8 +2124,42 @@ void Composer::writeMessageTypes(GraphI_t* graphI, std::ofstream& pkt_h)
     pkt_h << "#ifndef _MESSAGETYPES_H_\n";
     pkt_h << "#define _MESSAGETYPES_H_\n\n";
 
-    pkt_h << "#include <cstdint>\n";
+    pkt_h << "#include <cstdint>\n\n";
+    pkt_h << "// Global RTS flags and indicies for all device types\n";
     
+    // Write the global RTS Flags/indicies
+    WALKVECTOR(DevT_t*,graphI->pT->DevT_v,devT)
+    {
+        if((*devT)->devTyp == 'D')
+        {
+            std::string devTName = (*devT)->Name();  // grab copy of the name
+            if((*devT)->pPinTSO)
+            {
+                pkt_h << "const uint32_t RTS_SUPER_IMPLICIT_SEND_INDEX_";
+                pkt_h << devTName << " = " << (*devT)->PinTO_v.size();
+                pkt_h << ";\n";
+                
+                pkt_h << "const uint32_t RTS_SUPER_IMPLICIT_SEND_FLAG_";
+                pkt_h << devTName << " = 0x1 << " << (*devT)->PinTO_v.size();
+                pkt_h << ";\n";
+            }
+            // Other output pins
+            WALKVECTOR(PinT_t*,(*devT)->PinTO_v,pinO)
+            {
+                std::string pinOName = (*pinO)->Name();
+                uint32_t pinIdx = (*pinO)->Idx;
+            
+                pkt_h << "const uint32_t RTS_INDEX_" << devTName << "_";
+                pkt_h << pinOName << " = " << pinIdx << ";\n";
+                
+                pkt_h << "const uint32_t RTS_FLAG_" << devTName << "_";
+                pkt_h << pinOName << " = 0x1 << " << pinIdx << ";\n";
+            }
+        }
+    }
+    
+    // Now write all of the message structs
+    pkt_h <<  "\n// Message type structs for all messages\n";
     pkt_h <<  "#pragma pack(push,1)\n";
     
     WALKVECTOR(MsgT_t*,graphT->MsgT_v,msg)