Interleave loop pre-header with SWP prologue

llvm/lib/CodeGen/MachineScheduler.cpp

@@ -807,6 +807,8 @@ void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb,
                                      unsigned regioninstrs)
 {
   ScheduleDAGInstrs::enterRegion(bb, begin, end, regioninstrs);
+  CurrentTop = MachineBasicBlock::iterator();
+  CurrentBottom = MachineBasicBlock::iterator();
 
   SchedImpl->initPolicy(begin, end, regioninstrs);
 

llvm/lib/CodeGen/ScheduleDAGInstrs.cpp

@@ -197,6 +197,10 @@ void ScheduleDAGInstrs::enterRegion(MachineBasicBlock *bb,
   RegionBegin = begin;
   RegionEnd = end;
   NumRegionInstrs = regioninstrs;
+
+  // Ensure all instructions can be given a SUnit without re-allocation.
+  clearDAG();
+  SUnits.reserve(NumRegionInstrs);
 }
 
 void ScheduleDAGInstrs::exitRegion() {
@@ -633,8 +637,6 @@ std::optional<unsigned> ScheduleDAGInstrs::initSUnit(MachineInstr &MI) {
 
 void ScheduleDAGInstrs::initSUnits() {
   ScheduleDAG::clearDAG();
-  // Prevent reallocations for performance.
-  SUnits.reserve(NumRegionInstrs);
   // This loop creates SUnits for real instructions.
   for (MachineInstr &MI : make_range(RegionBegin, RegionEnd)) {
     initSUnit(MI);

llvm/lib/Target/AIE/AIE2InstrInfo.td

@@ -370,7 +370,7 @@ include "AIE2GenFixupInstrInfo.td"
 include "AIE2MultiSlotPseudoInstrInfo.td"
 
 //Intrinsics
-let hasSideEffects = 1, mayLoad = 0, mayStore = 0 in {
+let Itinerary = II_EVENT, hasSideEffects = 1, mayLoad = 0, mayStore = 0 in {
   def EVENT : AIE2_event_inst_alu<(outs), (ins t02u:$val), "event", "$val">;
 }
 def : Pat<(int_aie2_event t02u:$val), (EVENT t02u:$val)>;
@@ -500,7 +500,7 @@ let Uses = [SP] in
 def PseudoFI : Pseudo<(outs eP:$dst), (ins i32imm:$imm)>;
 
 // NOPs for each VLIW Slot
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isSlotNOP = true in {
+let Itinerary = II_NOP, hasSideEffects = 0, mayLoad = 0, mayStore = 0, isSlotNOP = true in {
   def NOP : AIE2_nop_nop_inst_nop<(outs), (ins), "nop">;
   def NOPA : AIE2_nop_lda_inst_lda<(outs), (ins), "nopa">;
   def NOPB : AIE2_nop_ldb_inst_ldb<(outs), (ins), "nopb">;

llvm/lib/Target/AIE/AIE2Schedule.td

@@ -236,6 +236,7 @@ def II_MOV_SS : InstrItinClass;
 def II_MUL : InstrItinClass;
 def II_NE : InstrItinClass;
 def II_NEZ : InstrItinClass;
+def II_NOP : InstrItinClass;
 def II_OR : InstrItinClass;
 def II_PADD : InstrItinClass;
 def II_PADD_2D : InstrItinClass;
@@ -634,6 +635,7 @@ InstrItinData<II_MOVd6, [SimpleCycle<P_RM_PORT>, EmptyCycles<5>, SimpleCycle<RS_
 InstrItinData<II_MUL, [EmptyCycles<1>, SimpleCycle<R_WX_PORT>], [2,1,1]>,
 InstrItinData<II_NE, [InstrStage<1,  [R_WX_PORT]>], [1,1,1,1]>,
 InstrItinData<II_NEZ, [InstrStage<1,  [R_WX_PORT]>], [1,1,1]>,
+InstrItinData<II_NOP, [], []>,
 InstrItinData<II_OR, [InstrStage<1,  [R_WX_PORT]>], [1,1,1,1]>,
 InstrItinData<II_PADD, [], [1,1,1]>,
 InstrItinData<II_PADD_2D, [], [1,1,1,1]>,

llvm/lib/Target/AIE/AIEBasePipelinerLoopInfo.cpp

@@ -28,11 +28,6 @@
 
 #define DEBUG_TYPE "aie-pipeliner"
 namespace llvm {
-cl::opt<int> LoopMinTripCount(
-    "aie-loop-min-tripcount",
-    cl::desc("Minimum number of loop iterations (warning: applies to all loop"
-             " pipelining candidates)"),
-    cl::init(-1), cl::Hidden);
 cl::opt<bool> TrackRegPressure(
     "aie-pipeliner-track-regpressure",
     cl::desc("Refuse SWP schedules likely to run into register spills"),
@@ -69,13 +64,7 @@ AIEBasePipelinerLoopInfo::AIEBasePipelinerLoopInfo(MachineInstr *EndLoop,
       AIELoopUtils::getMinTripCount(*LoopBlock);
   if (ParsedMinTripCount) {
     MinTripCount = *ParsedMinTripCount;
-    LLVM_DEBUG(dbgs() << "PLI: MinTripCount from pragma =  " << MinTripCount
-                      << "\n");
-  }
-
-  if (LoopMinTripCount > MinTripCount) {
-    MinTripCount = LoopMinTripCount;
-    LLVM_DEBUG(dbgs() << "PLI: MinTripCount from CL option =  " << MinTripCount
+    LLVM_DEBUG(dbgs() << "PLI: MinTripCount from pragma/CL =  " << MinTripCount
                       << "\n");
   }
 }
@@ -682,18 +671,11 @@ class ZeroOverheadLoop : public AIEBasePipelinerLoopInfo {
 };
 
 ZeroOverheadLoop::Assessment ZeroOverheadLoop::accept(MachineInstr *EndLoop) {
-  // We are using LoopMinTripCount below just for testing purposes.
-  // For MIR test cases without IR, we can't encode loop-related metadata.
-  if (!MinTripCount && LoopMinTripCount <= 0) {
+  if (!MinTripCount) {
     LLVM_DEBUG(dbgs() << "Unbounded loop detected!\n");
     return Assessment::UnboundedLoop;
   }
 
-  // Overwrite MinTripCount.
-  if (LoopMinTripCount > 0) {
-    MinTripCount = LoopMinTripCount;
-  }
-
   if (MinTripCount <= 1) {
     LLVM_DEBUG(dbgs() << "Not interesting MinTripCount (<=1)!\n");
     return Assessment::TooLowMinTripCount;

llvm/lib/Target/AIE/AIEBaseSubtarget.cpp

@@ -275,27 +275,14 @@ class RegionEndEdges : public ScheduleDAGMutation {
         assert(EdgeLatency < DelaySlots);
         EdgeLatency = DelaySlots + 1;
       }
+
       // Between writing Registers (lc, le, ls) and the end of the loop,
       // there must be a distance of 112 bytes in terms of PM addresses.
       // 112 bytes correspond to 7 fully-expanded 128-bit instructions and
       // hence adding a latency of 8 from LoopStart to the ExitSU.
-      // We can subtract the number of bundles that interblock pushed into
-      // BottomInsert
-      // FIXME: this holds as long as we insert them unconditionally. If we
-      // integrate them with the bottom region, we just need to keep 8 away
-      // from ExitSU
       if (TII->isZeroOverheadLoopSetupInstr(MI)) {
-        unsigned PatchCycles = 8;
-        if (DAG->getBB()) {
-          auto *Scheduler =
-              static_cast<AIEScheduleDAGMI *>(DAG)->getSchedImpl();
-          auto &InterBlock = Scheduler->getInterBlock();
-          unsigned InsertedCycles =
-              InterBlock.getBlockState(DAG->getBB()).BottomInsert.size();
-          PatchCycles =
-              PatchCycles >= InsertedCycles ? PatchCycles - InsertedCycles : 0;
-        }
-        EdgeLatency = std::max(EdgeLatency, PatchCycles);
+        const unsigned ZOLDistance = 8;
+        EdgeLatency = std::max(EdgeLatency, ZOLDistance);
       }
 
       ExitDep.setLatency(EdgeLatency);
@@ -318,6 +305,56 @@ class RegionEndEdges : public ScheduleDAGMutation {
   };
 };
 
+/// This Mutator is responsible for emitting "fixed" SUnits at the top or bottom
+/// of the region. These special SUnits require a specific cycle and cannot be
+/// placed freely by the scheduler.
+///
+/// Here, these special SUnits get created from Region::top_fixed_instrs() or
+/// Region::bot_fixed_instrs(), and dependencies are created between "free" and
+/// "fixed" SUnits.
+class EmitFixedSUnits : public ScheduleDAGMutation {
+public:
+  void apply(ScheduleDAGInstrs *DAG) override {
+    AIEPostRASchedStrategy *Scheduler =
+        static_cast<AIEScheduleDAGMI *>(DAG)->getSchedImpl();
+    auto *TII = static_cast<const AIEBaseInstrInfo *>(DAG->TII);
+    auto *ItinData = DAG->MF.getSubtarget().getInstrItineraryData();
+    const BlockState &BS =
+        Scheduler->getInterBlock().getBlockState(DAG->getBB());
+    const Region &CurRegion = BS.getCurrentRegion();
+
+    // First, create SUnits for all "fixed" instructions
+    unsigned DistToExitSU = 0;
+    for (MachineInstr &MI : reverse(CurRegion.bot_fixed_instrs())) {
+      Scheduler->addFixedSUnit(MI, /*IsTop=*/false, DistToExitSU);
+      ++DistToExitSU;
+    }
+    DAG->makeMaps();
+
+    // Then, create dependencies between "free" and "fixed" instructions
+    auto IsFreeSU = [Scheduler](const SUnit &SU) {
+      return Scheduler->isFreeSU(SU);
+    };
+    ArrayRef<AIE::MachineBundle> BotFixedBundles =
+        CurRegion.getBotFixedBundles();
+    for (SUnit &FreeSU : make_filter_range(DAG->SUnits, IsFreeSU)) {
+      const MachineInstr &MI = *FreeSU.getInstr();
+      MachineInstr *FixedDepMI =
+          AIE::findEarliestRef(MI, BotFixedBundles, BotFixedBundles.size()).MI;
+      if (!FixedDepMI)
+        continue;
+
+      SUnit *FixedDepSU =
+          DAG->getSUnit(&*getBundleStart(FixedDepMI->getIterator()));
+      assert(FixedDepSU && "Fixed Bundle has no corresponding SU.");
+      SDep Dep(&FreeSU, SDep::Artificial);
+      Dep.setLatency(
+          AIE::maxLatency(&MI, *TII, *ItinData, /*IncludeStages=*/true));
+      FixedDepSU->addPred(Dep, /*Required=*/true);
+    }
+  }
+};
+
 /// Collect all "weak" edges in a separate vector. This allows modifying
 /// \p SU.Preds without invalidating iterators.
 SmallVector<SDep, 4> getWeakPreds(SUnit &SU) {
@@ -543,11 +580,10 @@ class WAWEdges : public ScheduleDAGMutation {
     LiveRegs.init(*TRI);
     bool AddReservedRegs = true;
     if (Scheduler) {
+      assert(!Scheduler->doMBBSchedRegionsTopDown());
       MachineBasicBlock *MBB = DAG->getBB();
       const BlockState &BS = Scheduler->getInterBlock().getBlockState(MBB);
-      auto Region = BS.getCurrentRegion();
-      auto BottomRegion = BS.getBottom();
-      if (*Region.begin() == *BottomRegion.begin()) {
+      if (&BS.getCurrentRegion() == &BS.getBottom()) {
         // If the region is bottom region, liveouts of region are same as
         // liveouts of the MBB
         for (const MCPhysReg Reg : BS.LiveOuts) {
@@ -665,6 +701,7 @@ AIEBaseSubtarget::getPostRAMutationsImpl(const Triple &TT) {
     Mutations.emplace_back(std::make_unique<MemoryEdges>());
     Mutations.emplace_back(std::make_unique<MachineSchedWAWEdges>());
     Mutations.emplace_back(std::make_unique<BiasDepth>());
+    Mutations.emplace_back(std::make_unique<EmitFixedSUnits>());
   }
   return Mutations;
 }

llvm/lib/Target/AIE/AIEBundle.h

@@ -27,8 +27,6 @@
 
 #include "AIEBaseSubtarget.h"
 #include "MCTargetDesc/AIEMCFormats.h"
-#include "llvm-c/DebugInfo.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 
 #include <unordered_map>
@@ -78,6 +76,10 @@ template <class I> class Bundle {
       return true;
     }
 
+    if (InstOpCode == TargetOpcode::BUNDLE) {
+      return !BundleRoot;
+    }
+
     // if we have a standalone bundle, we can't add anything.
     if (isStandalone())
       return false;
@@ -103,6 +105,16 @@ template <class I> class Bundle {
       MetaInstrs.push_back(Instr);
       return;
     }
+
+    // Keep track of BUNDLE instructions. They need to be cleaned up when
+    // de-bundling before re-bundling. See applyBundles()
+    if (Instr->getOpcode() == TargetOpcode::BUNDLE) {
+      assert(!BundleRoot &&
+             "AIE::Bundle already has a root BUNDLE instruction");
+      BundleRoot = Instr;
+      return;
+    }
+
     // Check if the pre-condition is ensured
     assert((!ComputeSlots || !isStandalone()) &&
            "Tried to add an instruction in a standalone Bundle");
@@ -142,6 +154,7 @@ template <class I> class Bundle {
     Instrs.clear();
     MetaInstrs.clear();
     SlotMap.clear();
+    BundleRoot = nullptr;
   }
 
   /// Check if empty
@@ -192,6 +205,15 @@ template <class I> class Bundle {
     return false;
   }
 
+  /// Erase the BUNDLE root instruction from its parent MBB.
+  /// This does not remove the instructions within the BUNDLE, only the root.
+  void eraseRootFromBlock() {
+    if (BundleRoot) {
+      BundleRoot->eraseFromBundle();
+      BundleRoot = nullptr;
+    }
+  }
+
   bool isNOPBundle() const {
     const VLIWFormat *Format = getFormatOrNull();
     assert(Format);
@@ -228,6 +250,10 @@ template <class I> class Bundle {
 
   // Contained meta instructions (These will end up after the bundle)
   std::vector<I *> MetaInstrs;
+
+private:
+  /// A root BUNDLE instruction if it exists.
+  I *BundleRoot = nullptr;
 };
 
 template <class I> bool operator==(const Bundle<I> &B1, const Bundle<I> &B2) {