[AIEX] Use delayed MOV when possible

llvm/lib/Target/AIE/AIE2InstrInfo.cpp

@@ -983,6 +983,15 @@ bool AIE2InstrInfo::isSchedBarrier(const MachineInstr &MI) const {
           MI.getOpcode() == AIE2::MOV_CNTR || isDelayedSchedBarrier(MI));
 }
 
+bool AIE2InstrInfo::isScalarMove(unsigned OpCode) const {
+  switch (OpCode) {
+  case AIE2::MOV_mv_scl:
+    return true;
+  default:
+    return false;
+  }
+}
+
 unsigned AIE2InstrInfo::getNumReservedDelaySlots(const MachineInstr &MI) const {
   return 0;
 }
@@ -1419,6 +1428,11 @@ AIE2InstrInfo::getVExtractOpInfo(const MachineInstr &MI) const {
 
 unsigned AIE2InstrInfo::getMaxLoadStoreSize() const { return 256; }
 
+std::vector<unsigned> AIE2InstrInfo::getDelayedScalarMoveOpcode() const {
+  return {AIE2::MOV_D1, AIE2::MOV_D2, AIE2::MOV_D3,
+          AIE2::MOV_D4, AIE2::MOV_D5, AIE2::MOV_D6};
+}
+
 bool AIE2InstrInfo::canCombineWithLoadStore(const MachineInstr &MI) const {
 
   if (!isa<GIntrinsic>(MI))

llvm/lib/Target/AIE/AIE2InstrInfo.h

@@ -49,9 +49,12 @@ class AIE2InstrInfo : public AIE2GenInstrInfo {
   unsigned getGenericPadVectorOpcode() const override;
   unsigned getGenericUnpadVectorOpcode() const override;
   unsigned getCycleSeparatorOpcode() const override;
+  std::vector<unsigned> getDelayedScalarMoveOpcode() const override;
+
   bool isLock(unsigned Opc) const override;
   bool isDelayedSchedBarrier(const MachineInstr &MI) const override;
   bool isSchedBarrier(const MachineInstr &MI) const override;
+  bool isScalarMove(unsigned OpCode) const override;
 
   virtual unsigned
   getNumReservedDelaySlots(const MachineInstr &MI) const override;

llvm/lib/Target/AIE/AIE2RegisterInfo.cpp

@@ -141,6 +141,10 @@ bool AIE2RegisterInfo::isReservedStickyReg(MCRegister PhysReg) const {
   }
 }
 
+bool AIE2RegisterInfo::isRTypeReg(Register Reg) const {
+  return Reg.isPhysical() && AIE2::eRRegClass.contains(Reg);
+}
+
 const uint32_t *AIE2RegisterInfo::getNoPreservedMask() const {
   return CSR_NoRegs_RegMask;
 }

llvm/lib/Target/AIE/AIE2RegisterInfo.h

@@ -82,6 +82,7 @@ struct AIE2RegisterInfo : public AIE2GenRegisterInfo {
   getCoveringSubRegs(const TargetRegisterClass &RC) const override;
   bool isSimplifiableReservedReg(MCRegister PhysReg) const override;
   bool isReservedStickyReg(MCRegister PhysReg) const override;
+  bool isRTypeReg(Register Reg) const override;
 
   const TargetRegisterClass *get2DIteratorRegClass() const override {
     return &AIE2::eDRegClass;

llvm/lib/Target/AIE/AIEAlternateDescriptors.h

@@ -22,6 +22,9 @@
 
 namespace llvm {
 
+using MutateInstructionMap =
+    std::unordered_map<MachineInstr *,
+                       std::pair<MachineInstr *, const MCInstrDesc *>>;
 using MIAltDescsMap = std::unordered_map<MachineInstr *, const MCInstrDesc *>;
 
 class AIEAlternateDescriptors {
@@ -40,7 +43,11 @@ class AIEAlternateDescriptors {
     const AIEBaseSubtarget &STI = AIEBaseSubtarget::get(*MI->getMF());
     const AIEBaseInstrInfo *TII = STI.getInstrInfo();
 
-    AlternateDescs[MI] = &TII->get(AltInstOpcode);
+    setAlternateDescriptor(MI, &TII->get(AltInstOpcode));
+  }
+
+  void setAlternateDescriptor(MachineInstr *MI, const MCInstrDesc *AltDesc) {
+    AlternateDescs[MI] = AltDesc;
   }
 
   // Return the alternate descriptor for the given multi-opcode instruction.

llvm/lib/Target/AIE/AIEBaseInstrInfo.h

@@ -126,6 +126,9 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
   /// Check whether this is a scheduling barrier
   virtual bool isSchedBarrier(const MachineInstr &) const { return false; }
 
+  /// Check whether OpCode is a scalar move instruction
+  virtual bool isScalarMove(unsigned OpCode) const { return false; }
+
   /// Returns the number of delay slots that this instruction requires.
   /// This might be 0
   virtual unsigned
@@ -137,6 +140,12 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
   /// not filled in by the scheduler.
   virtual unsigned getNumReservedDelaySlots(const MachineInstr &MI) const;
 
+  /// Return Opcode for delayed scalar move insturction in increasing order of
+  /// delay
+  virtual std::vector<unsigned> getDelayedScalarMoveOpcode() const {
+    return std::vector<unsigned>();
+  }
+
   /// Check whether Opc represents a JNZ instruction. This is mainly for
   /// detecting a downcounting loop branch.
   virtual bool isJNZ(unsigned Opc) const { return false; }

llvm/lib/Target/AIE/AIEBaseRegisterInfo.h

@@ -58,6 +58,10 @@ struct AIEBaseRegisterInfo : public TargetRegisterInfo {
   virtual bool isReservedStickyReg(MCRegister PhysReg) const {
     llvm_unreachable("Target didn't implement isReservedStickyReg!");
   }
+
+  /// Check if Reg is part of the R-type register file
+  virtual bool isRTypeReg(Register Reg) const { return false; }
+
 #if 0
   /// Returns a BitVector of the intersection of GPR RegClass
   /// and CalleeSaved Registers

llvm/lib/Target/AIE/AIEHazardRecognizer.cpp

@@ -108,6 +108,15 @@ FuncUnitWrapper &FuncUnitWrapper::operator|=(const FuncUnitWrapper &Other) {
   return *this;
 }
 
+FuncUnitWrapper &FuncUnitWrapper::operator^=(const FuncUnitWrapper &Other) {
+  // XOR operation with the same FuncUnitWrapper will release resources.
+  Required ^= Other.Required;
+  Reserved ^= Other.Reserved;
+  Slots ^= Other.Slots;
+  MemoryBanks ^= Other.MemoryBanks;
+  return *this;
+}
+
 bool FuncUnitWrapper::conflict(const FuncUnitWrapper &Other) const {
   if ((Required & Other.Required) != 0 || (Slots & Other.Slots) != 0 ||
       (MemoryBanks & Other.MemoryBanks) != 0 ||
@@ -447,6 +456,14 @@ auto toHazardType(bool Conflict) {
 }
 } // namespace
 
+ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
+    const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
+    iterator_range<const MachineOperand *> MIOperands,
+    const MachineRegisterInfo &MRI, int DeltaCycles) {
+  return getHazardType(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
+                       DeltaCycles);
+}
+
 // These functions interpret the itinerary, translating InstrStages
 // to ResourceCycles to apply.
 // We deviate from the standard ScoreboardHazardRecognizer by not
@@ -470,10 +487,28 @@ ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
       FUDepthLimit));
 }
 
-bool AIEHazardRecognizer::checkConflict(
+ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
+                                                    int DeltaCycles) {
+  assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
+  return checkConflict(Scoreboard, MI, DeltaCycles);
+}
+
+ConflictTypeBits AIEHazardRecognizer::checkConflict(
     const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
     int DeltaCycles) const {
-  const MCInstrDesc &Desc = MI.getDesc();
+  assert(!TII->getFormatInterface()->getAlternateInstsOpcode(MI.getOpcode()));
+  return checkConflict(Scoreboard, MI, MI.getDesc(), DeltaCycles);
+}
+
+ConflictTypeBits AIEHazardRecognizer::checkConflict(MachineInstr &MI,
+                                                    const MCInstrDesc &Desc,
+                                                    int DeltaCycles) {
+  return checkConflict(Scoreboard, MI, Desc, DeltaCycles);
+}
+
+ConflictTypeBits AIEHazardRecognizer::checkConflict(
+    const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
+    const MCInstrDesc &Desc, int DeltaCycles) const {
   const unsigned SchedClass =
       TII->getSchedClass(Desc, MI.operands(), MI.getMF()->getRegInfo());
   const MemoryBankBits MemoryBanks = getMemoryBanks(&MI);
@@ -483,18 +518,41 @@ bool AIEHazardRecognizer::checkConflict(
       MemoryBanks, TII->getMemoryCycles(SchedClass), DeltaCycles, std::nullopt);
 }
 
-bool AIEHazardRecognizer::checkConflict(
+ConflictTypeBits AIEHazardRecognizer::checkConflict(
     const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
     const InstrItineraryData *ItinData, unsigned SchedClass, SlotBits SlotSet,
     MemoryBankBits MemoryBanks, SmallVector<int, 2> MemoryAccessCycles,
     int DeltaCycles, std::optional<int> FUDepthLimit) {
   assert(Scoreboard.isValidDelta(DeltaCycles));
+  ConflictTypeBits Conflict = static_cast<uint32_t>(ConflictType::NoConflict);
+
+  Conflict |= checkFormatConflict(Scoreboard, DeltaCycles, SlotSet);
+
+  Conflict |= checkMemoryBankConflict(MemoryAccessCycles, Scoreboard,
+                                      DeltaCycles, MemoryBanks);
+
+  Conflict |= checkFUConflict(ItinData, SchedClass, DeltaCycles, Scoreboard,
+                              FUDepthLimit);
 
+  return Conflict;
+}
+
+// Return true if there is a conflict due to format.
+ConflictTypeBits AIEHazardRecognizer::checkFormatConflict(
+    const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, int DeltaCycles,
+    unsigned SlotSet) {
   // Verify format hazards
   FuncUnitWrapper EmissionCycle(/*Req=*/0, /*Res=*/0, SlotSet);
-  if (EmissionCycle.conflict(Scoreboard[DeltaCycles]))
-    return true;
+  return static_cast<uint32_t>(EmissionCycle.conflict(Scoreboard[DeltaCycles])
+                                   ? ConflictType::Format
+                                   : ConflictType::NoConflict);
+}
 
+// Return true if there is a conflict due to memory banks.
+ConflictTypeBits AIEHazardRecognizer::checkMemoryBankConflict(
+    const SmallVector<int, 2> &MemoryAccessCycles,
+    const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, int DeltaCycles,
+    unsigned MemoryBanks) {
   // Verify memory bank hazards
   if (!MemoryAccessCycles.empty()) {
     FuncUnitWrapper MemoryBankAccessCycle(/*Req=*/0, /*Res=*/0, /*SlotSet=*/0,
@@ -506,39 +564,41 @@ bool AIEHazardRecognizer::checkConflict(
         LLVM_DEBUG(dbgs() << "*** Memory bank conflict in cycle=" << AccessCycle
                           << ":\n";
                    MemoryBankAccessCycle.dump(); dbgs() << "\n");
-        return true;
+        return static_cast<uint32_t>(ConflictType::MemoryBank);
       }
     }
   }
+  return static_cast<uint32_t>(ConflictType::NoConflict);
+}
+
+// Return true if there is a conflict in the functional units.
+ConflictTypeBits AIEHazardRecognizer::checkFUConflict(
+    const InstrItineraryData *ItinData, unsigned SchedClass, int DeltaCycles,
+    const ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
+    const std::optional<int> &FUDepthLimit) {
 
   // Note that Delta will be negative for bottom-up scheduling.
   // Cycle is 'our' cycle at which each stage of the itinerary starts.
   // It gets updated by the increment from the InstrStage.
   int Cycle = DeltaCycles;
   for (const InstrStage &IS : ItinData->getStages(SchedClass)) {
-    if (FUDepthLimit && (Cycle - DeltaCycles) >= *FUDepthLimit) {
+    if (FUDepthLimit && (Cycle - DeltaCycles) >= *FUDepthLimit)
       break;
-    }
     // Check availability of this stage's resources for the specified number
     // of cycles
     const FuncUnitWrapper ThisCycle(IS);
     for (unsigned int C = 0; C < IS.getCycles(); ++C) {
       int StageCycle = Cycle + (int)C;
       assert(StageCycle < Scoreboard.getDepth());
 
-      if (ThisCycle.conflict(Scoreboard[StageCycle])) {
-        LLVM_DEBUG(dbgs() << "*** Hazard in cycle=" << StageCycle
-                          << " EC=" << StageCycle - DeltaCycles << ":\n";
-                   ThisCycle.dump(); dbgs() << "\n");
-        return true;
-      }
+      if (ThisCycle.conflict(Scoreboard[StageCycle]))
+        return static_cast<uint32_t>(ConflictType::FuncUnit);
     }
 
     // Advance the cycle to the next stage.
     Cycle += IS.getNextCycles();
   }
-
-  return false;
+  return static_cast<uint32_t>(ConflictType::NoConflict);
 }
 
 void AIEHazardRecognizer::emitInScoreboard(
@@ -566,6 +626,26 @@ void AIEHazardRecognizer::emitInScoreboard(
                  TII->getMemoryCycles(SchedClass), DeltaCycles, FUDepthLimit);
 }
 
+void AIEHazardRecognizer::releaseFromScoreboard(
+    const MCInstrDesc &Desc, MemoryBankBits MemoryBanks,
+    iterator_range<const MachineOperand *> MIOperands,
+    const MachineRegisterInfo &MRI, int DeltaCycles) {
+  releaseFromScoreboard(Scoreboard, Desc, MemoryBanks, MIOperands, MRI,
+                        DeltaCycles);
+}
+
+void AIEHazardRecognizer::releaseFromScoreboard(
+    ResourceScoreboard<FuncUnitWrapper> &TheScoreboard, const MCInstrDesc &Desc,
+    MemoryBankBits MemoryBanks,
+    iterator_range<const MachineOperand *> MIOperands,
+    const MachineRegisterInfo &MRI, int DeltaCycles) const {
+  const unsigned SchedClass = TII->getSchedClass(Desc, MIOperands, MRI);
+  const SlotBits SlotSet =
+      getSlotSet(Desc, *TII->getFormatInterface(), IgnoreUnknownSlotSets);
+  releaseResources(TheScoreboard, ItinData, SchedClass, SlotSet, MemoryBanks,
+                   TII->getMemoryCycles(SchedClass), DeltaCycles, FUDepthLimit);
+}
+
 void AIEHazardRecognizer::enterResources(
     ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
     const InstrItineraryData *ItinData, unsigned SchedClass, SlotBits SlotSet,
@@ -607,6 +687,47 @@ void AIEHazardRecognizer::enterResources(
   });
 }
 
+void AIEHazardRecognizer::releaseResources(
+    ResourceScoreboard<FuncUnitWrapper> &Scoreboard,
+    const InstrItineraryData *ItinData, unsigned SchedClass, SlotBits SlotSet,
+    MemoryBankBits MemoryBanks, SmallVector<int, 2> MemoryAccessCycles,
+    int DeltaCycles, std::optional<int> FUDepthLimit) {
+  assert(Scoreboard.isValidDelta(DeltaCycles));
+
+  // Remove slot usage
+  FuncUnitWrapper EmissionCycle(/*Req=*/0, /*Res=*/0, SlotSet);
+  Scoreboard[DeltaCycles] ^= EmissionCycle;
+
+  // Remove memory bank usage
+  if (!MemoryAccessCycles.empty()) {
+    FuncUnitWrapper MemoryBankAccessCycle(/*Req=*/0, /*Res=*/0, /*SlotSet=*/0,
+                                          MemoryBanks);
+    for (int Cycles : MemoryAccessCycles) {
+      Scoreboard[DeltaCycles + Cycles - 1] ^= MemoryBankAccessCycle;
+    }
+  }
+
+  int Cycle = DeltaCycles;
+  Scoreboard[Cycle].IssueCount--;
+  for (const InstrStage &IS : ItinData->getStages(SchedClass)) {
+    if (FUDepthLimit && (Cycle - DeltaCycles) >= *FUDepthLimit) {
+      break;
+    }
+    const FuncUnitWrapper ResourceToRelease(IS);
+    for (unsigned int C = 0; C < IS.getCycles(); ++C) {
+      Scoreboard[Cycle + C] ^= ResourceToRelease;
+    }
+
+    // Advance the cycle to the next stage.
+    Cycle += IS.getNextCycles();
+  }
+
+  LLVM_DEBUG({
+    dbgs() << "Scoreboard after release resources:\n";
+    Scoreboard.dump();
+  });
+}
+
 unsigned AIEHazardRecognizer::getPipelineDepth() const { return PipelineDepth; }
 
 unsigned AIEHazardRecognizer::getMaxLatency() const { return MaxLatency; }