[SLP][NFC] Redesign schedule bundle, separate from schedule data, NFC #131625
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Created using spr 1.3.5
|
@llvm/pr-subscribers-llvm-transforms Author: Alexey Bataev (alexey-bataev) ChangesThat's the initial patch, intended to support revectorization of the Patch is 73.90 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/131625.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d450336cbc3ce..6766b68841a9a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1368,7 +1368,9 @@ namespace slpvectorizer {
/// Bottom Up SLP Vectorizer.
class BoUpSLP {
struct TreeEntry;
- struct ScheduleData;
+ class ScheduleEntity;
+ class ScheduleData;
+ class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
@@ -1433,7 +1435,8 @@ class BoUpSLP {
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
- InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {});
+ InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {},
+ InstructionCost ReductionCost = TTI::TCC_Free);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
@@ -3670,8 +3673,7 @@ class BoUpSLP {
#endif
/// Create a new VectorizableTree entry.
- TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
- std::optional<ScheduleData *> Bundle,
+ TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
@@ -3688,8 +3690,7 @@ class BoUpSLP {
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
TreeEntry::EntryState EntryState,
- std::optional<ScheduleData *> Bundle,
- const InstructionsState &S,
+ ScheduleBundle &Bundle, const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {}) {
@@ -3781,22 +3782,23 @@ class BoUpSLP {
}
}
// Update the scheduler bundle to point to this TreeEntry.
- ScheduleData *BundleMember = *Bundle;
- assert((BundleMember || isa<PHINode>(S.getMainOp()) ||
+ assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
doesNotNeedToSchedule(VL)) &&
"Bundle and VL out of sync");
- if (BundleMember) {
+ if (!Bundle.getBundle().empty()) {
+ auto *BundleMember = Bundle.getBundle().begin();
for (Value *V : VL) {
if (doesNotNeedToBeScheduled(V))
continue;
- if (!BundleMember)
+ if (BundleMember == Bundle.getBundle().end())
continue;
- BundleMember->TE = Last;
- BundleMember = BundleMember->NextInBundle;
+ ++BundleMember;
}
+ assert(BundleMember == Bundle.getBundle().end() &&
+ "Bundle and VL out of sync");
+ Bundle.setTreeEntry(Last);
}
- assert(!BundleMember && "Bundle and VL out of sync");
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
@@ -3946,16 +3948,17 @@ class BoUpSLP {
/// is invariant in the calling loop.
bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
Instruction *Inst2) {
- if (!Loc1.Ptr || !isSimple(Inst1) || !isSimple(Inst2))
+ assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instrucgion.");
+ if (!isSimple(Inst2))
return true;
// First check if the result is already in the cache.
AliasCacheKey Key = std::make_pair(Inst1, Inst2);
- auto It = AliasCache.find(Key);
- if (It != AliasCache.end())
- return It->second;
+ auto Res = AliasCache.try_emplace(Key);
+ if (!Res.second)
+ return Res.first->second;
bool Aliased = isModOrRefSet(BatchAA.getModRefInfo(Inst2, Loc1));
// Store the result in the cache.
- AliasCache.try_emplace(Key, Aliased);
+ Res.first->getSecond() = Aliased;
AliasCache.try_emplace(std::make_pair(Inst2, Inst1), Aliased);
return Aliased;
}
@@ -3964,7 +3967,7 @@ class BoUpSLP {
/// Cache for alias results.
/// TODO: consider moving this to the AliasAnalysis itself.
- DenseMap<AliasCacheKey, bool> AliasCache;
+ SmallDenseMap<AliasCacheKey, bool> AliasCache;
// Cache for pointerMayBeCaptured calls inside AA. This is preserved
// globally through SLP because we don't perform any action which
@@ -4011,26 +4014,55 @@ class BoUpSLP {
/// List of hashes of vector of loads, which are known to be non vectorizable.
DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
+ class ScheduleEntity {
+ friend class ScheduleBundle;
+ friend class ScheduleData;
+
+ protected:
+ enum class Kind { ScheduleData, ScheduleBundle };
+ Kind getKind() const { return K; }
+ ScheduleEntity(Kind K) : K(K) {}
+
+ private:
+ /// Used for getting a "good" final ordering of instructions.
+ int SchedulingPriority = 0;
+ /// The kind of the ScheduleEntity.
+ Kind K = Kind::ScheduleData;
+
+ public:
+ ScheduleEntity() = delete;
+ /// Gets/sets the scheduling priority.
+ void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
+ int getSchedulingPriority() const { return SchedulingPriority; }
+ bool isReady() const {
+ if (auto *SD = dyn_cast<ScheduleData>(this))
+ return SD->isReady();
+ return cast<ScheduleBundle>(this)->isReady();
+ }
+ static bool classof(const ScheduleEntity *) { return true; }
+ };
+
/// Contains all scheduling relevant data for an instruction.
/// A ScheduleData either represents a single instruction or a member of an
/// instruction bundle (= a group of instructions which is combined into a
/// vector instruction).
- struct ScheduleData {
+ class ScheduleData final : public ScheduleEntity {
+ public:
// The initial value for the dependency counters. It means that the
// dependencies are not calculated yet.
enum { InvalidDeps = -1 };
- ScheduleData() = default;
+ ScheduleData() : ScheduleEntity(Kind::ScheduleData) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleData;
+ }
void init(int BlockSchedulingRegionID, Instruction *I) {
- FirstInBundle = this;
- NextInBundle = nullptr;
NextLoadStore = nullptr;
IsScheduled = false;
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
- TE = nullptr;
}
/// Verify basic self consistency properties
@@ -4042,20 +4074,9 @@ class BoUpSLP {
}
if (IsScheduled) {
- assert(isSchedulingEntity() &&
- "unexpected scheduled state");
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- assert(BundleMember->hasValidDependencies() &&
- BundleMember->UnscheduledDeps == 0 &&
- "unexpected scheduled state");
- assert((BundleMember == this || !BundleMember->IsScheduled) &&
- "only bundle is marked scheduled");
- }
+ assert(hasValidDependencies() && UnscheduledDeps == 0 &&
+ "unexpected scheduled state");
}
-
- assert(Inst->getParent() == FirstInBundle->Inst->getParent() &&
- "all bundle members must be in same basic block");
}
/// Returns true if the dependency information has been calculated.
@@ -4063,23 +4084,9 @@ class BoUpSLP {
/// a single bundle.
bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
- /// Returns true for single instructions and for bundle representatives
- /// (= the head of a bundle).
- bool isSchedulingEntity() const { return FirstInBundle == this; }
-
- /// Returns true if it represents an instruction bundle and not only a
- /// single instruction.
- bool isPartOfBundle() const {
- return NextInBundle != nullptr || FirstInBundle != this || TE;
- }
-
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
- bool isReady() const {
- assert(isSchedulingEntity() &&
- "can't consider non-scheduling entity for ready list");
- return unscheduledDepsInBundle() == 0 && !IsScheduled;
- }
+ bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
@@ -4088,14 +4095,12 @@ class BoUpSLP {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
- return FirstInBundle->unscheduledDepsInBundle();
+ return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
- void resetUnscheduledDeps() {
- UnscheduledDeps = Dependencies;
- }
+ void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Clears all dependency information.
void clearDependencies() {
@@ -4103,78 +4108,76 @@ class BoUpSLP {
resetUnscheduledDeps();
MemoryDependencies.clear();
ControlDependencies.clear();
+ IsScheduled = false;
}
- int unscheduledDepsInBundle() const {
- assert(isSchedulingEntity() && "only meaningful on the bundle");
- int Sum = 0;
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- if (BundleMember->UnscheduledDeps == InvalidDeps)
- return InvalidDeps;
- Sum += BundleMember->UnscheduledDeps;
- }
- return Sum;
- }
+ /// Gets/sets if the bundle is scheduled.
+ bool isScheduled() const { return IsScheduled; }
+ void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
- void dump(raw_ostream &OS) const {
- if (isPartOfBundle()) {
- if (!isSchedulingEntity()) {
- OS << "/ " << *Inst << ", part of ";
- FirstInBundle->dump(OS);
- return;
- }
- OS << '[' << *Inst;
- ScheduleData *SD = NextInBundle;
- while (SD) {
- OS << ';' << *SD->Inst;
- SD = SD->NextInBundle;
- }
- OS << ']';
- } else {
- OS << *Inst;
- }
+ /// Gets the number of unscheduled dependencies.
+ int getUnscheduledDeps() const { return UnscheduledDeps; }
+ /// Gets the number of dependencies.
+ int getDependencies() const { return Dependencies; }
+ /// Initializes the number of dependencies.
+ void initDependencies() { Dependencies = 0; }
+ /// Increments the number of dependencies.
+ void incDependencies() { Dependencies++; }
+
+ /// Gets scheduling region ID.
+ int getSchedulingRegionID() const { return SchedulingRegionID; }
+
+ /// Gets the instruction.
+ Instruction *getInst() const { return Inst; }
+
+ /// Gets the list of memory dependencies.
+ ArrayRef<ScheduleData *> getMemoryDependencies() const {
+ return MemoryDependencies;
+ }
+ /// Adds a memory dependency.
+ void addMemoryDependency(ScheduleData *Dep) {
+ MemoryDependencies.push_back(Dep);
+ }
+ /// Gets the list of control dependencies.
+ ArrayRef<ScheduleData *> getControlDependencies() const {
+ return ControlDependencies;
}
+ /// Adds a control dependency.
+ void addControlDependency(ScheduleData *Dep) {
+ ControlDependencies.push_back(Dep);
+ }
+ /// Gets/sets the next load/store instruction in the block.
+ ScheduleData *getNextLoadStore() const { return NextLoadStore; }
+ void setNextLoadStore(ScheduleData *Next) { NextLoadStore = Next; }
+
+ void dump(raw_ostream &OS) const { OS << *Inst; }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
+ private:
Instruction *Inst = nullptr;
- /// The TreeEntry that this instruction corresponds to.
- TreeEntry *TE = nullptr;
-
- /// Points to the head in an instruction bundle (and always to this for
- /// single instructions).
- ScheduleData *FirstInBundle = nullptr;
-
- /// Single linked list of all instructions in a bundle. Null if it is a
- /// single instruction.
- ScheduleData *NextInBundle = nullptr;
-
/// Single linked list of all memory instructions (e.g. load, store, call)
/// in the block - until the end of the scheduling region.
ScheduleData *NextLoadStore = nullptr;
/// The dependent memory instructions.
/// This list is derived on demand in calculateDependencies().
- SmallVector<ScheduleData *, 4> MemoryDependencies;
+ SmallVector<ScheduleData *> MemoryDependencies;
/// List of instructions which this instruction could be control dependent
/// on. Allowing such nodes to be scheduled below this one could introduce
/// a runtime fault which didn't exist in the original program.
/// ex: this is a load or udiv following a readonly call which inf loops
- SmallVector<ScheduleData *, 4> ControlDependencies;
+ SmallVector<ScheduleData *> ControlDependencies;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
- /// Used for getting a "good" final ordering of instructions.
- int SchedulingPriority = 0;
-
/// The number of dependencies. Constitutes of the number of users of the
/// instruction plus the number of dependent memory instructions (if any).
/// This value is calculated on demand.
@@ -4200,6 +4203,112 @@ class BoUpSLP {
}
#endif
+ class ScheduleBundle final : public ScheduleEntity {
+ /// The schedule data for the instructions in the bundle.
+ SmallVector<ScheduleData *> Bundle;
+ /// True if this bundle is valid.
+ bool IsValid = true;
+ /// The TreeEntry that this instruction corresponds to.
+ TreeEntry *TE = nullptr;
+ ScheduleBundle(bool IsValid)
+ : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid) {}
+
+ public:
+ ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleBundle;
+ }
+
+ /// Verify basic self consistency properties
+ void verify() const {
+ for (const ScheduleData *SD : Bundle) {
+ if (SD->hasValidDependencies()) {
+ assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
+ "invariant");
+ } else {
+ assert(SD->getUnscheduledDeps() == SD->getDependencies() &&
+ "invariant");
+ }
+
+ if (isScheduled()) {
+ assert(SD->hasValidDependencies() && SD->getUnscheduledDeps() == 0 &&
+ "unexpected scheduled state");
+ }
+ }
+ }
+
+ bool isScheduled() const {
+ return all_of(Bundle,
+ [](const ScheduleData *SD) { return SD->isScheduled(); });
+ }
+
+ int unscheduledDepsInBundle() const {
+ assert(*this && "bundle must not be empty");
+ int Sum = 0;
+ for (const ScheduleData *BundleMember : Bundle) {
+ if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
+ return ScheduleData::InvalidDeps;
+ Sum += BundleMember->getUnscheduledDeps();
+ }
+ return Sum;
+ }
+
+ /// Returns true if the dependency information has been calculated.
+ /// Note that depenendency validity can vary between instructions within
+ /// a single bundle.
+ bool hasValidDependencies() const {
+ return all_of(Bundle, [](const ScheduleData *SD) {
+ return SD->hasValidDependencies();
+ });
+ }
+
+ /// Returns true if it is ready for scheduling, i.e. it has no more
+ /// unscheduled depending instructions/bundles.
+ bool isReady() const {
+ assert(*this && "bundle must not be empty");
+ return unscheduledDepsInBundle() == 0 && !isScheduled();
+ }
+
+ /// Returns the bundle of scheduling data, associated with the current
+ /// instruction.
+ ArrayRef<ScheduleData *> getBundle() { return Bundle; }
+ ArrayRef<const ScheduleData *> getBundle() const { return Bundle; }
+ /// Adds an instruction to the bundle.
+ void add(ScheduleData *SD) { Bundle.push_back(SD); }
+
+ /// Gets/sets the associated tree entry.
+ void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
+ TreeEntry *getTreeEntry() const { return TE; }
+
+ static ScheduleBundle invalid() { return {false}; }
+
+ operator bool() const { return IsValid; }
+
+ void dump(raw_ostream &OS) const {
+ if (!*this) {
+ OS << "[]";
+ return;
+ }
+ OS << '[';
+ interleaveComma(Bundle, OS,
+ [&](const ScheduleData *SD) { OS << *SD->getInst(); });
+ OS << ']';
+ }
+
+ LLVM_DUMP_METHOD void dump() const {
+ dump(dbgs());
+ dbgs() << '\n';
+ }
+ };
+
+#ifndef NDEBUG
+ friend inline raw_ostream &operator<<(raw_ostream &os,
+ const BoUpSLP::ScheduleBundle &Bundle) {
+ Bundle.dump(os);
+ return os;
+ }
+#endif
+
friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>;
@@ -4224,6 +4333,8 @@ class BoUpSLP {
: BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
void clear() {
+ ScheduledBundles.clear();
+ ScheduledBundlesList.clear();
ReadyInsts.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
@@ -4244,6 +4355,8 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Instruction *I) {
+ if (!I)
+ return nullptr;
if (BB != I->getParent())
// Avoid lookup if can't possibly be in map.
return nullptr;
@@ -4254,52 +4367,78 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Value *V) {
- if (auto *I = dyn_cast<Instruction>(V))
- return getScheduleData(I);
- return nullptr;
+ return getScheduleData(dyn_cast<Instruction>(V));
+ }
+
+ ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return {};
+ auto It = ScheduledBundles.find(I);
+ if (It == ScheduledBundles.end())
+ return {};
+ return It->getSecond();
}
bool isInSchedulingRegion(ScheduleData *SD) const {
- return SD->SchedulingRegionID == SchedulingRegionID;
+ return SD->getSchedulingRegionID() == SchedulingRegionID;
+ }
+
+ bool isInSchedulingRegion(const ScheduleBundle &Bundle) const {
+ return all_of(Bundle.getBundle(), [&](const ScheduleData *BundleMember) {
+ return BundleMember->getSchedulingRegionID() == SchedulingRegionID;
+ });
}
/// Marks an instruction as scheduled and puts all dependent ready
/// instructions into the ready-list.
template <typename ReadyListType>
- void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
- SD->IsScheduled = true;
- LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
-
- for (ScheduleData *BundleMember = SD; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
-
+ void schedule(ScheduleEntity *Data, ReadyListType &ReadyList) {
+ auto ProcessBundleMember = [&](ScheduleData *BundleMember,
+ ScheduleBundle *Bundle) {
// Handle the def-use chain dependencies.
// Decrement the unscheduled counter and insert to ready list if ready.
- auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
- ScheduleData *OpDef = getScheduleData(I);
- if (OpDef && OpDef->hasValidDependencies() &&
- OpDef->incrementUnscheduledDeps(-1) == 0) {
+ auto DecrUnsched = [&](ScheduleData *Data, bool IsControl = false) {
+ if ((IsControl || Data->hasValidDependencies()) &&
+ Data->incrementUnscheduledDeps(-1) == 0) {
...
[truncated]
|
|
@llvm/pr-subscribers-vectorizers Author: Alexey Bataev (alexey-bataev) ChangesThat's the initial patch, intended to support revectorization of the Patch is 73.90 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/131625.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d450336cbc3ce..6766b68841a9a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1368,7 +1368,9 @@ namespace slpvectorizer {
/// Bottom Up SLP Vectorizer.
class BoUpSLP {
struct TreeEntry;
- struct ScheduleData;
+ class ScheduleEntity;
+ class ScheduleData;
+ class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
@@ -1433,7 +1435,8 @@ class BoUpSLP {
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
- InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {});
+ InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {},
+ InstructionCost ReductionCost = TTI::TCC_Free);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
@@ -3670,8 +3673,7 @@ class BoUpSLP {
#endif
/// Create a new VectorizableTree entry.
- TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
- std::optional<ScheduleData *> Bundle,
+ TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
@@ -3688,8 +3690,7 @@ class BoUpSLP {
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
TreeEntry::EntryState EntryState,
- std::optional<ScheduleData *> Bundle,
- const InstructionsState &S,
+ ScheduleBundle &Bundle, const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {}) {
@@ -3781,22 +3782,23 @@ class BoUpSLP {
}
}
// Update the scheduler bundle to point to this TreeEntry.
- ScheduleData *BundleMember = *Bundle;
- assert((BundleMember || isa<PHINode>(S.getMainOp()) ||
+ assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
doesNotNeedToSchedule(VL)) &&
"Bundle and VL out of sync");
- if (BundleMember) {
+ if (!Bundle.getBundle().empty()) {
+ auto *BundleMember = Bundle.getBundle().begin();
for (Value *V : VL) {
if (doesNotNeedToBeScheduled(V))
continue;
- if (!BundleMember)
+ if (BundleMember == Bundle.getBundle().end())
continue;
- BundleMember->TE = Last;
- BundleMember = BundleMember->NextInBundle;
+ ++BundleMember;
}
+ assert(BundleMember == Bundle.getBundle().end() &&
+ "Bundle and VL out of sync");
+ Bundle.setTreeEntry(Last);
}
- assert(!BundleMember && "Bundle and VL out of sync");
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
@@ -3946,16 +3948,17 @@ class BoUpSLP {
/// is invariant in the calling loop.
bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
Instruction *Inst2) {
- if (!Loc1.Ptr || !isSimple(Inst1) || !isSimple(Inst2))
+ assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instrucgion.");
+ if (!isSimple(Inst2))
return true;
// First check if the result is already in the cache.
AliasCacheKey Key = std::make_pair(Inst1, Inst2);
- auto It = AliasCache.find(Key);
- if (It != AliasCache.end())
- return It->second;
+ auto Res = AliasCache.try_emplace(Key);
+ if (!Res.second)
+ return Res.first->second;
bool Aliased = isModOrRefSet(BatchAA.getModRefInfo(Inst2, Loc1));
// Store the result in the cache.
- AliasCache.try_emplace(Key, Aliased);
+ Res.first->getSecond() = Aliased;
AliasCache.try_emplace(std::make_pair(Inst2, Inst1), Aliased);
return Aliased;
}
@@ -3964,7 +3967,7 @@ class BoUpSLP {
/// Cache for alias results.
/// TODO: consider moving this to the AliasAnalysis itself.
- DenseMap<AliasCacheKey, bool> AliasCache;
+ SmallDenseMap<AliasCacheKey, bool> AliasCache;
// Cache for pointerMayBeCaptured calls inside AA. This is preserved
// globally through SLP because we don't perform any action which
@@ -4011,26 +4014,55 @@ class BoUpSLP {
/// List of hashes of vector of loads, which are known to be non vectorizable.
DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
+ class ScheduleEntity {
+ friend class ScheduleBundle;
+ friend class ScheduleData;
+
+ protected:
+ enum class Kind { ScheduleData, ScheduleBundle };
+ Kind getKind() const { return K; }
+ ScheduleEntity(Kind K) : K(K) {}
+
+ private:
+ /// Used for getting a "good" final ordering of instructions.
+ int SchedulingPriority = 0;
+ /// The kind of the ScheduleEntity.
+ Kind K = Kind::ScheduleData;
+
+ public:
+ ScheduleEntity() = delete;
+ /// Gets/sets the scheduling priority.
+ void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
+ int getSchedulingPriority() const { return SchedulingPriority; }
+ bool isReady() const {
+ if (auto *SD = dyn_cast<ScheduleData>(this))
+ return SD->isReady();
+ return cast<ScheduleBundle>(this)->isReady();
+ }
+ static bool classof(const ScheduleEntity *) { return true; }
+ };
+
/// Contains all scheduling relevant data for an instruction.
/// A ScheduleData either represents a single instruction or a member of an
/// instruction bundle (= a group of instructions which is combined into a
/// vector instruction).
- struct ScheduleData {
+ class ScheduleData final : public ScheduleEntity {
+ public:
// The initial value for the dependency counters. It means that the
// dependencies are not calculated yet.
enum { InvalidDeps = -1 };
- ScheduleData() = default;
+ ScheduleData() : ScheduleEntity(Kind::ScheduleData) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleData;
+ }
void init(int BlockSchedulingRegionID, Instruction *I) {
- FirstInBundle = this;
- NextInBundle = nullptr;
NextLoadStore = nullptr;
IsScheduled = false;
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
- TE = nullptr;
}
/// Verify basic self consistency properties
@@ -4042,20 +4074,9 @@ class BoUpSLP {
}
if (IsScheduled) {
- assert(isSchedulingEntity() &&
- "unexpected scheduled state");
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- assert(BundleMember->hasValidDependencies() &&
- BundleMember->UnscheduledDeps == 0 &&
- "unexpected scheduled state");
- assert((BundleMember == this || !BundleMember->IsScheduled) &&
- "only bundle is marked scheduled");
- }
+ assert(hasValidDependencies() && UnscheduledDeps == 0 &&
+ "unexpected scheduled state");
}
-
- assert(Inst->getParent() == FirstInBundle->Inst->getParent() &&
- "all bundle members must be in same basic block");
}
/// Returns true if the dependency information has been calculated.
@@ -4063,23 +4084,9 @@ class BoUpSLP {
/// a single bundle.
bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
- /// Returns true for single instructions and for bundle representatives
- /// (= the head of a bundle).
- bool isSchedulingEntity() const { return FirstInBundle == this; }
-
- /// Returns true if it represents an instruction bundle and not only a
- /// single instruction.
- bool isPartOfBundle() const {
- return NextInBundle != nullptr || FirstInBundle != this || TE;
- }
-
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
- bool isReady() const {
- assert(isSchedulingEntity() &&
- "can't consider non-scheduling entity for ready list");
- return unscheduledDepsInBundle() == 0 && !IsScheduled;
- }
+ bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
@@ -4088,14 +4095,12 @@ class BoUpSLP {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
- return FirstInBundle->unscheduledDepsInBundle();
+ return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
- void resetUnscheduledDeps() {
- UnscheduledDeps = Dependencies;
- }
+ void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Clears all dependency information.
void clearDependencies() {
@@ -4103,78 +4108,76 @@ class BoUpSLP {
resetUnscheduledDeps();
MemoryDependencies.clear();
ControlDependencies.clear();
+ IsScheduled = false;
}
- int unscheduledDepsInBundle() const {
- assert(isSchedulingEntity() && "only meaningful on the bundle");
- int Sum = 0;
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- if (BundleMember->UnscheduledDeps == InvalidDeps)
- return InvalidDeps;
- Sum += BundleMember->UnscheduledDeps;
- }
- return Sum;
- }
+ /// Gets/sets if the bundle is scheduled.
+ bool isScheduled() const { return IsScheduled; }
+ void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
- void dump(raw_ostream &OS) const {
- if (isPartOfBundle()) {
- if (!isSchedulingEntity()) {
- OS << "/ " << *Inst << ", part of ";
- FirstInBundle->dump(OS);
- return;
- }
- OS << '[' << *Inst;
- ScheduleData *SD = NextInBundle;
- while (SD) {
- OS << ';' << *SD->Inst;
- SD = SD->NextInBundle;
- }
- OS << ']';
- } else {
- OS << *Inst;
- }
+ /// Gets the number of unscheduled dependencies.
+ int getUnscheduledDeps() const { return UnscheduledDeps; }
+ /// Gets the number of dependencies.
+ int getDependencies() const { return Dependencies; }
+ /// Initializes the number of dependencies.
+ void initDependencies() { Dependencies = 0; }
+ /// Increments the number of dependencies.
+ void incDependencies() { Dependencies++; }
+
+ /// Gets scheduling region ID.
+ int getSchedulingRegionID() const { return SchedulingRegionID; }
+
+ /// Gets the instruction.
+ Instruction *getInst() const { return Inst; }
+
+ /// Gets the list of memory dependencies.
+ ArrayRef<ScheduleData *> getMemoryDependencies() const {
+ return MemoryDependencies;
+ }
+ /// Adds a memory dependency.
+ void addMemoryDependency(ScheduleData *Dep) {
+ MemoryDependencies.push_back(Dep);
+ }
+ /// Gets the list of control dependencies.
+ ArrayRef<ScheduleData *> getControlDependencies() const {
+ return ControlDependencies;
}
+ /// Adds a control dependency.
+ void addControlDependency(ScheduleData *Dep) {
+ ControlDependencies.push_back(Dep);
+ }
+ /// Gets/sets the next load/store instruction in the block.
+ ScheduleData *getNextLoadStore() const { return NextLoadStore; }
+ void setNextLoadStore(ScheduleData *Next) { NextLoadStore = Next; }
+
+ void dump(raw_ostream &OS) const { OS << *Inst; }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
+ private:
Instruction *Inst = nullptr;
- /// The TreeEntry that this instruction corresponds to.
- TreeEntry *TE = nullptr;
-
- /// Points to the head in an instruction bundle (and always to this for
- /// single instructions).
- ScheduleData *FirstInBundle = nullptr;
-
- /// Single linked list of all instructions in a bundle. Null if it is a
- /// single instruction.
- ScheduleData *NextInBundle = nullptr;
-
/// Single linked list of all memory instructions (e.g. load, store, call)
/// in the block - until the end of the scheduling region.
ScheduleData *NextLoadStore = nullptr;
/// The dependent memory instructions.
/// This list is derived on demand in calculateDependencies().
- SmallVector<ScheduleData *, 4> MemoryDependencies;
+ SmallVector<ScheduleData *> MemoryDependencies;
/// List of instructions which this instruction could be control dependent
/// on. Allowing such nodes to be scheduled below this one could introduce
/// a runtime fault which didn't exist in the original program.
/// ex: this is a load or udiv following a readonly call which inf loops
- SmallVector<ScheduleData *, 4> ControlDependencies;
+ SmallVector<ScheduleData *> ControlDependencies;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
- /// Used for getting a "good" final ordering of instructions.
- int SchedulingPriority = 0;
-
/// The number of dependencies. Constitutes of the number of users of the
/// instruction plus the number of dependent memory instructions (if any).
/// This value is calculated on demand.
@@ -4200,6 +4203,112 @@ class BoUpSLP {
}
#endif
+ class ScheduleBundle final : public ScheduleEntity {
+ /// The schedule data for the instructions in the bundle.
+ SmallVector<ScheduleData *> Bundle;
+ /// True if this bundle is valid.
+ bool IsValid = true;
+ /// The TreeEntry that this instruction corresponds to.
+ TreeEntry *TE = nullptr;
+ ScheduleBundle(bool IsValid)
+ : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid) {}
+
+ public:
+ ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleBundle;
+ }
+
+ /// Verify basic self consistency properties
+ void verify() const {
+ for (const ScheduleData *SD : Bundle) {
+ if (SD->hasValidDependencies()) {
+ assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
+ "invariant");
+ } else {
+ assert(SD->getUnscheduledDeps() == SD->getDependencies() &&
+ "invariant");
+ }
+
+ if (isScheduled()) {
+ assert(SD->hasValidDependencies() && SD->getUnscheduledDeps() == 0 &&
+ "unexpected scheduled state");
+ }
+ }
+ }
+
+ bool isScheduled() const {
+ return all_of(Bundle,
+ [](const ScheduleData *SD) { return SD->isScheduled(); });
+ }
+
+ int unscheduledDepsInBundle() const {
+ assert(*this && "bundle must not be empty");
+ int Sum = 0;
+ for (const ScheduleData *BundleMember : Bundle) {
+ if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
+ return ScheduleData::InvalidDeps;
+ Sum += BundleMember->getUnscheduledDeps();
+ }
+ return Sum;
+ }
+
+ /// Returns true if the dependency information has been calculated.
+ /// Note that depenendency validity can vary between instructions within
+ /// a single bundle.
+ bool hasValidDependencies() const {
+ return all_of(Bundle, [](const ScheduleData *SD) {
+ return SD->hasValidDependencies();
+ });
+ }
+
+ /// Returns true if it is ready for scheduling, i.e. it has no more
+ /// unscheduled depending instructions/bundles.
+ bool isReady() const {
+ assert(*this && "bundle must not be empty");
+ return unscheduledDepsInBundle() == 0 && !isScheduled();
+ }
+
+ /// Returns the bundle of scheduling data, associated with the current
+ /// instruction.
+ ArrayRef<ScheduleData *> getBundle() { return Bundle; }
+ ArrayRef<const ScheduleData *> getBundle() const { return Bundle; }
+ /// Adds an instruction to the bundle.
+ void add(ScheduleData *SD) { Bundle.push_back(SD); }
+
+ /// Gets/sets the associated tree entry.
+ void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
+ TreeEntry *getTreeEntry() const { return TE; }
+
+ static ScheduleBundle invalid() { return {false}; }
+
+ operator bool() const { return IsValid; }
+
+ void dump(raw_ostream &OS) const {
+ if (!*this) {
+ OS << "[]";
+ return;
+ }
+ OS << '[';
+ interleaveComma(Bundle, OS,
+ [&](const ScheduleData *SD) { OS << *SD->getInst(); });
+ OS << ']';
+ }
+
+ LLVM_DUMP_METHOD void dump() const {
+ dump(dbgs());
+ dbgs() << '\n';
+ }
+ };
+
+#ifndef NDEBUG
+ friend inline raw_ostream &operator<<(raw_ostream &os,
+ const BoUpSLP::ScheduleBundle &Bundle) {
+ Bundle.dump(os);
+ return os;
+ }
+#endif
+
friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>;
@@ -4224,6 +4333,8 @@ class BoUpSLP {
: BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
void clear() {
+ ScheduledBundles.clear();
+ ScheduledBundlesList.clear();
ReadyInsts.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
@@ -4244,6 +4355,8 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Instruction *I) {
+ if (!I)
+ return nullptr;
if (BB != I->getParent())
// Avoid lookup if can't possibly be in map.
return nullptr;
@@ -4254,52 +4367,78 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Value *V) {
- if (auto *I = dyn_cast<Instruction>(V))
- return getScheduleData(I);
- return nullptr;
+ return getScheduleData(dyn_cast<Instruction>(V));
+ }
+
+ ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return {};
+ auto It = ScheduledBundles.find(I);
+ if (It == ScheduledBundles.end())
+ return {};
+ return It->getSecond();
}
bool isInSchedulingRegion(ScheduleData *SD) const {
- return SD->SchedulingRegionID == SchedulingRegionID;
+ return SD->getSchedulingRegionID() == SchedulingRegionID;
+ }
+
+ bool isInSchedulingRegion(const ScheduleBundle &Bundle) const {
+ return all_of(Bundle.getBundle(), [&](const ScheduleData *BundleMember) {
+ return BundleMember->getSchedulingRegionID() == SchedulingRegionID;
+ });
}
/// Marks an instruction as scheduled and puts all dependent ready
/// instructions into the ready-list.
template <typename ReadyListType>
- void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
- SD->IsScheduled = true;
- LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
-
- for (ScheduleData *BundleMember = SD; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
-
+ void schedule(ScheduleEntity *Data, ReadyListType &ReadyList) {
+ auto ProcessBundleMember = [&](ScheduleData *BundleMember,
+ ScheduleBundle *Bundle) {
// Handle the def-use chain dependencies.
// Decrement the unscheduled counter and insert to ready list if ready.
- auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
- ScheduleData *OpDef = getScheduleData(I);
- if (OpDef && OpDef->hasValidDependencies() &&
- OpDef->incrementUnscheduledDeps(-1) == 0) {
+ auto DecrUnsched = [&](ScheduleData *Data, bool IsControl = false) {
+ if ((IsControl || Data->hasValidDependencies()) &&
+ Data->incrementUnscheduledDeps(-1) == 0) {
...
[truncated]
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
Created using spr 1.3.5
|
Ping! |
RKSimon
reviewed
Mar 19, 2025
| ArrayRef<ScheduleBundle *> Bundles = BS->getScheduleBundles(I); | ||
| if (!Bundles.empty()) { | ||
| for (ScheduleBundle *Bundle : Bundles) | ||
| assert(Bundle->isScheduled() && "must be scheduled at this point"); |
gbossu
reviewed
Mar 19, 2025
There was a problem hiding this comment.
I'm currently looking into the SLPVectorizer to add support for more "re-vectorization" and I saw this patch pop up so I thought I'd have a quick look at it. I'm completely new to the SLP code base, so feel free to ignore my comments. I just thought it might help me get a better understanding if I look at pull requests. 😄
As a vectorizer newbie, one thing which I feel would make the code easier to understand is to modularize, even just slightly. Do you maybe have some plans to make the SLPVectorizer act more like a driver, and move components like cost modelling or scheduling into separate files?
Modularization is always good, but I don't have plans for it right now, busy with missing lots of functional stuff |
gbossu
reviewed
Mar 19, 2025
Created using spr 1.3.5
gbossu
reviewed
Mar 20, 2025
gbossu
reviewed
Mar 20, 2025
gbossu
reviewed
Mar 20, 2025
gbossu
reviewed
Mar 20, 2025
Created using spr 1.3.5
alexey-bataev
deleted the
users/alexey-bataev/spr/slpnfc-redesign-schedule-bundle-separate-from-schedule-data-nfc
branch
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
Mar 21, 2025
…e data, NFC That's the initial patch, intended to support revectorization of the previously vectorized scalars. If the scalar is marked for the vectorization, it becomes a part of the schedule bundle, used to check dependencies and then schedule tree entry scalars into a single batch of instructions. Unfortunately, currently this info is part of the ScheduleData struct and it does not allow making scalars part of many bundles. The patch separates schedule bundles from the ScheduleData, introduces explicit class ScheduleBundle for bundles, allowing later to extend it to support revectorization of the previously vectorized scalars. Reviewers: hiraditya, RKSimon Reviewed By: RKSimon, hiraditya Pull Request: llvm/llvm-project#131625
|
Hi, |
Thanks for letting me know. The bug is not related to this patch, probably some non-functional side-effect. Will fix it ASAP |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
That's the initial patch, intended to support revectorization of the
previously vectorized scalars. If the scalar is marked for the
vectorization, it becomes a part of the schedule bundle, used to check
dependencies and then schedule tree entry scalars into a single batch of
instructions. Unfortunately, currently this info is part of the
ScheduleData struct and it does not allow making scalars part of many
bundles. The patch separates schedule bundles from the ScheduleData,
introduces explicit class ScheduleBundle for bundles, allowing later to
extend it to support revectorization of the previously vectorized
scalars.