diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h index 64090febc5a09..bec1cecf241f6 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h +++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/VecUtils.h @@ -111,10 +111,12 @@ class VecUtils { return LowestI; } /// \Returns the lowest instruction in \p Vals, or nullptr if no instructions - /// are found or if not in the same BB. - static Instruction *getLowest(ArrayRef Vals) { - // Find the first Instruction in Vals. - auto It = find_if(Vals, [](Value *V) { return isa(V); }); + /// are found. Skips instructions not in \p BB. + static Instruction *getLowest(ArrayRef Vals, BasicBlock *BB) { + // Find the first Instruction in Vals that is also in `BB`. + auto It = find_if(Vals, [BB](Value *V) { + return isa(V) && cast(V)->getParent() == BB; + }); // If we couldn't find an instruction return nullptr. if (It == Vals.end()) return nullptr; @@ -122,15 +124,14 @@ class VecUtils { // Now look for the lowest instruction in Vals starting from one position // after FirstI. Instruction *LowestI = FirstI; - auto *LowestBB = LowestI->getParent(); for (auto *V : make_range(std::next(It), Vals.end())) { auto *I = dyn_cast(V); // Skip non-instructions. if (I == nullptr) continue; - // If the instructions are in different BBs return nullptr. - if (I->getParent() != LowestBB) - return nullptr; + // Skips instructions not in \p BB. + if (I->getParent() != BB) + continue; // If `LowestI` comes before `I` then `I` is the new lowest. if (LowestI->comesBefore(I)) LowestI = I; diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp index b3a477c64a5cc..6f65657d29790 100644 --- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp +++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp @@ -54,7 +54,7 @@ static SmallVector getOperand(ArrayRef Bndl, /// of BB if no instruction found in \p Vals. static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef Vals, BasicBlock *BB) { - auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals)); + auto *BotI = VecUtils::getLastPHIOrSelf(VecUtils::getLowest(Vals, BB)); if (BotI == nullptr) // We are using BB->begin() (or after PHIs) as the fallback insert point. return BB->empty() diff --git a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll index e913fc5913ba7..6ec31060d7e0f 100644 --- a/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll +++ b/llvm/test/Transforms/SandboxVectorizer/cross_bbs.ll @@ -8,10 +8,10 @@ define void @cross_bbs(ptr %ptr) { ; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i32 1 ; CHECK-NEXT: [[L0:%.*]] = load i8, ptr [[PTR0]], align 1 ; CHECK-NEXT: [[L1:%.*]] = load i8, ptr [[PTR1]], align 1 -; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[L0]], i32 0 -; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[L1]], i32 1 ; CHECK-NEXT: br label %[[BB:.*]] ; CHECK: [[BB]]: +; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[L0]], i32 0 +; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[L1]], i32 1 ; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[PTR0]], align 1 ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SandboxVectorizer/pack.ll b/llvm/test/Transforms/SandboxVectorizer/pack.ll index 373ab743fb890..a0aa2a79a0ade 100644 --- a/llvm/test/Transforms/SandboxVectorizer/pack.ll +++ b/llvm/test/Transforms/SandboxVectorizer/pack.ll @@ -59,12 +59,12 @@ define void @packFromOtherBB(ptr %ptr, i8 %val) { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[ADD0:%.*]] = add i8 [[VAL]], 0 ; CHECK-NEXT: [[MUL1:%.*]] = mul i8 [[VAL]], 1 -; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0 -; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1 ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[PHI0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ] ; CHECK-NEXT: [[PHI1:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ] +; CHECK-NEXT: [[PACK:%.*]] = insertelement <2 x i8> poison, i8 [[ADD0]], i32 0 +; CHECK-NEXT: [[PACK1:%.*]] = insertelement <2 x i8> [[PACK]], i8 [[MUL1]], i32 1 ; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0 ; CHECK-NEXT: store <2 x i8> [[PACK1]], ptr [[GEP0]], align 1 ; CHECK-NEXT: br label %[[LOOP]] diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp index a46e47afea3c7..5c062df8112f6 100644 --- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp @@ -461,24 +461,33 @@ define void @foo(i8 %v) { // Check getLowest(ArrayRef) SmallVector C1Only({C1}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only), nullptr); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only, &BB), nullptr); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1Only, &BB0), nullptr); SmallVector AOnly({IA}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly, &BB), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AOnly, &BB0), nullptr); SmallVector AC1({IA, C1}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1, &BB), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1, &BB0), nullptr); SmallVector C1A({C1, IA}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A, &BB), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1A, &BB0), nullptr); SmallVector AC1B({IA, C1, IB}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B), IB); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B, &BB), IB); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1B, &BB0), nullptr); SmallVector ABC1({IA, IB, C1}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1), IB); + EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1, &BB), IB); + EXPECT_EQ(sandboxir::VecUtils::getLowest(ABC1, &BB0), nullptr); SmallVector AC1C2({IA, C1, C2}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2, &BB), IA); + EXPECT_EQ(sandboxir::VecUtils::getLowest(AC1C2, &BB0), nullptr); SmallVector C1C2C3({C1, C2, C3}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3), nullptr); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3, &BB), nullptr); + EXPECT_EQ(sandboxir::VecUtils::getLowest(C1C2C3, &BB0), nullptr); SmallVector DiffBBs({BB0I, IA}); - EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs), nullptr); + EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs, &BB0), BB0I); + EXPECT_EQ(sandboxir::VecUtils::getLowest(DiffBBs, &BB), IA); } TEST_F(VecUtilsTest, GetLastPHIOrSelf) {