diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index af3ad822e0b0d..2cad21b8b2a4f 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -563,6 +563,11 @@ class ScalarEvolution { /// Return an existing SCEV for V if there is one, otherwise return nullptr. const SCEV *getExistingSCEV(Value *V); + /// Return the AddRec for \p Ops and \p L if there is one, otherwise return + /// nullptr. + const SCEVAddRecExpr *getExistingAddRecExpr(ArrayRef Ops, + const Loop *L); + const SCEV *getConstant(ConstantInt *V); const SCEV *getConstant(const APInt &Val); const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); @@ -2124,8 +2129,10 @@ class ScalarEvolution { /// This is like \c isSCEVExprNeverPoison but it specifically works for /// instructions that will get mapped to SCEV add recurrences. Return true /// if \p I will never generate poison under the assumption that \p I is an - /// add recurrence on the loop \p L. - bool isAddRecNeverPoison(const Instruction *I, const Loop *L); + /// add recurrence on the loop \p L. Don't call \c isSCEVExprNeverPoison if \p + /// CheckSCEVScope is false. + bool isAddRecNeverPoison(const Instruction *I, const Loop *L, + bool CheckSCEVScope = true); /// Similar to createAddRecFromPHI, but with the additional flexibility of /// suggesting runtime overflow checks in case casts are encountered. diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index fd884f2a2f55b..fcfcb1c5af879 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -392,6 +392,10 @@ class SCEVAddRecExpr : public SCEVNAryExpr { SubclassData |= Flags; } + /// Set the flags for the recurrence to \p Flags. This overwrites any existing + /// flags. + void forceSetNoWrapFlags(NoWrapFlags Flags) { SubclassData = Flags; } + /// Return the value of this chain of recurrences at the specified /// iteration number. const SCEV *evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 2acb45837c480..660d555875da6 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -4520,6 +4520,18 @@ const SCEV *ScalarEvolution::getExistingSCEV(Value *V) { return nullptr; } +const SCEVAddRecExpr * +ScalarEvolution::getExistingAddRecExpr(ArrayRef Ops, + const Loop *L) { + FoldingSetNodeID ID; + ID.AddInteger(scAddRecExpr); + for (const SCEV *Op : Ops) + ID.AddPointer(Op); + ID.AddPointer(L); + void *IP = nullptr; + return static_cast(UniqueSCEVs.FindNodeOrInsertPos(ID, IP)); +} + /// Return a SCEV corresponding to -V = -1*V const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags) { @@ -5703,19 +5715,46 @@ const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN, return nullptr; SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap; - if (BO->IsNUW) + if (BO->IsNUW) { Flags = setFlags(Flags, SCEV::FlagNUW); - if (BO->IsNSW) + Flags = setFlags(Flags, SCEV::FlagNW); + } + if (BO->IsNSW) { Flags = setFlags(Flags, SCEV::FlagNSW); + Flags = setFlags(Flags, SCEV::FlagNW); + } const SCEV *StartVal = getSCEV(StartValueV); + SCEV::NoWrapFlags MinFlags = Flags; + auto *ExistingAR = getExistingAddRecExpr({StartVal, Accum}, L); + // If there's already an AddRec, use its flags to compute the minimal valid + // flags that hold for the users of the existing AddRec and the users of the + // current phi we are constructing an AddRec for. + if (ExistingAR) + MinFlags = maskFlags(ExistingAR->getNoWrapFlags(), Flags); + const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags); + + // Check if the AddRec can never be poison, but avoid constructing new SCEVs + // for PN's operands, as this would instantiate the SCEV for the increment + // earlier. + bool NeverPoison = isAddRecNeverPoison(PN, L, false); + // If the AddRec is never poison, it is safe the use the flags from the IR + // unconditionally. But if it may be poison, force the AddRec's flags to the + // minimum valid set for both the existing AddRec and the current context. + if (!NeverPoison && ExistingAR) { + forgetMemoizedResults(PHISCEV); + PHISCEV = getAddRecExpr(StartVal, Accum, L, MinFlags); + const_cast(ExistingAR)->forceSetNoWrapFlags(MinFlags); + } + insertValueToMap(PN, PHISCEV); if (auto *AR = dyn_cast(PHISCEV)) { - setNoWrapFlags(const_cast(AR), - (SCEV::NoWrapFlags)(AR->getNoWrapFlags() | - proveNoWrapViaConstantRanges(AR))); + SCEV::NoWrapFlags FlagsViaConstantRanges = proveNoWrapViaConstantRanges(AR); + setNoWrapFlags( + const_cast(AR), + (SCEV::NoWrapFlags)(AR->getNoWrapFlags() | FlagsViaConstantRanges)); } // We can add Flags to the post-inc expression only if we @@ -7269,9 +7308,10 @@ bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) { return isGuaranteedToTransferExecutionTo(DefI, I); } -bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) { +bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L, + bool CheckSCEVScope) { // If we know that \c I can never be poison period, then that's enough. - if (isSCEVExprNeverPoison(I)) + if (CheckSCEVScope && isSCEVExprNeverPoison(I)) return true; // If the loop only has one exit, then we know that, if the loop is entered, diff --git a/llvm/test/Analysis/ScalarEvolution/iv-poison.ll b/llvm/test/Analysis/ScalarEvolution/iv-poison.ll index 771a93f79cb11..e4da99e16a20a 100644 --- a/llvm/test/Analysis/ScalarEvolution/iv-poison.ll +++ b/llvm/test/Analysis/ScalarEvolution/iv-poison.ll @@ -8,7 +8,7 @@ define i2 @iv_nsw_poison(i2 %arg) { ; CHECK-NEXT: %.07 = phi i2 [ 1, %bb ], [ %i, %bb1 ] ; CHECK-NEXT: --> {1,+,1}<%bb1> U: [1,-2) S: [1,-2) Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %.0 = phi i2 [ 1, %bb ], [ %i2, %bb1 ] -; CHECK-NEXT: --> {1,+,1}<%bb1> U: [1,-2) S: [1,-2) Exits: <> LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: --> {1,+,1}<%bb1> U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %i = add nsw i2 %.07, 1 ; CHECK-NEXT: --> {-2,+,1}<%bb1> U: [-2,0) S: [-2,0) Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %i2 = add i2 %.0, 1 @@ -40,11 +40,11 @@ define i4 @iv_nsw_poison2(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: %iv.0 = phi i4 [ %start, %entry ], [ %iv.0.next, %loop ] ; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1 = phi i4 [ %start, %entry ], [ %iv.1.next, %loop ] -; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.0.next = add i4 %iv.0, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1.next = add nsw i4 %iv.1, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @iv_nsw_poison2 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. @@ -172,11 +172,11 @@ define i4 @iv_nsw_poison_extra_use(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: %iv.0 = phi i4 [ %start, %entry ], [ %iv.0.next, %loop ] ; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1 = phi i4 [ %start, %entry ], [ %iv.1.next, %loop ] -; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.0.next = add i4 %iv.0, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1.next = add nsw i4 %iv.1, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @iv_nsw_poison_extra_use ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. @@ -207,7 +207,7 @@ define i2 @iv_nuw_poison(i2 %arg, i2 %start) { ; CHECK-NEXT: %.07 = phi i2 [ %start, %bb ], [ %i, %bb1 ] ; CHECK-NEXT: --> {%start,+,1}<%bb1> U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %.0 = phi i2 [ %start, %bb ], [ %i2, %bb1 ] -; CHECK-NEXT: --> {%start,+,1}<%bb1> U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Computable } +; CHECK-NEXT: --> {%start,+,1}<%bb1> U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %i = add nuw i2 %.07, 1 ; CHECK-NEXT: --> {(1 + %start),+,1}<%bb1> U: full-set S: full-set Exits: <> LoopDispositions: { %bb1: Computable } ; CHECK-NEXT: %i2 = add i2 %.0, 1 @@ -239,11 +239,11 @@ define i4 @iv_nuw_poison2(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: %iv.0 = phi i4 [ %start, %entry ], [ %iv.0.next, %loop ] ; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1 = phi i4 [ %start, %entry ], [ %iv.1.next, %loop ] -; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.0.next = add i4 %iv.0, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1.next = add nuw i4 %iv.1, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @iv_nuw_poison2 ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. @@ -371,11 +371,11 @@ define i4 @iv_nuw_poison_extra_use(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: %iv.0 = phi i4 [ %start, %entry ], [ %iv.0.next, %loop ] ; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1 = phi i4 [ %start, %entry ], [ %iv.1.next, %loop ] -; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.0.next = add i4 %iv.0, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.1.next = add nuw i4 %iv.1, 1 -; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(1 + %start),+,1}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @iv_nuw_poison_extra_use ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. diff --git a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll index 6f659a88da2e2..0a2199b6efef4 100644 --- a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll +++ b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll @@ -928,27 +928,27 @@ define i32 @test16_unsigned_pos2(i32 %start, ptr %p, ptr %q, i32 %x) { ; CHECK: loop: ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ] ; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[INDVARS_IV]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[INDVARS_IV]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32 +; CHECK-NEXT: [[FOO:%.*]] = add i32 [[TMP1]], -1 ; CHECK-NEXT: br i1 [[COND]], label [[EXIT:%.*]], label [[GUARDED:%.*]] ; CHECK: guarded: -; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[X:%.*]] to i64 -; CHECK-NEXT: [[ICMP_USER_WIDE:%.*]] = icmp ne i64 [[TMP1]], [[TMP2]] -; CHECK-NEXT: br i1 [[ICMP_USER_WIDE]], label [[BACKEDGE]], label [[SIDE_EXIT:%.*]] +; CHECK-NEXT: [[ICMP_USER:%.*]] = icmp ne i32 [[FOO]], [[X:%.*]] +; CHECK-NEXT: br i1 [[ICMP_USER]], label [[BACKEDGE]], label [[SIDE_EXIT:%.*]] ; CHECK: backedge: -; CHECK-NEXT: [[STORE_ADDR:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]] +; CHECK-NEXT: [[INDEX:%.*]] = zext i32 [[FOO]] to i64 +; CHECK-NEXT: [[STORE_ADDR:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[INDEX]] ; CHECK-NEXT: store i32 1, ptr [[STORE_ADDR]], align 4 -; CHECK-NEXT: [[STOP:%.*]] = load i32, ptr [[Q:%.*]], align 4 +; CHECK-NEXT: [[LOAD_ADDR:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[STOP:%.*]] = load i32, ptr [[Q]], align 4 ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp eq i32 [[STOP]], 0 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1 ; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[FAILURE:%.*]] ; CHECK: exit: -; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 -1 to i32 -; CHECK-NEXT: call void @use(i32 [[TMP3]]) -; CHECK-NEXT: ret i32 [[TMP3]] +; CHECK-NEXT: call void @use(i32 -1) +; CHECK-NEXT: ret i32 -1 ; CHECK: failure: -; CHECK-NEXT: [[FOO_LCSSA2_WIDE:%.*]] = phi i64 [ [[TMP1]], [[BACKEDGE]] ] -; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[FOO_LCSSA2_WIDE]] to i32 -; CHECK-NEXT: call void @use(i32 [[TMP4]]) +; CHECK-NEXT: [[FOO_LCSSA2:%.*]] = phi i32 [ [[FOO]], [[BACKEDGE]] ] +; CHECK-NEXT: call void @use(i32 [[FOO_LCSSA2]]) ; CHECK-NEXT: unreachable ; CHECK: side_exit: ; CHECK-NEXT: ret i32 0 diff --git a/llvm/test/Transforms/IndVarSimplify/iv-poison.ll b/llvm/test/Transforms/IndVarSimplify/iv-poison.ll index 383599f614357..9acffea5669d7 100644 --- a/llvm/test/Transforms/IndVarSimplify/iv-poison.ll +++ b/llvm/test/Transforms/IndVarSimplify/iv-poison.ll @@ -338,7 +338,7 @@ define i4 @iv_hoist_nuw_poison_extra_use(i4 %0, i4 %end, i4 %start) { ; CHECK-NEXT: [[IV_0:%.*]] = phi i4 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[IV_0_NEXT:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[IV_0_NEXT]] = add i4 [[IV_0]], 1 ; CHECK-NEXT: call void @use(i4 [[IV_0_NEXT]]) -; CHECK-NEXT: [[DOTNOT_NOT:%.*]] = icmp ult i4 [[START]], [[END:%.*]] +; CHECK-NEXT: [[DOTNOT_NOT:%.*]] = icmp ult i4 [[IV_0]], [[END:%.*]] ; CHECK-NEXT: br i1 [[DOTNOT_NOT]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: ; CHECK-NEXT: [[IV_1_NEXT_LCSSA:%.*]] = phi i4 [ [[IV_0_NEXT]], [[LOOP]] ] diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/expander-crashes.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/expander-crashes.ll index 29c03b88c5fb1..9b6d7041aa8c2 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/X86/expander-crashes.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/X86/expander-crashes.ll @@ -17,7 +17,7 @@ define i64 @blam(ptr %start, ptr %end, ptr %ptr.2) { ; CHECK-NEXT: [[LSR_IV4:%.*]] = phi i64 [ [[LSR_IV_NEXT5:%.*]], [[LOOP_1_HEADER]] ], [ [[START1]], [[ENTRY:%.*]] ] ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], [[LOOP_1_HEADER]] ], [ [[START]], [[ENTRY]] ] ; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds [[STRUCT_HOGE:%.*]], ptr [[IV]], i64 1 -; CHECK-NEXT: [[LSR_IV_NEXT5]] = add nuw i64 [[LSR_IV4]], 16 +; CHECK-NEXT: [[LSR_IV_NEXT5]] = add i64 [[LSR_IV4]], 16 ; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV_NEXT]], [[END:%.*]] ; CHECK-NEXT: br i1 [[EC]], label [[LOOP_2_PH:%.*]], label [[LOOP_1_HEADER]] ; CHECK: loop.2.ph: