Skip to content

Commit 09325ad

Browse files
artagnonartagnon
committed
LAA: revert aggressively retrying with RT checks
1 parent afb657e commit 09325ad

8 files changed

+266
-335
lines changed

‎llvm/lib/Analysis/LoopAccessAnalysis.cpp

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2033,8 +2033,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
20332033
std::optional<uint64_t> CommonStride =
20342034
StrideA == StrideB ? std::make_optional(StrideA) : std::nullopt;
20352035
if (isa<SCEVCouldNotCompute>(Dist)) {
2036+
// TODO: Relax requirement that there is a common stride to retry with
2037+
// non-constant distance dependencies.
2038+
FoundNonConstantDistanceDependence |= CommonStride.has_value();
20362039
LLVM_DEBUG(dbgs() << "LAA: Dependence because of uncomputable distance.\n");
2037-
FoundNonConstantDistanceDependence = true;
20382040
return Dependence::Unknown;
20392041
}
20402042

@@ -2088,7 +2090,11 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
20882090

20892091
if (IsTrueDataDependence && EnableForwardingConflictDetection) {
20902092
if (!ConstDist) {
2091-
FoundNonConstantDistanceDependence = true;
2093+
// TODO: FoundNonConstantDistanceDependence is used as a necessary
2094+
// condition to consider retrying with runtime checks. Historically, we
2095+
// did not set it when strides were different but there is no inherent
2096+
// reason to.
2097+
FoundNonConstantDistanceDependence |= CommonStride.has_value();
20922098
return Dependence::Unknown;
20932099
}
20942100
if (couldPreventStoreLoadForward(
@@ -2106,12 +2112,20 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
21062112
int64_t MinDistance = SE.getSignedRangeMin(Dist).getSExtValue();
21072113
// Below we only handle strictly positive distances.
21082114
if (MinDistance <= 0) {
2109-
FoundNonConstantDistanceDependence = true;
2115+
FoundNonConstantDistanceDependence |= CommonStride.has_value();
21102116
return Dependence::Unknown;
21112117
}
21122118

21132119
if (!ConstDist)
2114-
FoundNonConstantDistanceDependence = true;
2120+
// Previously this case would be treated as Unknown, possibly setting
2121+
// FoundNonConstantDistanceDependence to force re-trying with runtime
2122+
// checks. Until the TODO below is addressed, set it here to preserve
2123+
// original behavior w.r.t. re-trying with runtime checks.
2124+
// TODO: FoundNonConstantDistanceDependence is used as a necessary
2125+
// condition to consider retrying with runtime checks. Historically, we
2126+
// did not set it when strides were different but there is no inherent
2127+
// reason to.
2128+
FoundNonConstantDistanceDependence |= CommonStride.has_value();
21152129

21162130
if (!CommonStride)
21172131
return Dependence::Unknown;

‎llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -109,21 +109,15 @@ exit:
109109
define void @unknown_dep_safe_with_rtchecks(ptr %A) {
110110
; CHECK-LABEL: 'unknown_dep_safe_with_rtchecks'
111111
; CHECK-NEXT: loop:
112-
; CHECK-NEXT: Memory dependences are safe with run-time checks
112+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
113+
; CHECK-NEXT: Unknown data dependence.
113114
; CHECK-NEXT: Dependences:
115+
; CHECK-NEXT: Unknown:
116+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
117+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
118+
; CHECK-EMPTY:
114119
; CHECK-NEXT: Run-time memory checks:
115-
; CHECK-NEXT: Check 0:
116-
; CHECK-NEXT: Comparing group ([[GRP1:0x[0-9a-f]+]]):
117-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.510, i64 %iv.mul.2
118-
; CHECK-NEXT: Against group ([[GRP2:0x[0-9a-f]+]]):
119-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
120120
; CHECK-NEXT: Grouped accesses:
121-
; CHECK-NEXT: Group [[GRP1]]:
122-
; CHECK-NEXT: (Low: (2040 + %A)<nuw> High: (4084 + %A))
123-
; CHECK-NEXT: Member: {(2040 + %A)<nuw>,+,8}<nuw><%loop>
124-
; CHECK-NEXT: Group [[GRP2]]:
125-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
126-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
127121
; CHECK-EMPTY:
128122
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
129123
; CHECK-NEXT: SCEV assumptions:

‎llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll

Lines changed: 36 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,15 @@ declare void @llvm.assume(i1)
88
define void @different_non_constant_strides_known_backward(ptr %A) {
99
; CHECK-LABEL: 'different_non_constant_strides_known_backward'
1010
; CHECK-NEXT: loop:
11-
; CHECK-NEXT: Memory dependences are safe with run-time checks
11+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
12+
; CHECK-NEXT: Unknown data dependence.
1213
; CHECK-NEXT: Dependences:
14+
; CHECK-NEXT: Unknown:
15+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
16+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
17+
; CHECK-EMPTY:
1318
; CHECK-NEXT: Run-time memory checks:
14-
; CHECK-NEXT: Check 0:
15-
; CHECK-NEXT: Comparing group ([[GRP1:0x[0-9a-f]+]]):
16-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A, i64 %iv.mul.2
17-
; CHECK-NEXT: Against group ([[GRP2:0x[0-9a-f]+]]):
18-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
1919
; CHECK-NEXT: Grouped accesses:
20-
; CHECK-NEXT: Group [[GRP1]]:
21-
; CHECK-NEXT: (Low: %A High: (2044 + %A))
22-
; CHECK-NEXT: Member: {%A,+,8}<nuw><%loop>
23-
; CHECK-NEXT: Group [[GRP2]]:
24-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
25-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
2620
; CHECK-EMPTY:
2721
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
2822
; CHECK-NEXT: SCEV assumptions:
@@ -51,21 +45,15 @@ exit:
5145
define void @different_non_constant_strides_known_backward_distance_larger_than_trip_count(ptr %A) {
5246
; CHECK-LABEL: 'different_non_constant_strides_known_backward_distance_larger_than_trip_count'
5347
; CHECK-NEXT: loop:
54-
; CHECK-NEXT: Memory dependences are safe with run-time checks
48+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
49+
; CHECK-NEXT: Unknown data dependence.
5550
; CHECK-NEXT: Dependences:
51+
; CHECK-NEXT: Unknown:
52+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
53+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
54+
; CHECK-EMPTY:
5655
; CHECK-NEXT: Run-time memory checks:
57-
; CHECK-NEXT: Check 0:
58-
; CHECK-NEXT: Comparing group ([[GRP3:0x[0-9a-f]+]]):
59-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.1024, i64 %iv.mul.2
60-
; CHECK-NEXT: Against group ([[GRP4:0x[0-9a-f]+]]):
61-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
6256
; CHECK-NEXT: Grouped accesses:
63-
; CHECK-NEXT: Group [[GRP3]]:
64-
; CHECK-NEXT: (Low: (1024 + %A)<nuw> High: (3068 + %A))
65-
; CHECK-NEXT: Member: {(1024 + %A)<nuw>,+,8}<nuw><%loop>
66-
; CHECK-NEXT: Group [[GRP4]]:
67-
; CHECK-NEXT: (Low: %A High: (1024 + %A)<nuw>)
68-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
6957
; CHECK-EMPTY:
7058
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
7159
; CHECK-NEXT: SCEV assumptions:
@@ -95,21 +83,15 @@ exit:
9583
define void @different_non_constant_strides_known_backward_min_distance_16(ptr %A) {
9684
; CHECK-LABEL: 'different_non_constant_strides_known_backward_min_distance_16'
9785
; CHECK-NEXT: loop:
98-
; CHECK-NEXT: Memory dependences are safe with run-time checks
86+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
87+
; CHECK-NEXT: Unknown data dependence.
9988
; CHECK-NEXT: Dependences:
89+
; CHECK-NEXT: Unknown:
90+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
91+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
92+
; CHECK-EMPTY:
10093
; CHECK-NEXT: Run-time memory checks:
101-
; CHECK-NEXT: Check 0:
102-
; CHECK-NEXT: Comparing group ([[GRP5:0x[0-9a-f]+]]):
103-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.16, i64 %iv.mul.2
104-
; CHECK-NEXT: Against group ([[GRP6:0x[0-9a-f]+]]):
105-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
10694
; CHECK-NEXT: Grouped accesses:
107-
; CHECK-NEXT: Group [[GRP5]]:
108-
; CHECK-NEXT: (Low: (16 + %A)<nuw> High: (2060 + %A))
109-
; CHECK-NEXT: Member: {(16 + %A)<nuw>,+,8}<nuw><%loop>
110-
; CHECK-NEXT: Group [[GRP6]]:
111-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
112-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
11395
; CHECK-EMPTY:
11496
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
11597
; CHECK-NEXT: SCEV assumptions:
@@ -139,21 +121,15 @@ exit:
139121
define void @different_non_constant_strides_known_backward_min_distance_15(ptr %A) {
140122
; CHECK-LABEL: 'different_non_constant_strides_known_backward_min_distance_15'
141123
; CHECK-NEXT: loop:
142-
; CHECK-NEXT: Memory dependences are safe with run-time checks
124+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
125+
; CHECK-NEXT: Unknown data dependence.
143126
; CHECK-NEXT: Dependences:
127+
; CHECK-NEXT: Unknown:
128+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
129+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
130+
; CHECK-EMPTY:
144131
; CHECK-NEXT: Run-time memory checks:
145-
; CHECK-NEXT: Check 0:
146-
; CHECK-NEXT: Comparing group ([[GRP7:0x[0-9a-f]+]]):
147-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.15, i64 %iv.mul.2
148-
; CHECK-NEXT: Against group ([[GRP8:0x[0-9a-f]+]]):
149-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
150132
; CHECK-NEXT: Grouped accesses:
151-
; CHECK-NEXT: Group [[GRP7]]:
152-
; CHECK-NEXT: (Low: (15 + %A)<nuw> High: (2059 + %A))
153-
; CHECK-NEXT: Member: {(15 + %A)<nuw>,+,8}<nuw><%loop>
154-
; CHECK-NEXT: Group [[GRP8]]:
155-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
156-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
157133
; CHECK-EMPTY:
158134
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
159135
; CHECK-NEXT: SCEV assumptions:
@@ -183,21 +159,15 @@ exit:
183159
define void @different_non_constant_strides_known_backward_min_distance_8(ptr %A) {
184160
; CHECK-LABEL: 'different_non_constant_strides_known_backward_min_distance_8'
185161
; CHECK-NEXT: loop:
186-
; CHECK-NEXT: Memory dependences are safe with run-time checks
162+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
163+
; CHECK-NEXT: Unknown data dependence.
187164
; CHECK-NEXT: Dependences:
165+
; CHECK-NEXT: Unknown:
166+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
167+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
168+
; CHECK-EMPTY:
188169
; CHECK-NEXT: Run-time memory checks:
189-
; CHECK-NEXT: Check 0:
190-
; CHECK-NEXT: Comparing group ([[GRP9:0x[0-9a-f]+]]):
191-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.8, i64 %iv.mul.2
192-
; CHECK-NEXT: Against group ([[GRP10:0x[0-9a-f]+]]):
193-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
194170
; CHECK-NEXT: Grouped accesses:
195-
; CHECK-NEXT: Group [[GRP9]]:
196-
; CHECK-NEXT: (Low: (8 + %A)<nuw> High: (2052 + %A))
197-
; CHECK-NEXT: Member: {(8 + %A)<nuw>,+,8}<nuw><%loop>
198-
; CHECK-NEXT: Group [[GRP10]]:
199-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
200-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
201171
; CHECK-EMPTY:
202172
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
203173
; CHECK-NEXT: SCEV assumptions:
@@ -227,21 +197,15 @@ exit:
227197
define void @different_non_constant_strides_known_backward_min_distance_3(ptr %A) {
228198
; CHECK-LABEL: 'different_non_constant_strides_known_backward_min_distance_3'
229199
; CHECK-NEXT: loop:
230-
; CHECK-NEXT: Memory dependences are safe with run-time checks
200+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
201+
; CHECK-NEXT: Unknown data dependence.
231202
; CHECK-NEXT: Dependences:
203+
; CHECK-NEXT: Unknown:
204+
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
205+
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
206+
; CHECK-EMPTY:
232207
; CHECK-NEXT: Run-time memory checks:
233-
; CHECK-NEXT: Check 0:
234-
; CHECK-NEXT: Comparing group ([[GRP11:0x[0-9a-f]+]]):
235-
; CHECK-NEXT: %gep.mul.2 = getelementptr inbounds i32, ptr %A.3, i64 %iv.mul.2
236-
; CHECK-NEXT: Against group ([[GRP12:0x[0-9a-f]+]]):
237-
; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %A, i64 %iv
238208
; CHECK-NEXT: Grouped accesses:
239-
; CHECK-NEXT: Group [[GRP11]]:
240-
; CHECK-NEXT: (Low: (3 + %A)<nuw> High: (2047 + %A))
241-
; CHECK-NEXT: Member: {(3 + %A)<nuw>,+,8}<nuw><%loop>
242-
; CHECK-NEXT: Group [[GRP12]]:
243-
; CHECK-NEXT: (Low: %A High: (1024 + %A))
244-
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop>
245209
; CHECK-EMPTY:
246210
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
247211
; CHECK-NEXT: SCEV assumptions:

‎llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,15 @@ exit:
4040
define void @test_distance_positive_backwards(ptr %A) {
4141
; CHECK-LABEL: 'test_distance_positive_backwards'
4242
; CHECK-NEXT: loop:
43-
; CHECK-NEXT: Memory dependences are safe with run-time checks
43+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
44+
; CHECK-NEXT: Unknown data dependence.
4445
; CHECK-NEXT: Dependences:
46+
; CHECK-NEXT: Unknown:
47+
; CHECK-NEXT: %l = load i8, ptr %gep.A, align 1 ->
48+
; CHECK-NEXT: store i32 %ext, ptr %gep.A.400, align 4
49+
; CHECK-EMPTY:
4550
; CHECK-NEXT: Run-time memory checks:
46-
; CHECK-NEXT: Check 0:
47-
; CHECK-NEXT: Comparing group ([[GRP1:0x[0-9a-f]+]]):
48-
; CHECK-NEXT: %gep.A.400 = getelementptr inbounds i32, ptr %A.1, i64 %iv
49-
; CHECK-NEXT: Against group ([[GRP2:0x[0-9a-f]+]]):
50-
; CHECK-NEXT: %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
5151
; CHECK-NEXT: Grouped accesses:
52-
; CHECK-NEXT: Group [[GRP1]]:
53-
; CHECK-NEXT: (Low: (1 + %A)<nuw> High: (401 + %A))
54-
; CHECK-NEXT: Member: {(1 + %A)<nuw>,+,4}<nuw><%loop>
55-
; CHECK-NEXT: Group [[GRP2]]:
56-
; CHECK-NEXT: (Low: %A High: (100 + %A))
57-
; CHECK-NEXT: Member: {%A,+,1}<nuw><%loop>
5852
; CHECK-EMPTY:
5953
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
6054
; CHECK-NEXT: SCEV assumptions:
@@ -83,21 +77,15 @@ exit:
8377
define void @test_distance_positive_via_assume(ptr %A, i64 %off) {
8478
; CHECK-LABEL: 'test_distance_positive_via_assume'
8579
; CHECK-NEXT: loop:
86-
; CHECK-NEXT: Memory dependences are safe with run-time checks
80+
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
81+
; CHECK-NEXT: Unknown data dependence.
8782
; CHECK-NEXT: Dependences:
83+
; CHECK-NEXT: Unknown:
84+
; CHECK-NEXT: %l = load i8, ptr %gep.A, align 1 ->
85+
; CHECK-NEXT: store i32 %ext, ptr %gep.A.400, align 4
86+
; CHECK-EMPTY:
8887
; CHECK-NEXT: Run-time memory checks:
89-
; CHECK-NEXT: Check 0:
90-
; CHECK-NEXT: Comparing group ([[GRP3:0x[0-9a-f]+]]):
91-
; CHECK-NEXT: %gep.A.400 = getelementptr inbounds i32, ptr %A.off, i64 %iv
92-
; CHECK-NEXT: Against group ([[GRP4:0x[0-9a-f]+]]):
93-
; CHECK-NEXT: %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
9488
; CHECK-NEXT: Grouped accesses:
95-
; CHECK-NEXT: Group [[GRP3]]:
96-
; CHECK-NEXT: (Low: (%off + %A) High: (400 + %off + %A))
97-
; CHECK-NEXT: Member: {(%off + %A),+,4}<nw><%loop>
98-
; CHECK-NEXT: Group [[GRP4]]:
99-
; CHECK-NEXT: (Low: %A High: (100 + %A))
100-
; CHECK-NEXT: Member: {%A,+,1}<nuw><%loop>
10189
; CHECK-EMPTY:
10290
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
10391
; CHECK-NEXT: SCEV assumptions:

0 commit comments

Comments
 (0)