Skip to content

Commit aca53f4

Browse files
fhahnfhahn
committed
[VPlan] Skip masked interleave groups in narrowInterleaveGroups.
8d29d09 exposed a crash due to incorrectly trying to handle masked interleave recipes. For now, the current code does not support masked interleave recipes. Bail out for them.
1 parent 6ceefbe commit aca53f4

File tree

2 files changed

+173
-1
lines changed

2 files changed

+173
-1
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4051,7 +4051,7 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
40514051
static std::optional<ElementCount> isConsecutiveInterleaveGroup(
40524052
VPInterleaveRecipe *InterleaveR, ArrayRef<ElementCount> VFs,
40534053
VPTypeAnalysis &TypeInfo, const TargetTransformInfo &TTI) {
4054-
if (!InterleaveR)
4054+
if (!InterleaveR || InterleaveR->getMask())
40554055
return std::nullopt;
40564056

40574057
Type *GroupElementTy = nullptr;

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,3 +210,175 @@ loop:
210210
exit:
211211
ret void
212212
}
213+
214+
define void @test_masked_interleave_group(i32 %N, ptr %mask, ptr %src, ptr %dst) {
215+
; IC1-LABEL: define void @test_masked_interleave_group(
216+
; IC1-SAME: i32 [[N:%.*]], ptr [[MASK:%.*]], ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
217+
; IC1-NEXT: [[ENTRY:.*:]]
218+
; IC1-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
219+
; IC1-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
220+
; IC1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
221+
; IC1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
222+
; IC1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 8)
223+
; IC1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
224+
; IC1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
225+
; IC1: [[VECTOR_MEMCHECK]]:
226+
; IC1-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64
227+
; IC1-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 4
228+
; IC1-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 16
229+
; IC1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
230+
; IC1-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP4]], 1
231+
; IC1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[TMP7]]
232+
; IC1-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
233+
; IC1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
234+
; IC1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[MASK]], [[SCEVGEP]]
235+
; IC1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
236+
; IC1-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
237+
; IC1-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
238+
; IC1-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
239+
; IC1-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
240+
; IC1-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
241+
; IC1: [[VECTOR_PH]]:
242+
; IC1-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
243+
; IC1-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
244+
; IC1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP9]]
245+
; IC1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
246+
; IC1-NEXT: [[TMP10:%.*]] = trunc i64 [[N_VEC]] to i32
247+
; IC1-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
248+
; IC1-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
249+
; IC1-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 16
250+
; IC1-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
251+
; IC1-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[N_VEC]]
252+
; IC1-NEXT: br label %[[VECTOR_BODY:.*]]
253+
; IC1: [[VECTOR_BODY]]:
254+
; IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
255+
; IC1-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
256+
; IC1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]
257+
; IC1-NEXT: [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
258+
; IC1-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX6]]
259+
; IC1-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX]]
260+
; IC1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[NEXT_GEP8]], align 1, !alias.scope [[META6:![0-9]+]]
261+
; IC1-NEXT: [[TMP16:%.*]] = icmp eq <vscale x 4 x i8> [[WIDE_LOAD]], zeroinitializer
262+
; IC1-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])
263+
; IC1-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x float> @llvm.masked.load.nxv16f32.p0(ptr align 4 [[NEXT_GEP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x float> poison), !alias.scope [[META9:![0-9]+]]
264+
; IC1-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave4.nxv16f32(<vscale x 16 x float> [[WIDE_MASKED_VEC]])
265+
; IC1-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
266+
; IC1-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
267+
; IC1-NEXT: [[TMP19:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 2
268+
; IC1-NEXT: [[TMP20:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 3
269+
; IC1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> [[TMP17]], <vscale x 4 x float> [[TMP18]], <vscale x 4 x float> [[TMP19]], <vscale x 4 x float> [[TMP20]])
270+
; IC1-NEXT: [[INTERLEAVED_MASK9:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])
271+
; IC1-NEXT: call void @llvm.masked.store.nxv16f32.p0(<vscale x 16 x float> [[INTERLEAVED_VEC]], ptr align 4 [[NEXT_GEP]], <vscale x 16 x i1> [[INTERLEAVED_MASK9]]), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
272+
; IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
273+
; IC1-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
274+
; IC1-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
275+
; IC1: [[MIDDLE_BLOCK]]:
276+
; IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
277+
; IC1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
278+
; IC1: [[SCALAR_PH]]:
279+
;
280+
; CHECK-LABEL: define void @test_masked_interleave_group(
281+
; CHECK-SAME: i32 [[N:%.*]], ptr [[MASK:%.*]], ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
282+
; CHECK-NEXT: [[ENTRY:.*:]]
283+
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
284+
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
285+
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
286+
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
287+
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 8)
288+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
289+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
290+
; CHECK: [[VECTOR_MEMCHECK]]:
291+
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[N]] to i64
292+
; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 4
293+
; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 16
294+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
295+
; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP4]], 1
296+
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[TMP7]]
297+
; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP6]]
298+
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
299+
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[MASK]], [[SCEVGEP]]
300+
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
301+
; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]
302+
; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
303+
; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
304+
; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
305+
; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
306+
; CHECK: [[VECTOR_PH]]:
307+
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
308+
; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
309+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP9]]
310+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
311+
; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[N_VEC]] to i32
312+
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
313+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
314+
; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 16
315+
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP13]]
316+
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[N_VEC]]
317+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
318+
; CHECK: [[VECTOR_BODY]]:
319+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
320+
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
321+
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX]]
322+
; CHECK-NEXT: [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
323+
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX6]]
324+
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[MASK]], i64 [[INDEX]]
325+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[NEXT_GEP8]], align 1, !alias.scope [[META6:![0-9]+]]
326+
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq <vscale x 4 x i8> [[WIDE_LOAD]], zeroinitializer
327+
; CHECK-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])
328+
; CHECK-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x float> @llvm.masked.load.nxv16f32.p0(ptr align 4 [[NEXT_GEP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x float> poison), !alias.scope [[META9:![0-9]+]]
329+
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave4.nxv16f32(<vscale x 16 x float> [[WIDE_MASKED_VEC]])
330+
; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
331+
; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
332+
; CHECK-NEXT: [[TMP19:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 2
333+
; CHECK-NEXT: [[TMP20:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 3
334+
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> [[TMP17]], <vscale x 4 x float> [[TMP18]], <vscale x 4 x float> [[TMP19]], <vscale x 4 x float> [[TMP20]])
335+
; CHECK-NEXT: [[INTERLEAVED_MASK9:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i1> [[TMP16]])
336+
; CHECK-NEXT: call void @llvm.masked.store.nxv16f32.p0(<vscale x 16 x float> [[INTERLEAVED_VEC]], ptr align 4 [[NEXT_GEP]], <vscale x 16 x i1> [[INTERLEAVED_MASK9]]), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
337+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
338+
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
339+
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
340+
; CHECK: [[MIDDLE_BLOCK]]:
341+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
342+
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
343+
; CHECK: [[SCALAR_PH]]:
344+
;
345+
entry:
346+
br label %loop.header
347+
348+
loop.header:
349+
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
350+
%dst.iv = phi ptr [ %dst, %entry ], [ %dst.iv.next, %loop.latch ]
351+
%src.iv = phi ptr [ %src, %entry ], [ %src.iv.next, %loop.latch ]
352+
%mask.iv = phi ptr [ %mask, %entry ], [ %mask.iv.next, %loop.latch ]
353+
%mask.iv.next = getelementptr i8, ptr %mask.iv, i64 1
354+
%mask.val = load i8, ptr %mask.iv, align 1
355+
%should.copy = icmp eq i8 %mask.val, 0
356+
br i1 %should.copy, label %then, label %loop.latch
357+
358+
then:
359+
%elem0 = load float, ptr %src.iv, align 4
360+
store float %elem0, ptr %dst.iv, align 4
361+
%src.1.ptr = getelementptr i8, ptr %src.iv, i64 4
362+
%s1 = load float, ptr %src.1.ptr, align 4
363+
%dst.1.ptr = getelementptr i8, ptr %dst.iv, i64 4
364+
store float %s1, ptr %dst.1.ptr, align 4
365+
%src.2.ptr = getelementptr i8, ptr %src.iv, i64 8
366+
%s2 = load float, ptr %src.2.ptr, align 4
367+
%dst.2.ptr = getelementptr i8, ptr %dst.iv, i64 8
368+
store float %s2, ptr %dst.2.ptr, align 4
369+
%src.3.ptr = getelementptr i8, ptr %src.iv, i64 12
370+
%s3 = load float, ptr %src.3.ptr, align 4
371+
%dst.3.ptr = getelementptr i8, ptr %dst.iv, i64 12
372+
store float %s3, ptr %dst.3.ptr, align 4
373+
br label %loop.latch
374+
375+
loop.latch:
376+
%iv.next = add i32 %iv, 1
377+
%src.iv.next = getelementptr i8, ptr %src.iv, i64 16
378+
%dst.iv.next = getelementptr i8, ptr %dst.iv, i64 16
379+
%ec = icmp eq i32 %iv, %N
380+
br i1 %ec, label %exit, label %loop.header
381+
382+
exit:
383+
ret void
384+
}

0 commit comments

Comments
 (0)