From 9d0f05af57e79ef573fd5506e082d831ba31dd81 Mon Sep 17 00:00:00 2001 From: David Green Date: Mon, 10 Jun 2024 16:06:00 +0100 Subject: [PATCH] [AArch64] Push mul into extend operands In a similar way to how we push vector adds into extends, this pushed 'mul(zext,zext)' into 'zext(mul(zext,zext))' if the extend can be done in two or more steps. https://alive2.llvm.org/ce/z/WjU7Kr --- .../Target/AArch64/AArch64ISelLowering.cpp | 80 +- llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll | 108 +- llvm/test/CodeGen/AArch64/addp-shuffle.ll | 32 +- llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 1122 ++++++++--------- llvm/test/CodeGen/AArch64/neon-extmul.ll | 106 +- llvm/test/CodeGen/AArch64/vecreduce-add.ll | 216 ++-- 6 files changed, 754 insertions(+), 910 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 48bf648b00522..87d737d7ffe61 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -17720,6 +17720,47 @@ static SDValue performMulVectorCmpZeroCombine(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(AArch64ISD::NVCAST, DL, VT, CM); } +// Transform vector add(zext i8 to i32, zext i8 to i32) +// into sext(add(zext(i8 to i16), zext(i8 to i16)) to i32) +// This allows extra uses of saddl/uaddl at the lower vector widths, and less +// extends. +static SDValue performVectorExtCombine(SDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 || + (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && + N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND) || + (N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && + N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND) || + N->getOperand(0).getOperand(0).getValueType() != + N->getOperand(1).getOperand(0).getValueType()) + return SDValue(); + + if (N->getOpcode() == ISD::MUL && + N->getOperand(0).getOpcode() != N->getOperand(1).getOpcode()) + return SDValue(); + + SDValue N0 = N->getOperand(0).getOperand(0); + SDValue N1 = N->getOperand(1).getOperand(0); + EVT InVT = N0.getValueType(); + + EVT S1 = InVT.getScalarType(); + EVT S2 = VT.getScalarType(); + if ((S2 == MVT::i32 && S1 == MVT::i8) || + (S2 == MVT::i64 && (S1 == MVT::i8 || S1 == MVT::i16))) { + SDLoc DL(N); + EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), + S2.getHalfSizedIntegerVT(*DAG.getContext()), + VT.getVectorElementCount()); + SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0); + SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1); + SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1); + return DAG.getNode(N->getOpcode() == ISD::MUL ? N->getOperand(0).getOpcode() + : (unsigned)ISD::SIGN_EXTEND, + DL, VT, NewOp); + } + return SDValue(); +} + static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -17728,6 +17769,8 @@ static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, return Ext; if (SDValue Ext = performMulVectorCmpZeroCombine(N, DAG)) return Ext; + if (SDValue Ext = performVectorExtCombine(N, DAG)) + return Ext; if (DCI.isBeforeLegalizeOps()) return SDValue(); @@ -19604,41 +19647,6 @@ static SDValue foldADCToCINC(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond); } -// Transform vector add(zext i8 to i32, zext i8 to i32) -// into sext(add(zext(i8 to i16), zext(i8 to i16)) to i32) -// This allows extra uses of saddl/uaddl at the lower vector widths, and less -// extends. -static SDValue performVectorAddSubExtCombine(SDNode *N, SelectionDAG &DAG) { - EVT VT = N->getValueType(0); - if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 || - (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && - N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND) || - (N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && - N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND) || - N->getOperand(0).getOperand(0).getValueType() != - N->getOperand(1).getOperand(0).getValueType()) - return SDValue(); - - SDValue N0 = N->getOperand(0).getOperand(0); - SDValue N1 = N->getOperand(1).getOperand(0); - EVT InVT = N0.getValueType(); - - EVT S1 = InVT.getScalarType(); - EVT S2 = VT.getScalarType(); - if ((S2 == MVT::i32 && S1 == MVT::i8) || - (S2 == MVT::i64 && (S1 == MVT::i8 || S1 == MVT::i16))) { - SDLoc DL(N); - EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), - S2.getHalfSizedIntegerVT(*DAG.getContext()), - VT.getVectorElementCount()); - SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0); - SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1); - SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1); - return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewOp); - } - return SDValue(); -} - static SDValue performBuildVectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { @@ -20260,7 +20268,7 @@ static SDValue performAddSubCombine(SDNode *N, return Val; if (SDValue Val = performNegCSelCombine(N, DCI.DAG)) return Val; - if (SDValue Val = performVectorAddSubExtCombine(N, DCI.DAG)) + if (SDValue Val = performVectorExtCombine(N, DCI.DAG)) return Val; if (SDValue Val = performAddCombineForShiftedOperands(N, DCI.DAG)) return Val; diff --git a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll index 4c0d1efb99498..410c2d9021d6d 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll @@ -28,14 +28,12 @@ entry: define <16 x i32> @mul_i32(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-LABEL: mul_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll v4.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll2 v5.8h, v0.16b, #0 -; CHECK-SD-NEXT: ushll2 v6.8h, v1.16b, #0 -; CHECK-SD-NEXT: umull v0.4s, v2.4h, v4.4h -; CHECK-SD-NEXT: umull2 v1.4s, v2.8h, v4.8h -; CHECK-SD-NEXT: umull2 v3.4s, v5.8h, v6.8h -; CHECK-SD-NEXT: umull v2.4s, v5.4h, v6.4h +; CHECK-SD-NEXT: umull v2.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: umull2 v4.8h, v0.16b, v1.16b +; CHECK-SD-NEXT: ushll v0.4s, v2.4h, #0 +; CHECK-SD-NEXT: ushll2 v3.4s, v4.8h, #0 +; CHECK-SD-NEXT: ushll2 v1.4s, v2.8h, #0 +; CHECK-SD-NEXT: ushll v2.4s, v4.4h, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: mul_i32: @@ -59,26 +57,20 @@ entry: define <16 x i64> @mul_i64(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-LABEL: mul_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-SD-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-SD-NEXT: ushll v4.4s, v2.4h, #0 -; CHECK-SD-NEXT: ushll v5.4s, v0.4h, #0 -; CHECK-SD-NEXT: ushll v6.4s, v3.4h, #0 +; CHECK-SD-NEXT: umull v2.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: umull2 v0.8h, v0.16b, v1.16b +; CHECK-SD-NEXT: ushll v3.4s, v2.4h, #0 ; CHECK-SD-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-SD-NEXT: ushll v16.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll2 v7.4s, v3.8h, #0 -; CHECK-SD-NEXT: ushll2 v17.4s, v0.8h, #0 -; CHECK-SD-NEXT: ushll2 v18.4s, v1.8h, #0 -; CHECK-SD-NEXT: umull2 v1.2d, v4.4s, v6.4s -; CHECK-SD-NEXT: umull v0.2d, v4.2s, v6.2s -; CHECK-SD-NEXT: umull2 v3.2d, v2.4s, v7.4s -; CHECK-SD-NEXT: umull v2.2d, v2.2s, v7.2s -; CHECK-SD-NEXT: umull v4.2d, v5.2s, v16.2s -; CHECK-SD-NEXT: umull2 v7.2d, v17.4s, v18.4s -; CHECK-SD-NEXT: umull2 v5.2d, v5.4s, v16.4s -; CHECK-SD-NEXT: umull v6.2d, v17.2s, v18.2s +; CHECK-SD-NEXT: ushll v5.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll2 v6.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll2 v1.2d, v3.4s, #0 +; CHECK-SD-NEXT: ushll v0.2d, v3.2s, #0 +; CHECK-SD-NEXT: ushll2 v3.2d, v2.4s, #0 +; CHECK-SD-NEXT: ushll v2.2d, v2.2s, #0 +; CHECK-SD-NEXT: ushll v4.2d, v5.2s, #0 +; CHECK-SD-NEXT: ushll2 v7.2d, v6.4s, #0 +; CHECK-SD-NEXT: ushll2 v5.2d, v5.4s, #0 +; CHECK-SD-NEXT: ushll v6.2d, v6.2s, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: mul_i64: @@ -139,17 +131,12 @@ entry: define <16 x i32> @mla_i32(<16 x i8> %a, <16 x i8> %b, <16 x i32> %c) { ; CHECK-SD-LABEL: mla_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v6.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll v7.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-SD-NEXT: umlal v2.4s, v6.4h, v7.4h -; CHECK-SD-NEXT: umlal2 v3.4s, v6.8h, v7.8h -; CHECK-SD-NEXT: umlal2 v5.4s, v0.8h, v1.8h -; CHECK-SD-NEXT: umlal v4.4s, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b -; CHECK-SD-NEXT: mov v2.16b, v4.16b +; CHECK-SD-NEXT: umull2 v7.8h, v0.16b, v1.16b +; CHECK-SD-NEXT: umull v6.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: uaddw2 v5.4s, v5.4s, v7.8h +; CHECK-SD-NEXT: uaddw v0.4s, v2.4s, v6.4h +; CHECK-SD-NEXT: uaddw2 v1.4s, v3.4s, v6.8h +; CHECK-SD-NEXT: uaddw v2.4s, v4.4s, v7.4h ; CHECK-SD-NEXT: mov v3.16b, v5.16b ; CHECK-SD-NEXT: ret ; @@ -179,35 +166,22 @@ entry: define <16 x i64> @mla_i64(<16 x i8> %a, <16 x i8> %b, <16 x i64> %c) { ; CHECK-SD-LABEL: mla_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: mov v17.16b, v7.16b -; CHECK-SD-NEXT: mov v16.16b, v6.16b -; CHECK-SD-NEXT: ushll v6.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-SD-NEXT: ushll v7.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-SD-NEXT: ushll v18.4s, v6.4h, #0 -; CHECK-SD-NEXT: ushll2 v21.4s, v6.8h, #0 -; CHECK-SD-NEXT: ushll v19.4s, v0.4h, #0 -; CHECK-SD-NEXT: ushll v20.4s, v7.4h, #0 -; CHECK-SD-NEXT: ushll v22.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll2 v23.4s, v7.8h, #0 -; CHECK-SD-NEXT: ldp q6, q7, [sp] -; CHECK-SD-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-SD-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-SD-NEXT: umlal2 v3.2d, v18.4s, v20.4s -; CHECK-SD-NEXT: umlal v2.2d, v18.2s, v20.2s -; CHECK-SD-NEXT: umlal v16.2d, v19.2s, v22.2s -; CHECK-SD-NEXT: umlal2 v5.2d, v21.4s, v23.4s -; CHECK-SD-NEXT: umlal v4.2d, v21.2s, v23.2s -; CHECK-SD-NEXT: umlal2 v17.2d, v19.4s, v22.4s -; CHECK-SD-NEXT: umlal2 v7.2d, v0.4s, v1.4s -; CHECK-SD-NEXT: umlal v6.2d, v0.2s, v1.2s -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b -; CHECK-SD-NEXT: mov v2.16b, v4.16b -; CHECK-SD-NEXT: mov v3.16b, v5.16b -; CHECK-SD-NEXT: mov v4.16b, v16.16b -; CHECK-SD-NEXT: mov v5.16b, v17.16b +; CHECK-SD-NEXT: umull v16.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: umull2 v0.8h, v0.16b, v1.16b +; CHECK-SD-NEXT: ldp q20, q21, [sp] +; CHECK-SD-NEXT: ushll v17.4s, v16.4h, #0 +; CHECK-SD-NEXT: ushll2 v16.4s, v16.8h, #0 +; CHECK-SD-NEXT: ushll2 v19.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v18.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddw2 v1.2d, v3.2d, v17.4s +; CHECK-SD-NEXT: uaddw v0.2d, v2.2d, v17.2s +; CHECK-SD-NEXT: uaddw2 v3.2d, v5.2d, v16.4s +; CHECK-SD-NEXT: uaddw v2.2d, v4.2d, v16.2s +; CHECK-SD-NEXT: uaddw2 v16.2d, v21.2d, v19.4s +; CHECK-SD-NEXT: uaddw v4.2d, v6.2d, v18.2s +; CHECK-SD-NEXT: uaddw2 v5.2d, v7.2d, v18.4s +; CHECK-SD-NEXT: uaddw v6.2d, v20.2d, v19.2s +; CHECK-SD-NEXT: mov v7.16b, v16.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: mla_i64: diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll index 7cc5041d59f49..fb96d11acc275 100644 --- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll @@ -136,15 +136,13 @@ define <4 x double> @deinterleave_shuffle_v8f64(<8 x double> %a) { define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: udot: ; CHECK: // %bb.0: -; CHECK-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-NEXT: ushll v4.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: umull2 v5.4s, v3.8h, v4.8h -; CHECK-NEXT: umull v3.4s, v3.4h, v4.4h -; CHECK-NEXT: umull2 v4.4s, v1.8h, v2.8h -; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h -; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s +; CHECK-NEXT: umull v3.8h, v1.8b, v2.8b +; CHECK-NEXT: umull2 v1.8h, v1.16b, v2.16b +; CHECK-NEXT: ushll2 v2.4s, v3.8h, #0 +; CHECK-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-NEXT: ushll2 v4.4s, v1.8h, #0 +; CHECK-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s ; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s ; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s @@ -165,15 +163,13 @@ define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { define <4 x i32> @sdot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: sdot: ; CHECK: // %bb.0: -; CHECK-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-NEXT: sshll v4.8h, v2.8b, #0 -; CHECK-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: smull2 v5.4s, v3.8h, v4.8h -; CHECK-NEXT: smull v3.4s, v3.4h, v4.4h -; CHECK-NEXT: smull2 v4.4s, v1.8h, v2.8h -; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h -; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s +; CHECK-NEXT: smull v3.8h, v1.8b, v2.8b +; CHECK-NEXT: smull2 v1.8h, v1.16b, v2.16b +; CHECK-NEXT: sshll2 v2.4s, v3.8h, #0 +; CHECK-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-NEXT: sshll2 v4.4s, v1.8h, #0 +; CHECK-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s ; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s ; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll index 40b8a47f92aa7..33245a2b120ea 100644 --- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll +++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll @@ -132,13 +132,12 @@ define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] -; CHECK-NEXT: movi v3.2d, #0000000000000000 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: umull2 v2.4s, v1.8h, v0.8h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: umlal v3.4s, v1.4h, v0.4h -; CHECK-NEXT: addv s0, v3.4s +; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0 +; CHECK-NEXT: mov v1.s[0], v2.s[0] +; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h +; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 ; CHECK-NEXT: ret @@ -176,13 +175,12 @@ define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] -; CHECK-NEXT: movi v3.2d, #0000000000000000 -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: smlal v3.4s, v1.4h, v0.4h -; CHECK-NEXT: addv s0, v3.4s +; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0 +; CHECK-NEXT: mov v1.s[0], v2.s[0] +; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h +; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 ; CHECK-NEXT: ret @@ -200,19 +198,17 @@ entry: define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) { ; CHECK-LABEL: test_sdot_v5i8_double: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: movi v6.2d, #0000000000000000 -; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h -; CHECK-NEXT: smull2 v7.4s, v2.8h, v3.8h -; CHECK-NEXT: mov v6.s[0], v4.s[0] -; CHECK-NEXT: mov v5.s[0], v7.s[0] -; CHECK-NEXT: smlal v6.4s, v0.4h, v1.4h -; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h -; CHECK-NEXT: add v0.4s, v6.4s, v5.4s +; CHECK-NEXT: smull v2.8h, v2.8b, v3.8b +; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v4.4s, v0.8h, #0 +; CHECK-NEXT: sshll2 v5.4s, v2.8h, #0 +; CHECK-NEXT: mov v3.s[0], v4.s[0] +; CHECK-NEXT: mov v1.s[0], v5.s[0] +; CHECK-NEXT: saddw v0.4s, v3.4s, v0.4h +; CHECK-NEXT: saddw v1.4s, v1.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret @@ -998,27 +994,21 @@ entry: define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q0, [x0] -; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: ushll v6.8h, v2.8b, #0 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 -; CHECK-NEXT: ushll v7.8h, v5.8b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: umull v3.4s, v4.4h, v3.4h -; CHECK-NEXT: movi v4.2d, #0000000000000000 -; CHECK-NEXT: umull2 v16.4s, v7.8h, v6.8h -; CHECK-NEXT: umull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v4.s[0], v3.s[0] -; CHECK-NEXT: ushll2 v3.8h, v5.16b, #0 -; CHECK-NEXT: umlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: umlal v6.4s, v1.4h, v0.4h -; CHECK-NEXT: umlal v4.4s, v3.4h, v2.4h -; CHECK-NEXT: umlal2 v16.4s, v3.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v4.4s -; CHECK-NEXT: add v0.4s, v0.4s, v16.4s +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: ushll v3.4s, v4.4h, #0 +; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1063,27 +1053,21 @@ entry: define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_sdot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q0, [x0] -; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: sshll v6.8h, v2.8b, #0 -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 -; CHECK-NEXT: sshll v7.8h, v5.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h -; CHECK-NEXT: movi v4.2d, #0000000000000000 -; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h -; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v4.s[0], v3.s[0] -; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0 -; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h -; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h -; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v4.4s -; CHECK-NEXT: add v0.4s, v0.4s, v16.4s +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: sshll v3.4s, v4.4h, #0 +; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1105,222 +1089,210 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ldr b1, [sp, #16] -; CHECK-NEXT: ldr b0, [sp, #80] -; CHECK-NEXT: add x11, sp, #24 -; CHECK-NEXT: ldr b3, [sp, #216] -; CHECK-NEXT: add x10, sp, #88 -; CHECK-NEXT: ldr b2, [sp, #280] -; CHECK-NEXT: ld1 { v1.b }[1], [x11] -; CHECK-NEXT: add x11, sp, #224 -; CHECK-NEXT: ldr b4, [sp, #152] -; CHECK-NEXT: ldr b6, [sp, #480] -; CHECK-NEXT: ld1 { v0.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #288 -; CHECK-NEXT: add x12, sp, #160 -; CHECK-NEXT: ld1 { v3.b }[1], [x11] -; CHECK-NEXT: add x11, sp, #488 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] -; CHECK-NEXT: ld1 { v4.b }[1], [x12] -; CHECK-NEXT: ld1 { v6.b }[1], [x11] -; CHECK-NEXT: add x11, sp, #32 -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: add x8, sp, #104 -; CHECK-NEXT: ld1 { v1.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #232 -; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: ldr b0, [sp, #280] +; CHECK-NEXT: add x8, sp, #288 +; CHECK-NEXT: ldr b1, [sp, #80] +; CHECK-NEXT: ldr b2, [sp, #152] ; CHECK-NEXT: add x9, sp, #296 -; CHECK-NEXT: ld1 { v3.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #168 +; CHECK-NEXT: ldr b4, [sp, #216] +; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #88 +; CHECK-NEXT: add x10, sp, #320 +; CHECK-NEXT: ld1 { v1.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #160 +; CHECK-NEXT: add x12, sp, #192 +; CHECK-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: add x11, sp, #328 +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #96 +; CHECK-NEXT: ldr b5, [sp, #16] +; CHECK-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #168 +; CHECK-NEXT: ldr b6, [sp, #680] ; CHECK-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-NEXT: ld1 { v4.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #40 -; CHECK-NEXT: ld1 { v1.b }[3], [x11] +; CHECK-NEXT: add x9, sp, #104 +; CHECK-NEXT: ldr b7, [sp, #480] ; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #304 -; CHECK-NEXT: add x10, sp, #112 +; CHECK-NEXT: add x8, sp, #312 +; CHECK-NEXT: fmov s3, w0 +; CHECK-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #176 +; CHECK-NEXT: ldr b19, [sp, #552] +; CHECK-NEXT: ld1 { v2.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #112 +; CHECK-NEXT: ldr b22, [sp, #744] +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: add x8, sp, #336 +; CHECK-NEXT: mov v3.b[1], w1 +; CHECK-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #184 +; CHECK-NEXT: ldr b23, [sp, #544] +; CHECK-NEXT: ld1 { v2.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #224 +; CHECK-NEXT: ldr b20, [sp, #352] +; CHECK-NEXT: ld1 { v0.b }[5], [x10] +; CHECK-NEXT: ld1 { v4.b }[1], [x9] +; CHECK-NEXT: add x10, sp, #120 +; CHECK-NEXT: ld1 { v1.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #128 +; CHECK-NEXT: add x9, sp, #136 +; CHECK-NEXT: ld1 { v2.b }[5], [x12] +; CHECK-NEXT: add x12, sp, #232 +; CHECK-NEXT: mov v3.b[2], w2 +; CHECK-NEXT: ld1 { v0.b }[6], [x11] +; CHECK-NEXT: ld1 { v4.b }[2], [x12] ; CHECK-NEXT: add x11, sp, #240 -; CHECK-NEXT: add x13, sp, #56 -; CHECK-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #48 -; CHECK-NEXT: ld1 { v3.b }[3], [x11] -; CHECK-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-NEXT: ld1 { v0.b }[4], [x10] -; CHECK-NEXT: add x15, sp, #312 -; CHECK-NEXT: add x12, sp, #120 +; CHECK-NEXT: add x12, sp, #24 +; CHECK-NEXT: ld1 { v1.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #200 +; CHECK-NEXT: ld1 { v5.b }[1], [x12] +; CHECK-NEXT: ld1 { v2.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #256 +; CHECK-NEXT: ld1 { v0.b }[7], [x8] +; CHECK-NEXT: ld1 { v4.b }[3], [x11] +; CHECK-NEXT: add x8, sp, #688 +; CHECK-NEXT: ld1 { v6.b }[1], [x8] +; CHECK-NEXT: add x11, sp, #32 ; CHECK-NEXT: add x8, sp, #248 -; CHECK-NEXT: add x11, sp, #64 -; CHECK-NEXT: ld1 { v2.b }[4], [x15] -; CHECK-NEXT: ld1 { v3.b }[4], [x8] -; CHECK-NEXT: add x15, sp, #320 -; CHECK-NEXT: ld1 { v1.b }[5], [x13] -; CHECK-NEXT: ld1 { v0.b }[5], [x12] -; CHECK-NEXT: ldr b18, [sp, #552] -; CHECK-NEXT: add x14, sp, #128 -; CHECK-NEXT: add x16, sp, #256 -; CHECK-NEXT: ldr b16, [sp, #352] -; CHECK-NEXT: ld1 { v2.b }[5], [x15] -; CHECK-NEXT: add x15, sp, #176 -; CHECK-NEXT: ld1 { v3.b }[5], [x16] -; CHECK-NEXT: ld1 { v1.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #560 -; CHECK-NEXT: ld1 { v0.b }[6], [x14] -; CHECK-NEXT: add x16, sp, #360 -; CHECK-NEXT: ld1 { v4.b }[3], [x15] -; CHECK-NEXT: ld1 { v18.b }[1], [x11] +; CHECK-NEXT: ld1 { v5.b }[2], [x11] +; CHECK-NEXT: ld1 { v1.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #40 +; CHECK-NEXT: ld1 { v4.b }[4], [x8] +; CHECK-NEXT: add x8, sp, #696 +; CHECK-NEXT: ldr b21, [sp, #616] +; CHECK-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-NEXT: add x8, sp, #208 +; CHECK-NEXT: smull v23.8h, v23.8b, v22.8b +; CHECK-NEXT: ld1 { v5.b }[3], [x9] +; CHECK-NEXT: ld1 { v2.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #704 +; CHECK-NEXT: ld1 { v4.b }[5], [x10] +; CHECK-NEXT: add x9, sp, #48 +; CHECK-NEXT: add x10, sp, #56 +; CHECK-NEXT: ld1 { v6.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #264 +; CHECK-NEXT: ldr b22, [sp, #416] +; CHECK-NEXT: ld1 { v5.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #488 +; CHECK-NEXT: mov v3.b[3], w3 +; CHECK-NEXT: ld1 { v4.b }[6], [x8] +; CHECK-NEXT: add x8, sp, #712 +; CHECK-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-NEXT: ld1 { v6.b }[4], [x8] +; CHECK-NEXT: add x9, sp, #720 +; CHECK-NEXT: add x8, sp, #64 +; CHECK-NEXT: ld1 { v5.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #496 +; CHECK-NEXT: add x11, sp, #576 +; CHECK-NEXT: ld1 { v7.b }[2], [x10] ; CHECK-NEXT: add x10, sp, #72 -; CHECK-NEXT: ld1 { v16.b }[1], [x16] -; CHECK-NEXT: add x9, sp, #136 -; CHECK-NEXT: add x14, sp, #184 -; CHECK-NEXT: ld1 { v1.b }[7], [x10] +; CHECK-NEXT: mov v3.b[4], w4 +; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #272 +; CHECK-NEXT: ldr b16, [sp, #344] +; CHECK-NEXT: ld1 { v5.b }[6], [x8] +; CHECK-NEXT: add x8, sp, #728 +; CHECK-NEXT: ld1 { v4.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #504 +; CHECK-NEXT: ldr b17, [sp, #144] +; CHECK-NEXT: sshll v23.4s, v23.4h, #0 +; CHECK-NEXT: ld1 { v6.b }[6], [x8] +; CHECK-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-NEXT: add x8, sp, #736 +; CHECK-NEXT: add x9, sp, #512 +; CHECK-NEXT: ld1 { v5.b }[7], [x10] ; CHECK-NEXT: add x10, sp, #568 -; CHECK-NEXT: ld1 { v0.b }[7], [x9] -; CHECK-NEXT: ld1 { v4.b }[4], [x14] +; CHECK-NEXT: mov v3.b[5], w5 +; CHECK-NEXT: smull v16.8h, v17.8b, v16.8b +; CHECK-NEXT: movi v17.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v6.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #560 +; CHECK-NEXT: ld1 { v7.b }[4], [x9] +; CHECK-NEXT: ld1 { v19.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #360 +; CHECK-NEXT: add x9, sp, #424 +; CHECK-NEXT: ld1 { v20.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #624 +; CHECK-NEXT: ld1 { v22.b }[1], [x9] +; CHECK-NEXT: ld1 { v21.b }[1], [x8] ; CHECK-NEXT: add x9, sp, #368 -; CHECK-NEXT: ld1 { v18.b }[2], [x10] -; CHECK-NEXT: add x11, sp, #496 -; CHECK-NEXT: ld1 { v16.b }[2], [x9] -; CHECK-NEXT: fmov s5, w0 -; CHECK-NEXT: add x9, sp, #192 -; CHECK-NEXT: ld1 { v6.b }[2], [x11] -; CHECK-NEXT: add x10, sp, #576 -; CHECK-NEXT: ld1 { v4.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #376 -; CHECK-NEXT: ld1 { v18.b }[3], [x10] -; CHECK-NEXT: add x11, sp, #504 -; CHECK-NEXT: ld1 { v16.b }[3], [x9] -; CHECK-NEXT: mov v5.b[1], w1 -; CHECK-NEXT: ldr b7, [sp, #144] -; CHECK-NEXT: ldr b17, [sp, #344] -; CHECK-NEXT: add x9, sp, #200 -; CHECK-NEXT: ld1 { v6.b }[3], [x11] +; CHECK-NEXT: add x8, sp, #520 +; CHECK-NEXT: ld1 { v19.b }[2], [x10] +; CHECK-NEXT: add x10, sp, #432 +; CHECK-NEXT: ld1 { v7.b }[5], [x8] +; CHECK-NEXT: ld1 { v20.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #632 +; CHECK-NEXT: ld1 { v22.b }[2], [x10] +; CHECK-NEXT: ld1 { v21.b }[2], [x9] +; CHECK-NEXT: add x8, sp, #376 +; CHECK-NEXT: add x9, sp, #440 +; CHECK-NEXT: ld1 { v19.b }[3], [x11] ; CHECK-NEXT: add x10, sp, #584 -; CHECK-NEXT: ld1 { v4.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #384 -; CHECK-NEXT: ld1 { v18.b }[4], [x10] -; CHECK-NEXT: sshll v7.8h, v7.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: add x11, sp, #512 -; CHECK-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-NEXT: ld1 { v6.b }[4], [x11] ; CHECK-NEXT: add x11, sp, #592 -; CHECK-NEXT: mov v5.b[2], w2 -; CHECK-NEXT: add x10, sp, #392 -; CHECK-NEXT: ldr b19, [sp, #680] -; CHECK-NEXT: ld1 { v18.b }[5], [x11] -; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h -; CHECK-NEXT: ldr b17, [sp, #416] -; CHECK-NEXT: ld1 { v16.b }[5], [x10] -; CHECK-NEXT: add x10, sp, #688 -; CHECK-NEXT: add x12, sp, #328 -; CHECK-NEXT: add x9, sp, #424 -; CHECK-NEXT: ld1 { v19.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #600 -; CHECK-NEXT: ldr b20, [sp, #616] -; CHECK-NEXT: ld1 { v2.b }[6], [x12] -; CHECK-NEXT: ld1 { v17.b }[1], [x9] -; CHECK-NEXT: add x11, sp, #400 -; CHECK-NEXT: ld1 { v18.b }[6], [x10] -; CHECK-NEXT: add x12, sp, #624 -; CHECK-NEXT: mov v5.b[3], w3 -; CHECK-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #696 -; CHECK-NEXT: ld1 { v20.b }[1], [x12] -; CHECK-NEXT: add x9, sp, #432 -; CHECK-NEXT: ld1 { v19.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #608 -; CHECK-NEXT: ld1 { v17.b }[2], [x9] -; CHECK-NEXT: add x10, sp, #408 -; CHECK-NEXT: ld1 { v18.b }[7], [x11] -; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: ld1 { v16.b }[7], [x10] -; CHECK-NEXT: ld1 { v20.b }[2], [x11] -; CHECK-NEXT: mov v5.b[4], w4 -; CHECK-NEXT: add x10, sp, #704 -; CHECK-NEXT: add x12, sp, #440 -; CHECK-NEXT: ld1 { v19.b }[3], [x10] +; CHECK-NEXT: ld1 { v20.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #640 +; CHECK-NEXT: ld1 { v22.b }[3], [x9] +; CHECK-NEXT: ld1 { v21.b }[3], [x8] +; CHECK-NEXT: add x9, sp, #384 +; CHECK-NEXT: add x8, sp, #528 +; CHECK-NEXT: ld1 { v19.b }[4], [x10] ; CHECK-NEXT: add x10, sp, #448 -; CHECK-NEXT: ld1 { v17.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #640 -; CHECK-NEXT: sshll v21.8h, v16.8b, #0 -; CHECK-NEXT: ld1 { v20.b }[3], [x12] -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: add x11, sp, #712 -; CHECK-NEXT: mov v5.b[5], w5 -; CHECK-NEXT: ld1 { v19.b }[4], [x11] -; CHECK-NEXT: add x9, sp, #520 -; CHECK-NEXT: ld1 { v17.b }[4], [x10] -; CHECK-NEXT: add x10, sp, #648 -; CHECK-NEXT: ldr b22, [sp, #544] -; CHECK-NEXT: ld1 { v20.b }[4], [x10] -; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h -; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h -; CHECK-NEXT: ldr b21, [sp, #744] -; CHECK-NEXT: add x11, sp, #720 -; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: ld1 { v7.b }[6], [x8] +; CHECK-NEXT: ld1 { v20.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #648 +; CHECK-NEXT: ld1 { v22.b }[4], [x10] +; CHECK-NEXT: ld1 { v21.b }[4], [x9] +; CHECK-NEXT: add x8, sp, #392 ; CHECK-NEXT: add x9, sp, #456 ; CHECK-NEXT: ld1 { v19.b }[5], [x11] -; CHECK-NEXT: mov v5.b[6], w6 -; CHECK-NEXT: ld1 { v17.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #656 -; CHECK-NEXT: sshll v22.8h, v22.8b, #0 -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: ld1 { v20.b }[5], [x9] -; CHECK-NEXT: add x10, sp, #528 -; CHECK-NEXT: add x11, sp, #728 -; CHECK-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-NEXT: mov v3.b[6], w6 +; CHECK-NEXT: add x10, sp, #600 +; CHECK-NEXT: ld1 { v20.b }[5], [x8] +; CHECK-NEXT: add x8, sp, #656 +; CHECK-NEXT: ld1 { v22.b }[5], [x9] +; CHECK-NEXT: ld1 { v21.b }[5], [x8] +; CHECK-NEXT: add x9, sp, #400 +; CHECK-NEXT: add x8, sp, #536 +; CHECK-NEXT: ld1 { v19.b }[6], [x10] ; CHECK-NEXT: add x10, sp, #464 -; CHECK-NEXT: ld1 { v19.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #664 -; CHECK-NEXT: ld1 { v17.b }[6], [x10] -; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h -; CHECK-NEXT: movi v22.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v20.b }[6], [x11] -; CHECK-NEXT: mov v5.b[7], w7 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: add x10, sp, #736 -; CHECK-NEXT: add x11, sp, #208 -; CHECK-NEXT: add x13, sp, #264 -; CHECK-NEXT: ld1 { v6.b }[7], [x9] -; CHECK-NEXT: ld1 { v19.b }[7], [x10] -; CHECK-NEXT: ld1 { v4.b }[7], [x11] +; CHECK-NEXT: ld1 { v7.b }[7], [x8] +; CHECK-NEXT: ld1 { v20.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #664 +; CHECK-NEXT: ld1 { v22.b }[6], [x10] +; CHECK-NEXT: ld1 { v21.b }[6], [x9] +; CHECK-NEXT: add x8, sp, #408 +; CHECK-NEXT: mov v3.b[7], w7 +; CHECK-NEXT: sshll v18.4s, v16.4h, #0 +; CHECK-NEXT: movi v16.2d, #0000000000000000 +; CHECK-NEXT: add x11, sp, #608 +; CHECK-NEXT: ld1 { v20.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #672 ; CHECK-NEXT: add x9, sp, #472 -; CHECK-NEXT: add x10, sp, #672 -; CHECK-NEXT: ld1 { v3.b }[6], [x13] -; CHECK-NEXT: ld1 { v17.b }[7], [x9] -; CHECK-NEXT: ld1 { v20.b }[7], [x10] -; CHECK-NEXT: add x8, sp, #336 -; CHECK-NEXT: mov v22.s[0], v21.s[0] -; CHECK-NEXT: movi v21.2d, #0000000000000000 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: sshll v19.8h, v19.8b, #0 -; CHECK-NEXT: ld1 { v2.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #272 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: ld1 { v3.b }[7], [x8] -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v20.8h, v20.8b, #0 -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h -; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h -; CHECK-NEXT: mov v21.s[0], v7.s[0] -; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h -; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h -; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h -; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h -; CHECK-NEXT: add v5.4s, v18.4s, v22.4s -; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v21.4s -; CHECK-NEXT: add v2.4s, v5.4s, v16.4s -; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: ld1 { v19.b }[7], [x11] +; CHECK-NEXT: ld1 { v21.b }[7], [x8] +; CHECK-NEXT: ld1 { v22.b }[7], [x9] +; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: smull v2.8h, v5.8b, v4.8b +; CHECK-NEXT: mov v17.s[0], v18.s[0] +; CHECK-NEXT: smull v3.8h, v7.8b, v6.8b +; CHECK-NEXT: mov v16.s[0], v23.s[0] +; CHECK-NEXT: smull v4.8h, v20.8b, v19.8b +; CHECK-NEXT: smull v5.8h, v22.8b, v21.8b +; CHECK-NEXT: saddl v7.4s, v1.4h, v0.4h +; CHECK-NEXT: saddl2 v0.4s, v1.8h, v0.8h +; CHECK-NEXT: saddw v6.4s, v17.4s, v2.4h +; CHECK-NEXT: saddl v1.4s, v4.4h, v3.4h +; CHECK-NEXT: saddl2 v3.4s, v4.8h, v3.8h +; CHECK-NEXT: saddw v4.4s, v16.4s, v5.4h +; CHECK-NEXT: saddw2 v0.4s, v0.4s, v2.8h +; CHECK-NEXT: add v6.4s, v7.4s, v6.4s +; CHECK-NEXT: add v1.4s, v1.4s, v4.4s +; CHECK-NEXT: saddw2 v2.4s, v3.4s, v5.8h +; CHECK-NEXT: add v0.4s, v6.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -1586,32 +1558,24 @@ define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] -; CHECK-NEXT: ldp q2, q4, [x0] -; CHECK-NEXT: ldp q3, q6, [x1] -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v5.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: ushll2 v16.8h, v4.16b, #0 -; CHECK-NEXT: ushll v7.8h, v3.8b, #0 -; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0 -; CHECK-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-NEXT: umull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ushll2 v19.8h, v6.16b, #0 -; CHECK-NEXT: ushll v6.8h, v6.8b, #0 -; CHECK-NEXT: umull2 v17.4s, v7.8h, v5.8h -; CHECK-NEXT: umull2 v18.4s, v3.8h, v2.8h -; CHECK-NEXT: mov v1.s[0], v0.s[0] -; CHECK-NEXT: umull v0.4s, v3.4h, v2.4h -; CHECK-NEXT: umlal2 v18.4s, v19.8h, v16.8h -; CHECK-NEXT: umlal2 v17.4s, v6.8h, v4.8h -; CHECK-NEXT: umlal v1.4s, v7.4h, v5.4h -; CHECK-NEXT: umlal v0.4s, v19.4h, v16.4h -; CHECK-NEXT: add v2.4s, v17.4s, v18.4s -; CHECK-NEXT: umlal v1.4s, v6.4h, v4.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q4, q2, [x1] +; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-NEXT: ldp q3, q1, [x0] +; CHECK-NEXT: umull v6.8h, v2.8b, v1.8b +; CHECK-NEXT: umull2 v1.8h, v2.16b, v1.16b +; CHECK-NEXT: umull v2.8h, v4.8b, v3.8b +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: umull2 v3.8h, v4.16b, v3.16b +; CHECK-NEXT: mov v5.s[0], v0.s[0] +; CHECK-NEXT: uaddl2 v4.4s, v2.8h, v6.8h +; CHECK-NEXT: uaddl2 v0.4s, v3.8h, v1.8h +; CHECK-NEXT: uaddl v1.4s, v3.4h, v1.4h +; CHECK-NEXT: add v0.4s, v4.4s, v0.4s +; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: uaddw v2.4s, v2.4s, v6.4h ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1662,32 +1626,24 @@ define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] -; CHECK-NEXT: ldp q2, q4, [x0] -; CHECK-NEXT: ldp q3, q6, [x1] -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v5.8h, v2.8b, #0 -; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: sshll2 v16.8h, v4.16b, #0 -; CHECK-NEXT: sshll v7.8h, v3.8b, #0 -; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: sshll2 v19.8h, v6.16b, #0 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h -; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h -; CHECK-NEXT: mov v1.s[0], v0.s[0] -; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h -; CHECK-NEXT: smlal2 v18.4s, v19.8h, v16.8h -; CHECK-NEXT: smlal2 v17.4s, v6.8h, v4.8h -; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h -; CHECK-NEXT: smlal v0.4s, v19.4h, v16.4h -; CHECK-NEXT: add v2.4s, v17.4s, v18.4s -; CHECK-NEXT: smlal v1.4s, v6.4h, v4.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q4, q2, [x1] +; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-NEXT: ldp q3, q1, [x0] +; CHECK-NEXT: smull v6.8h, v2.8b, v1.8b +; CHECK-NEXT: smull2 v1.8h, v2.16b, v1.16b +; CHECK-NEXT: smull v2.8h, v4.8b, v3.8b +; CHECK-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-NEXT: smull2 v3.8h, v4.16b, v3.16b +; CHECK-NEXT: mov v5.s[0], v0.s[0] +; CHECK-NEXT: saddl2 v4.4s, v2.8h, v6.8h +; CHECK-NEXT: saddl2 v0.4s, v3.8h, v1.8h +; CHECK-NEXT: saddl v1.4s, v3.4h, v1.4h +; CHECK-NEXT: add v0.4s, v4.4s, v0.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: saddw v2.4s, v2.4s, v6.4h ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1709,291 +1665,275 @@ define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: fmov s4, w0 -; CHECK-NEXT: ldr b0, [sp, #80] -; CHECK-NEXT: add x8, sp, #88 -; CHECK-NEXT: ldr b1, [sp, #144] -; CHECK-NEXT: add x10, sp, #152 -; CHECK-NEXT: ldr b6, [sp, #16] +; CHECK-NEXT: ldr b0, [sp, #344] +; CHECK-NEXT: add x8, sp, #352 +; CHECK-NEXT: ldr b2, [sp, #80] +; CHECK-NEXT: add x9, sp, #88 +; CHECK-NEXT: ldr b3, [sp, #216] +; CHECK-NEXT: add x10, sp, #232 ; CHECK-NEXT: ld1 { v0.b }[1], [x8] -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: ldr b2, [sp, #344] -; CHECK-NEXT: mov v4.b[1], w1 -; CHECK-NEXT: ld1 { v1.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #24 -; CHECK-NEXT: ld1 { v6.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #352 -; CHECK-NEXT: add x8, sp, #104 +; CHECK-NEXT: add x8, sp, #224 +; CHECK-NEXT: ld1 { v2.b }[1], [x9] +; CHECK-NEXT: add x9, sp, #360 +; CHECK-NEXT: ld1 { v3.b }[1], [x8] +; CHECK-NEXT: add x8, sp, #96 +; CHECK-NEXT: add x11, sp, #376 +; CHECK-NEXT: ldr b4, [sp, #408] +; CHECK-NEXT: add x12, sp, #384 ; CHECK-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-NEXT: add x9, sp, #160 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] -; CHECK-NEXT: ld1 { v1.b }[2], [x9] -; CHECK-NEXT: add x10, sp, #32 -; CHECK-NEXT: add x11, sp, #112 -; CHECK-NEXT: mov v4.b[2], w2 -; CHECK-NEXT: ld1 { v6.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #168 -; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: ldr b5, [sp, #216] -; CHECK-NEXT: add x13, sp, #224 -; CHECK-NEXT: ld1 { v1.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #40 -; CHECK-NEXT: add x12, sp, #120 -; CHECK-NEXT: ld1 { v6.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #176 -; CHECK-NEXT: ld1 { v5.b }[1], [x13] -; CHECK-NEXT: mov v4.b[3], w3 -; CHECK-NEXT: ld1 { v0.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #48 -; CHECK-NEXT: add x8, sp, #360 -; CHECK-NEXT: ld1 { v1.b }[4], [x10] -; CHECK-NEXT: add x13, sp, #56 -; CHECK-NEXT: ld1 { v6.b }[4], [x11] -; CHECK-NEXT: ldr b7, [sp, #280] +; CHECK-NEXT: add x9, sp, #368 ; CHECK-NEXT: ld1 { v2.b }[2], [x8] -; CHECK-NEXT: add x15, sp, #232 +; CHECK-NEXT: ld1 { v3.b }[2], [x10] +; CHECK-NEXT: add x8, sp, #104 +; CHECK-NEXT: add x14, sp, #248 +; CHECK-NEXT: add x10, sp, #392 +; CHECK-NEXT: ldr b5, [sp, #144] +; CHECK-NEXT: ldr b6, [sp, #280] +; CHECK-NEXT: ld1 { v0.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #240 +; CHECK-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-NEXT: ld1 { v3.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #112 +; CHECK-NEXT: add x8, sp, #400 +; CHECK-NEXT: add x13, sp, #128 +; CHECK-NEXT: ldr b17, [sp, #744] +; CHECK-NEXT: ldr b19, [sp, #480] +; CHECK-NEXT: ld1 { v0.b }[4], [x11] +; CHECK-NEXT: ld1 { v2.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #416 +; CHECK-NEXT: ld1 { v4.b }[1], [x9] +; CHECK-NEXT: ld1 { v3.b }[4], [x14] +; CHECK-NEXT: add x11, sp, #120 +; CHECK-NEXT: add x9, sp, #136 +; CHECK-NEXT: ldr b21, [sp, #936] +; CHECK-NEXT: ldr b22, [sp, #672] ; CHECK-NEXT: ld1 { v0.b }[5], [x12] -; CHECK-NEXT: add x14, sp, #184 -; CHECK-NEXT: mov v4.b[4], w4 -; CHECK-NEXT: ld1 { v5.b }[2], [x15] -; CHECK-NEXT: add x9, sp, #128 -; CHECK-NEXT: ld1 { v6.b }[5], [x13] -; CHECK-NEXT: add x13, sp, #288 -; CHECK-NEXT: add x10, sp, #368 -; CHECK-NEXT: ld1 { v7.b }[1], [x13] -; CHECK-NEXT: ld1 { v1.b }[5], [x14] -; CHECK-NEXT: ld1 { v2.b }[3], [x10] -; CHECK-NEXT: add x15, sp, #240 -; CHECK-NEXT: ld1 { v0.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #296 -; CHECK-NEXT: mov v4.b[5], w5 -; CHECK-NEXT: add x11, sp, #192 -; CHECK-NEXT: ld1 { v5.b }[3], [x15] -; CHECK-NEXT: ldr b3, [sp, #408] -; CHECK-NEXT: ld1 { v7.b }[2], [x9] -; CHECK-NEXT: add x12, sp, #64 -; CHECK-NEXT: add x13, sp, #376 -; CHECK-NEXT: ld1 { v1.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #416 -; CHECK-NEXT: ld1 { v6.b }[6], [x12] -; CHECK-NEXT: add x12, sp, #248 -; CHECK-NEXT: ld1 { v3.b }[1], [x11] -; CHECK-NEXT: mov v4.b[6], w6 -; CHECK-NEXT: ld1 { v2.b }[4], [x13] -; CHECK-NEXT: add x11, sp, #304 -; CHECK-NEXT: ld1 { v5.b }[4], [x12] -; CHECK-NEXT: ld1 { v7.b }[3], [x11] -; CHECK-NEXT: add x8, sp, #136 -; CHECK-NEXT: add x15, sp, #384 -; CHECK-NEXT: add x9, sp, #424 -; CHECK-NEXT: ld1 { v0.b }[7], [x8] -; CHECK-NEXT: ld1 { v3.b }[2], [x9] -; CHECK-NEXT: ld1 { v2.b }[5], [x15] -; CHECK-NEXT: add x8, sp, #312 -; CHECK-NEXT: mov v4.b[7], w7 -; CHECK-NEXT: add x9, sp, #256 -; CHECK-NEXT: add x10, sp, #200 -; CHECK-NEXT: ld1 { v7.b }[4], [x8] -; CHECK-NEXT: ld1 { v5.b }[5], [x9] -; CHECK-NEXT: add x14, sp, #72 -; CHECK-NEXT: ld1 { v1.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #432 -; CHECK-NEXT: add x8, sp, #392 -; CHECK-NEXT: ld1 { v6.b }[7], [x14] -; CHECK-NEXT: ld1 { v3.b }[3], [x10] -; CHECK-NEXT: ld1 { v2.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #320 -; CHECK-NEXT: add x9, sp, #264 -; CHECK-NEXT: sshll v21.8h, v4.8b, #0 -; CHECK-NEXT: ldr b4, [sp, #208] -; CHECK-NEXT: ld1 { v7.b }[5], [x8] -; CHECK-NEXT: ld1 { v5.b }[6], [x9] +; CHECK-NEXT: ld1 { v2.b }[5], [x11] +; CHECK-NEXT: add x11, sp, #424 +; CHECK-NEXT: add x12, sp, #256 +; CHECK-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #152 +; CHECK-NEXT: ld1 { v3.b }[5], [x12] +; CHECK-NEXT: ld1 { v5.b }[1], [x11] +; CHECK-NEXT: add x11, sp, #432 +; CHECK-NEXT: ld1 { v0.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #264 +; CHECK-NEXT: ld1 { v2.b }[6], [x13] +; CHECK-NEXT: ld1 { v4.b }[3], [x11] +; CHECK-NEXT: add x11, sp, #160 +; CHECK-NEXT: ldr b7, [sp, #472] +; CHECK-NEXT: ld1 { v3.b }[6], [x10] +; CHECK-NEXT: ld1 { v5.b }[2], [x11] ; CHECK-NEXT: add x10, sp, #440 -; CHECK-NEXT: add x8, sp, #400 -; CHECK-NEXT: sshll v16.8h, v6.8b, #0 -; CHECK-NEXT: sshll v6.8h, v4.8b, #0 -; CHECK-NEXT: ld1 { v3.b }[4], [x10] -; CHECK-NEXT: ld1 { v2.b }[7], [x8] +; CHECK-NEXT: ld1 { v0.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #288 +; CHECK-NEXT: add x11, sp, #168 +; CHECK-NEXT: ld1 { v6.b }[1], [x8] ; CHECK-NEXT: add x8, sp, #272 -; CHECK-NEXT: add x9, sp, #328 -; CHECK-NEXT: ldr b4, [sp, #608] -; CHECK-NEXT: ld1 { v7.b }[6], [x9] -; CHECK-NEXT: ld1 { v5.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #616 -; CHECK-NEXT: add x10, sp, #448 -; CHECK-NEXT: ld1 { v4.b }[1], [x8] -; CHECK-NEXT: ldr b18, [sp, #480] -; CHECK-NEXT: ld1 { v3.b }[5], [x10] -; CHECK-NEXT: add x9, sp, #336 -; CHECK-NEXT: ldr b17, [sp, #472] -; CHECK-NEXT: add x8, sp, #488 -; CHECK-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #624 -; CHECK-NEXT: ld1 { v18.b }[1], [x8] -; CHECK-NEXT: sshll v22.8h, v5.8b, #0 -; CHECK-NEXT: add x8, sp, #456 -; CHECK-NEXT: sshll v5.8h, v17.8b, #0 -; CHECK-NEXT: ld1 { v4.b }[2], [x9] -; CHECK-NEXT: ld1 { v3.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #496 -; CHECK-NEXT: sshll v17.8h, v7.8b, #0 -; CHECK-NEXT: add x10, sp, #632 -; CHECK-NEXT: ld1 { v18.b }[2], [x8] -; CHECK-NEXT: add x9, sp, #464 -; CHECK-NEXT: add x8, sp, #504 -; CHECK-NEXT: smull v19.4s, v6.4h, v5.4h -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v4.b }[3], [x10] -; CHECK-NEXT: ld1 { v3.b }[7], [x9] -; CHECK-NEXT: smull v6.4s, v16.4h, v17.4h -; CHECK-NEXT: add x9, sp, #640 -; CHECK-NEXT: ld1 { v18.b }[3], [x8] -; CHECK-NEXT: smull2 v16.4s, v16.8h, v17.8h -; CHECK-NEXT: ldr b17, [sp, #672] -; CHECK-NEXT: ld1 { v4.b }[4], [x9] -; CHECK-NEXT: add x9, sp, #680 -; CHECK-NEXT: ldr b20, [sp, #544] -; CHECK-NEXT: mov v5.s[0], v19.s[0] -; CHECK-NEXT: add x8, sp, #512 -; CHECK-NEXT: ld1 { v17.b }[1], [x9] -; CHECK-NEXT: add x11, sp, #552 -; CHECK-NEXT: add x10, sp, #648 -; CHECK-NEXT: ld1 { v18.b }[4], [x8] -; CHECK-NEXT: ld1 { v20.b }[1], [x11] -; CHECK-NEXT: ld1 { v4.b }[5], [x10] -; CHECK-NEXT: add x10, sp, #688 -; CHECK-NEXT: add x9, sp, #520 -; CHECK-NEXT: ld1 { v17.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #560 -; CHECK-NEXT: smull2 v7.4s, v21.8h, v22.8h -; CHECK-NEXT: ld1 { v18.b }[5], [x9] -; CHECK-NEXT: smlal v5.4s, v21.4h, v22.4h -; CHECK-NEXT: ld1 { v20.b }[2], [x10] -; CHECK-NEXT: ldr b21, [sp, #736] -; CHECK-NEXT: ldr b22, [sp, #1000] -; CHECK-NEXT: add x8, sp, #656 -; CHECK-NEXT: add x9, sp, #696 -; CHECK-NEXT: add x11, sp, #568 -; CHECK-NEXT: ld1 { v4.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #528 -; CHECK-NEXT: ld1 { v17.b }[3], [x9] -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: sshll v24.8h, v22.8b, #0 -; CHECK-NEXT: ld1 { v18.b }[6], [x8] -; CHECK-NEXT: ld1 { v20.b }[3], [x11] -; CHECK-NEXT: add x10, sp, #704 -; CHECK-NEXT: ldr b23, [sp, #808] -; CHECK-NEXT: movi v19.2d, #0000000000000000 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: ld1 { v17.b }[4], [x10] -; CHECK-NEXT: add x10, sp, #576 -; CHECK-NEXT: ldr b22, [sp, #744] -; CHECK-NEXT: add x11, sp, #816 -; CHECK-NEXT: smull v24.4s, v21.4h, v24.4h -; CHECK-NEXT: ld1 { v18.b }[7], [x9] -; CHECK-NEXT: ld1 { v20.b }[4], [x10] +; CHECK-NEXT: ld1 { v4.b }[4], [x10] +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #296 +; CHECK-NEXT: ld1 { v5.b }[3], [x11] +; CHECK-NEXT: ld1 { v2.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #448 +; CHECK-NEXT: add x10, sp, #176 +; CHECK-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: ld1 { v5.b }[4], [x10] +; CHECK-NEXT: add x9, sp, #456 +; CHECK-NEXT: add x10, sp, #184 +; CHECK-NEXT: add x11, sp, #192 +; CHECK-NEXT: ldr b16, [sp, #208] +; CHECK-NEXT: add x12, sp, #784 +; CHECK-NEXT: ld1 { v6.b }[3], [x8] +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #312 +; CHECK-NEXT: ld1 { v5.b }[5], [x10] ; CHECK-NEXT: add x10, sp, #752 -; CHECK-NEXT: ld1 { v23.b }[1], [x11] -; CHECK-NEXT: add x9, sp, #712 +; CHECK-NEXT: smull v7.8h, v16.8b, v7.8b +; CHECK-NEXT: ld1 { v17.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #760 +; CHECK-NEXT: ldr b16, [sp, #16] +; CHECK-NEXT: ld1 { v6.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #320 +; CHECK-NEXT: ldr b18, [sp, #1000] +; CHECK-NEXT: ld1 { v5.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #768 +; CHECK-NEXT: ldr b20, [sp, #736] +; CHECK-NEXT: ld1 { v17.b }[2], [x10] +; CHECK-NEXT: add x10, sp, #680 +; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #488 ; CHECK-NEXT: ld1 { v22.b }[1], [x10] -; CHECK-NEXT: ld1 { v17.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #584 -; CHECK-NEXT: add x10, sp, #824 -; CHECK-NEXT: sshll v21.8h, v18.8b, #0 -; CHECK-NEXT: ld1 { v20.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #760 -; CHECK-NEXT: ldr b18, [sp, #936] -; CHECK-NEXT: ld1 { v23.b }[2], [x10] -; CHECK-NEXT: mov v19.s[0], v24.s[0] -; CHECK-NEXT: ldr b24, [sp, #872] -; CHECK-NEXT: ld1 { v22.b }[2], [x9] +; CHECK-NEXT: ld1 { v19.b }[1], [x9] ; CHECK-NEXT: add x9, sp, #944 -; CHECK-NEXT: add x11, sp, #880 -; CHECK-NEXT: add x10, sp, #768 -; CHECK-NEXT: ld1 { v18.b }[1], [x9] -; CHECK-NEXT: add x9, sp, #832 -; CHECK-NEXT: ld1 { v24.b }[1], [x11] -; CHECK-NEXT: ld1 { v23.b }[3], [x9] -; CHECK-NEXT: ld1 { v22.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #952 -; CHECK-NEXT: add x12, sp, #888 -; CHECK-NEXT: add x9, sp, #592 +; CHECK-NEXT: add x10, sp, #688 +; CHECK-NEXT: ld1 { v21.b }[1], [x9] +; CHECK-NEXT: add x9, sp, #496 +; CHECK-NEXT: ld1 { v17.b }[3], [x11] +; CHECK-NEXT: ld1 { v22.b }[2], [x10] ; CHECK-NEXT: add x11, sp, #776 -; CHECK-NEXT: ld1 { v18.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #840 -; CHECK-NEXT: ld1 { v24.b }[2], [x12] -; CHECK-NEXT: ld1 { v23.b }[4], [x10] -; CHECK-NEXT: ld1 { v22.b }[4], [x11] -; CHECK-NEXT: ld1 { v20.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #960 -; CHECK-NEXT: add x11, sp, #896 -; CHECK-NEXT: add x10, sp, #784 -; CHECK-NEXT: ld1 { v18.b }[3], [x9] -; CHECK-NEXT: add x9, sp, #848 -; CHECK-NEXT: ld1 { v24.b }[3], [x11] -; CHECK-NEXT: ld1 { v23.b }[5], [x9] -; CHECK-NEXT: ld1 { v22.b }[5], [x10] -; CHECK-NEXT: add x10, sp, #968 -; CHECK-NEXT: add x12, sp, #904 -; CHECK-NEXT: add x9, sp, #600 +; CHECK-NEXT: add x10, sp, #504 +; CHECK-NEXT: ld1 { v19.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #952 +; CHECK-NEXT: smull v20.8h, v20.8b, v18.8b +; CHECK-NEXT: ld1 { v21.b }[2], [x9] +; CHECK-NEXT: ld1 { v17.b }[4], [x11] +; CHECK-NEXT: add x11, sp, #696 +; CHECK-NEXT: add x9, sp, #24 +; CHECK-NEXT: ld1 { v22.b }[3], [x11] ; CHECK-NEXT: add x11, sp, #792 -; CHECK-NEXT: ld1 { v18.b }[4], [x10] -; CHECK-NEXT: add x10, sp, #856 -; CHECK-NEXT: ld1 { v24.b }[4], [x12] -; CHECK-NEXT: ld1 { v23.b }[6], [x10] -; CHECK-NEXT: ld1 { v22.b }[6], [x11] -; CHECK-NEXT: ld1 { v20.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #976 -; CHECK-NEXT: add x11, sp, #912 -; CHECK-NEXT: add x10, sp, #800 -; CHECK-NEXT: ld1 { v18.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #864 -; CHECK-NEXT: ld1 { v24.b }[5], [x11] -; CHECK-NEXT: ld1 { v23.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #720 -; CHECK-NEXT: ld1 { v22.b }[7], [x10] +; CHECK-NEXT: ld1 { v19.b }[3], [x10] +; CHECK-NEXT: add x10, sp, #960 +; CHECK-NEXT: ld1 { v16.b }[1], [x9] +; CHECK-NEXT: ld1 { v21.b }[3], [x10] +; CHECK-NEXT: add x9, sp, #512 +; CHECK-NEXT: ld1 { v17.b }[5], [x12] +; CHECK-NEXT: add x10, sp, #704 +; CHECK-NEXT: add x12, sp, #800 +; CHECK-NEXT: movi v18.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v19.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #968 +; CHECK-NEXT: ld1 { v22.b }[4], [x10] +; CHECK-NEXT: ld1 { v21.b }[4], [x9] +; CHECK-NEXT: add x10, sp, #520 +; CHECK-NEXT: ld1 { v17.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #712 +; CHECK-NEXT: add x9, sp, #32 +; CHECK-NEXT: sshll v23.4s, v20.4h, #0 +; CHECK-NEXT: ld1 { v19.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #976 +; CHECK-NEXT: ld1 { v22.b }[5], [x11] +; CHECK-NEXT: ld1 { v21.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #528 +; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: ld1 { v16.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #536 +; CHECK-NEXT: ld1 { v17.b }[7], [x12] +; CHECK-NEXT: ld1 { v19.b }[6], [x10] ; CHECK-NEXT: add x10, sp, #984 -; CHECK-NEXT: ld1 { v17.b }[6], [x9] +; CHECK-NEXT: ld1 { v22.b }[6], [x11] +; CHECK-NEXT: ld1 { v21.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #992 +; CHECK-NEXT: add x11, sp, #728 +; CHECK-NEXT: mov v1.b[1], w1 +; CHECK-NEXT: ldr b20, [sp, #872] +; CHECK-NEXT: mov v18.s[0], v23.s[0] +; CHECK-NEXT: ld1 { v19.b }[7], [x9] +; CHECK-NEXT: ld1 { v22.b }[7], [x11] +; CHECK-NEXT: add x9, sp, #328 +; CHECK-NEXT: ld1 { v21.b }[7], [x10] +; CHECK-NEXT: add x10, sp, #40 +; CHECK-NEXT: ldr b23, [sp, #608] +; CHECK-NEXT: ld1 { v16.b }[3], [x10] +; CHECK-NEXT: add x10, sp, #816 +; CHECK-NEXT: add x11, sp, #552 +; CHECK-NEXT: smull v17.8h, v19.8b, v17.8b +; CHECK-NEXT: ld1 { v6.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #880 +; CHECK-NEXT: smull v19.8h, v22.8b, v21.8b +; CHECK-NEXT: ldr b21, [sp, #808] +; CHECK-NEXT: ldr b22, [sp, #544] +; CHECK-NEXT: add x12, sp, #616 +; CHECK-NEXT: mov v1.b[2], w2 +; CHECK-NEXT: ld1 { v20.b }[1], [x9] +; CHECK-NEXT: ld1 { v21.b }[1], [x10] +; CHECK-NEXT: ld1 { v22.b }[1], [x11] +; CHECK-NEXT: ld1 { v23.b }[1], [x12] +; CHECK-NEXT: add x11, sp, #824 +; CHECK-NEXT: add x12, sp, #560 +; CHECK-NEXT: add x9, sp, #888 +; CHECK-NEXT: add x13, sp, #624 +; CHECK-NEXT: add x10, sp, #48 +; CHECK-NEXT: ld1 { v20.b }[2], [x9] +; CHECK-NEXT: ld1 { v21.b }[2], [x11] +; CHECK-NEXT: ld1 { v22.b }[2], [x12] +; CHECK-NEXT: ld1 { v23.b }[2], [x13] +; CHECK-NEXT: mov v1.b[3], w3 +; CHECK-NEXT: ld1 { v16.b }[4], [x10] +; CHECK-NEXT: add x10, sp, #832 +; CHECK-NEXT: add x11, sp, #568 +; CHECK-NEXT: add x9, sp, #896 +; CHECK-NEXT: add x12, sp, #632 +; CHECK-NEXT: ld1 { v21.b }[3], [x10] +; CHECK-NEXT: ld1 { v22.b }[3], [x11] +; CHECK-NEXT: ld1 { v20.b }[3], [x9] +; CHECK-NEXT: ld1 { v23.b }[3], [x12] +; CHECK-NEXT: add x11, sp, #840 +; CHECK-NEXT: add x12, sp, #576 +; CHECK-NEXT: mov v1.b[4], w4 +; CHECK-NEXT: add x9, sp, #904 +; CHECK-NEXT: add x13, sp, #640 +; CHECK-NEXT: ld1 { v21.b }[4], [x11] +; CHECK-NEXT: ld1 { v22.b }[4], [x12] +; CHECK-NEXT: add x10, sp, #56 +; CHECK-NEXT: ld1 { v20.b }[4], [x9] +; CHECK-NEXT: ld1 { v23.b }[4], [x13] +; CHECK-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #848 +; CHECK-NEXT: add x11, sp, #584 +; CHECK-NEXT: add x9, sp, #912 +; CHECK-NEXT: add x12, sp, #648 +; CHECK-NEXT: ld1 { v21.b }[5], [x10] +; CHECK-NEXT: ld1 { v22.b }[5], [x11] +; CHECK-NEXT: mov v1.b[5], w5 +; CHECK-NEXT: ld1 { v20.b }[5], [x9] +; CHECK-NEXT: ld1 { v23.b }[5], [x12] +; CHECK-NEXT: add x11, sp, #856 +; CHECK-NEXT: add x12, sp, #592 ; CHECK-NEXT: add x9, sp, #920 -; CHECK-NEXT: ld1 { v18.b }[6], [x10] -; CHECK-NEXT: ld1 { v24.b }[6], [x9] -; CHECK-NEXT: add x10, sp, #728 -; CHECK-NEXT: add x8, sp, #664 -; CHECK-NEXT: sshll v20.8h, v20.8b, #0 -; CHECK-NEXT: sshll v22.8h, v22.8b, #0 -; CHECK-NEXT: sshll v23.8h, v23.8b, #0 -; CHECK-NEXT: add x9, sp, #992 -; CHECK-NEXT: ld1 { v17.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #928 -; CHECK-NEXT: ld1 { v18.b }[7], [x9] +; CHECK-NEXT: add x13, sp, #656 +; CHECK-NEXT: ld1 { v21.b }[6], [x11] +; CHECK-NEXT: ld1 { v22.b }[6], [x12] +; CHECK-NEXT: add x10, sp, #64 +; CHECK-NEXT: ld1 { v20.b }[6], [x9] +; CHECK-NEXT: ld1 { v23.b }[6], [x13] +; CHECK-NEXT: mov v1.b[6], w6 +; CHECK-NEXT: ld1 { v16.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #864 +; CHECK-NEXT: add x11, sp, #600 +; CHECK-NEXT: add x9, sp, #928 +; CHECK-NEXT: add x12, sp, #664 +; CHECK-NEXT: ld1 { v21.b }[7], [x10] +; CHECK-NEXT: ld1 { v22.b }[7], [x11] +; CHECK-NEXT: add x8, sp, #464 +; CHECK-NEXT: ld1 { v20.b }[7], [x9] +; CHECK-NEXT: ld1 { v23.b }[7], [x12] ; CHECK-NEXT: ld1 { v4.b }[7], [x8] -; CHECK-NEXT: ld1 { v24.b }[7], [x10] -; CHECK-NEXT: smlal v19.4s, v21.4h, v22.4h -; CHECK-NEXT: smull2 v21.4s, v21.8h, v22.8h -; CHECK-NEXT: smull v22.4s, v20.4h, v23.4h -; CHECK-NEXT: smull2 v20.4s, v20.8h, v23.8h -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: sshll v23.8h, v24.8b, #0 -; CHECK-NEXT: smlal2 v16.4s, v1.8h, v3.8h -; CHECK-NEXT: smlal v6.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal2 v7.4s, v0.8h, v2.8h -; CHECK-NEXT: smlal v5.4s, v0.4h, v2.4h -; CHECK-NEXT: smlal2 v20.4s, v17.8h, v18.8h -; CHECK-NEXT: smlal v22.4s, v17.4h, v18.4h -; CHECK-NEXT: smlal2 v21.4s, v4.8h, v23.8h -; CHECK-NEXT: smlal v19.4s, v4.4h, v23.4h -; CHECK-NEXT: add v0.4s, v7.4s, v16.4s -; CHECK-NEXT: add v1.4s, v5.4s, v6.4s -; CHECK-NEXT: add v2.4s, v21.4s, v20.4s -; CHECK-NEXT: add v3.4s, v19.4s, v22.4s -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: add v1.4s, v3.4s, v2.4s +; CHECK-NEXT: add x8, sp, #200 +; CHECK-NEXT: mov v1.b[7], w7 +; CHECK-NEXT: add x10, sp, #336 +; CHECK-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #72 +; CHECK-NEXT: smull v21.8h, v22.8b, v21.8b +; CHECK-NEXT: movi v22.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v6.b }[7], [x10] +; CHECK-NEXT: ld1 { v16.b }[7], [x8] +; CHECK-NEXT: smull v20.8h, v23.8b, v20.8b +; CHECK-NEXT: sshll v7.4s, v7.4h, #0 +; CHECK-NEXT: smull v0.8h, v2.8b, v0.8b +; CHECK-NEXT: saddw v2.4s, v18.4s, v17.4h +; CHECK-NEXT: smull v1.8h, v1.8b, v3.8b +; CHECK-NEXT: smull v3.8h, v5.8b, v4.8b +; CHECK-NEXT: smull v4.8h, v16.8b, v6.8b +; CHECK-NEXT: saddl2 v5.4s, v21.8h, v19.8h +; CHECK-NEXT: mov v22.s[0], v7.s[0] +; CHECK-NEXT: saddl v7.4s, v21.4h, v19.4h +; CHECK-NEXT: saddl2 v6.4s, v17.8h, v20.8h +; CHECK-NEXT: saddw v2.4s, v2.4s, v20.4h +; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h +; CHECK-NEXT: saddl2 v16.4s, v4.8h, v3.8h +; CHECK-NEXT: saddl v3.4s, v4.4h, v3.4h +; CHECK-NEXT: saddw v1.4s, v22.4s, v1.4h +; CHECK-NEXT: add v5.4s, v6.4s, v5.4s +; CHECK-NEXT: add v2.4s, v2.4s, v7.4s +; CHECK-NEXT: add v6.4s, v17.4s, v16.4s +; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h +; CHECK-NEXT: add v1.4s, v2.4s, v5.4s +; CHECK-NEXT: add v0.4s, v0.4s, v3.4s +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 diff --git a/llvm/test/CodeGen/AArch64/neon-extmul.ll b/llvm/test/CodeGen/AArch64/neon-extmul.ll index 502673fa76694..3dbc033dfab96 100644 --- a/llvm/test/CodeGen/AArch64/neon-extmul.ll +++ b/llvm/test/CodeGen/AArch64/neon-extmul.ll @@ -5,10 +5,9 @@ define <8 x i32> @extmuls_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-SD-LABEL: extmuls_v8i8_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: sshll v2.8h, v1.8b, #0 -; CHECK-SD-NEXT: smull2 v1.4s, v0.8h, v2.8h -; CHECK-SD-NEXT: smull v0.4s, v0.4h, v2.4h +; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmuls_v8i8_i32: @@ -28,10 +27,9 @@ entry: define <8 x i32> @extmulu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-SD-LABEL: extmulu_v8i8_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll v2.8h, v1.8b, #0 -; CHECK-SD-NEXT: umull2 v1.4s, v0.8h, v2.8h -; CHECK-SD-NEXT: umull v0.4s, v0.4h, v2.4h +; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmulu_v8i8_i32: @@ -78,12 +76,9 @@ entry: define <8 x i32> @extmuladds_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) { ; CHECK-SD-LABEL: extmuladds_v8i8_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: smlal2 v3.4s, v0.8h, v1.8h -; CHECK-SD-NEXT: smlal v2.4s, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b +; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: saddw2 v1.4s, v3.4s, v0.8h +; CHECK-SD-NEXT: saddw v0.4s, v2.4s, v0.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmuladds_v8i8_i32: @@ -106,12 +101,9 @@ entry: define <8 x i32> @extmuladdu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) { ; CHECK-SD-LABEL: extmuladdu_v8i8_i32: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: umlal2 v3.4s, v0.8h, v1.8h -; CHECK-SD-NEXT: umlal v2.4s, v0.4h, v1.4h -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b +; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: uaddw2 v1.4s, v3.4s, v0.8h +; CHECK-SD-NEXT: uaddw v0.4s, v2.4s, v0.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmuladdu_v8i8_i32: @@ -168,16 +160,13 @@ entry: define <8 x i64> @extmuls_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-SD-LABEL: extmuls_v8i8_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: sshll v2.4s, v0.4h, #0 -; CHECK-SD-NEXT: sshll v4.4s, v1.4h, #0 -; CHECK-SD-NEXT: sshll2 v5.4s, v0.8h, #0 -; CHECK-SD-NEXT: sshll2 v6.4s, v1.8h, #0 -; CHECK-SD-NEXT: smull v0.2d, v2.2s, v4.2s -; CHECK-SD-NEXT: smull2 v1.2d, v2.4s, v4.4s -; CHECK-SD-NEXT: smull2 v3.2d, v5.4s, v6.4s -; CHECK-SD-NEXT: smull v2.2d, v5.2s, v6.2s +; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v0.2d, v1.2s, #0 +; CHECK-SD-NEXT: sshll2 v3.2d, v2.4s, #0 +; CHECK-SD-NEXT: sshll2 v1.2d, v1.4s, #0 +; CHECK-SD-NEXT: sshll v2.2d, v2.2s, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmuls_v8i8_i64: @@ -203,16 +192,13 @@ entry: define <8 x i64> @extmulu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-SD-LABEL: extmulu_v8i8_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll v2.4s, v0.4h, #0 -; CHECK-SD-NEXT: ushll v4.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll2 v5.4s, v0.8h, #0 -; CHECK-SD-NEXT: ushll2 v6.4s, v1.8h, #0 -; CHECK-SD-NEXT: umull v0.2d, v2.2s, v4.2s -; CHECK-SD-NEXT: umull2 v1.2d, v2.4s, v4.4s -; CHECK-SD-NEXT: umull2 v3.2d, v5.4s, v6.4s -; CHECK-SD-NEXT: umull v2.2d, v5.2s, v6.2s +; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v0.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll2 v3.2d, v2.4s, #0 +; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #0 +; CHECK-SD-NEXT: ushll v2.2d, v2.2s, #0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: extmulu_v8i8_i64: @@ -309,19 +295,13 @@ entry: define <8 x i64> @extmuladds_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) { ; CHECK-SD-LABEL: extmuladds_v8i8_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: sshll v6.4s, v0.4h, #0 -; CHECK-SD-NEXT: sshll v7.4s, v1.4h, #0 -; CHECK-SD-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-SD-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-SD-NEXT: smlal v2.2d, v6.2s, v7.2s -; CHECK-SD-NEXT: smlal2 v3.2d, v6.4s, v7.4s -; CHECK-SD-NEXT: smlal2 v5.2d, v0.4s, v1.4s -; CHECK-SD-NEXT: smlal v4.2d, v0.2s, v1.2s -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b -; CHECK-SD-NEXT: mov v2.16b, v4.16b +; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: sshll2 v6.4s, v0.8h, #0 +; CHECK-SD-NEXT: sshll v1.4s, v0.4h, #0 +; CHECK-SD-NEXT: saddw2 v5.2d, v5.2d, v6.4s +; CHECK-SD-NEXT: saddw v0.2d, v2.2d, v1.2s +; CHECK-SD-NEXT: saddw2 v1.2d, v3.2d, v1.4s +; CHECK-SD-NEXT: saddw v2.2d, v4.2d, v6.2s ; CHECK-SD-NEXT: mov v3.16b, v5.16b ; CHECK-SD-NEXT: ret ; @@ -353,19 +333,13 @@ entry: define <8 x i64> @extmuladdu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) { ; CHECK-SD-LABEL: extmuladdu_v8i8_i64: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-NEXT: ushll v6.4s, v0.4h, #0 -; CHECK-SD-NEXT: ushll v7.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-SD-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-SD-NEXT: umlal v2.2d, v6.2s, v7.2s -; CHECK-SD-NEXT: umlal2 v3.2d, v6.4s, v7.4s -; CHECK-SD-NEXT: umlal2 v5.2d, v0.4s, v1.4s -; CHECK-SD-NEXT: umlal v4.2d, v0.2s, v1.2s -; CHECK-SD-NEXT: mov v0.16b, v2.16b -; CHECK-SD-NEXT: mov v1.16b, v3.16b -; CHECK-SD-NEXT: mov v2.16b, v4.16b +; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: ushll2 v6.4s, v0.8h, #0 +; CHECK-SD-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-SD-NEXT: uaddw2 v5.2d, v5.2d, v6.4s +; CHECK-SD-NEXT: uaddw v0.2d, v2.2d, v1.2s +; CHECK-SD-NEXT: uaddw2 v1.2d, v3.2d, v1.4s +; CHECK-SD-NEXT: uaddw v2.2d, v4.2d, v6.2s ; CHECK-SD-NEXT: mov v3.16b, v5.16b ; CHECK-SD-NEXT: ret ; diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll index c9fe89aec8ad9..c81fd26a77525 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll @@ -1925,11 +1925,8 @@ entry: define i32 @test_udot_v8i8(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-BASE-LABEL: test_udot_v8i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: addv s0, v2.4s +; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; @@ -1969,15 +1966,11 @@ entry: define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-BASE-LABEL: test_udot_v16i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: umull v4.4s, v3.4h, v2.4h -; CHECK-SD-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h -; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: umlal v4.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v4.4s, v2.4s +; CHECK-SD-BASE-NEXT: umull2 v2.8h, v1.16b, v0.16b +; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: uaddl2 v1.4s, v0.8h, v2.8h +; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2025,21 +2018,16 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) { ; CHECK-SD-BASE: // %bb.0: // %entry ; CHECK-SD-BASE-NEXT: ldr q0, [x0] ; CHECK-SD-BASE-NEXT: ldr q1, [x1] -; CHECK-SD-BASE-NEXT: ldr d4, [x0, #16] -; CHECK-SD-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-SD-BASE-NEXT: ushll v2.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: umull v6.4s, v3.4h, v2.4h -; CHECK-SD-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h -; CHECK-SD-BASE-NEXT: ushll v3.8h, v4.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v4.8h, v5.8b, #0 -; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v4.8h, v3.8h -; CHECK-SD-BASE-NEXT: umlal v6.4s, v4.4h, v3.4h -; CHECK-SD-BASE-NEXT: umlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: umlal v6.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-SD-BASE-NEXT: ldr d2, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr d3, [x1, #16] +; CHECK-SD-BASE-NEXT: umull v2.8h, v3.8b, v2.8b +; CHECK-SD-BASE-NEXT: umull v3.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b +; CHECK-SD-BASE-NEXT: uaddl2 v1.4s, v3.8h, v2.8h +; CHECK-SD-BASE-NEXT: uaddl v2.4s, v3.4h, v2.4h +; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v1.4s, v0.8h +; CHECK-SD-BASE-NEXT: uaddw v0.4s, v2.4s, v0.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2125,37 +2113,27 @@ entry: define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { ; CHECK-SD-BASE-LABEL: test_udot_v48i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ldp q0, q4, [x1] -; CHECK-SD-BASE-NEXT: ldr q2, [x0, #32] -; CHECK-SD-BASE-NEXT: ldp q1, q3, [x0] -; CHECK-SD-BASE-NEXT: ldr q7, [x1, #32] -; CHECK-SD-BASE-NEXT: ushll2 v16.8h, v2.16b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v6.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v17.8h, v7.16b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v5.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: umull2 v18.4s, v6.8h, v5.8h -; CHECK-SD-BASE-NEXT: umull v19.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: umull v5.4s, v6.4h, v5.4h -; CHECK-SD-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h -; CHECK-SD-BASE-NEXT: ushll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v2.8h, v7.8b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v6.8h, v3.16b, #0 -; CHECK-SD-BASE-NEXT: ushll2 v7.8h, v4.16b, #0 -; CHECK-SD-BASE-NEXT: umlal2 v18.4s, v17.8h, v16.8h -; CHECK-SD-BASE-NEXT: umlal v5.4s, v17.4h, v16.4h -; CHECK-SD-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h -; CHECK-SD-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h -; CHECK-SD-BASE-NEXT: ushll v1.8h, v3.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v2.8h, v4.8b, #0 -; CHECK-SD-BASE-NEXT: umlal2 v18.4s, v7.8h, v6.8h -; CHECK-SD-BASE-NEXT: umlal v5.4s, v7.4h, v6.4h -; CHECK-SD-BASE-NEXT: umlal v19.4s, v2.4h, v1.4h -; CHECK-SD-BASE-NEXT: umlal2 v0.4s, v2.8h, v1.8h -; CHECK-SD-BASE-NEXT: add v1.4s, v19.4s, v5.4s -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v18.4s -; CHECK-SD-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-BASE-NEXT: ldp q4, q0, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr q2, [x1, #32] +; CHECK-SD-BASE-NEXT: ldp q1, q5, [x1] +; CHECK-SD-BASE-NEXT: ldr q3, [x0] +; CHECK-SD-BASE-NEXT: umull2 v6.8h, v2.16b, v0.16b +; CHECK-SD-BASE-NEXT: umull v0.8h, v2.8b, v0.8b +; CHECK-SD-BASE-NEXT: umull2 v7.8h, v1.16b, v3.16b +; CHECK-SD-BASE-NEXT: umull v1.8h, v1.8b, v3.8b +; CHECK-SD-BASE-NEXT: umull2 v2.8h, v5.16b, v4.16b +; CHECK-SD-BASE-NEXT: umull v3.8h, v5.8b, v4.8b +; CHECK-SD-BASE-NEXT: uaddl2 v4.4s, v7.8h, v6.8h +; CHECK-SD-BASE-NEXT: uaddl2 v5.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: uaddl v6.4s, v7.4h, v6.4h +; CHECK-SD-BASE-NEXT: uaddl v0.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v4.4s, v2.8h +; CHECK-SD-BASE-NEXT: uaddw2 v4.4s, v5.4s, v3.8h +; CHECK-SD-BASE-NEXT: uaddw v2.4s, v6.4s, v2.4h +; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v3.4h +; CHECK-SD-BASE-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2275,11 +2253,8 @@ entry: define i32 @test_sdot_v8i8(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-BASE-LABEL: test_sdot_v8i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: smull v2.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: addv s0, v2.4s +; CHECK-SD-BASE-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret ; @@ -2319,15 +2294,11 @@ entry: define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) { ; CHECK-SD-BASE-LABEL: test_sdot_v16i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: smull v4.4s, v3.4h, v2.4h -; CHECK-SD-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h -; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: smlal v4.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v4.4s, v2.4s +; CHECK-SD-BASE-NEXT: smull2 v2.8h, v1.16b, v0.16b +; CHECK-SD-BASE-NEXT: smull v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: saddl2 v1.4s, v0.8h, v2.8h +; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2375,21 +2346,16 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) { ; CHECK-SD-BASE: // %bb.0: // %entry ; CHECK-SD-BASE-NEXT: ldr q0, [x0] ; CHECK-SD-BASE-NEXT: ldr q1, [x1] -; CHECK-SD-BASE-NEXT: ldr d4, [x0, #16] -; CHECK-SD-BASE-NEXT: ldr d5, [x1, #16] -; CHECK-SD-BASE-NEXT: sshll v2.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v3.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: smull v6.4s, v3.4h, v2.4h -; CHECK-SD-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h -; CHECK-SD-BASE-NEXT: sshll v3.8h, v4.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v4.8h, v5.8b, #0 -; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v4.8h, v3.8h -; CHECK-SD-BASE-NEXT: smlal v6.4s, v4.4h, v3.4h -; CHECK-SD-BASE-NEXT: smlal2 v2.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: smlal v6.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v6.4s, v2.4s +; CHECK-SD-BASE-NEXT: ldr d2, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr d3, [x1, #16] +; CHECK-SD-BASE-NEXT: smull v2.8h, v3.8b, v2.8b +; CHECK-SD-BASE-NEXT: smull v3.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b +; CHECK-SD-BASE-NEXT: saddl2 v1.4s, v3.8h, v2.8h +; CHECK-SD-BASE-NEXT: saddl v2.4s, v3.4h, v2.4h +; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v1.4s, v0.8h +; CHECK-SD-BASE-NEXT: saddw v0.4s, v2.4s, v0.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2475,37 +2441,27 @@ entry: define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { ; CHECK-SD-BASE-LABEL: test_sdot_v48i8: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ldp q0, q4, [x1] -; CHECK-SD-BASE-NEXT: ldr q2, [x0, #32] -; CHECK-SD-BASE-NEXT: ldp q1, q3, [x0] -; CHECK-SD-BASE-NEXT: ldr q7, [x1, #32] -; CHECK-SD-BASE-NEXT: sshll2 v16.8h, v2.16b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v6.8h, v0.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v17.8h, v7.16b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v5.8h, v1.16b, #0 -; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: smull2 v18.4s, v6.8h, v5.8h -; CHECK-SD-BASE-NEXT: smull v19.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: smull v5.4s, v6.4h, v5.4h -; CHECK-SD-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h -; CHECK-SD-BASE-NEXT: sshll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v2.8h, v7.8b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v6.8h, v3.16b, #0 -; CHECK-SD-BASE-NEXT: sshll2 v7.8h, v4.16b, #0 -; CHECK-SD-BASE-NEXT: smlal2 v18.4s, v17.8h, v16.8h -; CHECK-SD-BASE-NEXT: smlal v5.4s, v17.4h, v16.4h -; CHECK-SD-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h -; CHECK-SD-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h -; CHECK-SD-BASE-NEXT: sshll v1.8h, v3.8b, #0 -; CHECK-SD-BASE-NEXT: sshll v2.8h, v4.8b, #0 -; CHECK-SD-BASE-NEXT: smlal2 v18.4s, v7.8h, v6.8h -; CHECK-SD-BASE-NEXT: smlal v5.4s, v7.4h, v6.4h -; CHECK-SD-BASE-NEXT: smlal v19.4s, v2.4h, v1.4h -; CHECK-SD-BASE-NEXT: smlal2 v0.4s, v2.8h, v1.8h -; CHECK-SD-BASE-NEXT: add v1.4s, v19.4s, v5.4s -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v18.4s -; CHECK-SD-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-BASE-NEXT: ldp q4, q0, [x0, #16] +; CHECK-SD-BASE-NEXT: ldr q2, [x1, #32] +; CHECK-SD-BASE-NEXT: ldp q1, q5, [x1] +; CHECK-SD-BASE-NEXT: ldr q3, [x0] +; CHECK-SD-BASE-NEXT: smull2 v6.8h, v2.16b, v0.16b +; CHECK-SD-BASE-NEXT: smull v0.8h, v2.8b, v0.8b +; CHECK-SD-BASE-NEXT: smull2 v7.8h, v1.16b, v3.16b +; CHECK-SD-BASE-NEXT: smull v1.8h, v1.8b, v3.8b +; CHECK-SD-BASE-NEXT: smull2 v2.8h, v5.16b, v4.16b +; CHECK-SD-BASE-NEXT: smull v3.8h, v5.8b, v4.8b +; CHECK-SD-BASE-NEXT: saddl2 v4.4s, v7.8h, v6.8h +; CHECK-SD-BASE-NEXT: saddl2 v5.4s, v1.8h, v0.8h +; CHECK-SD-BASE-NEXT: saddl v6.4s, v7.4h, v6.4h +; CHECK-SD-BASE-NEXT: saddl v0.4s, v1.4h, v0.4h +; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v4.4s, v2.8h +; CHECK-SD-BASE-NEXT: saddw2 v4.4s, v5.4s, v3.8h +; CHECK-SD-BASE-NEXT: saddw v2.4s, v6.4s, v2.4h +; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v3.4h +; CHECK-SD-BASE-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -2626,26 +2582,22 @@ entry: define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-BASE-LABEL: test_udot_v8i8_multi_use: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-BASE-NEXT: umull v2.4s, v1.4h, v0.4h -; CHECK-SD-BASE-NEXT: mov v3.16b, v2.16b -; CHECK-SD-BASE-NEXT: fmov w8, s2 -; CHECK-SD-BASE-NEXT: umlal2 v3.4s, v1.8h, v0.8h -; CHECK-SD-BASE-NEXT: addv s0, v3.4s +; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b +; CHECK-SD-BASE-NEXT: uaddlv s1, v0.8h +; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-SD-BASE-NEXT: fmov w9, s0 -; CHECK-SD-BASE-NEXT: add w0, w9, w8 +; CHECK-SD-BASE-NEXT: fmov w8, s1 +; CHECK-SD-BASE-NEXT: add w0, w8, w9 ; CHECK-SD-BASE-NEXT: ret ; ; CHECK-SD-DOT-LABEL: test_udot_v8i8_multi_use: ; CHECK-SD-DOT: // %bb.0: // %entry ; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 -; CHECK-SD-DOT-NEXT: ushll v3.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll v4.8h, v1.8b, #0 +; CHECK-SD-DOT-NEXT: umull v3.8h, v1.8b, v0.8b ; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b -; CHECK-SD-DOT-NEXT: umull v0.4s, v4.4h, v3.4h -; CHECK-SD-DOT-NEXT: addp v1.2s, v2.2s, v2.2s +; CHECK-SD-DOT-NEXT: ushll v0.4s, v3.4h, #0 ; CHECK-SD-DOT-NEXT: fmov w9, s0 +; CHECK-SD-DOT-NEXT: addp v1.2s, v2.2s, v2.2s ; CHECK-SD-DOT-NEXT: fmov w8, s1 ; CHECK-SD-DOT-NEXT: add w0, w8, w9 ; CHECK-SD-DOT-NEXT: ret