diff options
author | LemonBoy <thatlemon@gmail.com> | 2021-03-05 16:01:45 +0100 |
---|---|---|
committer | Tom Stellard <tstellar@redhat.com> | 2021-03-08 20:14:33 -0800 |
commit | 46a1b0655666e21c56fa79560e9baee87405d4f4 (patch) | |
tree | cbe9ed5a1b17f009566b8e35a92eb607a9df856b | |
parent | [OpenMP] Fixed a crash when offloading to x86_64 with target nowait (diff) | |
download | llvm-project-46a1b0655666e21c56fa79560e9baee87405d4f4.tar.gz llvm-project-46a1b0655666e21c56fa79560e9baee87405d4f4.tar.bz2 llvm-project-46a1b0655666e21c56fa79560e9baee87405d4f4.zip |
[AArch64] Legalize horizontal fmax/fmin reductions on f16 vectors
Expand the horizontal reduction during the instruction selection phase, but only if the target doesn't support the full fp16 instruction set.
Fixes https://bugs.llvm.org/show_bug.cgi?id=49401
Reviewed By: aemerson
Differential Revision: https://reviews.llvm.org/D97840
(cherry picked from commit 8725b24c6d4abaa97425e704652a13dacb35fe3f)
-rw-r--r-- | llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 7 | ||||
-rw-r--r-- | llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll | 62 | ||||
-rw-r--r-- | llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll | 62 |
3 files changed, 126 insertions, 5 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 1be09186dc0a..1451151f4dc5 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1017,11 +1017,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, // Vector reductions for (MVT VT : { MVT::v4f16, MVT::v2f32, MVT::v8f16, MVT::v4f32, MVT::v2f64 }) { - setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); + if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) { + setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); + setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); - if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) setOperationAction(ISD::VECREDUCE_FADD, VT, Legal); + } } for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll index f1ebd8fa85ea..d26db2aefee0 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll @@ -1,11 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK +; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP +; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon,+fullfp16 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP declare half @llvm.vector.reduce.fmax.v1f16(<1 x half> %a) declare float @llvm.vector.reduce.fmax.v1f32(<1 x float> %a) declare double @llvm.vector.reduce.fmax.v1f64(<1 x double> %a) declare fp128 @llvm.vector.reduce.fmax.v1f128(<1 x fp128> %a) +declare half @llvm.vector.reduce.fmax.v4f16(<4 x half> %a) declare float @llvm.vector.reduce.fmax.v3f32(<3 x float> %a) declare fp128 @llvm.vector.reduce.fmax.v2f128(<2 x fp128> %a) declare float @llvm.vector.reduce.fmax.v16f32(<16 x float> %a) @@ -44,6 +46,64 @@ define fp128 @test_v1f128(<1 x fp128> %a) nounwind { ret fp128 %b } +define half @test_v4f16(<4 x half> %a) nounwind { +; CHECK-NOFP-LABEL: test_v4f16: +; CHECK-NOFP: // %bb.0: +; CHECK-NOFP-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NOFP-NEXT: mov h3, v0.h[1] +; CHECK-NOFP-NEXT: mov h1, v0.h[3] +; CHECK-NOFP-NEXT: mov h2, v0.h[2] +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s3, h3 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s3 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s2, h2 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s2 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s1, h1 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s1 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: ret +; +; CHECK-FP-LABEL: test_v4f16: +; CHECK-FP: // %bb.0: +; CHECK-FP-NEXT: fmaxnmv h0, v0.4h +; CHECK-FP-NEXT: ret + %b = call nnan half @llvm.vector.reduce.fmax.v4f16(<4 x half> %a) + ret half %b +} + +define half @test_v4f16_ninf(<4 x half> %a) nounwind { +; CHECK-NOFP-LABEL: test_v4f16_ninf: +; CHECK-NOFP: // %bb.0: +; CHECK-NOFP-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NOFP-NEXT: mov h3, v0.h[1] +; CHECK-NOFP-NEXT: mov h1, v0.h[3] +; CHECK-NOFP-NEXT: mov h2, v0.h[2] +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s3, h3 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s3 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s2, h2 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s2 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s1, h1 +; CHECK-NOFP-NEXT: fmaxnm s0, s0, s1 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: ret +; +; CHECK-FP-LABEL: test_v4f16_ninf: +; CHECK-FP: // %bb.0: +; CHECK-FP-NEXT: fmaxnmv h0, v0.4h +; CHECK-FP-NEXT: ret + %b = call nnan ninf half @llvm.vector.reduce.fmax.v4f16(<4 x half> %a) + ret half %b +} + define float @test_v3f32(<3 x float> %a) nounwind { ; CHECK-LABEL: test_v3f32: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll index 4129fa80b13e..52d6e9773ab2 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll @@ -1,11 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK +; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP +; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon,+fullfp16 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP declare half @llvm.vector.reduce.fmin.v1f16(<1 x half> %a) declare float @llvm.vector.reduce.fmin.v1f32(<1 x float> %a) declare double @llvm.vector.reduce.fmin.v1f64(<1 x double> %a) declare fp128 @llvm.vector.reduce.fmin.v1f128(<1 x fp128> %a) +declare half @llvm.vector.reduce.fmin.v4f16(<4 x half> %a) declare float @llvm.vector.reduce.fmin.v3f32(<3 x float> %a) declare fp128 @llvm.vector.reduce.fmin.v2f128(<2 x fp128> %a) declare float @llvm.vector.reduce.fmin.v16f32(<16 x float> %a) @@ -44,6 +46,64 @@ define fp128 @test_v1f128(<1 x fp128> %a) nounwind { ret fp128 %b } +define half @test_v4f16(<4 x half> %a) nounwind { +; CHECK-NOFP-LABEL: test_v4f16: +; CHECK-NOFP: // %bb.0: +; CHECK-NOFP-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NOFP-NEXT: mov h3, v0.h[1] +; CHECK-NOFP-NEXT: mov h1, v0.h[3] +; CHECK-NOFP-NEXT: mov h2, v0.h[2] +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s3, h3 +; CHECK-NOFP-NEXT: fminnm s0, s0, s3 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s2, h2 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fminnm s0, s0, s2 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s1, h1 +; CHECK-NOFP-NEXT: fminnm s0, s0, s1 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: ret +; +; CHECK-FP-LABEL: test_v4f16: +; CHECK-FP: // %bb.0: +; CHECK-FP-NEXT: fminnmv h0, v0.4h +; CHECK-FP-NEXT: ret + %b = call nnan half @llvm.vector.reduce.fmin.v4f16(<4 x half> %a) + ret half %b +} + +define half @test_v4f16_ninf(<4 x half> %a) nounwind { +; CHECK-NOFP-LABEL: test_v4f16_ninf: +; CHECK-NOFP: // %bb.0: +; CHECK-NOFP-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NOFP-NEXT: mov h3, v0.h[1] +; CHECK-NOFP-NEXT: mov h1, v0.h[3] +; CHECK-NOFP-NEXT: mov h2, v0.h[2] +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s3, h3 +; CHECK-NOFP-NEXT: fminnm s0, s0, s3 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s2, h2 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fminnm s0, s0, s2 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: fcvt s0, h0 +; CHECK-NOFP-NEXT: fcvt s1, h1 +; CHECK-NOFP-NEXT: fminnm s0, s0, s1 +; CHECK-NOFP-NEXT: fcvt h0, s0 +; CHECK-NOFP-NEXT: ret +; +; CHECK-FP-LABEL: test_v4f16_ninf: +; CHECK-FP: // %bb.0: +; CHECK-FP-NEXT: fminnmv h0, v0.4h +; CHECK-FP-NEXT: ret + %b = call nnan ninf half @llvm.vector.reduce.fmin.v4f16(<4 x half> %a) + ret half %b +} + define float @test_v3f32(<3 x float> %a) nounwind { ; CHECK-LABEL: test_v3f32: ; CHECK: // %bb.0: |