From bfbcf279bbf3523bb3cfc6d82076f409d9d6c132 Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 10:48:23 -0600 Subject: [PATCH 1/7] Neon Absolute Value fns --- crates/core_arch/src/aarch64/neon/mod.rs | 53 ++++++++++ crates/core_arch/src/arm/neon/mod.rs | 118 +++++++++++++++++++++++ 2 files changed, 171 insertions(+) diff --git a/crates/core_arch/src/aarch64/neon/mod.rs b/crates/core_arch/src/aarch64/neon/mod.rs index 532c5b4d5a..c581784551 100644 --- a/crates/core_arch/src/aarch64/neon/mod.rs +++ b/crates/core_arch/src/aarch64/neon/mod.rs @@ -71,6 +71,14 @@ pub struct poly8x16x4_t( #[allow(improper_ctypes)] extern "C" { + // absolute value + #[link_name = "llvm.aarch64.neon.abs.i64"] + fn vabsd_s64_(a: i64) -> i64; + #[link_name = "llvm.aarch64.neon.abs.v1i64"] + fn vabs_s64_(a: int64x1_t) -> int64x1_t; + #[link_name = "llvm.aarch64.neon.abs.v2i64"] + fn vabsq_s64_(a: int64x2_t) -> int64x2_t; + #[link_name = "llvm.aarch64.neon.pmull64"] fn vmull_p64_(a: i64, b: i64) -> int8x16_t; @@ -246,6 +254,28 @@ extern "C" { ) -> int8x16_t; } +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +pub unsafe fn vabsd_s64(a: i64) -> i64 { + vabsd_s64_(a) +} +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +pub unsafe fn vabs_s64(a: int64x1_t) -> int64x1_t { + vabs_s64_(a) +} +/// Absolute Value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(test, assert_instr(abs))] +pub unsafe fn vabsq_s64(a: int64x2_t) -> int64x2_t { + vabsq_s64_(a) +} + /// Add pairwise #[inline] #[target_feature(enable = "neon")] @@ -2503,6 +2533,29 @@ mod tests { unsafe fn test_vsubq_f64() { testq_ari_f64(|i, j| vsubq_f64(i, j), |a: f64, b: f64| -> f64 { a - b }); } + + #[simd_test(enable = "neon")] + unsafe fn test_vabsd_s64() { + assert_eq!(vabsd_s64(-1), 1); + assert_eq!(vabsd_s64(0), 0); + assert_eq!(vabsd_s64(1), 1); + assert_eq!(vabsd_s64(i64::MIN), i64::MIN); + assert_eq!(vabsd_s64(i64::MIN + 1), i64::MAX); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s64() { + let a = i64x2::new(i64::MIN, i64::MIN + 1); + let r: i64x2 = transmute(vabs_s64(transmute(a))); + let e = i64x2::new(i64::MIN, i64::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s64() { + let a = i64x4::new(i64::MIN, i64::MIN + 1, 0, -1); + let r: i64x4 = transmute(vabsq_s64(transmute(a))); + let e = i64x4::new(i64::MIN, i64::MAX, 0, 1); + assert_eq!(r, e); + } } #[cfg(test)] diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index 5c3ccc9fc2..1bdff2af03 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -102,6 +102,27 @@ pub struct poly8x8x4_t(pub poly8x8_t, pub poly8x8_t, pub poly8x8_t, pub poly8x8_ #[allow(improper_ctypes)] extern "C" { + // absolute value (64-bit) + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i8")] + fn vabs_s8_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i16")] + fn vabs_s16_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v2i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v2i32")] + fn vabs_s32_(a: int16x4_t) -> int16x4_t; + // absolute value (128-bit) + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v16i8")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v16i8")] + fn vabsq_s8_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i16")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i16")] + fn vabsq_s16_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i32")] + #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i32")] + fn vabsq_s32_(a: int16x4_t) -> int16x4_t; + #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] fn frsqrte_v2f32(a: float32x2_t) -> float32x2_t; @@ -185,6 +206,61 @@ extern "C" { ) -> int8x8_t; } +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabs_s8(a: int8x8_t) -> int8x8_t { + vabs_s8_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabs_s16(a: int16x4_t) -> int16x4_t { + vabs_s16_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabs_s32(a: int32x2_t) -> int32x2_t { + vabs_s32(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabsq_s8(a: int8x16_t) -> int8x16_t { + vabsq_s8_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabsq_s16(a: int16x8_t) -> int16x8_t { + vabsq_s16_(a) +} +/// Absolute value (wrapping). +#[inline] +#[target_feature(enable = "neon")] +#[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +#[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] +#[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] +pub unsafe fn vabsq_s32(a: int32x4_t) -> int32x4_t { + vabsq_s32(a) +} + /// Unsigned saturating extract narrow. #[inline] #[target_feature(enable = "neon")] @@ -3945,6 +4021,48 @@ mod tests { let e = i8x16::new(-1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); assert_eq!(r, e) } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s8() { + let a = i8x8::new(-1, 0, 1, -2, 0, 2, -128, 127); + let r: i8x8 = transmute(vabs_s8(transmute(a))); + let e = i8x8::new(1, 0, 1, 2, 0, 2, -128, 127); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s8() { + let a = i8x16::new(-1, 0, 1, -2, 0, 2, -128, 127, -1, 0, 1, -2, 0, 2, -128, 127); + let r: i8x16 = transmute(vabsq_s8(transmute(a))); + let e = i8x16::new(1, 0, 1, 2, 0, 2, -128, 127, 1, 0, 1, 2, 0, 2, -128, 127); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s16() { + let a = i16x4::new(-1, 0, i16::MIN, i16::MAX); + let r: i16x4 = transmute(vabs_s16(transmute(a))); + let e = i16x4::new(-1, 0, i16::MIN, i16::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s16() { + let a = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); + let r: i16x8 = transmute(vabsq_s16(transmute(a))); + let e = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabs_s32() { + let a = i32x2::new(i32::MIN, i32::MIN + 1); + let r: i32x2 = transmute(vabs_s32(transmute(a))); + let e = i32x2::new(i32::MIN, i32::MAX); + assert_eq!(r, e); + } + #[simd_test(enable = "neon")] + unsafe fn test_vabsq_s32() { + let a = i32x4::new(i32::MIN, i32::MIN + 1, 0, -1); + let r: i32x4 = transmute(vabsq_s32(transmute(a))); + let e = i32x4::new(i32::MIN, i32::MAX, 0, 1); + assert_eq!(r, e); + } } #[cfg(test)] From 66c33eafafefc1699c03fda65ec6a0c0be3ea291 Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 15:39:56 -0600 Subject: [PATCH 2/7] fix extern declaration errors. --- crates/core_arch/src/arm/neon/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index 1bdff2af03..795af0b6b3 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -105,23 +105,23 @@ extern "C" { // absolute value (64-bit) #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i8")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i8")] - fn vabs_s8_(a: int16x4_t) -> int16x4_t; + fn vabs_s8_(a: int8x8_t) -> int8x8_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i16")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i16")] fn vabs_s16_(a: int16x4_t) -> int16x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v2i32")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v2i32")] - fn vabs_s32_(a: int16x4_t) -> int16x4_t; + fn vabs_s32_(a: int32x2_t) -> int32x2_t; // absolute value (128-bit) #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v16i8")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v16i8")] - fn vabsq_s8_(a: int16x4_t) -> int16x4_t; + fn vabsq_s8_(a: int8x16_t) -> int8x16_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i16")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i16")] fn vabsq_s16_(a: int16x4_t) -> int16x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i32")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i32")] - fn vabsq_s32_(a: int16x4_t) -> int16x4_t; + fn vabsq_s32_(a: int32x4_t) -> int32x4_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vrsqrte.v2f32")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.frsqrte.v2f32")] From 797623dc3574775ade77c70ff3f09791db95fc71 Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 15:43:48 -0600 Subject: [PATCH 3/7] as before --- crates/core_arch/src/arm/neon/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index 795af0b6b3..e96a31ee22 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -118,7 +118,7 @@ extern "C" { fn vabsq_s8_(a: int8x16_t) -> int8x16_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v8i16")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v8i16")] - fn vabsq_s16_(a: int16x4_t) -> int16x4_t; + fn vabsq_s16_(a: int16x8_t) -> int16x8_t; #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vabs.v4i32")] #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.neon.abs.v4i32")] fn vabsq_s32_(a: int32x4_t) -> int32x4_t; From 3dfb4bc78504db04289a3f77f647d0fa1392bcff Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 15:47:28 -0600 Subject: [PATCH 4/7] call the correct extern. --- crates/core_arch/src/arm/neon/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index e96a31ee22..7555b24f15 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -231,7 +231,7 @@ pub unsafe fn vabs_s16(a: int16x4_t) -> int16x4_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] pub unsafe fn vabs_s32(a: int32x2_t) -> int32x2_t { - vabs_s32(a) + vabs_s32_(a) } /// Absolute value (wrapping). #[inline] @@ -258,7 +258,7 @@ pub unsafe fn vabsq_s16(a: int16x8_t) -> int16x8_t { #[cfg_attr(all(test, target_arch = "arm"), assert_instr(vabs))] #[cfg_attr(all(test, target_arch = "aarch64"), assert_instr(abs))] pub unsafe fn vabsq_s32(a: int32x4_t) -> int32x4_t { - vabsq_s32(a) + vabsq_s32_(a) } /// Unsigned saturating extract narrow. From 20a85927003ceb6a91f7d0f4ffd2eb6577bc353b Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 15:53:43 -0600 Subject: [PATCH 5/7] obviously it helps if the test expects the correct values. --- crates/core_arch/src/arm/neon/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index 7555b24f15..163acf6169 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -4039,14 +4039,14 @@ mod tests { unsafe fn test_vabs_s16() { let a = i16x4::new(-1, 0, i16::MIN, i16::MAX); let r: i16x4 = transmute(vabs_s16(transmute(a))); - let e = i16x4::new(-1, 0, i16::MIN, i16::MAX); + let e = i16x4::new(1, 0, i16::MIN, i16::MAX); assert_eq!(r, e); } #[simd_test(enable = "neon")] unsafe fn test_vabsq_s16() { let a = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); let r: i16x8 = transmute(vabsq_s16(transmute(a))); - let e = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); + let e = i16x8::new(1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); assert_eq!(r, e); } #[simd_test(enable = "neon")] From 7e5d1fe30e3ed26c8daea56f149a17f870b6466c Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 16:17:46 -0600 Subject: [PATCH 6/7] . --- crates/core_arch/src/arm/neon/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_arch/src/arm/neon/mod.rs b/crates/core_arch/src/arm/neon/mod.rs index 163acf6169..b39b1191e3 100644 --- a/crates/core_arch/src/arm/neon/mod.rs +++ b/crates/core_arch/src/arm/neon/mod.rs @@ -4046,7 +4046,7 @@ mod tests { unsafe fn test_vabsq_s16() { let a = i16x8::new(-1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); let r: i16x8 = transmute(vabsq_s16(transmute(a))); - let e = i16x8::new(1, 0, i16::MIN, i16::MAX, -1, 0, i16::MIN, i16::MAX); + let e = i16x8::new(1, 0, i16::MIN, i16::MAX, 1, 0, i16::MIN, i16::MAX); assert_eq!(r, e); } #[simd_test(enable = "neon")] From 966d115bddc7ae3427528edeb2489f511dc492c7 Mon Sep 17 00:00:00 2001 From: Lokathor Date: Sun, 19 Jul 2020 16:30:59 -0600 Subject: [PATCH 7/7] corrent i64 test sizes --- crates/core_arch/src/aarch64/neon/mod.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/core_arch/src/aarch64/neon/mod.rs b/crates/core_arch/src/aarch64/neon/mod.rs index c581784551..66da36463a 100644 --- a/crates/core_arch/src/aarch64/neon/mod.rs +++ b/crates/core_arch/src/aarch64/neon/mod.rs @@ -2544,16 +2544,20 @@ mod tests { } #[simd_test(enable = "neon")] unsafe fn test_vabs_s64() { - let a = i64x2::new(i64::MIN, i64::MIN + 1); - let r: i64x2 = transmute(vabs_s64(transmute(a))); - let e = i64x2::new(i64::MIN, i64::MAX); + let a = i64x1::new(i64::MIN); + let r: i64x1 = transmute(vabs_s64(transmute(a))); + let e = i64x1::new(i64::MIN); + assert_eq!(r, e); + let a = i64x1::new(i64::MIN + 1); + let r: i64x1 = transmute(vabs_s64(transmute(a))); + let e = i64x1::new(i64::MAX); assert_eq!(r, e); } #[simd_test(enable = "neon")] unsafe fn test_vabsq_s64() { - let a = i64x4::new(i64::MIN, i64::MIN + 1, 0, -1); - let r: i64x4 = transmute(vabsq_s64(transmute(a))); - let e = i64x4::new(i64::MIN, i64::MAX, 0, 1); + let a = i64x2::new(i64::MIN, i64::MIN + 1); + let r: i64x2 = transmute(vabsq_s64(transmute(a))); + let e = i64x2::new(i64::MIN, i64::MAX); assert_eq!(r, e); } }