12 #ifndef EIGEN_PACKET_MATH_NEON_H 13 #define EIGEN_PACKET_MATH_NEON_H 19 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 20 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 23 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD 24 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 27 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 29 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 31 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16 35 #if EIGEN_COMP_MSVC_STRICT 41 typedef eigen_packet_wrapper<float32x2_t,0> Packet2f;
42 typedef eigen_packet_wrapper<float32x4_t,1> Packet4f;
43 typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
44 typedef eigen_packet_wrapper<int8x8_t ,3> Packet8c;
45 typedef eigen_packet_wrapper<int8x16_t ,4> Packet16c;
46 typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
47 typedef eigen_packet_wrapper<uint8x8_t ,6> Packet8uc;
48 typedef eigen_packet_wrapper<uint8x16_t ,7> Packet16uc;
49 typedef eigen_packet_wrapper<int16x4_t ,8> Packet4s;
50 typedef eigen_packet_wrapper<int16x8_t ,9> Packet8s;
51 typedef eigen_packet_wrapper<uint16x4_t ,10> Packet4us;
52 typedef eigen_packet_wrapper<uint16x8_t ,11> Packet8us;
53 typedef eigen_packet_wrapper<int32x2_t ,12> Packet2i;
54 typedef eigen_packet_wrapper<int32x4_t ,13> Packet4i;
55 typedef eigen_packet_wrapper<uint32x2_t ,14> Packet2ui;
56 typedef eigen_packet_wrapper<uint32x4_t ,15> Packet4ui;
57 typedef eigen_packet_wrapper<int64x2_t ,16> Packet2l;
58 typedef eigen_packet_wrapper<uint64x2_t ,17> Packet2ul;
62 typedef float32x2_t Packet2f;
63 typedef float32x4_t Packet4f;
64 typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
65 typedef int8x8_t Packet8c;
66 typedef int8x16_t Packet16c;
67 typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
68 typedef uint8x8_t Packet8uc;
69 typedef uint8x16_t Packet16uc;
70 typedef int16x4_t Packet4s;
71 typedef int16x8_t Packet8s;
72 typedef uint16x4_t Packet4us;
73 typedef uint16x8_t Packet8us;
74 typedef int32x2_t Packet2i;
75 typedef int32x4_t Packet4i;
76 typedef uint32x2_t Packet2ui;
77 typedef uint32x4_t Packet4ui;
78 typedef int64x2_t Packet2l;
79 typedef uint64x2_t Packet2ul;
81 #endif // EIGEN_COMP_MSVC_STRICT 83 EIGEN_STRONG_INLINE Packet4f shuffle1(
const Packet4f& m,
int mask){
84 const float* a =
reinterpret_cast<const float*
>(&m);
85 Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3))};
93 template<
bool interleave>
94 EIGEN_STRONG_INLINE Packet4f shuffle2(
const Packet4f &m,
const Packet4f &n,
int mask)
96 const float* a =
reinterpret_cast<const float*
>(&m);
97 const float* b =
reinterpret_cast<const float*
>(&n);
98 Packet4f res = {*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))};
103 EIGEN_STRONG_INLINE Packet4f shuffle2<true>(
const Packet4f &m,
const Packet4f &n,
int mask)
105 const float* a =
reinterpret_cast<const float*
>(&m);
106 const float* b =
reinterpret_cast<const float*
>(&n);
107 Packet4f res = {*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))};
111 EIGEN_STRONG_INLINE
static int eigen_neon_shuffle_mask(
int p,
int q,
int r,
int s) {
return ((s)<<6|(r)<<4|(q)<<2|(p));}
113 EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(
const Packet4f& a,
int p,
int q,
int r,
int s)
115 return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
117 EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(
const Packet4f& a,
const Packet4f& b,
int p,
int q,
int r,
int s)
119 return shuffle2<false>(a,b,eigen_neon_shuffle_mask(p, q, r, s));
121 EIGEN_STRONG_INLINE Packet4f vec4f_movelh(
const Packet4f& a,
const Packet4f& b)
123 return shuffle2<false>(a,b,eigen_neon_shuffle_mask(0, 1, 0, 1));
125 EIGEN_STRONG_INLINE Packet4f vec4f_movehl(
const Packet4f& a,
const Packet4f& b)
127 return shuffle2<false>(b,a,eigen_neon_shuffle_mask(2, 3, 2, 3));
129 EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(
const Packet4f& a,
const Packet4f& b)
131 return shuffle2<true>(a,b,eigen_neon_shuffle_mask(0, 0, 1, 1));
133 EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(
const Packet4f& a,
const Packet4f& b)
135 return shuffle2<true>(a,b,eigen_neon_shuffle_mask(2, 2, 3, 3));
137 #define vec4f_duplane(a, p) \ 138 vdupq_lane_f32(vget_low_f32(a), p) 140 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ 141 const Packet4f p4f_##NAME = pset1<Packet4f>(X) 143 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ 144 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X)) 146 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ 147 const Packet4i p4i_##NAME = pset1<Packet4i>(X) 153 #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : ); 154 #elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC 155 #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR); 157 #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR) 158 #elif EIGEN_ARCH_ARM32 159 #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : ); 162 #define EIGEN_ARM_PREFETCH(ADDR) 166 struct packet_traits<float> : default_packet_traits
168 typedef Packet4f type;
169 typedef Packet2f half;
197 HasSin = EIGEN_FAST_MATH,
198 HasCos = EIGEN_FAST_MATH,
203 HasTanh = EIGEN_FAST_MATH,
204 HasErf = EIGEN_FAST_MATH,
211 struct packet_traits<int8_t> : default_packet_traits
213 typedef Packet16c type;
214 typedef Packet8c half;
240 struct packet_traits<uint8_t> : default_packet_traits
242 typedef Packet16uc type;
243 typedef Packet8uc half;
271 struct packet_traits<int16_t> : default_packet_traits
273 typedef Packet8s type;
274 typedef Packet4s half;
300 struct packet_traits<uint16_t> : default_packet_traits
302 typedef Packet8us type;
303 typedef Packet4us half;
330 struct packet_traits<int32_t> : default_packet_traits
332 typedef Packet4i type;
333 typedef Packet2i half;
359 struct packet_traits<uint32_t> : default_packet_traits
361 typedef Packet4ui type;
362 typedef Packet2ui half;
390 struct packet_traits<int64_t> : default_packet_traits
392 typedef Packet2l type;
393 typedef Packet2l half;
420 struct packet_traits<uint64_t> : default_packet_traits
422 typedef Packet2ul type;
423 typedef Packet2ul half;
449 #if EIGEN_GNUC_AT_MOST(4, 4) && !EIGEN_COMP_LLVM 451 EIGEN_STRONG_INLINE float32x4_t vld1q_f32(
const float* x) { return ::vld1q_f32((
const float32_t*)x); }
452 EIGEN_STRONG_INLINE float32x2_t vld1_f32(
const float* x) { return ::vld1_f32 ((
const float32_t*)x); }
453 EIGEN_STRONG_INLINE float32x2_t vld1_dup_f32(
const float* x) { return ::vld1_dup_f32 ((
const float32_t*)x); }
454 EIGEN_STRONG_INLINE
void vst1q_f32(
float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
455 EIGEN_STRONG_INLINE
void vst1_f32 (
float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
458 template<>
struct unpacket_traits<Packet2f>
461 typedef Packet2f half;
462 typedef Packet2i integer_packet;
468 masked_load_available =
false,
469 masked_store_available =
false 472 template<>
struct unpacket_traits<Packet4f>
475 typedef Packet2f half;
476 typedef Packet4i integer_packet;
482 masked_load_available =
false,
483 masked_store_available =
false 486 template<>
struct unpacket_traits<Packet4c>
489 typedef Packet4c half;
495 masked_load_available =
false,
496 masked_store_available =
false 499 template<>
struct unpacket_traits<Packet8c>
502 typedef Packet4c half;
508 masked_load_available =
false,
509 masked_store_available =
false 512 template<>
struct unpacket_traits<Packet16c>
515 typedef Packet8c half;
521 masked_load_available =
false,
522 masked_store_available =
false 525 template<>
struct unpacket_traits<Packet4uc>
527 typedef uint8_t type;
528 typedef Packet4uc half;
534 masked_load_available =
false,
535 masked_store_available =
false 538 template<>
struct unpacket_traits<Packet8uc>
540 typedef uint8_t type;
541 typedef Packet4uc half;
547 masked_load_available =
false,
548 masked_store_available =
false 551 template<>
struct unpacket_traits<Packet16uc>
553 typedef uint8_t type;
554 typedef Packet8uc half;
560 masked_load_available =
false,
561 masked_store_available =
false};
563 template<>
struct unpacket_traits<Packet4s>
565 typedef int16_t type;
566 typedef Packet4s half;
572 masked_load_available =
false,
573 masked_store_available =
false 576 template<>
struct unpacket_traits<Packet8s>
578 typedef int16_t type;
579 typedef Packet4s half;
585 masked_load_available =
false,
586 masked_store_available =
false 589 template<>
struct unpacket_traits<Packet4us>
591 typedef uint16_t type;
592 typedef Packet4us half;
598 masked_load_available =
false,
599 masked_store_available =
false 602 template<>
struct unpacket_traits<Packet8us>
604 typedef uint16_t type;
605 typedef Packet4us half;
611 masked_load_available =
false,
612 masked_store_available =
false 615 template<>
struct unpacket_traits<Packet2i>
617 typedef int32_t type;
618 typedef Packet2i half;
624 masked_load_available =
false,
625 masked_store_available =
false 628 template<>
struct unpacket_traits<Packet4i>
630 typedef int32_t type;
631 typedef Packet2i half;
637 masked_load_available =
false,
638 masked_store_available =
false 641 template<>
struct unpacket_traits<Packet2ui>
643 typedef uint32_t type;
644 typedef Packet2ui half;
650 masked_load_available =
false,
651 masked_store_available =
false 654 template<>
struct unpacket_traits<Packet4ui>
656 typedef uint32_t type;
657 typedef Packet2ui half;
663 masked_load_available =
false,
664 masked_store_available =
false 667 template<>
struct unpacket_traits<Packet2l>
669 typedef int64_t type;
670 typedef Packet2l half;
676 masked_load_available =
false,
677 masked_store_available =
false 680 template<>
struct unpacket_traits<Packet2ul>
682 typedef uint64_t type;
683 typedef Packet2ul half;
689 masked_load_available =
false,
690 masked_store_available =
false 694 template<> EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(
const float& from) {
return vdup_n_f32(from); }
695 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
return vdupq_n_f32(from); }
696 template<> EIGEN_STRONG_INLINE Packet4c pset1<Packet4c>(
const int8_t& from)
697 {
return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0); }
698 template<> EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(
const int8_t& from) {
return vdup_n_s8(from); }
699 template<> EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(
const int8_t& from) {
return vdupq_n_s8(from); }
700 template<> EIGEN_STRONG_INLINE Packet4uc pset1<Packet4uc>(
const uint8_t& from)
701 {
return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0); }
702 template<> EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(
const uint8_t& from) {
return vdup_n_u8(from); }
703 template<> EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(
const uint8_t& from) {
return vdupq_n_u8(from); }
704 template<> EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(
const int16_t& from) {
return vdup_n_s16(from); }
705 template<> EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(
const int16_t& from) {
return vdupq_n_s16(from); }
706 template<> EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(
const uint16_t& from) {
return vdup_n_u16(from); }
707 template<> EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(
const uint16_t& from) {
return vdupq_n_u16(from); }
708 template<> EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(
const int32_t& from) {
return vdup_n_s32(from); }
709 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int32_t& from) {
return vdupq_n_s32(from); }
710 template<> EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(
const uint32_t& from) {
return vdup_n_u32(from); }
711 template<> EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(
const uint32_t& from) {
return vdupq_n_u32(from); }
712 template<> EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const int64_t& from) {
return vdupq_n_s64(from); }
713 template<> EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(
const uint64_t& from) {
return vdupq_n_u64(from); }
715 template<> EIGEN_STRONG_INLINE Packet2f pset1frombits<Packet2f>(
unsigned int from)
716 {
return vreinterpret_f32_u32(vdup_n_u32(from)); }
717 template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(
unsigned int from)
718 {
return vreinterpretq_f32_u32(vdupq_n_u32(from)); }
720 template<> EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(
const float& a)
722 const float c[] = {0.0f,1.0f};
723 return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
725 template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a)
727 const float c[] = {0.0f,1.0f,2.0f,3.0f};
728 return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
730 template<> EIGEN_STRONG_INLINE Packet4c plset<Packet4c>(
const int8_t& a)
731 {
return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0); }
732 template<> EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(
const int8_t& a)
734 const int8_t c[] = {0,1,2,3,4,5,6,7};
735 return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
737 template<> EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(
const int8_t& a)
739 const int8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
740 return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
742 template<> EIGEN_STRONG_INLINE Packet4uc plset<Packet4uc>(
const uint8_t& a)
743 {
return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0); }
744 template<> EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(
const uint8_t& a)
746 const uint8_t c[] = {0,1,2,3,4,5,6,7};
747 return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
749 template<> EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(
const uint8_t& a)
751 const uint8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
752 return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
754 template<> EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(
const int16_t& a)
756 const int16_t c[] = {0,1,2,3};
757 return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
759 template<> EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(
const uint16_t& a)
761 const uint16_t c[] = {0,1,2,3};
762 return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
764 template<> EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(
const int16_t& a)
766 const int16_t c[] = {0,1,2,3,4,5,6,7};
767 return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
769 template<> EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(
const uint16_t& a)
771 const uint16_t c[] = {0,1,2,3,4,5,6,7};
772 return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
774 template<> EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(
const int32_t& a)
776 const int32_t c[] = {0,1};
777 return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
779 template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int32_t& a)
781 const int32_t c[] = {0,1,2,3};
782 return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
784 template<> EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(
const uint32_t& a)
786 const uint32_t c[] = {0,1};
787 return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
789 template<> EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(
const uint32_t& a)
791 const uint32_t c[] = {0,1,2,3};
792 return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
794 template<> EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(
const int64_t& a)
796 const int64_t c[] = {0,1};
797 return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
799 template<> EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(
const uint64_t& a)
801 const uint64_t c[] = {0,1};
802 return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
805 template<> EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vadd_f32(a,b); }
806 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vaddq_f32(a,b); }
807 template<> EIGEN_STRONG_INLINE Packet4c padd<Packet4c>(
const Packet4c& a,
const Packet4c& b)
809 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(
810 vreinterpret_s8_s32(vdup_n_s32(a)),
811 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
813 template<> EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vadd_s8(a,b); }
814 template<> EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vaddq_s8(a,b); }
815 template<> EIGEN_STRONG_INLINE Packet4uc padd<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
817 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(
818 vreinterpret_u8_u32(vdup_n_u32(a)),
819 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
821 template<> EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
return vadd_u8(a,b); }
822 template<> EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
return vaddq_u8(a,b); }
823 template<> EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vadd_s16(a,b); }
824 template<> EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vaddq_s16(a,b); }
825 template<> EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
return vadd_u16(a,b); }
826 template<> EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
return vaddq_u16(a,b); }
827 template<> EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vadd_s32(a,b); }
828 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vaddq_s32(a,b); }
829 template<> EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
return vadd_u32(a,b); }
830 template<> EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
return vaddq_u32(a,b); }
831 template<> EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
return vaddq_s64(a,b); }
832 template<> EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
return vaddq_u64(a,b); }
834 template<> EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vsub_f32(a,b); }
835 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vsubq_f32(a,b); }
836 template<> EIGEN_STRONG_INLINE Packet4c psub<Packet4c>(
const Packet4c& a,
const Packet4c& b)
838 return vget_lane_s32(vreinterpret_s32_s8(vsub_s8(
839 vreinterpret_s8_s32(vdup_n_s32(a)),
840 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
842 template<> EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vsub_s8(a,b); }
843 template<> EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vsubq_s8(a,b); }
844 template<> EIGEN_STRONG_INLINE Packet4uc psub<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
846 return vget_lane_u32(vreinterpret_u32_u8(vsub_u8(
847 vreinterpret_u8_u32(vdup_n_u32(a)),
848 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
850 template<> EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
return vsub_u8(a,b); }
851 template<> EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
return vsubq_u8(a,b); }
852 template<> EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vsub_s16(a,b); }
853 template<> EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vsubq_s16(a,b); }
854 template<> EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
return vsub_u16(a,b); }
855 template<> EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
return vsubq_u16(a,b); }
856 template<> EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vsub_s32(a,b); }
857 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vsubq_s32(a,b); }
858 template<> EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
return vsub_u32(a,b); }
859 template<> EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
return vsubq_u32(a,b); }
860 template<> EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
return vsubq_s64(a,b); }
861 template<> EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
return vsubq_u64(a,b); }
863 template<> EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(
const Packet2f& a,
const Packet2f& b);
864 template<> EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(
const Packet2f& a,
const Packet2f & b) {
865 Packet2f mask = {numext::bit_cast<
float>(0x80000000u), 0.0f};
866 return padd(a, pxor(mask, b));
868 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b);
869 template<> EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
870 Packet4f mask = {numext::bit_cast<
float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f};
871 return padd(a, pxor(mask, b));
874 template<> EIGEN_STRONG_INLINE Packet2f pnegate(
const Packet2f& a) {
return vneg_f32(a); }
875 template<> EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
return vnegq_f32(a); }
876 template<> EIGEN_STRONG_INLINE Packet4c pnegate(
const Packet4c& a)
877 {
return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
878 template<> EIGEN_STRONG_INLINE Packet8c pnegate(
const Packet8c& a) {
return vneg_s8(a); }
879 template<> EIGEN_STRONG_INLINE Packet16c pnegate(
const Packet16c& a) {
return vnegq_s8(a); }
880 template<> EIGEN_STRONG_INLINE Packet4s pnegate(
const Packet4s& a) {
return vneg_s16(a); }
881 template<> EIGEN_STRONG_INLINE Packet8s pnegate(
const Packet8s& a) {
return vnegq_s16(a); }
882 template<> EIGEN_STRONG_INLINE Packet2i pnegate(
const Packet2i& a) {
return vneg_s32(a); }
883 template<> EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
return vnegq_s32(a); }
884 template<> EIGEN_STRONG_INLINE Packet2l pnegate(
const Packet2l& a) {
889 vdup_n_s64(-vgetq_lane_s64(a, 0)),
890 vdup_n_s64(-vgetq_lane_s64(a, 1)));
894 template<> EIGEN_STRONG_INLINE Packet2f pconj(
const Packet2f& a) {
return a; }
895 template<> EIGEN_STRONG_INLINE Packet4f pconj(
const Packet4f& a) {
return a; }
896 template<> EIGEN_STRONG_INLINE Packet4c pconj(
const Packet4c& a) {
return a; }
897 template<> EIGEN_STRONG_INLINE Packet8c pconj(
const Packet8c& a) {
return a; }
898 template<> EIGEN_STRONG_INLINE Packet16c pconj(
const Packet16c& a) {
return a; }
899 template<> EIGEN_STRONG_INLINE Packet4uc pconj(
const Packet4uc& a) {
return a; }
900 template<> EIGEN_STRONG_INLINE Packet8uc pconj(
const Packet8uc& a) {
return a; }
901 template<> EIGEN_STRONG_INLINE Packet16uc pconj(
const Packet16uc& a) {
return a; }
902 template<> EIGEN_STRONG_INLINE Packet4s pconj(
const Packet4s& a) {
return a; }
903 template<> EIGEN_STRONG_INLINE Packet8s pconj(
const Packet8s& a) {
return a; }
904 template<> EIGEN_STRONG_INLINE Packet4us pconj(
const Packet4us& a) {
return a; }
905 template<> EIGEN_STRONG_INLINE Packet8us pconj(
const Packet8us& a) {
return a; }
906 template<> EIGEN_STRONG_INLINE Packet2i pconj(
const Packet2i& a) {
return a; }
907 template<> EIGEN_STRONG_INLINE Packet4i pconj(
const Packet4i& a) {
return a; }
908 template<> EIGEN_STRONG_INLINE Packet2ui pconj(
const Packet2ui& a) {
return a; }
909 template<> EIGEN_STRONG_INLINE Packet4ui pconj(
const Packet4ui& a) {
return a; }
910 template<> EIGEN_STRONG_INLINE Packet2l pconj(
const Packet2l& a) {
return a; }
911 template<> EIGEN_STRONG_INLINE Packet2ul pconj(
const Packet2ul& a) {
return a; }
913 template<> EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vmul_f32(a,b); }
914 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmulq_f32(a,b); }
915 template<> EIGEN_STRONG_INLINE Packet4c pmul<Packet4c>(
const Packet4c& a,
const Packet4c& b)
917 return vget_lane_s32(vreinterpret_s32_s8(vmul_s8(
918 vreinterpret_s8_s32(vdup_n_s32(a)),
919 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
921 template<> EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vmul_s8(a,b); }
922 template<> EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vmulq_s8(a,b); }
923 template<> EIGEN_STRONG_INLINE Packet4uc pmul<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
925 return vget_lane_u32(vreinterpret_u32_u8(vmul_u8(
926 vreinterpret_u8_u32(vdup_n_u32(a)),
927 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
929 template<> EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
return vmul_u8(a,b); }
930 template<> EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
return vmulq_u8(a,b); }
931 template<> EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vmul_s16(a,b); }
932 template<> EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vmulq_s16(a,b); }
933 template<> EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
return vmul_u16(a,b); }
934 template<> EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
return vmulq_u16(a,b); }
935 template<> EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vmul_s32(a,b); }
936 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmulq_s32(a,b); }
937 template<> EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
return vmul_u32(a,b); }
938 template<> EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
return vmulq_u32(a,b); }
939 template<> EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
941 vdup_n_s64(vgetq_lane_s64(a, 0)*vgetq_lane_s64(b, 0)),
942 vdup_n_s64(vgetq_lane_s64(a, 1)*vgetq_lane_s64(b, 1)));
944 template<> EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
946 vdup_n_u64(vgetq_lane_u64(a, 0)*vgetq_lane_u64(b, 0)),
947 vdup_n_u64(vgetq_lane_u64(a, 1)*vgetq_lane_u64(b, 1)));
950 template<> EIGEN_STRONG_INLINE Packet2f pdiv<Packet2f>(
const Packet2f& a,
const Packet2f& b)
953 return vdiv_f32(a,b);
955 Packet2f inv, restep, div;
966 restep = vrecps_f32(b, inv);
967 inv = vmul_f32(restep, inv);
970 div = vmul_f32(a, inv);
975 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b)
978 return vdivq_f32(a,b);
980 Packet4f inv, restep, div;
987 inv = vrecpeq_f32(b);
991 restep = vrecpsq_f32(b, inv);
992 inv = vmulq_f32(restep, inv);
995 div = vmulq_f32(a, inv);
1001 template<> EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(
const Packet4c& ,
const Packet4c& )
1003 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1004 return pset1<Packet4c>(0);
1006 template<> EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(
const Packet8c& ,
const Packet8c& )
1008 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1009 return pset1<Packet8c>(0);
1011 template<> EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(
const Packet16c& ,
const Packet16c& )
1013 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1014 return pset1<Packet16c>(0);
1016 template<> EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(
const Packet4uc& ,
const Packet4uc& )
1018 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1019 return pset1<Packet4uc>(0);
1021 template<> EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(
const Packet8uc& ,
const Packet8uc& )
1023 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1024 return pset1<Packet8uc>(0);
1026 template<> EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(
const Packet16uc& ,
const Packet16uc& )
1028 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1029 return pset1<Packet16uc>(0);
1031 template<> EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(
const Packet4s& ,
const Packet4s& )
1033 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1034 return pset1<Packet4s>(0);
1036 template<> EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(
const Packet8s& ,
const Packet8s& )
1038 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1039 return pset1<Packet8s>(0);
1041 template<> EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(
const Packet4us& ,
const Packet4us& )
1043 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1044 return pset1<Packet4us>(0);
1046 template<> EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(
const Packet8us& ,
const Packet8us& )
1048 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1049 return pset1<Packet8us>(0);
1051 template<> EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(
const Packet2i& ,
const Packet2i& )
1053 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1054 return pset1<Packet2i>(0);
1056 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& )
1058 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1059 return pset1<Packet4i>(0);
1061 template<> EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(
const Packet2ui& ,
const Packet2ui& )
1063 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1064 return pset1<Packet2ui>(0);
1066 template<> EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(
const Packet4ui& ,
const Packet4ui& )
1068 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1069 return pset1<Packet4ui>(0);
1071 template<> EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(
const Packet2l& ,
const Packet2l& )
1073 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1074 return pset1<Packet2l>(0LL);
1076 template<> EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(
const Packet2ul& ,
const Packet2ul& )
1078 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1079 return pset1<Packet2ul>(0ULL);
1083 #ifdef __ARM_FEATURE_FMA 1084 template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c)
1085 {
return vfmaq_f32(c,a,b); }
1086 template<> EIGEN_STRONG_INLINE Packet2f pmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c)
1087 {
return vfma_f32(c,a,b); }
1089 template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c)
1091 return vmlaq_f32(c,a,b);
1093 template<> EIGEN_STRONG_INLINE Packet2f pmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c)
1095 return vmla_f32(c,a,b);
1100 template<> EIGEN_STRONG_INLINE Packet4c pmadd(
const Packet4c& a,
const Packet4c& b,
const Packet4c& c)
1102 return vget_lane_s32(vreinterpret_s32_s8(vmla_s8(
1103 vreinterpret_s8_s32(vdup_n_s32(c)),
1104 vreinterpret_s8_s32(vdup_n_s32(a)),
1105 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1107 template<> EIGEN_STRONG_INLINE Packet8c pmadd(
const Packet8c& a,
const Packet8c& b,
const Packet8c& c)
1108 {
return vmla_s8(c,a,b); }
1109 template<> EIGEN_STRONG_INLINE Packet16c pmadd(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c)
1110 {
return vmlaq_s8(c,a,b); }
1111 template<> EIGEN_STRONG_INLINE Packet4uc pmadd(
const Packet4uc& a,
const Packet4uc& b,
const Packet4uc& c)
1113 return vget_lane_u32(vreinterpret_u32_u8(vmla_u8(
1114 vreinterpret_u8_u32(vdup_n_u32(c)),
1115 vreinterpret_u8_u32(vdup_n_u32(a)),
1116 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1118 template<> EIGEN_STRONG_INLINE Packet8uc pmadd(
const Packet8uc& a,
const Packet8uc& b,
const Packet8uc& c)
1119 {
return vmla_u8(c,a,b); }
1120 template<> EIGEN_STRONG_INLINE Packet16uc pmadd(
const Packet16uc& a,
const Packet16uc& b,
const Packet16uc& c)
1121 {
return vmlaq_u8(c,a,b); }
1122 template<> EIGEN_STRONG_INLINE Packet4s pmadd(
const Packet4s& a,
const Packet4s& b,
const Packet4s& c)
1123 {
return vmla_s16(c,a,b); }
1124 template<> EIGEN_STRONG_INLINE Packet8s pmadd(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c)
1125 {
return vmlaq_s16(c,a,b); }
1126 template<> EIGEN_STRONG_INLINE Packet4us pmadd(
const Packet4us& a,
const Packet4us& b,
const Packet4us& c)
1127 {
return vmla_u16(c,a,b); }
1128 template<> EIGEN_STRONG_INLINE Packet8us pmadd(
const Packet8us& a,
const Packet8us& b,
const Packet8us& c)
1129 {
return vmlaq_u16(c,a,b); }
1130 template<> EIGEN_STRONG_INLINE Packet2i pmadd(
const Packet2i& a,
const Packet2i& b,
const Packet2i& c)
1131 {
return vmla_s32(c,a,b); }
1132 template<> EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c)
1133 {
return vmlaq_s32(c,a,b); }
1134 template<> EIGEN_STRONG_INLINE Packet2ui pmadd(
const Packet2ui& a,
const Packet2ui& b,
const Packet2ui& c)
1135 {
return vmla_u32(c,a,b); }
1136 template<> EIGEN_STRONG_INLINE Packet4ui pmadd(
const Packet4ui& a,
const Packet4ui& b,
const Packet4ui& c)
1137 {
return vmlaq_u32(c,a,b); }
1139 template<> EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1140 {
return vabd_f32(a,b); }
1141 template<> EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1142 {
return vabdq_f32(a,b); }
1143 template<> EIGEN_STRONG_INLINE Packet4c pabsdiff<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1145 return vget_lane_s32(vreinterpret_s32_s8(vabd_s8(
1146 vreinterpret_s8_s32(vdup_n_s32(a)),
1147 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1149 template<> EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1150 {
return vabd_s8(a,b); }
1151 template<> EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1152 {
return vabdq_s8(a,b); }
1153 template<> EIGEN_STRONG_INLINE Packet4uc pabsdiff<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1155 return vget_lane_u32(vreinterpret_u32_u8(vabd_u8(
1156 vreinterpret_u8_u32(vdup_n_u32(a)),
1157 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1159 template<> EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1160 {
return vabd_u8(a,b); }
1161 template<> EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1162 {
return vabdq_u8(a,b); }
1163 template<> EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1164 {
return vabd_s16(a,b); }
1165 template<> EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1166 {
return vabdq_s16(a,b); }
1167 template<> EIGEN_STRONG_INLINE Packet4us pabsdiff<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1168 {
return vabd_u16(a,b); }
1169 template<> EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1170 {
return vabdq_u16(a,b); }
1171 template<> EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(
const Packet2i& a,
const Packet2i& b)
1172 {
return vabd_s32(a,b); }
1173 template<> EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(
const Packet4i& a,
const Packet4i& b)
1174 {
return vabdq_s32(a,b); }
1175 template<> EIGEN_STRONG_INLINE Packet2ui pabsdiff<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1176 {
return vabd_u32(a,b); }
1177 template<> EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1178 {
return vabdq_u32(a,b); }
1180 template<> EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vmin_f32(a,b); }
1181 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vminq_f32(a,b); }
1183 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN 1185 template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vminnmq_f32(a, b); }
1186 template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vminnm_f32(a, b); }
1189 template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return pmin<Packet4f>(a, b); }
1191 template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNaN, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return pmin<Packet2f>(a, b); }
1193 template<> EIGEN_STRONG_INLINE Packet4c pmin<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1195 return vget_lane_s32(vreinterpret_s32_s8(vmin_s8(
1196 vreinterpret_s8_s32(vdup_n_s32(a)),
1197 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1199 template<> EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vmin_s8(a,b); }
1200 template<> EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vminq_s8(a,b); }
1201 template<> EIGEN_STRONG_INLINE Packet4uc pmin<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1203 return vget_lane_u32(vreinterpret_u32_u8(vmin_u8(
1204 vreinterpret_u8_u32(vdup_n_u32(a)),
1205 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1207 template<> EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
return vmin_u8(a,b); }
1208 template<> EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
return vminq_u8(a,b); }
1209 template<> EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vmin_s16(a,b); }
1210 template<> EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vminq_s16(a,b); }
1211 template<> EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
return vmin_u16(a,b); }
1212 template<> EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
return vminq_u16(a,b); }
1213 template<> EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vmin_s32(a,b); }
1214 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vminq_s32(a,b); }
1215 template<> EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
return vmin_u32(a,b); }
1216 template<> EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
return vminq_u32(a,b); }
1217 template<> EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1218 return vcombine_s64(
1219 vdup_n_s64((std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1220 vdup_n_s64((std::min)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1222 template<> EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1223 return vcombine_u64(
1224 vdup_n_u64((std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1225 vdup_n_u64((std::min)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1228 template<> EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vmax_f32(a,b); }
1229 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmaxq_f32(a,b); }
1231 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN 1233 template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vmaxnmq_f32(a, b); }
1234 template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return vmaxnm_f32(a, b); }
1237 template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNaN, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return pmax<Packet4f>(a, b); }
1239 template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNaN, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
return pmax<Packet2f>(a, b); }
1241 template<> EIGEN_STRONG_INLINE Packet4c pmax<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1243 return vget_lane_s32(vreinterpret_s32_s8(vmax_s8(
1244 vreinterpret_s8_s32(vdup_n_s32(a)),
1245 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1247 template<> EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vmax_s8(a,b); }
1248 template<> EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vmaxq_s8(a,b); }
1249 template<> EIGEN_STRONG_INLINE Packet4uc pmax<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1251 return vget_lane_u32(vreinterpret_u32_u8(vmax_u8(
1252 vreinterpret_u8_u32(vdup_n_u32(a)),
1253 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1255 template<> EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
return vmax_u8(a,b); }
1256 template<> EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
return vmaxq_u8(a,b); }
1257 template<> EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vmax_s16(a,b); }
1258 template<> EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vmaxq_s16(a,b); }
1259 template<> EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
return vmax_u16(a,b); }
1260 template<> EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
return vmaxq_u16(a,b); }
1261 template<> EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vmax_s32(a,b); }
1262 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vmaxq_s32(a,b); }
1263 template<> EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
return vmax_u32(a,b); }
1264 template<> EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
return vmaxq_u32(a,b); }
1265 template<> EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1266 return vcombine_s64(
1267 vdup_n_s64((std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1268 vdup_n_s64((std::max)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1270 template<> EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1271 return vcombine_u64(
1272 vdup_n_u64((std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1273 vdup_n_u64((std::max)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1276 template<> EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1277 {
return vreinterpret_f32_u32(vcle_f32(a,b)); }
1278 template<> EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1279 {
return vreinterpretq_f32_u32(vcleq_f32(a,b)); }
1280 template<> EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1282 return vget_lane_s32(vreinterpret_s32_u8(vcle_s8(
1283 vreinterpret_s8_s32(vdup_n_s32(a)),
1284 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1286 template<> EIGEN_STRONG_INLINE Packet8c pcmp_le<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1287 {
return vreinterpret_s8_u8(vcle_s8(a,b)); }
1288 template<> EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1289 {
return vreinterpretq_s8_u8(vcleq_s8(a,b)); }
1290 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_le<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1292 return vget_lane_u32(vreinterpret_u32_u8(vcle_u8(
1293 vreinterpret_u8_u32(vdup_n_u32(a)),
1294 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1296 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1297 {
return vcle_u8(a,b); }
1298 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1299 {
return vcleq_u8(a,b); }
1300 template<> EIGEN_STRONG_INLINE Packet4s pcmp_le<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1301 {
return vreinterpret_s16_u16(vcle_s16(a,b)); }
1302 template<> EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1303 {
return vreinterpretq_s16_u16(vcleq_s16(a,b)); }
1304 template<> EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1305 {
return vcle_u16(a,b); }
1306 template<> EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1307 {
return vcleq_u16(a,b); }
1308 template<> EIGEN_STRONG_INLINE Packet2i pcmp_le<Packet2i>(
const Packet2i& a,
const Packet2i& b)
1309 {
return vreinterpret_s32_u32(vcle_s32(a,b)); }
1310 template<> EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(
const Packet4i& a,
const Packet4i& b)
1311 {
return vreinterpretq_s32_u32(vcleq_s32(a,b)); }
1312 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1313 {
return vcle_u32(a,b); }
1314 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1315 {
return vcleq_u32(a,b); }
1316 template<> EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1318 #if EIGEN_ARCH_ARM64 1319 return vreinterpretq_s64_u64(vcleq_s64(a,b));
1321 return vcombine_s64(
1322 vdup_n_s64(vgetq_lane_s64(a, 0) <= vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1323 vdup_n_s64(vgetq_lane_s64(a, 1) <= vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1326 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1328 #if EIGEN_ARCH_ARM64 1329 return vcleq_u64(a,b);
1331 return vcombine_u64(
1332 vdup_n_u64(vgetq_lane_u64(a, 0) <= vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1333 vdup_n_u64(vgetq_lane_u64(a, 1) <= vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1337 template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1338 {
return vreinterpret_f32_u32(vclt_f32(a,b)); }
1339 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1340 {
return vreinterpretq_f32_u32(vcltq_f32(a,b)); }
1341 template<> EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1343 return vget_lane_s32(vreinterpret_s32_u8(vclt_s8(
1344 vreinterpret_s8_s32(vdup_n_s32(a)),
1345 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1347 template<> EIGEN_STRONG_INLINE Packet8c pcmp_lt<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1348 {
return vreinterpret_s8_u8(vclt_s8(a,b)); }
1349 template<> EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1350 {
return vreinterpretq_s8_u8(vcltq_s8(a,b)); }
1351 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_lt<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1353 return vget_lane_u32(vreinterpret_u32_u8(vclt_u8(
1354 vreinterpret_u8_u32(vdup_n_u32(a)),
1355 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1357 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1358 {
return vclt_u8(a,b); }
1359 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1360 {
return vcltq_u8(a,b); }
1361 template<> EIGEN_STRONG_INLINE Packet4s pcmp_lt<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1362 {
return vreinterpret_s16_u16(vclt_s16(a,b)); }
1363 template<> EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1364 {
return vreinterpretq_s16_u16(vcltq_s16(a,b)); }
1365 template<> EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1366 {
return vclt_u16(a,b); }
1367 template<> EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1368 {
return vcltq_u16(a,b); }
1369 template<> EIGEN_STRONG_INLINE Packet2i pcmp_lt<Packet2i>(
const Packet2i& a,
const Packet2i& b)
1370 {
return vreinterpret_s32_u32(vclt_s32(a,b)); }
1371 template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(
const Packet4i& a,
const Packet4i& b)
1372 {
return vreinterpretq_s32_u32(vcltq_s32(a,b)); }
1373 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1374 {
return vclt_u32(a,b); }
1375 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1376 {
return vcltq_u32(a,b); }
1377 template<> EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1379 #if EIGEN_ARCH_ARM64 1380 return vreinterpretq_s64_u64(vcltq_s64(a,b));
1382 return vcombine_s64(
1383 vdup_n_s64(vgetq_lane_s64(a, 0) < vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1384 vdup_n_s64(vgetq_lane_s64(a, 1) < vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1387 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1389 #if EIGEN_ARCH_ARM64 1390 return vcltq_u64(a,b);
1392 return vcombine_u64(
1393 vdup_n_u64(vgetq_lane_u64(a, 0) < vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1394 vdup_n_u64(vgetq_lane_u64(a, 1) < vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1398 template<> EIGEN_STRONG_INLINE Packet2f pcmp_eq<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1399 {
return vreinterpret_f32_u32(vceq_f32(a,b)); }
1400 template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1401 {
return vreinterpretq_f32_u32(vceqq_f32(a,b)); }
1402 template<> EIGEN_STRONG_INLINE Packet4c pcmp_eq<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1404 return vget_lane_s32(vreinterpret_s32_u8(vceq_s8(
1405 vreinterpret_s8_s32(vdup_n_s32(a)),
1406 vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1408 template<> EIGEN_STRONG_INLINE Packet8c pcmp_eq<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1409 {
return vreinterpret_s8_u8(vceq_s8(a,b)); }
1410 template<> EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1411 {
return vreinterpretq_s8_u8(vceqq_s8(a,b)); }
1412 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_eq<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1414 return vget_lane_u32(vreinterpret_u32_u8(vceq_u8(
1415 vreinterpret_u8_u32(vdup_n_u32(a)),
1416 vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1418 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1419 {
return vceq_u8(a,b); }
1420 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1421 {
return vceqq_u8(a,b); }
1422 template<> EIGEN_STRONG_INLINE Packet4s pcmp_eq<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1423 {
return vreinterpret_s16_u16(vceq_s16(a,b)); }
1424 template<> EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1425 {
return vreinterpretq_s16_u16(vceqq_s16(a,b)); }
1426 template<> EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1427 {
return vceq_u16(a,b); }
1428 template<> EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1429 {
return vceqq_u16(a,b); }
1430 template<> EIGEN_STRONG_INLINE Packet2i pcmp_eq<Packet2i>(
const Packet2i& a,
const Packet2i& b)
1431 {
return vreinterpret_s32_u32(vceq_s32(a,b)); }
1432 template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(
const Packet4i& a,
const Packet4i& b)
1433 {
return vreinterpretq_s32_u32(vceqq_s32(a,b)); }
1434 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1435 {
return vceq_u32(a,b); }
1436 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1437 {
return vceqq_u32(a,b); }
1438 template<> EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1440 #if EIGEN_ARCH_ARM64 1441 return vreinterpretq_s64_u64(vceqq_s64(a,b));
1443 return vcombine_s64(
1444 vdup_n_s64(vgetq_lane_s64(a, 0) == vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1445 vdup_n_s64(vgetq_lane_s64(a, 1) == vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1448 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1450 #if EIGEN_ARCH_ARM64 1451 return vceqq_u64(a,b);
1453 return vcombine_u64(
1454 vdup_n_u64(vgetq_lane_u64(a, 0) == vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1455 vdup_n_u64(vgetq_lane_u64(a, 1) == vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1459 template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1460 {
return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a,b))); }
1461 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1462 {
return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a,b))); }
1465 template<> EIGEN_STRONG_INLINE Packet2f pand<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1466 {
return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1467 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1468 {
return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1469 template<> EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1471 template<> EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1472 {
return vand_s8(a,b); }
1473 template<> EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1474 {
return vandq_s8(a,b); }
1475 template<> EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1477 template<> EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1478 {
return vand_u8(a,b); }
1479 template<> EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1480 {
return vandq_u8(a,b); }
1481 template<> EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return vand_s16(a,b); }
1482 template<> EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return vandq_s16(a,b); }
1483 template<> EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1484 {
return vand_u16(a,b); }
1485 template<> EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1486 {
return vandq_u16(a,b); }
1487 template<> EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vand_s32(a,b); }
1488 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vandq_s32(a,b); }
1489 template<> EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1490 {
return vand_u32(a,b); }
1491 template<> EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1492 {
return vandq_u32(a,b); }
1493 template<> EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
return vandq_s64(a,b); }
1494 template<> EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1495 {
return vandq_u64(a,b); }
1497 template<> EIGEN_STRONG_INLINE Packet2f por<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1498 {
return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1499 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1500 {
return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1501 template<> EIGEN_STRONG_INLINE Packet4c por<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1503 template<> EIGEN_STRONG_INLINE Packet8c por<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vorr_s8(a,b); }
1504 template<> EIGEN_STRONG_INLINE Packet16c por<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1505 {
return vorrq_s8(a,b); }
1506 template<> EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1508 template<> EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1509 {
return vorr_u8(a,b); }
1510 template<> EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1511 {
return vorrq_u8(a,b); }
1512 template<> EIGEN_STRONG_INLINE Packet4s por<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1513 {
return vorr_s16(a,b); }
1514 template<> EIGEN_STRONG_INLINE Packet8s por<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1515 {
return vorrq_s16(a,b); }
1516 template<> EIGEN_STRONG_INLINE Packet4us por<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1517 {
return vorr_u16(a,b); }
1518 template<> EIGEN_STRONG_INLINE Packet8us por<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1519 {
return vorrq_u16(a,b); }
1520 template<> EIGEN_STRONG_INLINE Packet2i por<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return vorr_s32(a,b); }
1521 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vorrq_s32(a,b); }
1522 template<> EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1523 {
return vorr_u32(a,b); }
1524 template<> EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1525 {
return vorrq_u32(a,b); }
1526 template<> EIGEN_STRONG_INLINE Packet2l por<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1527 {
return vorrq_s64(a,b); }
1528 template<> EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1529 {
return vorrq_u64(a,b); }
1531 template<> EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1532 {
return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1533 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1534 {
return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1535 template<> EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1537 template<> EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(
const Packet8c& a,
const Packet8c& b)
1538 {
return veor_s8(a,b); }
1539 template<> EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(
const Packet16c& a,
const Packet16c& b)
1540 {
return veorq_s8(a,b); }
1541 template<> EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1543 template<> EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1544 {
return veor_u8(a,b); }
1545 template<> EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1546 {
return veorq_u8(a,b); }
1547 template<> EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
return veor_s16(a,b); }
1548 template<> EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
return veorq_s16(a,b); }
1549 template<> EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1550 {
return veor_u16(a,b); }
1551 template<> EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1552 {
return veorq_u16(a,b); }
1553 template<> EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
return veor_s32(a,b); }
1554 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return veorq_s32(a,b); }
1555 template<> EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1556 {
return veor_u32(a,b); }
1557 template<> EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1558 {
return veorq_u32(a,b); }
1559 template<> EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1560 {
return veorq_s64(a,b); }
1561 template<> EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1562 {
return veorq_u64(a,b); }
1564 template<> EIGEN_STRONG_INLINE Packet2f pandnot<Packet2f>(
const Packet2f& a,
const Packet2f& b)
1565 {
return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1566 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b)
1567 {
return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1568 template<> EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(
const Packet4c& a,
const Packet4c& b)
1570 template<> EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
return vbic_s8(a,b); }
1571 template<> EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
return vbicq_s8(a,b); }
1572 template<> EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b)
1574 template<> EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b)
1575 {
return vbic_u8(a,b); }
1576 template<> EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b)
1577 {
return vbicq_u8(a,b); }
1578 template<> EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(
const Packet4s& a,
const Packet4s& b)
1579 {
return vbic_s16(a,b); }
1580 template<> EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(
const Packet8s& a,
const Packet8s& b)
1581 {
return vbicq_s16(a,b); }
1582 template<> EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(
const Packet4us& a,
const Packet4us& b)
1583 {
return vbic_u16(a,b); }
1584 template<> EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(
const Packet8us& a,
const Packet8us& b)
1585 {
return vbicq_u16(a,b); }
1586 template<> EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(
const Packet2i& a,
const Packet2i& b)
1587 {
return vbic_s32(a,b); }
1588 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b)
1589 {
return vbicq_s32(a,b); }
1590 template<> EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b)
1591 {
return vbic_u32(a,b); }
1592 template<> EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b)
1593 {
return vbicq_u32(a,b); }
1594 template<> EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(
const Packet2l& a,
const Packet2l& b)
1595 {
return vbicq_s64(a,b); }
1596 template<> EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b)
1597 {
return vbicq_u64(a,b); }
1600 template<
int N> EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(Packet4c& a)
1601 {
return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); }
1602 template<
int N> EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(Packet8c a) {
return vshr_n_s8(a,N); }
1603 template<
int N> EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(Packet16c a) {
return vshrq_n_s8(a,N); }
1604 template<
int N> EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(Packet4uc& a)
1605 {
return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); }
1606 template<
int N> EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(Packet8uc a) {
return vshr_n_u8(a,N); }
1607 template<
int N> EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(Packet16uc a) {
return vshrq_n_u8(a,N); }
1608 template<
int N> EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(Packet4s a) {
return vshr_n_s16(a,N); }
1609 template<
int N> EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) {
return vshrq_n_s16(a,N); }
1610 template<
int N> EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(Packet4us a) {
return vshr_n_u16(a,N); }
1611 template<
int N> EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(Packet8us a) {
return vshrq_n_u16(a,N); }
1612 template<
int N> EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(Packet2i a) {
return vshr_n_s32(a,N); }
1613 template<
int N> EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(Packet4i a) {
return vshrq_n_s32(a,N); }
1614 template<
int N> EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(Packet2ui a) {
return vshr_n_u32(a,N); }
1615 template<
int N> EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(Packet4ui a) {
return vshrq_n_u32(a,N); }
1616 template<
int N> EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(Packet2l a) {
return vshrq_n_s64(a,N); }
1617 template<
int N> EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(Packet2ul a) {
return vshrq_n_u64(a,N); }
1619 template<
int N> EIGEN_STRONG_INLINE Packet4c plogical_shift_right(Packet4c& a)
1620 {
return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0); }
1621 template<
int N> EIGEN_STRONG_INLINE Packet8c plogical_shift_right(Packet8c a)
1622 {
return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a),N)); }
1623 template<
int N> EIGEN_STRONG_INLINE Packet16c plogical_shift_right(Packet16c a)
1624 {
return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a),N)); }
1625 template<
int N> EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(Packet4uc& a)
1626 {
return vget_lane_u32(vreinterpret_u32_s8(vshr_n_s8(vreinterpret_s8_u32(vdup_n_u32(a)), N)), 0); }
1627 template<
int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(Packet8uc a) {
return vshr_n_u8(a,N); }
1628 template<
int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(Packet16uc a) {
return vshrq_n_u8(a,N); }
1629 template<
int N> EIGEN_STRONG_INLINE Packet4s plogical_shift_right(Packet4s a)
1630 {
return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a),N)); }
1631 template<
int N> EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a)
1632 {
return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a),N)); }
1633 template<
int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_right(Packet4us a) {
return vshr_n_u16(a,N); }
1634 template<
int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_right(Packet8us a) {
return vshrq_n_u16(a,N); }
1635 template<
int N> EIGEN_STRONG_INLINE Packet2i plogical_shift_right(Packet2i a)
1636 {
return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a),N)); }
1637 template<
int N> EIGEN_STRONG_INLINE Packet4i plogical_shift_right(Packet4i a)
1638 {
return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a),N)); }
1639 template<
int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(Packet2ui a) {
return vshr_n_u32(a,N); }
1640 template<
int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(Packet4ui a) {
return vshrq_n_u32(a,N); }
1641 template<
int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_right(Packet2l a)
1642 {
return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a),N)); }
1643 template<
int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(Packet2ul a) {
return vshrq_n_u64(a,N); }
1645 template<
int N> EIGEN_STRONG_INLINE Packet4c plogical_shift_left(Packet4c& a)
1646 {
return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); }
1647 template<
int N> EIGEN_STRONG_INLINE Packet8c plogical_shift_left(Packet8c a) {
return vshl_n_s8(a,N); }
1648 template<
int N> EIGEN_STRONG_INLINE Packet16c plogical_shift_left(Packet16c a) {
return vshlq_n_s8(a,N); }
1649 template<
int N> EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(Packet4uc& a)
1650 {
return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); }
1651 template<
int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(Packet8uc a) {
return vshl_n_u8(a,N); }
1652 template<
int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(Packet16uc a) {
return vshlq_n_u8(a,N); }
1653 template<
int N> EIGEN_STRONG_INLINE Packet4s plogical_shift_left(Packet4s a) {
return vshl_n_s16(a,N); }
1654 template<
int N> EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) {
return vshlq_n_s16(a,N); }
1655 template<
int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_left(Packet4us a) {
return vshl_n_u16(a,N); }
1656 template<
int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_left(Packet8us a) {
return vshlq_n_u16(a,N); }
1657 template<
int N> EIGEN_STRONG_INLINE Packet2i plogical_shift_left(Packet2i a) {
return vshl_n_s32(a,N); }
1658 template<
int N> EIGEN_STRONG_INLINE Packet4i plogical_shift_left(Packet4i a) {
return vshlq_n_s32(a,N); }
1659 template<
int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(Packet2ui a) {
return vshl_n_u32(a,N); }
1660 template<
int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(Packet4ui a) {
return vshlq_n_u32(a,N); }
1661 template<
int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_left(Packet2l a) {
return vshlq_n_s64(a,N); }
1662 template<
int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(Packet2ul a) {
return vshlq_n_u64(a,N); }
1664 template<> EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(
const float* from)
1665 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_f32(from); }
1666 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from)
1667 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f32(from); }
1668 template<> EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(
const int8_t* from)
1671 memcpy(&res, from,
sizeof(Packet4c));
1674 template<> EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(
const int8_t* from)
1675 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s8(from); }
1676 template<> EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(
const int8_t* from)
1677 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s8(from); }
1678 template<> EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(
const uint8_t* from)
1681 memcpy(&res, from,
sizeof(Packet4uc));
1684 template<> EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(
const uint8_t* from)
1685 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u8(from); }
1686 template<> EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(
const uint8_t* from)
1687 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u8(from); }
1688 template<> EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(
const int16_t* from)
1689 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s16(from); }
1690 template<> EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(
const int16_t* from)
1691 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s16(from); }
1692 template<> EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(
const uint16_t* from)
1693 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u16(from); }
1694 template<> EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(
const uint16_t* from)
1695 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u16(from); }
1696 template<> EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(
const int32_t* from)
1697 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s32(from); }
1698 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int32_t* from)
1699 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s32(from); }
1700 template<> EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(
const uint32_t* from)
1701 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u32(from); }
1702 template<> EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(
const uint32_t* from)
1703 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u32(from); }
1704 template<> EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(
const int64_t* from)
1705 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s64(from); }
1706 template<> EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(
const uint64_t* from)
1707 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u64(from); }
1709 template<> EIGEN_STRONG_INLINE Packet2f ploadu<Packet2f>(
const float* from)
1710 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_f32(from); }
1711 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from)
1712 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f32(from); }
1713 template<> EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(
const int8_t* from)
1716 memcpy(&res, from,
sizeof(Packet4c));
1719 template<> EIGEN_STRONG_INLINE Packet8c ploadu<Packet8c>(
const int8_t* from)
1720 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s8(from); }
1721 template<> EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(
const int8_t* from)
1722 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s8(from); }
1723 template<> EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(
const uint8_t* from)
1726 memcpy(&res, from,
sizeof(Packet4uc));
1729 template<> EIGEN_STRONG_INLINE Packet8uc ploadu<Packet8uc>(
const uint8_t* from)
1730 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u8(from); }
1731 template<> EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(
const uint8_t* from)
1732 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u8(from); }
1733 template<> EIGEN_STRONG_INLINE Packet4s ploadu<Packet4s>(
const int16_t* from)
1734 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s16(from); }
1735 template<> EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(
const int16_t* from)
1736 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s16(from); }
1737 template<> EIGEN_STRONG_INLINE Packet4us ploadu<Packet4us>(
const uint16_t* from)
1738 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u16(from); }
1739 template<> EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(
const uint16_t* from)
1740 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u16(from); }
1741 template<> EIGEN_STRONG_INLINE Packet2i ploadu<Packet2i>(
const int32_t* from)
1742 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s32(from); }
1743 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int32_t* from)
1744 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s32(from); }
1745 template<> EIGEN_STRONG_INLINE Packet2ui ploadu<Packet2ui>(
const uint32_t* from)
1746 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u32(from); }
1747 template<> EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(
const uint32_t* from)
1748 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u32(from); }
1749 template<> EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(
const int64_t* from)
1750 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s64(from); }
1751 template<> EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(
const uint64_t* from)
1752 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u64(from); }
1754 template<> EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(
const float* from)
1755 {
return vld1_dup_f32(from); }
1756 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from)
1757 {
return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from+1)); }
1758 template<> EIGEN_STRONG_INLINE Packet4c ploaddup<Packet4c>(
const int8_t* from)
1760 const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
1761 return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a,a).val[0]), 0);
1763 template<> EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(
const int8_t* from)
1765 const int8x8_t a = vld1_s8(from);
1766 return vzip_s8(a,a).val[0];
1768 template<> EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(
const int8_t* from)
1770 const int8x8_t a = vld1_s8(from);
1771 const int8x8x2_t b = vzip_s8(a,a);
1772 return vcombine_s8(b.val[0], b.val[1]);
1774 template<> EIGEN_STRONG_INLINE Packet4uc ploaddup<Packet4uc>(
const uint8_t* from)
1776 const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
1777 return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a,a).val[0]), 0);
1779 template<> EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(
const uint8_t* from)
1781 const uint8x8_t a = vld1_u8(from);
1782 return vzip_u8(a,a).val[0];
1784 template<> EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(
const uint8_t* from)
1786 const uint8x8_t a = vld1_u8(from);
1787 const uint8x8x2_t b = vzip_u8(a,a);
1788 return vcombine_u8(b.val[0], b.val[1]);
1790 template<> EIGEN_STRONG_INLINE Packet4s ploaddup<Packet4s>(
const int16_t* from)
1792 return vreinterpret_s16_u32(vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)),
1793 vreinterpret_u32_s16(vld1_dup_s16(from+1))).val[0]);
1795 template<> EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(
const int16_t* from)
1797 const int16x4_t a = vld1_s16(from);
1798 const int16x4x2_t b = vzip_s16(a,a);
1799 return vcombine_s16(b.val[0], b.val[1]);
1801 template<> EIGEN_STRONG_INLINE Packet4us ploaddup<Packet4us>(
const uint16_t* from)
1803 return vreinterpret_u16_u32(vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)),
1804 vreinterpret_u32_u16(vld1_dup_u16(from+1))).val[0]);
1806 template<> EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(
const uint16_t* from)
1808 const uint16x4_t a = vld1_u16(from);
1809 const uint16x4x2_t b = vzip_u16(a,a);
1810 return vcombine_u16(b.val[0], b.val[1]);
1812 template<> EIGEN_STRONG_INLINE Packet2i ploaddup<Packet2i>(
const int32_t* from)
1813 {
return vld1_dup_s32(from); }
1814 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int32_t* from)
1815 {
return vcombine_s32(vld1_dup_s32(from), vld1_dup_s32(from+1)); }
1816 template<> EIGEN_STRONG_INLINE Packet2ui ploaddup<Packet2ui>(
const uint32_t* from)
1817 {
return vld1_dup_u32(from); }
1818 template<> EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(
const uint32_t* from)
1819 {
return vcombine_u32(vld1_dup_u32(from), vld1_dup_u32(from+1)); }
1820 template<> EIGEN_STRONG_INLINE Packet2l ploaddup<Packet2l>(
const int64_t* from)
1821 {
return vld1q_dup_s64(from); }
1822 template<> EIGEN_STRONG_INLINE Packet2ul ploaddup<Packet2ul>(
const uint64_t* from)
1823 {
return vld1q_dup_u64(from); }
1825 template<> EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(
const float* from) {
return vld1q_dup_f32(from); }
1826 template<> EIGEN_STRONG_INLINE Packet4c ploadquad<Packet4c>(
const int8_t* from)
1827 {
return vget_lane_s32(vreinterpret_s32_s8(vld1_dup_s8(from)), 0); }
1828 template<> EIGEN_STRONG_INLINE Packet8c ploadquad<Packet8c>(
const int8_t* from)
1830 return vreinterpret_s8_u32(vzip_u32(
1831 vreinterpret_u32_s8(vld1_dup_s8(from)),
1832 vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]);
1834 template<> EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(
const int8_t* from)
1836 const int8x8_t a = vreinterpret_s8_u32(vzip_u32(
1837 vreinterpret_u32_s8(vld1_dup_s8(from)),
1838 vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]);
1839 const int8x8_t b = vreinterpret_s8_u32(vzip_u32(
1840 vreinterpret_u32_s8(vld1_dup_s8(from+2)),
1841 vreinterpret_u32_s8(vld1_dup_s8(from+3))).val[0]);
1842 return vcombine_s8(a,b);
1844 template<> EIGEN_STRONG_INLINE Packet4uc ploadquad<Packet4uc>(
const uint8_t* from)
1845 {
return vget_lane_u32(vreinterpret_u32_u8(vld1_dup_u8(from)), 0); }
1846 template<> EIGEN_STRONG_INLINE Packet8uc ploadquad<Packet8uc>(
const uint8_t* from)
1848 return vreinterpret_u8_u32(vzip_u32(
1849 vreinterpret_u32_u8(vld1_dup_u8(from)),
1850 vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]);
1852 template<> EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(
const uint8_t* from)
1854 const uint8x8_t a = vreinterpret_u8_u32(vzip_u32(
1855 vreinterpret_u32_u8(vld1_dup_u8(from)),
1856 vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]);
1857 const uint8x8_t b = vreinterpret_u8_u32(vzip_u32(
1858 vreinterpret_u32_u8(vld1_dup_u8(from+2)),
1859 vreinterpret_u32_u8(vld1_dup_u8(from+3))).val[0]);
1860 return vcombine_u8(a,b);
1862 template<> EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(
const int16_t* from)
1863 {
return vcombine_s16(vld1_dup_s16(from), vld1_dup_s16(from+1)); }
1864 template<> EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(
const uint16_t* from)
1865 {
return vcombine_u16(vld1_dup_u16(from), vld1_dup_u16(from+1)); }
1866 template<> EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(
const int32_t* from) {
return vld1q_dup_s32(from); }
1867 template<> EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(
const uint32_t* from) {
return vld1q_dup_u32(from); }
1869 template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet2f& from)
1870 { EIGEN_DEBUG_ALIGNED_STORE vst1_f32(to,from); }
1871 template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from)
1872 { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to,from); }
1873 template<> EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet4c& from)
1874 { memcpy(to, &from,
sizeof(from)); }
1875 template<> EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet8c& from)
1876 { EIGEN_DEBUG_ALIGNED_STORE vst1_s8(to,from); }
1877 template<> EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet16c& from)
1878 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(to,from); }
1879 template<> EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet4uc& from)
1880 { memcpy(to, &from,
sizeof(from)); }
1881 template<> EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet8uc& from)
1882 { EIGEN_DEBUG_ALIGNED_STORE vst1_u8(to,from); }
1883 template<> EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet16uc& from)
1884 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(to,from); }
1885 template<> EIGEN_STRONG_INLINE
void pstore<int16_t>(int16_t* to,
const Packet4s& from)
1886 { EIGEN_DEBUG_ALIGNED_STORE vst1_s16(to,from); }
1887 template<> EIGEN_STRONG_INLINE
void pstore<int16_t>(int16_t* to,
const Packet8s& from)
1888 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(to,from); }
1889 template<> EIGEN_STRONG_INLINE
void pstore<uint16_t>(uint16_t* to,
const Packet4us& from)
1890 { EIGEN_DEBUG_ALIGNED_STORE vst1_u16(to,from); }
1891 template<> EIGEN_STRONG_INLINE
void pstore<uint16_t>(uint16_t* to,
const Packet8us& from)
1892 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(to,from); }
1893 template<> EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet2i& from)
1894 { EIGEN_DEBUG_ALIGNED_STORE vst1_s32(to,from); }
1895 template<> EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet4i& from)
1896 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to,from); }
1897 template<> EIGEN_STRONG_INLINE
void pstore<uint32_t>(uint32_t* to,
const Packet2ui& from)
1898 { EIGEN_DEBUG_ALIGNED_STORE vst1_u32(to,from); }
1899 template<> EIGEN_STRONG_INLINE
void pstore<uint32_t>(uint32_t* to,
const Packet4ui& from)
1900 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(to,from); }
1901 template<> EIGEN_STRONG_INLINE
void pstore<int64_t>(int64_t* to,
const Packet2l& from)
1902 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(to,from); }
1903 template<> EIGEN_STRONG_INLINE
void pstore<uint64_t>(uint64_t* to,
const Packet2ul& from)
1904 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(to,from); }
1906 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet2f& from)
1907 { EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to,from); }
1908 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from)
1909 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to,from); }
1910 template<> EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet4c& from)
1911 { memcpy(to, &from,
sizeof(from)); }
1912 template<> EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet8c& from)
1913 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to,from); }
1914 template<> EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet16c& from)
1915 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s8(to,from); }
1916 template<> EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet4uc& from)
1917 { memcpy(to, &from,
sizeof(from)); }
1918 template<> EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet8uc& from)
1919 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to,from); }
1920 template<> EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet16uc& from)
1921 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u8(to,from); }
1922 template<> EIGEN_STRONG_INLINE
void pstoreu<int16_t>(int16_t* to,
const Packet4s& from)
1923 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to,from); }
1924 template<> EIGEN_STRONG_INLINE
void pstoreu<int16_t>(int16_t* to,
const Packet8s& from)
1925 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s16(to,from); }
1926 template<> EIGEN_STRONG_INLINE
void pstoreu<uint16_t>(uint16_t* to,
const Packet4us& from)
1927 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to,from); }
1928 template<> EIGEN_STRONG_INLINE
void pstoreu<uint16_t>(uint16_t* to,
const Packet8us& from)
1929 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(to,from); }
1930 template<> EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet2i& from)
1931 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to,from); }
1932 template<> EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet4i& from)
1933 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to,from); }
1934 template<> EIGEN_STRONG_INLINE
void pstoreu<uint32_t>(uint32_t* to,
const Packet2ui& from)
1935 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to,from); }
1936 template<> EIGEN_STRONG_INLINE
void pstoreu<uint32_t>(uint32_t* to,
const Packet4ui& from)
1937 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u32(to,from); }
1938 template<> EIGEN_STRONG_INLINE
void pstoreu<int64_t>(int64_t* to,
const Packet2l& from)
1939 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s64(to,from); }
1940 template<> EIGEN_STRONG_INLINE
void pstoreu<uint64_t>(uint64_t* to,
const Packet2ul& from)
1941 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to,from); }
1943 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather<float, Packet2f>(
const float* from,
Index stride)
1945 Packet2f res = vld1_dup_f32(from);
1946 res = vld1_lane_f32(from + 1*stride, res, 1);
1949 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(
const float* from,
Index stride)
1951 Packet4f res = vld1q_dup_f32(from);
1952 res = vld1q_lane_f32(from + 1*stride, res, 1);
1953 res = vld1q_lane_f32(from + 2*stride, res, 2);
1954 res = vld1q_lane_f32(from + 3*stride, res, 3);
1957 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather<int8_t, Packet4c>(
const int8_t* from,
Index stride)
1960 for (
int i = 0; i != 4; i++)
1961 reinterpret_cast<int8_t*>(&res)[i] = *(from + i * stride);
1964 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather<int8_t, Packet8c>(
const int8_t* from,
Index stride)
1966 Packet8c res = vld1_dup_s8(from);
1967 res = vld1_lane_s8(from + 1*stride, res, 1);
1968 res = vld1_lane_s8(from + 2*stride, res, 2);
1969 res = vld1_lane_s8(from + 3*stride, res, 3);
1970 res = vld1_lane_s8(from + 4*stride, res, 4);
1971 res = vld1_lane_s8(from + 5*stride, res, 5);
1972 res = vld1_lane_s8(from + 6*stride, res, 6);
1973 res = vld1_lane_s8(from + 7*stride, res, 7);
1976 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(
const int8_t* from,
Index stride)
1978 Packet16c res = vld1q_dup_s8(from);
1979 res = vld1q_lane_s8(from + 1*stride, res, 1);
1980 res = vld1q_lane_s8(from + 2*stride, res, 2);
1981 res = vld1q_lane_s8(from + 3*stride, res, 3);
1982 res = vld1q_lane_s8(from + 4*stride, res, 4);
1983 res = vld1q_lane_s8(from + 5*stride, res, 5);
1984 res = vld1q_lane_s8(from + 6*stride, res, 6);
1985 res = vld1q_lane_s8(from + 7*stride, res, 7);
1986 res = vld1q_lane_s8(from + 8*stride, res, 8);
1987 res = vld1q_lane_s8(from + 9*stride, res, 9);
1988 res = vld1q_lane_s8(from + 10*stride, res, 10);
1989 res = vld1q_lane_s8(from + 11*stride, res, 11);
1990 res = vld1q_lane_s8(from + 12*stride, res, 12);
1991 res = vld1q_lane_s8(from + 13*stride, res, 13);
1992 res = vld1q_lane_s8(from + 14*stride, res, 14);
1993 res = vld1q_lane_s8(from + 15*stride, res, 15);
1996 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather<uint8_t, Packet4uc>(
const uint8_t* from,
Index stride)
1999 for (
int i = 0; i != 4; i++)
2000 reinterpret_cast<uint8_t*>(&res)[i] = *(from + i * stride);
2003 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather<uint8_t, Packet8uc>(
const uint8_t* from,
Index stride)
2005 Packet8uc res = vld1_dup_u8(from);
2006 res = vld1_lane_u8(from + 1*stride, res, 1);
2007 res = vld1_lane_u8(from + 2*stride, res, 2);
2008 res = vld1_lane_u8(from + 3*stride, res, 3);
2009 res = vld1_lane_u8(from + 4*stride, res, 4);
2010 res = vld1_lane_u8(from + 5*stride, res, 5);
2011 res = vld1_lane_u8(from + 6*stride, res, 6);
2012 res = vld1_lane_u8(from + 7*stride, res, 7);
2015 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(
const uint8_t* from,
Index stride)
2017 Packet16uc res = vld1q_dup_u8(from);
2018 res = vld1q_lane_u8(from + 1*stride, res, 1);
2019 res = vld1q_lane_u8(from + 2*stride, res, 2);
2020 res = vld1q_lane_u8(from + 3*stride, res, 3);
2021 res = vld1q_lane_u8(from + 4*stride, res, 4);
2022 res = vld1q_lane_u8(from + 5*stride, res, 5);
2023 res = vld1q_lane_u8(from + 6*stride, res, 6);
2024 res = vld1q_lane_u8(from + 7*stride, res, 7);
2025 res = vld1q_lane_u8(from + 8*stride, res, 8);
2026 res = vld1q_lane_u8(from + 9*stride, res, 9);
2027 res = vld1q_lane_u8(from + 10*stride, res, 10);
2028 res = vld1q_lane_u8(from + 11*stride, res, 11);
2029 res = vld1q_lane_u8(from + 12*stride, res, 12);
2030 res = vld1q_lane_u8(from + 13*stride, res, 13);
2031 res = vld1q_lane_u8(from + 14*stride, res, 14);
2032 res = vld1q_lane_u8(from + 15*stride, res, 15);
2035 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather<int16_t, Packet4s>(
const int16_t* from,
Index stride)
2037 Packet4s res = vld1_dup_s16(from);
2038 res = vld1_lane_s16(from + 1*stride, res, 1);
2039 res = vld1_lane_s16(from + 2*stride, res, 2);
2040 res = vld1_lane_s16(from + 3*stride, res, 3);
2043 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(
const int16_t* from,
Index stride)
2045 Packet8s res = vld1q_dup_s16(from);
2046 res = vld1q_lane_s16(from + 1*stride, res, 1);
2047 res = vld1q_lane_s16(from + 2*stride, res, 2);
2048 res = vld1q_lane_s16(from + 3*stride, res, 3);
2049 res = vld1q_lane_s16(from + 4*stride, res, 4);
2050 res = vld1q_lane_s16(from + 5*stride, res, 5);
2051 res = vld1q_lane_s16(from + 6*stride, res, 6);
2052 res = vld1q_lane_s16(from + 7*stride, res, 7);
2055 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather<uint16_t, Packet4us>(
const uint16_t* from,
Index stride)
2057 Packet4us res = vld1_dup_u16(from);
2058 res = vld1_lane_u16(from + 1*stride, res, 1);
2059 res = vld1_lane_u16(from + 2*stride, res, 2);
2060 res = vld1_lane_u16(from + 3*stride, res, 3);
2063 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(
const uint16_t* from,
Index stride)
2065 Packet8us res = vld1q_dup_u16(from);
2066 res = vld1q_lane_u16(from + 1*stride, res, 1);
2067 res = vld1q_lane_u16(from + 2*stride, res, 2);
2068 res = vld1q_lane_u16(from + 3*stride, res, 3);
2069 res = vld1q_lane_u16(from + 4*stride, res, 4);
2070 res = vld1q_lane_u16(from + 5*stride, res, 5);
2071 res = vld1q_lane_u16(from + 6*stride, res, 6);
2072 res = vld1q_lane_u16(from + 7*stride, res, 7);
2075 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather<int32_t, Packet2i>(
const int32_t* from,
Index stride)
2077 Packet2i res = vld1_dup_s32(from);
2078 res = vld1_lane_s32(from + 1*stride, res, 1);
2081 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(
const int32_t* from,
Index stride)
2083 Packet4i res = vld1q_dup_s32(from);
2084 res = vld1q_lane_s32(from + 1*stride, res, 1);
2085 res = vld1q_lane_s32(from + 2*stride, res, 2);
2086 res = vld1q_lane_s32(from + 3*stride, res, 3);
2089 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather<uint32_t, Packet2ui>(
const uint32_t* from,
Index stride)
2091 Packet2ui res = vld1_dup_u32(from);
2092 res = vld1_lane_u32(from + 1*stride, res, 1);
2095 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(
const uint32_t* from,
Index stride)
2097 Packet4ui res = vld1q_dup_u32(from);
2098 res = vld1q_lane_u32(from + 1*stride, res, 1);
2099 res = vld1q_lane_u32(from + 2*stride, res, 2);
2100 res = vld1q_lane_u32(from + 3*stride, res, 3);
2103 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(
const int64_t* from,
Index stride)
2105 Packet2l res = vld1q_dup_s64(from);
2106 res = vld1q_lane_s64(from + 1*stride, res, 1);
2109 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(
const uint64_t* from,
Index stride)
2111 Packet2ul res = vld1q_dup_u64(from);
2112 res = vld1q_lane_u64(from + 1*stride, res, 1);
2116 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<float, Packet2f>(
float* to,
const Packet2f& from,
Index stride)
2118 vst1_lane_f32(to + stride*0, from, 0);
2119 vst1_lane_f32(to + stride*1, from, 1);
2121 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<float, Packet4f>(
float* to,
const Packet4f& from,
Index stride)
2123 vst1q_lane_f32(to + stride*0, from, 0);
2124 vst1q_lane_f32(to + stride*1, from, 1);
2125 vst1q_lane_f32(to + stride*2, from, 2);
2126 vst1q_lane_f32(to + stride*3, from, 3);
2128 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet4c>(int8_t* to,
const Packet4c& from,
Index stride)
2130 for (
int i = 0; i != 4; i++)
2131 *(to + i * stride) =
reinterpret_cast<const int8_t*
>(&from)[i];
2133 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet8c>(int8_t* to,
const Packet8c& from,
Index stride)
2135 vst1_lane_s8(to + stride*0, from, 0);
2136 vst1_lane_s8(to + stride*1, from, 1);
2137 vst1_lane_s8(to + stride*2, from, 2);
2138 vst1_lane_s8(to + stride*3, from, 3);
2139 vst1_lane_s8(to + stride*4, from, 4);
2140 vst1_lane_s8(to + stride*5, from, 5);
2141 vst1_lane_s8(to + stride*6, from, 6);
2142 vst1_lane_s8(to + stride*7, from, 7);
2144 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet16c>(int8_t* to,
const Packet16c& from,
Index stride)
2146 vst1q_lane_s8(to + stride*0, from, 0);
2147 vst1q_lane_s8(to + stride*1, from, 1);
2148 vst1q_lane_s8(to + stride*2, from, 2);
2149 vst1q_lane_s8(to + stride*3, from, 3);
2150 vst1q_lane_s8(to + stride*4, from, 4);
2151 vst1q_lane_s8(to + stride*5, from, 5);
2152 vst1q_lane_s8(to + stride*6, from, 6);
2153 vst1q_lane_s8(to + stride*7, from, 7);
2154 vst1q_lane_s8(to + stride*8, from, 8);
2155 vst1q_lane_s8(to + stride*9, from, 9);
2156 vst1q_lane_s8(to + stride*10, from, 10);
2157 vst1q_lane_s8(to + stride*11, from, 11);
2158 vst1q_lane_s8(to + stride*12, from, 12);
2159 vst1q_lane_s8(to + stride*13, from, 13);
2160 vst1q_lane_s8(to + stride*14, from, 14);
2161 vst1q_lane_s8(to + stride*15, from, 15);
2163 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet4uc>(uint8_t* to,
const Packet4uc& from,
Index stride)
2165 for (
int i = 0; i != 4; i++)
2166 *(to + i * stride) =
reinterpret_cast<const uint8_t*
>(&from)[i];
2168 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet8uc>(uint8_t* to,
const Packet8uc& from,
Index stride)
2170 vst1_lane_u8(to + stride*0, from, 0);
2171 vst1_lane_u8(to + stride*1, from, 1);
2172 vst1_lane_u8(to + stride*2, from, 2);
2173 vst1_lane_u8(to + stride*3, from, 3);
2174 vst1_lane_u8(to + stride*4, from, 4);
2175 vst1_lane_u8(to + stride*5, from, 5);
2176 vst1_lane_u8(to + stride*6, from, 6);
2177 vst1_lane_u8(to + stride*7, from, 7);
2179 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet16uc>(uint8_t* to,
const Packet16uc& from,
Index stride)
2181 vst1q_lane_u8(to + stride*0, from, 0);
2182 vst1q_lane_u8(to + stride*1, from, 1);
2183 vst1q_lane_u8(to + stride*2, from, 2);
2184 vst1q_lane_u8(to + stride*3, from, 3);
2185 vst1q_lane_u8(to + stride*4, from, 4);
2186 vst1q_lane_u8(to + stride*5, from, 5);
2187 vst1q_lane_u8(to + stride*6, from, 6);
2188 vst1q_lane_u8(to + stride*7, from, 7);
2189 vst1q_lane_u8(to + stride*8, from, 8);
2190 vst1q_lane_u8(to + stride*9, from, 9);
2191 vst1q_lane_u8(to + stride*10, from, 10);
2192 vst1q_lane_u8(to + stride*11, from, 11);
2193 vst1q_lane_u8(to + stride*12, from, 12);
2194 vst1q_lane_u8(to + stride*13, from, 13);
2195 vst1q_lane_u8(to + stride*14, from, 14);
2196 vst1q_lane_u8(to + stride*15, from, 15);
2198 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int16_t, Packet4s>(int16_t* to,
const Packet4s& from,
Index stride)
2200 vst1_lane_s16(to + stride*0, from, 0);
2201 vst1_lane_s16(to + stride*1, from, 1);
2202 vst1_lane_s16(to + stride*2, from, 2);
2203 vst1_lane_s16(to + stride*3, from, 3);
2205 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int16_t, Packet8s>(int16_t* to,
const Packet8s& from,
Index stride)
2207 vst1q_lane_s16(to + stride*0, from, 0);
2208 vst1q_lane_s16(to + stride*1, from, 1);
2209 vst1q_lane_s16(to + stride*2, from, 2);
2210 vst1q_lane_s16(to + stride*3, from, 3);
2211 vst1q_lane_s16(to + stride*4, from, 4);
2212 vst1q_lane_s16(to + stride*5, from, 5);
2213 vst1q_lane_s16(to + stride*6, from, 6);
2214 vst1q_lane_s16(to + stride*7, from, 7);
2216 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint16_t, Packet4us>(uint16_t* to,
const Packet4us& from,
Index stride)
2218 vst1_lane_u16(to + stride*0, from, 0);
2219 vst1_lane_u16(to + stride*1, from, 1);
2220 vst1_lane_u16(to + stride*2, from, 2);
2221 vst1_lane_u16(to + stride*3, from, 3);
2223 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint16_t, Packet8us>(uint16_t* to,
const Packet8us& from,
Index stride)
2225 vst1q_lane_u16(to + stride*0, from, 0);
2226 vst1q_lane_u16(to + stride*1, from, 1);
2227 vst1q_lane_u16(to + stride*2, from, 2);
2228 vst1q_lane_u16(to + stride*3, from, 3);
2229 vst1q_lane_u16(to + stride*4, from, 4);
2230 vst1q_lane_u16(to + stride*5, from, 5);
2231 vst1q_lane_u16(to + stride*6, from, 6);
2232 vst1q_lane_u16(to + stride*7, from, 7);
2234 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int32_t, Packet2i>(int32_t* to,
const Packet2i& from,
Index stride)
2236 vst1_lane_s32(to + stride*0, from, 0);
2237 vst1_lane_s32(to + stride*1, from, 1);
2239 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int32_t, Packet4i>(int32_t* to,
const Packet4i& from,
Index stride)
2241 vst1q_lane_s32(to + stride*0, from, 0);
2242 vst1q_lane_s32(to + stride*1, from, 1);
2243 vst1q_lane_s32(to + stride*2, from, 2);
2244 vst1q_lane_s32(to + stride*3, from, 3);
2246 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint32_t, Packet2ui>(uint32_t* to,
const Packet2ui& from,
Index stride)
2248 vst1_lane_u32(to + stride*0, from, 0);
2249 vst1_lane_u32(to + stride*1, from, 1);
2251 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint32_t, Packet4ui>(uint32_t* to,
const Packet4ui& from,
Index stride)
2253 vst1q_lane_u32(to + stride*0, from, 0);
2254 vst1q_lane_u32(to + stride*1, from, 1);
2255 vst1q_lane_u32(to + stride*2, from, 2);
2256 vst1q_lane_u32(to + stride*3, from, 3);
2258 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int64_t, Packet2l>(int64_t* to,
const Packet2l& from,
Index stride)
2260 vst1q_lane_s64(to + stride*0, from, 0);
2261 vst1q_lane_s64(to + stride*1, from, 1);
2263 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint64_t, Packet2ul>(uint64_t* to,
const Packet2ul& from,
Index stride)
2265 vst1q_lane_u64(to + stride*0, from, 0);
2266 vst1q_lane_u64(to + stride*1, from, 1);
2269 template<> EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) { EIGEN_ARM_PREFETCH(addr); }
2270 template<> EIGEN_STRONG_INLINE
void prefetch<int8_t>(
const int8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2271 template<> EIGEN_STRONG_INLINE
void prefetch<uint8_t>(
const uint8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2272 template<> EIGEN_STRONG_INLINE
void prefetch<int16_t>(
const int16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2273 template<> EIGEN_STRONG_INLINE
void prefetch<uint16_t>(
const uint16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2274 template<> EIGEN_STRONG_INLINE
void prefetch<int32_t>(
const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2275 template<> EIGEN_STRONG_INLINE
void prefetch<uint32_t>(
const uint32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2276 template<> EIGEN_STRONG_INLINE
void prefetch<int64_t>(
const int64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2277 template<> EIGEN_STRONG_INLINE
void prefetch<uint64_t>(
const uint64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2279 template<> EIGEN_STRONG_INLINE
float pfirst<Packet2f>(
const Packet2f& a) {
return vget_lane_f32(a,0); }
2280 template<> EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
return vgetq_lane_f32(a,0); }
2281 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet4c>(
const Packet4c& a) {
return static_cast<int8_t
>(a & 0xff); }
2282 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet8c>(
const Packet8c& a) {
return vget_lane_s8(a,0); }
2283 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(
const Packet16c& a) {
return vgetq_lane_s8(a,0); }
2284 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet4uc>(
const Packet4uc& a) {
return static_cast<uint8_t
>(a & 0xff); }
2285 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet8uc>(
const Packet8uc& a) {
return vget_lane_u8(a,0); }
2286 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(
const Packet16uc& a) {
return vgetq_lane_u8(a,0); }
2287 template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet4s>(
const Packet4s& a) {
return vget_lane_s16(a,0); }
2288 template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(
const Packet8s& a) {
return vgetq_lane_s16(a,0); }
2289 template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet4us>(
const Packet4us& a) {
return vget_lane_u16(a,0); }
2290 template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(
const Packet8us& a) {
return vgetq_lane_u16(a,0); }
2291 template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet2i>(
const Packet2i& a) {
return vget_lane_s32(a,0); }
2292 template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(
const Packet4i& a) {
return vgetq_lane_s32(a,0); }
2293 template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet2ui>(
const Packet2ui& a) {
return vget_lane_u32(a,0); }
2294 template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(
const Packet4ui& a) {
return vgetq_lane_u32(a,0); }
2295 template<> EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(
const Packet2l& a) {
return vgetq_lane_s64(a,0); }
2296 template<> EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(
const Packet2ul& a) {
return vgetq_lane_u64(a,0); }
2298 template<> EIGEN_STRONG_INLINE Packet2f preverse(
const Packet2f& a) {
return vrev64_f32(a); }
2299 template<> EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a)
2301 const float32x4_t a_r64 = vrev64q_f32(a);
2302 return vcombine_f32(vget_high_f32(a_r64), vget_low_f32(a_r64));
2304 template<> EIGEN_STRONG_INLINE Packet4c preverse(
const Packet4c& a)
2305 {
return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
2306 template<> EIGEN_STRONG_INLINE Packet8c preverse(
const Packet8c& a) {
return vrev64_s8(a); }
2307 template<> EIGEN_STRONG_INLINE Packet16c preverse(
const Packet16c& a)
2309 const int8x16_t a_r64 = vrev64q_s8(a);
2310 return vcombine_s8(vget_high_s8(a_r64), vget_low_s8(a_r64));
2312 template<> EIGEN_STRONG_INLINE Packet4uc preverse(
const Packet4uc& a)
2313 {
return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0); }
2314 template<> EIGEN_STRONG_INLINE Packet8uc preverse(
const Packet8uc& a) {
return vrev64_u8(a); }
2315 template<> EIGEN_STRONG_INLINE Packet16uc preverse(
const Packet16uc& a)
2317 const uint8x16_t a_r64 = vrev64q_u8(a);
2318 return vcombine_u8(vget_high_u8(a_r64), vget_low_u8(a_r64));
2320 template<> EIGEN_STRONG_INLINE Packet4s preverse(
const Packet4s& a) {
return vrev64_s16(a); }
2321 template<> EIGEN_STRONG_INLINE Packet8s preverse(
const Packet8s& a)
2323 const int16x8_t a_r64 = vrev64q_s16(a);
2324 return vcombine_s16(vget_high_s16(a_r64), vget_low_s16(a_r64));
2326 template<> EIGEN_STRONG_INLINE Packet4us preverse(
const Packet4us& a) {
return vrev64_u16(a); }
2327 template<> EIGEN_STRONG_INLINE Packet8us preverse(
const Packet8us& a)
2329 const uint16x8_t a_r64 = vrev64q_u16(a);
2330 return vcombine_u16(vget_high_u16(a_r64), vget_low_u16(a_r64));
2332 template<> EIGEN_STRONG_INLINE Packet2i preverse(
const Packet2i& a) {
return vrev64_s32(a); }
2333 template<> EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a)
2335 const int32x4_t a_r64 = vrev64q_s32(a);
2336 return vcombine_s32(vget_high_s32(a_r64), vget_low_s32(a_r64));
2338 template<> EIGEN_STRONG_INLINE Packet2ui preverse(
const Packet2ui& a) {
return vrev64_u32(a); }
2339 template<> EIGEN_STRONG_INLINE Packet4ui preverse(
const Packet4ui& a)
2341 const uint32x4_t a_r64 = vrev64q_u32(a);
2342 return vcombine_u32(vget_high_u32(a_r64), vget_low_u32(a_r64));
2344 template<> EIGEN_STRONG_INLINE Packet2l preverse(
const Packet2l& a)
2345 {
return vcombine_s64(vget_high_s64(a), vget_low_s64(a)); }
2346 template<> EIGEN_STRONG_INLINE Packet2ul preverse(
const Packet2ul& a)
2347 {
return vcombine_u64(vget_high_u64(a), vget_low_u64(a)); }
2349 template<> EIGEN_STRONG_INLINE Packet2f pabs(
const Packet2f& a) {
return vabs_f32(a); }
2350 template<> EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
return vabsq_f32(a); }
2351 template<> EIGEN_STRONG_INLINE Packet4c pabs<Packet4c>(
const Packet4c& a)
2352 {
return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
2353 template<> EIGEN_STRONG_INLINE Packet8c pabs(
const Packet8c& a) {
return vabs_s8(a); }
2354 template<> EIGEN_STRONG_INLINE Packet16c pabs(
const Packet16c& a) {
return vabsq_s8(a); }
2355 template<> EIGEN_STRONG_INLINE Packet4uc pabs(
const Packet4uc& a) {
return a; }
2356 template<> EIGEN_STRONG_INLINE Packet8uc pabs(
const Packet8uc& a) {
return a; }
2357 template<> EIGEN_STRONG_INLINE Packet16uc pabs(
const Packet16uc& a) {
return a; }
2358 template<> EIGEN_STRONG_INLINE Packet4s pabs(
const Packet4s& a) {
return vabs_s16(a); }
2359 template<> EIGEN_STRONG_INLINE Packet8s pabs(
const Packet8s& a) {
return vabsq_s16(a); }
2360 template<> EIGEN_STRONG_INLINE Packet4us pabs(
const Packet4us& a) {
return a; }
2361 template<> EIGEN_STRONG_INLINE Packet8us pabs(
const Packet8us& a) {
return a; }
2362 template<> EIGEN_STRONG_INLINE Packet2i pabs(
const Packet2i& a) {
return vabs_s32(a); }
2363 template<> EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
return vabsq_s32(a); }
2364 template<> EIGEN_STRONG_INLINE Packet2ui pabs(
const Packet2ui& a) {
return a; }
2365 template<> EIGEN_STRONG_INLINE Packet4ui pabs(
const Packet4ui& a) {
return a; }
2366 template<> EIGEN_STRONG_INLINE Packet2l pabs(
const Packet2l& a) {
2367 #if EIGEN_ARCH_ARM64 2368 return vabsq_s64(a);
2370 return vcombine_s64(
2371 vdup_n_s64((std::abs)(vgetq_lane_s64(a, 0))),
2372 vdup_n_s64((std::abs)(vgetq_lane_s64(a, 1))));
2375 template<> EIGEN_STRONG_INLINE Packet2ul pabs(
const Packet2ul& a) {
return a; }
2377 template<> EIGEN_STRONG_INLINE Packet2f pfrexp<Packet2f>(
const Packet2f& a, Packet2f& exponent)
2378 {
return pfrexp_generic(a,exponent); }
2379 template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(
const Packet4f& a, Packet4f& exponent)
2380 {
return pfrexp_generic(a,exponent); }
2382 template<> EIGEN_STRONG_INLINE Packet2f pldexp<Packet2f>(
const Packet2f& a,
const Packet2f& exponent)
2383 {
return pldexp_generic(a,exponent); }
2384 template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent)
2385 {
return pldexp_generic(a,exponent); }
2387 template<> EIGEN_STRONG_INLINE
float predux<Packet2f>(
const Packet2f& a) {
return vget_lane_f32(vpadd_f32(a,a), 0); }
2388 template<> EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a)
2390 const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a));
2391 return vget_lane_f32(vpadd_f32(sum, sum), 0);
2393 template<> EIGEN_STRONG_INLINE int8_t predux<Packet4c>(
const Packet4c& a)
2395 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2396 int8x8_t sum = vpadd_s8(a_dup, a_dup);
2397 sum = vpadd_s8(sum, sum);
2398 return vget_lane_s8(sum, 0);
2400 template<> EIGEN_STRONG_INLINE int8_t predux<Packet8c>(
const Packet8c& a)
2402 int8x8_t sum = vpadd_s8(a,a);
2403 sum = vpadd_s8(sum, sum);
2404 sum = vpadd_s8(sum, sum);
2405 return vget_lane_s8(sum, 0);
2407 template<> EIGEN_STRONG_INLINE int8_t predux<Packet16c>(
const Packet16c& a)
2409 int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a));
2410 sum = vpadd_s8(sum, sum);
2411 sum = vpadd_s8(sum, sum);
2412 sum = vpadd_s8(sum, sum);
2413 return vget_lane_s8(sum, 0);
2415 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet4uc>(
const Packet4uc& a)
2417 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2418 uint8x8_t sum = vpadd_u8(a_dup, a_dup);
2419 sum = vpadd_u8(sum, sum);
2420 return vget_lane_u8(sum, 0);
2422 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(
const Packet8uc& a)
2424 uint8x8_t sum = vpadd_u8(a,a);
2425 sum = vpadd_u8(sum, sum);
2426 sum = vpadd_u8(sum, sum);
2427 return vget_lane_u8(sum, 0);
2429 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(
const Packet16uc& a)
2431 uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a));
2432 sum = vpadd_u8(sum, sum);
2433 sum = vpadd_u8(sum, sum);
2434 sum = vpadd_u8(sum, sum);
2435 return vget_lane_u8(sum, 0);
2437 template<> EIGEN_STRONG_INLINE int16_t predux<Packet4s>(
const Packet4s& a)
2439 const int16x4_t sum = vpadd_s16(a,a);
2440 return vget_lane_s16(vpadd_s16(sum, sum), 0);
2442 template<> EIGEN_STRONG_INLINE int16_t predux<Packet8s>(
const Packet8s& a)
2444 int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a));
2445 sum = vpadd_s16(sum, sum);
2446 sum = vpadd_s16(sum, sum);
2447 return vget_lane_s16(sum, 0);
2449 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(
const Packet4us& a)
2451 const uint16x4_t sum = vpadd_u16(a,a);
2452 return vget_lane_u16(vpadd_u16(sum, sum), 0);
2454 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(
const Packet8us& a)
2456 uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a));
2457 sum = vpadd_u16(sum, sum);
2458 sum = vpadd_u16(sum, sum);
2459 return vget_lane_u16(sum, 0);
2461 template<> EIGEN_STRONG_INLINE int32_t predux<Packet2i>(
const Packet2i& a) {
return vget_lane_s32(vpadd_s32(a,a), 0); }
2462 template<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(
const Packet4i& a)
2464 const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a));
2465 return vget_lane_s32(vpadd_s32(sum, sum), 0);
2467 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(
const Packet2ui& a) {
return vget_lane_u32(vpadd_u32(a,a), 0); }
2468 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(
const Packet4ui& a)
2470 const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a));
2471 return vget_lane_u32(vpadd_u32(sum, sum), 0);
2473 template<> EIGEN_STRONG_INLINE int64_t predux<Packet2l>(
const Packet2l& a)
2474 {
return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1); }
2475 template<> EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(
const Packet2ul& a)
2476 {
return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); }
2478 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(
const Packet8c& a)
2480 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a,
2481 vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0);
2483 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half_dowto4(
const Packet16c& a)
2484 {
return vadd_s8(vget_high_s8(a), vget_low_s8(a)); }
2485 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half_dowto4(
const Packet8uc& a)
2487 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a,
2488 vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0);
2490 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half_dowto4(
const Packet16uc& a)
2491 {
return vadd_u8(vget_high_u8(a), vget_low_u8(a)); }
2492 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half_dowto4(
const Packet8s& a)
2493 {
return vadd_s16(vget_high_s16(a), vget_low_s16(a)); }
2494 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half_dowto4(
const Packet8us& a)
2495 {
return vadd_u16(vget_high_u16(a), vget_low_u16(a)); }
2499 template<> EIGEN_STRONG_INLINE
float predux_mul<Packet2f>(
const Packet2f& a)
2500 {
return vget_lane_f32(a, 0) * vget_lane_f32(a, 1); }
2501 template<> EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a)
2502 {
return predux_mul(vmul_f32(vget_low_f32(a), vget_high_f32(a))); }
2503 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet4c>(
const Packet4c& a)
2505 int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a));
2506 prod = vmul_s8(prod, vrev16_s8(prod));
2507 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2);
2509 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet8c>(
const Packet8c& a)
2511 int8x8_t prod = vmul_s8(a, vrev16_s8(a));
2512 prod = vmul_s8(prod, vrev32_s8(prod));
2513 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
2515 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(
const Packet16c& a)
2516 {
return predux_mul(vmul_s8(vget_low_s8(a), vget_high_s8(a))); }
2517 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet4uc>(
const Packet4uc& a)
2519 uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a));
2520 prod = vmul_u8(prod, vrev16_u8(prod));
2521 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2);
2523 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet8uc>(
const Packet8uc& a)
2525 uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
2526 prod = vmul_u8(prod, vrev32_u8(prod));
2527 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
2529 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(
const Packet16uc& a)
2530 {
return predux_mul(vmul_u8(vget_low_u8(a), vget_high_u8(a))); }
2531 template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet4s>(
const Packet4s& a)
2533 const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
2534 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2536 template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(
const Packet8s& a)
2541 prod = vmul_s16(vget_low_s16(a), vget_high_s16(a));
2543 prod = vmul_s16(prod, vrev32_s16(prod));
2545 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2547 template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet4us>(
const Packet4us& a)
2549 const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
2550 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2552 template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(
const Packet8us& a)
2557 prod = vmul_u16(vget_low_u16(a), vget_high_u16(a));
2559 prod = vmul_u16(prod, vrev32_u16(prod));
2561 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2563 template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet2i>(
const Packet2i& a)
2564 {
return vget_lane_s32(a, 0) * vget_lane_s32(a, 1); }
2565 template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(
const Packet4i& a)
2566 {
return predux_mul(vmul_s32(vget_low_s32(a), vget_high_s32(a))); }
2567 template<> EIGEN_STRONG_INLINE uint32_t predux_mul<Packet2ui>(
const Packet2ui& a)
2568 {
return vget_lane_u32(a, 0) * vget_lane_u32(a, 1); }
2569 template<> EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(
const Packet4ui& a)
2570 {
return predux_mul(vmul_u32(vget_low_u32(a), vget_high_u32(a))); }
2571 template<> EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(
const Packet2l& a)
2572 {
return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1); }
2573 template<> EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(
const Packet2ul& a)
2574 {
return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1); }
2577 template<> EIGEN_STRONG_INLINE
float predux_min<Packet2f>(
const Packet2f& a)
2578 {
return vget_lane_f32(vpmin_f32(a,a), 0); }
2579 template<> EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a)
2581 const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a));
2582 return vget_lane_f32(vpmin_f32(min, min), 0);
2584 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet4c>(
const Packet4c& a)
2586 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2587 int8x8_t min = vpmin_s8(a_dup, a_dup);
2588 min = vpmin_s8(min, min);
2589 return vget_lane_s8(min, 0);
2591 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(
const Packet8c& a)
2593 int8x8_t min = vpmin_s8(a,a);
2594 min = vpmin_s8(min, min);
2595 min = vpmin_s8(min, min);
2596 return vget_lane_s8(min, 0);
2598 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(
const Packet16c& a)
2600 int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a));
2601 min = vpmin_s8(min, min);
2602 min = vpmin_s8(min, min);
2603 min = vpmin_s8(min, min);
2604 return vget_lane_s8(min, 0);
2606 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet4uc>(
const Packet4uc& a)
2608 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2609 uint8x8_t min = vpmin_u8(a_dup, a_dup);
2610 min = vpmin_u8(min, min);
2611 return vget_lane_u8(min, 0);
2613 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(
const Packet8uc& a)
2615 uint8x8_t min = vpmin_u8(a,a);
2616 min = vpmin_u8(min, min);
2617 min = vpmin_u8(min, min);
2618 return vget_lane_u8(min, 0);
2620 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(
const Packet16uc& a)
2622 uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a));
2623 min = vpmin_u8(min, min);
2624 min = vpmin_u8(min, min);
2625 min = vpmin_u8(min, min);
2626 return vget_lane_u8(min, 0);
2628 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(
const Packet4s& a)
2630 const int16x4_t min = vpmin_s16(a,a);
2631 return vget_lane_s16(vpmin_s16(min, min), 0);
2633 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(
const Packet8s& a)
2635 int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a));
2636 min = vpmin_s16(min, min);
2637 min = vpmin_s16(min, min);
2638 return vget_lane_s16(min, 0);
2640 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(
const Packet4us& a)
2642 const uint16x4_t min = vpmin_u16(a,a);
2643 return vget_lane_u16(vpmin_u16(min, min), 0);
2645 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(
const Packet8us& a)
2647 uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a));
2648 min = vpmin_u16(min, min);
2649 min = vpmin_u16(min, min);
2650 return vget_lane_u16(min, 0);
2652 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(
const Packet2i& a)
2653 {
return vget_lane_s32(vpmin_s32(a,a), 0); }
2654 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(
const Packet4i& a)
2656 const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a));
2657 return vget_lane_s32(vpmin_s32(min, min), 0);
2659 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(
const Packet2ui& a)
2660 {
return vget_lane_u32(vpmin_u32(a,a), 0); }
2661 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(
const Packet4ui& a)
2663 const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a));
2664 return vget_lane_u32(vpmin_u32(min, min), 0);
2666 template<> EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(
const Packet2l& a)
2667 {
return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2668 template<> EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(
const Packet2ul& a)
2669 {
return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2672 template<> EIGEN_STRONG_INLINE
float predux_max<Packet2f>(
const Packet2f& a)
2673 {
return vget_lane_f32(vpmax_f32(a,a), 0); }
2674 template<> EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a)
2676 const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a));
2677 return vget_lane_f32(vpmax_f32(max, max), 0);
2679 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet4c>(
const Packet4c& a)
2681 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2682 int8x8_t max = vpmax_s8(a_dup, a_dup);
2683 max = vpmax_s8(max, max);
2684 return vget_lane_s8(max, 0);
2686 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(
const Packet8c& a)
2688 int8x8_t max = vpmax_s8(a,a);
2689 max = vpmax_s8(max, max);
2690 max = vpmax_s8(max, max);
2691 return vget_lane_s8(max, 0);
2693 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(
const Packet16c& a)
2695 int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a));
2696 max = vpmax_s8(max, max);
2697 max = vpmax_s8(max, max);
2698 max = vpmax_s8(max, max);
2699 return vget_lane_s8(max, 0);
2701 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet4uc>(
const Packet4uc& a)
2703 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2704 uint8x8_t max = vpmax_u8(a_dup, a_dup);
2705 max = vpmax_u8(max, max);
2706 return vget_lane_u8(max, 0);
2708 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(
const Packet8uc& a)
2710 uint8x8_t max = vpmax_u8(a,a);
2711 max = vpmax_u8(max, max);
2712 max = vpmax_u8(max, max);
2713 return vget_lane_u8(max, 0);
2715 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(
const Packet16uc& a)
2717 uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a));
2718 max = vpmax_u8(max, max);
2719 max = vpmax_u8(max, max);
2720 max = vpmax_u8(max, max);
2721 return vget_lane_u8(max, 0);
2723 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(
const Packet4s& a)
2725 const int16x4_t max = vpmax_s16(a,a);
2726 return vget_lane_s16(vpmax_s16(max, max), 0);
2728 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(
const Packet8s& a)
2730 int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a));
2731 max = vpmax_s16(max, max);
2732 max = vpmax_s16(max, max);
2733 return vget_lane_s16(max, 0);
2735 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(
const Packet4us& a)
2737 const uint16x4_t max = vpmax_u16(a,a);
2738 return vget_lane_u16(vpmax_u16(max, max), 0);
2740 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(
const Packet8us& a)
2742 uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a));
2743 max = vpmax_u16(max, max);
2744 max = vpmax_u16(max, max);
2745 return vget_lane_u16(max, 0);
2747 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(
const Packet2i& a)
2748 {
return vget_lane_s32(vpmax_s32(a,a), 0); }
2749 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(
const Packet4i& a)
2751 const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a));
2752 return vget_lane_s32(vpmax_s32(max, max), 0);
2754 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(
const Packet2ui& a)
2755 {
return vget_lane_u32(vpmax_u32(a,a), 0); }
2756 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(
const Packet4ui& a)
2758 const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a));
2759 return vget_lane_u32(vpmax_u32(max, max), 0);
2761 template<> EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(
const Packet2l& a)
2762 {
return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2763 template<> EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(
const Packet2ul& a)
2764 {
return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2766 template<> EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& x)
2768 uint32x2_t tmp = vorr_u32(vget_low_u32( vreinterpretq_u32_f32(x)),
2769 vget_high_u32(vreinterpretq_u32_f32(x)));
2770 return vget_lane_u32(vpmax_u32(tmp, tmp), 0);
2776 template<
typename Packet>
2777 void zip_in_place(Packet& p1, Packet& p2);
2780 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2f>(Packet2f& p1, Packet2f& p2) {
2781 const float32x2x2_t tmp = vzip_f32(p1, p2);
2787 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4f>(Packet4f& p1, Packet4f& p2) {
2788 const float32x4x2_t tmp = vzipq_f32(p1, p2);
2794 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8c>(Packet8c& p1, Packet8c& p2) {
2795 const int8x8x2_t tmp = vzip_s8(p1, p2);
2801 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet16c>(Packet16c& p1, Packet16c& p2) {
2802 const int8x16x2_t tmp = vzipq_s8(p1, p2);
2808 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8uc>(Packet8uc& p1, Packet8uc& p2) {
2809 const uint8x8x2_t tmp = vzip_u8(p1, p2);
2815 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet16uc>(Packet16uc& p1, Packet16uc& p2) {
2816 const uint8x16x2_t tmp = vzipq_u8(p1, p2);
2822 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2i>(Packet2i& p1, Packet2i& p2) {
2823 const int32x2x2_t tmp = vzip_s32(p1, p2);
2829 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4i>(Packet4i& p1, Packet4i& p2) {
2830 const int32x4x2_t tmp = vzipq_s32(p1, p2);
2836 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2ui>(Packet2ui& p1, Packet2ui& p2) {
2837 const uint32x2x2_t tmp = vzip_u32(p1, p2);
2843 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4ui>(Packet4ui& p1, Packet4ui& p2) {
2844 const uint32x4x2_t tmp = vzipq_u32(p1, p2);
2850 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4s>(Packet4s& p1, Packet4s& p2) {
2851 const int16x4x2_t tmp = vzip_s16(p1, p2);
2857 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8s>(Packet8s& p1, Packet8s& p2) {
2858 const int16x8x2_t tmp = vzipq_s16(p1, p2);
2864 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4us>(Packet4us& p1, Packet4us& p2) {
2865 const uint16x4x2_t tmp = vzip_u16(p1, p2);
2871 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8us>(Packet8us& p1, Packet8us& p2) {
2872 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
2877 template<
typename Packet>
2878 EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
2879 zip_in_place(kernel.packet[0], kernel.packet[1]);
2882 template<
typename Packet>
2883 EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
2884 zip_in_place(kernel.packet[0], kernel.packet[2]);
2885 zip_in_place(kernel.packet[1], kernel.packet[3]);
2886 zip_in_place(kernel.packet[0], kernel.packet[1]);
2887 zip_in_place(kernel.packet[2], kernel.packet[3]);
2890 template<
typename Packet>
2891 EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
2892 zip_in_place(kernel.packet[0], kernel.packet[4]);
2893 zip_in_place(kernel.packet[1], kernel.packet[5]);
2894 zip_in_place(kernel.packet[2], kernel.packet[6]);
2895 zip_in_place(kernel.packet[3], kernel.packet[7]);
2897 zip_in_place(kernel.packet[0], kernel.packet[2]);
2898 zip_in_place(kernel.packet[1], kernel.packet[3]);
2899 zip_in_place(kernel.packet[4], kernel.packet[6]);
2900 zip_in_place(kernel.packet[5], kernel.packet[7]);
2902 zip_in_place(kernel.packet[0], kernel.packet[1]);
2903 zip_in_place(kernel.packet[2], kernel.packet[3]);
2904 zip_in_place(kernel.packet[4], kernel.packet[5]);
2905 zip_in_place(kernel.packet[6], kernel.packet[7]);
2908 template<
typename Packet>
2909 EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
2911 for (
int i=0; i<4; ++i) {
2912 const int m = (1 << i);
2914 for (
int j=0; j<m; ++j) {
2915 const int n = (1 << (3-i));
2917 for (
int k=0; k<n; ++k) {
2918 const int idx = 2*j*n+k;
2919 zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
2927 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2f, 2>& kernel) {
2928 detail::ptranspose_impl(kernel);
2930 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
2931 detail::ptranspose_impl(kernel);
2934 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4c, 4>& kernel)
2936 const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
2937 const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
2939 const int8x8x2_t zip8 = vzip_s8(a,b);
2940 const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
2942 kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
2943 kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
2944 kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
2945 kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
2947 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8c, 8>& kernel) {
2948 detail::ptranspose_impl(kernel);
2950 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8c, 4>& kernel) {
2951 detail::ptranspose_impl(kernel);
2953 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2954 detail::ptranspose_impl(kernel);
2956 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
2957 detail::ptranspose_impl(kernel);
2959 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2960 detail::ptranspose_impl(kernel);
2963 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4uc, 4>& kernel)
2965 const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
2966 const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
2968 const uint8x8x2_t zip8 = vzip_u8(a,b);
2969 const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
2971 kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
2972 kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
2973 kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
2974 kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
2976 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8uc, 8>& kernel) {
2977 detail::ptranspose_impl(kernel);
2979 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8uc, 4>& kernel) {
2980 detail::ptranspose_impl(kernel);
2982 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
2983 detail::ptranspose_impl(kernel);
2985 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
2986 detail::ptranspose_impl(kernel);
2988 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2989 detail::ptranspose_impl(kernel);
2992 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4s, 4>& kernel) {
2993 detail::ptranspose_impl(kernel);
2995 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2996 detail::ptranspose_impl(kernel);
2998 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2999 detail::ptranspose_impl(kernel);
3002 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4us, 4>& kernel) {
3003 detail::ptranspose_impl(kernel);
3005 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
3006 detail::ptranspose_impl(kernel);
3008 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
3009 detail::ptranspose_impl(kernel);
3012 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2i, 2>& kernel) {
3013 detail::ptranspose_impl(kernel);
3015 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
3016 detail::ptranspose_impl(kernel);
3018 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2ui, 2>& kernel) {
3019 detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
3021 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
3022 detail::ptranspose_impl(kernel);
3025 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void 3026 ptranspose(PacketBlock<Packet2l, 2>& kernel)
3028 #if EIGEN_ARCH_ARM64 3029 const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]);
3030 kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]);
3031 kernel.packet[0] = tmp1;
3033 const int64x1_t tmp[2][2] = {
3034 { vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0]) },
3035 { vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1]) }
3038 kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]);
3039 kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]);
3042 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void 3043 ptranspose(PacketBlock<Packet2ul, 2>& kernel)
3045 #if EIGEN_ARCH_ARM64 3046 const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]);
3047 kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]);
3048 kernel.packet[0] = tmp1;
3050 const uint64x1_t tmp[2][2] = {
3051 { vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0]) },
3052 { vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1]) }
3055 kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]);
3056 kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]);
3060 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect(
const Packet2f& mask,
const Packet2f& a,
const Packet2f& b)
3061 {
return vbsl_f32(vreinterpret_u32_f32(mask), a, b); }
3062 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(
const Packet4f& mask,
const Packet4f& a,
const Packet4f& b)
3063 {
return vbslq_f32(vreinterpretq_u32_f32(mask), a, b); }
3064 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(
const Packet8c& mask,
const Packet8c& a,
const Packet8c& b)
3065 {
return vbsl_s8(vreinterpret_u8_s8(mask), a, b); }
3066 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(
const Packet16c& mask,
const Packet16c& a,
const Packet16c& b)
3067 {
return vbslq_s8(vreinterpretq_u8_s8(mask), a, b); }
3068 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(
const Packet8uc& mask,
const Packet8uc& a,
const Packet8uc& b)
3069 {
return vbsl_u8(mask, a, b); }
3070 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(
const Packet16uc& mask,
const Packet16uc& a,
const Packet16uc& b)
3071 {
return vbslq_u8(mask, a, b); }
3072 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(
const Packet4s& mask,
const Packet4s& a,
const Packet4s& b)
3073 {
return vbsl_s16(vreinterpret_u16_s16(mask), a, b); }
3074 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(
const Packet8s& mask,
const Packet8s& a,
const Packet8s& b)
3075 {
return vbslq_s16(vreinterpretq_u16_s16(mask), a, b); }
3076 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(
const Packet4us& mask,
const Packet4us& a,
const Packet4us& b)
3077 {
return vbsl_u16(mask, a, b); }
3078 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(
const Packet8us& mask,
const Packet8us& a,
const Packet8us& b)
3079 {
return vbslq_u16(mask, a, b); }
3080 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(
const Packet2i& mask,
const Packet2i& a,
const Packet2i& b)
3081 {
return vbsl_s32(vreinterpret_u32_s32(mask), a, b); }
3082 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(
const Packet4i& mask,
const Packet4i& a,
const Packet4i& b)
3083 {
return vbslq_s32(vreinterpretq_u32_s32(mask), a, b); }
3084 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(
const Packet2ui& mask,
const Packet2ui& a,
const Packet2ui& b)
3085 {
return vbsl_u32(mask, a, b); }
3086 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(
const Packet4ui& mask,
const Packet4ui& a,
const Packet4ui& b)
3087 {
return vbslq_u32(mask, a, b); }
3088 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(
const Packet2l& mask,
const Packet2l& a,
const Packet2l& b)
3089 {
return vbslq_s64(vreinterpretq_u64_s64(mask), a, b); }
3090 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(
const Packet2ul& mask,
const Packet2ul& a,
const Packet2ul& b)
3091 {
return vbslq_u64(mask, a, b); }
3094 #if EIGEN_ARCH_ARMV8 3095 template<> EIGEN_STRONG_INLINE Packet2f print<Packet2f>(
const Packet2f& a)
3096 {
return vrndn_f32(a); }
3098 template<> EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a)
3099 {
return vrndnq_f32(a); }
3101 template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(
const Packet2f& a)
3102 {
return vrndm_f32(a); }
3104 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a)
3105 {
return vrndmq_f32(a); }
3107 template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(
const Packet2f& a)
3108 {
return vrndp_f32(a); }
3110 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a)
3111 {
return vrndpq_f32(a); }
3115 template<> EIGEN_STRONG_INLINE Packet4f print(
const Packet4f& a) {
3117 const Packet4f limit = pset1<Packet4f>(
static_cast<float>(1<<23));
3118 const Packet4f abs_a = pabs(a);
3119 Packet4f r = padd(abs_a, limit);
3121 EIGEN_OPTIMIZATION_BARRIER(r);
3124 r = pselect(pcmp_lt(abs_a, limit),
3125 pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3129 template<> EIGEN_STRONG_INLINE Packet2f print(
const Packet2f& a) {
3131 const Packet2f limit = pset1<Packet2f>(
static_cast<float>(1<<23));
3132 const Packet2f abs_a = pabs(a);
3133 Packet2f r = padd(abs_a, limit);
3135 EIGEN_OPTIMIZATION_BARRIER(r);
3138 r = pselect(pcmp_lt(abs_a, limit),
3139 pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3143 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a)
3145 const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3146 Packet4f tmp = print<Packet4f>(a);
3148 Packet4f mask = pcmp_lt(a, tmp);
3149 mask = pand(mask, cst_1);
3150 return psub(tmp, mask);
3153 template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(
const Packet2f& a)
3155 const Packet2f cst_1 = pset1<Packet2f>(1.0f);
3156 Packet2f tmp = print<Packet2f>(a);
3158 Packet2f mask = pcmp_lt(a, tmp);
3159 mask = pand(mask, cst_1);
3160 return psub(tmp, mask);
3163 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a)
3165 const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3166 Packet4f tmp = print<Packet4f>(a);
3168 Packet4f mask = pcmp_lt(tmp, a);
3169 mask = pand(mask, cst_1);
3170 return padd(tmp, mask);
3173 template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(
const Packet2f& a)
3175 const Packet2f cst_1 = pset1<Packet2f>(1.0);
3176 Packet2f tmp = print<Packet2f>(a);
3178 Packet2f mask = pcmp_lt(tmp, a);
3179 mask = pand(mask, cst_1);
3180 return padd(tmp, mask);
3191 template<> EIGEN_STRONG_INLINE Packet4uc psqrt(
const Packet4uc& a) {
3192 uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
3193 uint8x8_t res = vdup_n_u8(0);
3194 uint8x8_t add = vdup_n_u8(0x8);
3195 for (
int i = 0; i < 4; i++)
3197 const uint8x8_t temp = vorr_u8(res, add);
3198 res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
3199 add = vshr_n_u8(add, 1);
3201 return vget_lane_u32(vreinterpret_u32_u8(res), 0);
3204 template<> EIGEN_STRONG_INLINE Packet8uc psqrt(
const Packet8uc& a) {
3205 uint8x8_t res = vdup_n_u8(0);
3206 uint8x8_t add = vdup_n_u8(0x8);
3207 for (
int i = 0; i < 4; i++)
3209 const uint8x8_t temp = vorr_u8(res, add);
3210 res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
3211 add = vshr_n_u8(add, 1);
3216 template<> EIGEN_STRONG_INLINE Packet16uc psqrt(
const Packet16uc& a) {
3217 uint8x16_t res = vdupq_n_u8(0);
3218 uint8x16_t add = vdupq_n_u8(0x8);
3219 for (
int i = 0; i < 4; i++)
3221 const uint8x16_t temp = vorrq_u8(res, add);
3222 res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res);
3223 add = vshrq_n_u8(add, 1);
3228 template<> EIGEN_STRONG_INLINE Packet4us psqrt(
const Packet4us& a) {
3229 uint16x4_t res = vdup_n_u16(0);
3230 uint16x4_t add = vdup_n_u16(0x80);
3231 for (
int i = 0; i < 8; i++)
3233 const uint16x4_t temp = vorr_u16(res, add);
3234 res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
3235 add = vshr_n_u16(add, 1);
3240 template<> EIGEN_STRONG_INLINE Packet8us psqrt(
const Packet8us& a) {
3241 uint16x8_t res = vdupq_n_u16(0);
3242 uint16x8_t add = vdupq_n_u16(0x80);
3243 for (
int i = 0; i < 8; i++)
3245 const uint16x8_t temp = vorrq_u16(res, add);
3246 res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res);
3247 add = vshrq_n_u16(add, 1);
3252 template<> EIGEN_STRONG_INLINE Packet2ui psqrt(
const Packet2ui& a) {
3253 uint32x2_t res = vdup_n_u32(0);
3254 uint32x2_t add = vdup_n_u32(0x8000);
3255 for (
int i = 0; i < 16; i++)
3257 const uint32x2_t temp = vorr_u32(res, add);
3258 res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
3259 add = vshr_n_u32(add, 1);
3264 template<> EIGEN_STRONG_INLINE Packet4ui psqrt(
const Packet4ui& a) {
3265 uint32x4_t res = vdupq_n_u32(0);
3266 uint32x4_t add = vdupq_n_u32(0x8000);
3267 for (
int i = 0; i < 16; i++)
3269 const uint32x4_t temp = vorrq_u32(res, add);
3270 res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res);
3271 add = vshrq_n_u32(add, 1);
3276 template<> EIGEN_STRONG_INLINE Packet4f prsqrt(
const Packet4f& a) {
3278 Packet4f x = vrsqrteq_f32(a);
3280 x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x);
3281 x = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, x), x), x);
3282 const Packet4f infinity = pset1<Packet4f>(NumTraits<float>::infinity());
3283 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3286 template<> EIGEN_STRONG_INLINE Packet2f prsqrt(
const Packet2f& a) {
3288 Packet2f x = vrsqrte_f32(a);
3290 x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x);
3291 x = vmul_f32(vrsqrts_f32(vmul_f32(a, x), x), x);
3292 const Packet2f infinity = pset1<Packet2f>(NumTraits<float>::infinity());
3293 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3297 #if EIGEN_ARCH_ARM64 3298 template<> EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& _x){
return vsqrtq_f32(_x);}
3299 template<> EIGEN_STRONG_INLINE Packet2f psqrt(
const Packet2f& _x){
return vsqrt_f32(_x); }
3301 template<> EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& a) {
3302 const Packet4f infinity = pset1<Packet4f>(NumTraits<float>::infinity());
3303 const Packet4f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity));
3304 return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a)));
3306 template<> EIGEN_STRONG_INLINE Packet2f psqrt(
const Packet2f& a) {
3307 const Packet2f infinity = pset1<Packet2f>(NumTraits<float>::infinity());
3308 const Packet2f is_zero_or_inf = por(pcmp_eq(a, pzero(a)), pcmp_eq(a, infinity));
3309 return pselect(is_zero_or_inf, a, pmul(a, prsqrt(a)));
3317 typedef eigen_packet_wrapper<uint16x4_t, 19> Packet4bf;
3319 template<>
struct is_arithmetic<Packet4bf> {
enum { value =
true }; };
3321 template<>
struct packet_traits<bfloat16> : default_packet_traits
3323 typedef Packet4bf type;
3324 typedef Packet4bf half;
3328 AlignedOnScalar = 1,
3352 HasSin = EIGEN_FAST_MATH,
3353 HasCos = EIGEN_FAST_MATH,
3357 HasTanh = EIGEN_FAST_MATH,
3358 HasErf = EIGEN_FAST_MATH,
3364 template<>
struct unpacket_traits<Packet4bf>
3366 typedef bfloat16 type;
3367 typedef Packet4bf half;
3372 vectorizable =
true,
3373 masked_load_available =
false,
3374 masked_store_available =
false 3380 EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) {
3381 const uint16x4x2_t tmp = vzip_u16(p1, p2);
3387 EIGEN_STRONG_INLINE Packet4bf F32ToBf16(
const Packet4f& p)
3391 Packet4ui input =
reinterpret_cast<Packet4ui
>(p);
3394 Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1));
3397 Packet4ui rounding_bias = vaddq_u32(lsb, vdupq_n_u32(0x7fff));
3400 input = vaddq_u32(input, rounding_bias);
3403 input = vshrq_n_u32(input, 16);
3406 const Packet4ui bf16_nan = vdupq_n_u32(0x7fc0);
3407 const Packet4ui mask = vceqq_f32(p, p);
3408 input = vbslq_u32(mask, input, bf16_nan);
3411 return vmovn_u32(input);
3414 EIGEN_STRONG_INLINE Packet4f Bf16ToF32(
const Packet4bf& p)
3416 return reinterpret_cast<Packet4f
>(vshlq_n_u32(vmovl_u16(p), 16));
3419 EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(
const Packet4f& p) {
3420 return vmovn_u32(vreinterpretq_u32_f32(p));
3423 template<> EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(
const bfloat16& from) {
3424 return pset1<Packet4us>(from.value);
3427 template<> EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(
const Packet4bf& from) {
3428 return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(from)));
3431 template<> EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(
const bfloat16* from)
3433 return pload<Packet4us>(
reinterpret_cast<const uint16_t*
>(from));
3436 template<> EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(
const bfloat16* from)
3438 return ploadu<Packet4us>(
reinterpret_cast<const uint16_t*
>(from));
3441 template<> EIGEN_STRONG_INLINE
void pstore<bfloat16>(bfloat16* to,
const Packet4bf& from)
3443 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3446 template<> EIGEN_STRONG_INLINE
void pstoreu<bfloat16>(bfloat16* to,
const Packet4bf& from)
3448 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3451 template<> EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(
const bfloat16* from)
3453 return ploaddup<Packet4us>(
reinterpret_cast<const uint16_t*
>(from));
3456 template <> EIGEN_STRONG_INLINE Packet4bf pabs(
const Packet4bf& a) {
3457 return F32ToBf16(pabs<Packet4f>(Bf16ToF32(a)));
3460 template <> EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(
const Packet4bf &a,
3463 return F32ToBf16(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3465 template <> EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(
const Packet4bf &a,
3468 return F32ToBf16(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3471 template <> EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(
const Packet4bf &a,
3474 return F32ToBf16(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3477 template <> EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(
const Packet4bf &a,
3480 return F32ToBf16(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3482 template <> EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(
const Packet4bf &a,
3485 return F32ToBf16(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3488 template <> EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(
const Packet4bf &a,
3491 return F32ToBf16(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3494 template<> EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(
const bfloat16& a)
3496 return F32ToBf16(plset<Packet4f>(static_cast<float>(a)));
3499 template<> EIGEN_STRONG_INLINE Packet4bf por(
const Packet4bf& a,
const Packet4bf& b) {
3500 return por<Packet4us>(a, b);
3503 template<> EIGEN_STRONG_INLINE Packet4bf pxor(
const Packet4bf& a,
const Packet4bf& b) {
3504 return pxor<Packet4us>(a, b);
3507 template<> EIGEN_STRONG_INLINE Packet4bf pand(
const Packet4bf& a,
const Packet4bf& b) {
3508 return pand<Packet4us>(a, b);
3511 template<> EIGEN_STRONG_INLINE Packet4bf pandnot(
const Packet4bf& a,
const Packet4bf& b) {
3512 return pandnot<Packet4us>(a, b);
3515 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(
const Packet4bf& mask,
const Packet4bf& a,
3518 return pselect<Packet4us>(mask, a, b);
3521 template<> EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(
const Packet4bf& a)
3523 return F32ToBf16(print<Packet4f>(Bf16ToF32(a)));
3526 template<> EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(
const Packet4bf& a)
3528 return F32ToBf16(pfloor<Packet4f>(Bf16ToF32(a)));
3531 template<> EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(
const Packet4bf& a)
3533 return F32ToBf16(pceil<Packet4f>(Bf16ToF32(a)));
3536 template<> EIGEN_STRONG_INLINE Packet4bf pconj(
const Packet4bf& a) {
return a; }
3538 template<> EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
3539 return F32ToBf16(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3542 template<> EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
3543 return F32ToBf16(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3546 template<> EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
3547 return F32ToBf16(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3550 template<> EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
3551 return F32ToBf16(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3555 EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(
const bfloat16* from,
Index stride)
3557 return pgather<uint16_t, Packet4us>(
reinterpret_cast<const uint16_t*
>(from), stride);
3561 EIGEN_STRONG_INLINE
void pscatter<bfloat16, Packet4bf>(bfloat16* to,
const Packet4bf& from,
Index stride)
3563 pscatter<uint16_t, Packet4us>(
reinterpret_cast<uint16_t*
>(to), from, stride);
3566 template<> EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(
const Packet4bf& a)
3568 return static_cast<bfloat16
>(predux<Packet4f>(Bf16ToF32(a)));
3571 template<> EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(
const Packet4bf& a)
3573 return static_cast<bfloat16
>(predux_max<Packet4f>(Bf16ToF32(a)));
3576 template<> EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(
const Packet4bf& a)
3578 return static_cast<bfloat16
>(predux_min<Packet4f>(Bf16ToF32(a)));
3581 template<> EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(
const Packet4bf& a)
3583 return static_cast<bfloat16
>(predux_mul<Packet4f>(Bf16ToF32(a)));
3586 template<> EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(
const Packet4bf& a)
3588 return preverse<Packet4us>(a);
3591 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4bf, 4>& kernel)
3593 detail::ptranspose_impl(kernel);
3596 template<> EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b)
3598 return F32ToBf16(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3601 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b)
3603 return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3606 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b)
3608 return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3611 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b)
3613 return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3616 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b)
3618 return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3621 template<> EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(
const Packet4bf& a)
3623 return pxor<Packet4us>(a, pset1<Packet4us>(
static_cast<uint16_t
>(0x8000)));
3630 #ifdef __apple_build_version__ 3634 #define EIGEN_APPLE_DOUBLE_NEON_BUG (__apple_build_version__ < 6010000) 3636 #define EIGEN_APPLE_DOUBLE_NEON_BUG 0 3639 #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG 3645 template <
typename T> uint64x2_t vreinterpretq_u64_f64(T a) {
return (uint64x2_t) a; }
3647 template <
typename T> float64x2_t vreinterpretq_f64_u64(T a) {
return (float64x2_t) a; }
3649 typedef float64x2_t Packet2d;
3650 typedef float64x1_t Packet1d;
3655 EIGEN_STRONG_INLINE Packet2d shuffle(
const Packet2d& m,
const Packet2d& n,
int mask)
3657 const double* a =
reinterpret_cast<const double*
>(&m);
3658 const double* b =
reinterpret_cast<const double*
>(&n);
3659 Packet2d res = {*(a + (mask & 1)), *(b + ((mask >> 1) & 1))};
3663 EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(
const Packet2d& a,
const Packet2d& b,
int mask)
3665 return shuffle(a, b, mask);
3667 EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(
const Packet2d& a,
const Packet2d& b)
3669 return shuffle(a, b, 0);
3671 EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(
const Packet2d& a,
const Packet2d& b)
3673 return shuffle(a, b, 3);
3675 #define vec2d_duplane(a, p) \ 3676 vdupq_laneq_f64(a, p) 3678 template<>
struct packet_traits<double> : default_packet_traits
3680 typedef Packet2d type;
3681 typedef Packet2d half;
3685 AlignedOnScalar = 1,
3721 template<>
struct unpacket_traits<Packet2d>
3723 typedef double type;
3724 typedef Packet2d half;
3725 typedef Packet2l integer_packet;
3730 vectorizable =
true,
3731 masked_load_available =
false,
3732 masked_store_available =
false 3736 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
return vdupq_n_f64(from); }
3738 template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a)
3740 const double c[] = {0.0,1.0};
3741 return vaddq_f64(pset1<Packet2d>(a), vld1q_f64(c));
3744 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vaddq_f64(a,b); }
3746 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vsubq_f64(a,b); }
3748 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& ,
const Packet2d& );
3749 template<> EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(
const Packet2d& a,
const Packet2d& b){
3750 const Packet2d mask = {numext::bit_cast<
double>(0x8000000000000000ull),0.0};
3751 return padd(a, pxor(mask, b));
3754 template<> EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
return vnegq_f64(a); }
3756 template<> EIGEN_STRONG_INLINE Packet2d pconj(
const Packet2d& a) {
return a; }
3758 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vmulq_f64(a,b); }
3760 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vdivq_f64(a,b); }
3762 #ifdef __ARM_FEATURE_FMA 3764 template<> EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c)
3765 {
return vfmaq_f64(c,a,b); }
3767 template<> EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c)
3768 {
return vmlaq_f64(c,a,b); }
3771 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vminq_f64(a,b); }
3773 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN 3775 template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vminnmq_f64(a, b); }
3776 template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vmaxnmq_f64(a, b); }
3780 template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNaN, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return pmin<Packet2d>(a, b); }
3782 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vmaxq_f64(a,b); }
3785 template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNaN, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return pmax<Packet2d>(a, b); }
3788 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b)
3789 {
return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3791 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b)
3792 {
return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3794 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b)
3795 {
return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3797 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b)
3798 {
return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3800 template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(
const Packet2d& a,
const Packet2d& b)
3801 {
return vreinterpretq_f64_u64(vcleq_f64(a,b)); }
3803 template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(
const Packet2d& a,
const Packet2d& b)
3804 {
return vreinterpretq_f64_u64(vcltq_f64(a,b)); }
3806 template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(
const Packet2d& a,
const Packet2d& b)
3807 {
return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a,b)))); }
3809 template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(
const Packet2d& a,
const Packet2d& b)
3810 {
return vreinterpretq_f64_u64(vceqq_f64(a,b)); }
3812 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from)
3813 { EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f64(from); }
3815 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from)
3816 { EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f64(from); }
3818 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from) {
return vld1q_dup_f64(from); }
3819 template<> EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from)
3820 { EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to,from); }
3822 template<> EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from)
3823 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to,from); }
3825 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(
const double* from,
Index stride)
3827 Packet2d res = pset1<Packet2d>(0.0);
3828 res = vld1q_lane_f64(from + 0*stride, res, 0);
3829 res = vld1q_lane_f64(from + 1*stride, res, 1);
3833 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<double, Packet2d>(
double* to,
const Packet2d& from,
Index stride)
3835 vst1q_lane_f64(to + stride*0, from, 0);
3836 vst1q_lane_f64(to + stride*1, from, 1);
3839 template<> EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) { EIGEN_ARM_PREFETCH(addr); }
3842 template<> EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
return vgetq_lane_f64(a,0); }
3844 template<> EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a)
3845 {
return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); }
3847 template<> EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
return vabsq_f64(a); }
3849 #if EIGEN_COMP_CLANG && defined(__apple_build_version__) 3851 template<> EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a)
3852 {
return (vget_low_f64(a) + vget_high_f64(a))[0]; }
3854 template<> EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a)
3855 {
return vget_lane_f64(vget_low_f64(a) + vget_high_f64(a), 0); }
3860 #if EIGEN_COMP_CLANG && defined(__apple_build_version__) 3861 template<> EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a)
3862 {
return (vget_low_f64(a) * vget_high_f64(a))[0]; }
3864 template<> EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a)
3865 {
return vget_lane_f64(vget_low_f64(a) * vget_high_f64(a), 0); }
3869 template<> EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a)
3870 {
return vgetq_lane_f64(vpminq_f64(a,a), 0); }
3873 template<> EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a)
3874 {
return vgetq_lane_f64(vpmaxq_f64(a,a), 0); }
3877 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void 3878 ptranspose(PacketBlock<Packet2d, 2>& kernel)
3880 const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]);
3881 const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]);
3883 kernel.packet[0] = tmp1;
3884 kernel.packet[1] = tmp2;
3887 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(
const Packet2d& mask,
const Packet2d& a,
const Packet2d& b)
3888 {
return vbslq_f64(vreinterpretq_u64_f64(mask), a, b); }
3890 template<> EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a)
3891 {
return vrndnq_f64(a); }
3893 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a)
3894 {
return vrndmq_f64(a); }
3896 template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a)
3897 {
return vrndpq_f64(a); }
3899 template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent)
3900 {
return pldexp_generic(a, exponent); }
3902 template<> EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(
const Packet2d& a, Packet2d& exponent)
3903 {
return pfrexp_generic(a,exponent); }
3905 template<> EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from)
3906 {
return vreinterpretq_f64_u64(vdupq_n_u64(from)); }
3908 template<> EIGEN_STRONG_INLINE Packet2d prsqrt(
const Packet2d& a) {
3910 Packet2d x = vrsqrteq_f64(a);
3912 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3913 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3914 x = vmulq_f64(vrsqrtsq_f64(vmulq_f64(a, x), x), x);
3915 const Packet2d infinity = pset1<Packet2d>(NumTraits<double>::infinity());
3916 return pselect(pcmp_eq(a, pzero(a)), infinity, x);
3919 template<> EIGEN_STRONG_INLINE Packet2d psqrt(
const Packet2d& _x){
return vsqrtq_f64(_x); }
3921 #endif // EIGEN_ARCH_ARM64 3924 #if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 3925 typedef float16x4_t Packet4hf;
3926 typedef float16x8_t Packet8hf;
3929 struct packet_traits<
Eigen::half> : default_packet_traits {
3930 typedef Packet8hf type;
3931 typedef Packet4hf half;
3934 AlignedOnScalar = 1,
3966 HasErf = EIGEN_FAST_MATH,
3973 struct unpacket_traits<Packet4hf> {
3974 typedef Eigen::half type;
3975 typedef Packet4hf half;
3979 vectorizable =
true,
3980 masked_load_available =
false,
3981 masked_store_available =
false 3986 struct unpacket_traits<Packet8hf> {
3987 typedef Eigen::half type;
3988 typedef Packet4hf half;
3992 vectorizable =
true,
3993 masked_load_available =
false,
3994 masked_store_available =
false 3999 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half_dowto4<Packet8hf>(
const Packet8hf& a) {
4000 return vadd_f16(vget_low_f16(a), vget_high_f16(a));
4004 EIGEN_STRONG_INLINE Packet8hf pset1<Packet8hf>(
const Eigen::half& from) {
4005 return vdupq_n_f16(from.x);
4009 EIGEN_STRONG_INLINE Packet4hf pset1<Packet4hf>(
const Eigen::half& from) {
4010 return vdup_n_f16(from.x);
4014 EIGEN_STRONG_INLINE Packet8hf plset<Packet8hf>(
const Eigen::half& a) {
4015 const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7};
4016 Packet8hf countdown = vld1q_f16(f);
4017 return vaddq_f16(pset1<Packet8hf>(a), countdown);
4021 EIGEN_STRONG_INLINE Packet4hf plset<Packet4hf>(
const Eigen::half& a) {
4022 const float16_t f[] = {0, 1, 2, 3};
4023 Packet4hf countdown = vld1_f16(f);
4024 return vadd_f16(pset1<Packet4hf>(a), countdown);
4028 EIGEN_STRONG_INLINE Packet8hf padd<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4029 return vaddq_f16(a, b);
4033 EIGEN_STRONG_INLINE Packet4hf padd<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4034 return vadd_f16(a, b);
4038 EIGEN_STRONG_INLINE Packet8hf psub<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4039 return vsubq_f16(a, b);
4043 EIGEN_STRONG_INLINE Packet4hf psub<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4044 return vsub_f16(a, b);
4048 EIGEN_STRONG_INLINE Packet8hf pnegate(
const Packet8hf& a) {
4049 return vnegq_f16(a);
4053 EIGEN_STRONG_INLINE Packet4hf pnegate(
const Packet4hf& a) {
4058 EIGEN_STRONG_INLINE Packet8hf pconj(
const Packet8hf& a) {
4063 EIGEN_STRONG_INLINE Packet4hf pconj(
const Packet4hf& a) {
4068 EIGEN_STRONG_INLINE Packet8hf pmul<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4069 return vmulq_f16(a, b);
4073 EIGEN_STRONG_INLINE Packet4hf pmul<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4074 return vmul_f16(a, b);
4078 EIGEN_STRONG_INLINE Packet8hf pdiv<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4079 return vdivq_f16(a, b);
4083 EIGEN_STRONG_INLINE Packet4hf pdiv<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4084 return vdiv_f16(a, b);
4088 EIGEN_STRONG_INLINE Packet8hf pmadd(
const Packet8hf& a,
const Packet8hf& b,
const Packet8hf& c) {
4089 return vfmaq_f16(c, a, b);
4093 EIGEN_STRONG_INLINE Packet4hf pmadd(
const Packet4hf& a,
const Packet4hf& b,
const Packet4hf& c) {
4094 return vfma_f16(c, a, b);
4098 EIGEN_STRONG_INLINE Packet8hf pmin<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4099 return vminq_f16(a, b);
4103 EIGEN_STRONG_INLINE Packet4hf pmin<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4104 return vmin_f16(a, b);
4107 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN 4109 template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNumbers, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
return vminnm_f16(a, b); }
4110 template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNumbers, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
return vminnmq_f16(a, b); }
4113 template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNaN, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
return pmin<Packet4hf>(a, b); }
4115 template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNaN, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
return pmin<Packet8hf>(a, b); }
4118 EIGEN_STRONG_INLINE Packet8hf pmax<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4119 return vmaxq_f16(a, b);
4123 EIGEN_STRONG_INLINE Packet4hf pmax<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4124 return vmax_f16(a, b);
4127 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN 4129 template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNumbers, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
return vmaxnm_f16(a, b); }
4130 template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNumbers, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
return vmaxnmq_f16(a, b); }
4133 template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNaN, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
return pmax<Packet4hf>(a, b); }
4135 template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNaN, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
return pmax<Packet8hf>(a, b); }
4137 #define EIGEN_MAKE_ARM_FP16_CMP_8(name) \ 4139 EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \ 4140 return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \ 4143 #define EIGEN_MAKE_ARM_FP16_CMP_4(name) \ 4145 EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \ 4146 return vreinterpret_f16_u16(vc##name##_f16(a, b)); \ 4149 EIGEN_MAKE_ARM_FP16_CMP_8(eq)
4150 EIGEN_MAKE_ARM_FP16_CMP_8(lt)
4151 EIGEN_MAKE_ARM_FP16_CMP_8(le)
4153 EIGEN_MAKE_ARM_FP16_CMP_4(eq)
4154 EIGEN_MAKE_ARM_FP16_CMP_4(lt)
4155 EIGEN_MAKE_ARM_FP16_CMP_4(le)
4157 #undef EIGEN_MAKE_ARM_FP16_CMP_8 4158 #undef EIGEN_MAKE_ARM_FP16_CMP_4 4161 EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4162 return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b)));
4166 EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4167 return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b)));
4171 EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(
const Packet8hf& a)
4172 {
return vrndnq_f16(a); }
4175 EIGEN_STRONG_INLINE Packet4hf print<Packet4hf>(
const Packet4hf& a)
4176 {
return vrndn_f16(a); }
4179 EIGEN_STRONG_INLINE Packet8hf pfloor<Packet8hf>(
const Packet8hf& a)
4180 {
return vrndmq_f16(a); }
4183 EIGEN_STRONG_INLINE Packet4hf pfloor<Packet4hf>(
const Packet4hf& a)
4184 {
return vrndm_f16(a); }
4187 EIGEN_STRONG_INLINE Packet8hf pceil<Packet8hf>(
const Packet8hf& a)
4188 {
return vrndpq_f16(a); }
4191 EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(
const Packet4hf& a)
4192 {
return vrndp_f16(a); }
4195 EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(
const Packet8hf& a) {
4196 return vsqrtq_f16(a);
4200 EIGEN_STRONG_INLINE Packet4hf psqrt<Packet4hf>(
const Packet4hf& a) {
4201 return vsqrt_f16(a);
4205 EIGEN_STRONG_INLINE Packet8hf pand<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4206 return vreinterpretq_f16_u16(vandq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4210 EIGEN_STRONG_INLINE Packet4hf pand<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4211 return vreinterpret_f16_u16(vand_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4215 EIGEN_STRONG_INLINE Packet8hf por<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4216 return vreinterpretq_f16_u16(vorrq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4220 EIGEN_STRONG_INLINE Packet4hf por<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4221 return vreinterpret_f16_u16(vorr_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4225 EIGEN_STRONG_INLINE Packet8hf pxor<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4226 return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4230 EIGEN_STRONG_INLINE Packet4hf pxor<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4231 return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4235 EIGEN_STRONG_INLINE Packet8hf pandnot<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
4236 return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4240 EIGEN_STRONG_INLINE Packet4hf pandnot<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
4241 return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4245 EIGEN_STRONG_INLINE Packet8hf pload<Packet8hf>(
const Eigen::half* from) {
4246 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4250 EIGEN_STRONG_INLINE Packet4hf pload<Packet4hf>(
const Eigen::half* from) {
4251 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_f16(reinterpret_cast<const float16_t*>(from));
4255 EIGEN_STRONG_INLINE Packet8hf ploadu<Packet8hf>(
const Eigen::half* from) {
4256 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4260 EIGEN_STRONG_INLINE Packet4hf ploadu<Packet4hf>(
const Eigen::half* from) {
4261 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_f16(reinterpret_cast<const float16_t*>(from));
4265 EIGEN_STRONG_INLINE Packet8hf ploaddup<Packet8hf>(
const Eigen::half* from) {
4267 packet[0] = from[0].x;
4268 packet[1] = from[0].x;
4269 packet[2] = from[1].x;
4270 packet[3] = from[1].x;
4271 packet[4] = from[2].x;
4272 packet[5] = from[2].x;
4273 packet[6] = from[3].x;
4274 packet[7] = from[3].x;
4279 EIGEN_STRONG_INLINE Packet4hf ploaddup<Packet4hf>(
const Eigen::half* from) {
4282 tmp = (float16_t*)&packet;
4291 EIGEN_STRONG_INLINE Packet8hf ploadquad<Packet8hf>(
const Eigen::half* from) {
4293 lo = vld1_dup_f16(reinterpret_cast<const float16_t*>(from));
4294 hi = vld1_dup_f16(reinterpret_cast<const float16_t*>(from+1));
4295 return vcombine_f16(lo, hi);
4298 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(
const Packet8hf& a, Eigen::half b) {
return vsetq_lane_f16(b.x, a, 0); }
4300 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(
const Packet4hf& a, Eigen::half b) {
return vset_lane_f16(b.x, a, 0); }
4303 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(
const Packet8hf& mask,
const Packet8hf& a,
const Packet8hf& b) {
4304 return vbslq_f16(vreinterpretq_u16_f16(mask), a, b);
4308 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(
const Packet4hf& mask,
const Packet4hf& a,
const Packet4hf& b) {
4309 return vbsl_f16(vreinterpret_u16_f16(mask), a, b);
4312 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(
const Packet8hf& a, Eigen::half b) {
return vsetq_lane_f16(b.x, a, 7); }
4314 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(
const Packet4hf& a, Eigen::half b) {
return vset_lane_f16(b.x, a, 3); }
4317 EIGEN_STRONG_INLINE
void pstore<Eigen::half>(Eigen::half* to,
const Packet8hf& from) {
4318 EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4322 EIGEN_STRONG_INLINE
void pstore<Eigen::half>(Eigen::half* to,
const Packet4hf& from) {
4323 EIGEN_DEBUG_ALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4327 EIGEN_STRONG_INLINE
void pstoreu<Eigen::half>(Eigen::half* to,
const Packet8hf& from) {
4328 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4332 EIGEN_STRONG_INLINE
void pstoreu<Eigen::half>(Eigen::half* to,
const Packet4hf& from) {
4333 EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4337 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather<Eigen::half, Packet8hf>(
const Eigen::half* from,
Index stride) {
4338 Packet8hf res = pset1<Packet8hf>(Eigen::half(0.f));
4339 res = vsetq_lane_f16(from[0 * stride].x, res, 0);
4340 res = vsetq_lane_f16(from[1 * stride].x, res, 1);
4341 res = vsetq_lane_f16(from[2 * stride].x, res, 2);
4342 res = vsetq_lane_f16(from[3 * stride].x, res, 3);
4343 res = vsetq_lane_f16(from[4 * stride].x, res, 4);
4344 res = vsetq_lane_f16(from[5 * stride].x, res, 5);
4345 res = vsetq_lane_f16(from[6 * stride].x, res, 6);
4346 res = vsetq_lane_f16(from[7 * stride].x, res, 7);
4351 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather<Eigen::half, Packet4hf>(
const Eigen::half* from,
Index stride) {
4352 Packet4hf res = pset1<Packet4hf>(Eigen::half(0.f));
4353 res = vset_lane_f16(from[0 * stride].x, res, 0);
4354 res = vset_lane_f16(from[1 * stride].x, res, 1);
4355 res = vset_lane_f16(from[2 * stride].x, res, 2);
4356 res = vset_lane_f16(from[3 * stride].x, res, 3);
4361 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<Eigen::half, Packet8hf>(Eigen::half* to,
const Packet8hf& from,
Index stride) {
4362 to[stride * 0].x = vgetq_lane_f16(from, 0);
4363 to[stride * 1].x = vgetq_lane_f16(from, 1);
4364 to[stride * 2].x = vgetq_lane_f16(from, 2);
4365 to[stride * 3].x = vgetq_lane_f16(from, 3);
4366 to[stride * 4].x = vgetq_lane_f16(from, 4);
4367 to[stride * 5].x = vgetq_lane_f16(from, 5);
4368 to[stride * 6].x = vgetq_lane_f16(from, 6);
4369 to[stride * 7].x = vgetq_lane_f16(from, 7);
4373 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<Eigen::half, Packet4hf>(Eigen::half* to,
const Packet4hf& from,
Index stride) {
4374 to[stride * 0].x = vget_lane_f16(from, 0);
4375 to[stride * 1].x = vget_lane_f16(from, 1);
4376 to[stride * 2].x = vget_lane_f16(from, 2);
4377 to[stride * 3].x = vget_lane_f16(from, 3);
4381 EIGEN_STRONG_INLINE
void prefetch<Eigen::half>(
const Eigen::half* addr) {
4382 EIGEN_ARM_PREFETCH(addr);
4386 EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8hf>(
const Packet8hf& a) {
4395 EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4hf>(
const Packet4hf& a) {
4403 template<> EIGEN_STRONG_INLINE Packet8hf preverse(
const Packet8hf& a) {
4404 float16x4_t a_lo, a_hi;
4407 a_r64 = vrev64q_f16(a);
4408 a_lo = vget_low_f16(a_r64);
4409 a_hi = vget_high_f16(a_r64);
4410 return vcombine_f16(a_hi, a_lo);
4414 EIGEN_STRONG_INLINE Packet4hf preverse<Packet4hf>(
const Packet4hf& a) {
4415 return vrev64_f16(a);
4419 EIGEN_STRONG_INLINE Packet8hf pabs<Packet8hf>(
const Packet8hf& a) {
4420 return vabsq_f16(a);
4424 EIGEN_STRONG_INLINE Packet4hf pabs<Packet4hf>(
const Packet4hf& a) {
4429 EIGEN_STRONG_INLINE Eigen::half predux<Packet8hf>(
const Packet8hf& a) {
4430 float16x4_t a_lo, a_hi, sum;
4432 a_lo = vget_low_f16(a);
4433 a_hi = vget_high_f16(a);
4434 sum = vpadd_f16(a_lo, a_hi);
4435 sum = vpadd_f16(sum, sum);
4436 sum = vpadd_f16(sum, sum);
4439 h.x = vget_lane_f16(sum, 0);
4444 EIGEN_STRONG_INLINE Eigen::half predux<Packet4hf>(
const Packet4hf& a) {
4447 sum = vpadd_f16(a, a);
4448 sum = vpadd_f16(sum, sum);
4450 h.x = vget_lane_f16(sum, 0);
4455 EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8hf>(
const Packet8hf& a) {
4456 float16x4_t a_lo, a_hi, prod;
4458 a_lo = vget_low_f16(a);
4459 a_hi = vget_high_f16(a);
4460 prod = vmul_f16(a_lo, a_hi);
4461 prod = vmul_f16(prod, vrev64_f16(prod));
4464 h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4469 EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4hf>(
const Packet4hf& a) {
4471 prod = vmul_f16(a, vrev64_f16(a));
4473 h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4478 EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8hf>(
const Packet8hf& a) {
4479 float16x4_t a_lo, a_hi, min;
4481 a_lo = vget_low_f16(a);
4482 a_hi = vget_high_f16(a);
4483 min = vpmin_f16(a_lo, a_hi);
4484 min = vpmin_f16(min, min);
4485 min = vpmin_f16(min, min);
4488 h.x = vget_lane_f16(min, 0);
4493 EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4hf>(
const Packet4hf& a) {
4495 tmp = vpmin_f16(a, a);
4496 tmp = vpmin_f16(tmp, tmp);
4498 h.x = vget_lane_f16(tmp, 0);
4503 EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8hf>(
const Packet8hf& a) {
4504 float16x4_t a_lo, a_hi, max;
4506 a_lo = vget_low_f16(a);
4507 a_hi = vget_high_f16(a);
4508 max = vpmax_f16(a_lo, a_hi);
4509 max = vpmax_f16(max, max);
4510 max = vpmax_f16(max, max);
4513 h.x = vget_lane_f16(max, 0);
4518 EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4hf>(
const Packet4hf& a) {
4520 tmp = vpmax_f16(a, a);
4521 tmp = vpmax_f16(tmp, tmp);
4523 h.x = vget_lane_f16(tmp, 0);
4527 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8hf, 4>& kernel)
4529 const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]);
4530 const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]);
4532 const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0]));
4533 const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1]));
4535 kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]);
4536 kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]);
4537 kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]);
4538 kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]);
4541 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4hf, 4>& kernel) {
4542 EIGEN_ALIGN16 float16x4x4_t tmp_x4;
4543 float16_t* tmp = (float16_t*)&kernel;
4544 tmp_x4 = vld4_f16(tmp);
4546 kernel.packet[0] = tmp_x4.val[0];
4547 kernel.packet[1] = tmp_x4.val[1];
4548 kernel.packet[2] = tmp_x4.val[2];
4549 kernel.packet[3] = tmp_x4.val[3];
4552 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8hf, 8>& kernel) {
4553 float16x8x2_t T_1[4];
4555 T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]);
4556 T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]);
4557 T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]);
4558 T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]);
4560 float16x8x2_t T_2[4];
4561 T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]);
4562 T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]);
4563 T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]);
4564 T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]);
4566 float16x8x2_t T_3[4];
4567 T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]);
4568 T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]);
4569 T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]);
4570 T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]);
4572 kernel.packet[0] = T_3[0].val[0];
4573 kernel.packet[1] = T_3[2].val[0];
4574 kernel.packet[2] = T_3[1].val[0];
4575 kernel.packet[3] = T_3[3].val[0];
4576 kernel.packet[4] = T_3[0].val[1];
4577 kernel.packet[5] = T_3[2].val[1];
4578 kernel.packet[6] = T_3[1].val[1];
4579 kernel.packet[7] = T_3[3].val[1];
4581 #endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC 4587 #endif // EIGEN_PACKET_MATH_NEON_H Definition: Constants.h:235
Namespace containing all symbols from the Eigen library.
Definition: Core:141
Definition: Constants.h:233
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74
Definition: Eigen_Colamd.h:50