Please, help us to better know about our user community by answering the following short survey: https://forms.gle/wpyrxWi18ox9Z5ae9
Eigen  3.4.0
NEON/TypeCasting.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com>
5 // Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6 //
7 // This Source Code Form is subject to the terms of the Mozilla
8 // Public License v. 2.0. If a copy of the MPL was not distributed
9 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10 
11 #ifndef EIGEN_TYPE_CASTING_NEON_H
12 #define EIGEN_TYPE_CASTING_NEON_H
13 
14 namespace Eigen {
15 
16 namespace internal {
17 
18 //==============================================================================
19 // pcast, SrcType = float
20 //==============================================================================
21 template <>
22 struct type_casting_traits<float, float> {
23  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
24 };
25 template <>
26 EIGEN_STRONG_INLINE Packet4f pcast<Packet4f, Packet4f>(const Packet4f& a) {
27  return a;
28 }
29 template <>
30 EIGEN_STRONG_INLINE Packet2f pcast<Packet2f, Packet2f>(const Packet2f& a) {
31  return a;
32 }
33 
34 template <>
35 struct type_casting_traits<float, numext::int64_t> {
36  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
37 };
38 template <>
39 struct type_casting_traits<float, numext::uint64_t> {
40  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
41 };
42 // If float64 exists, first convert to that to keep as much precision as possible.
43 #if EIGEN_ARCH_ARM64
44 template <>
45 EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
46  // Discard second half of input.
47  return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
48 }
49 template <>
50 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
51  // Discard second half of input.
52  return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
53 }
54 #else
55 template <>
56 EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
57  // Discard second half of input.
58  return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
59 }
60 template <>
61 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
62  // Discard second half of input.
63  return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
64 }
65 #endif // EIGEN_ARCH_ARM64
66 
67 template <>
68 struct type_casting_traits<float, numext::int32_t> {
69  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
70 };
71 template <>
72 EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
73  return vcvtq_s32_f32(a);
74 }
75 template <>
76 EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
77  return vcvt_s32_f32(a);
78 }
79 
80 template <>
81 struct type_casting_traits<float, numext::uint32_t> {
82  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
83 };
84 template <>
85 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
86  return vcvtq_u32_f32(a);
87 }
88 template <>
89 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
90  return vcvt_u32_f32(a);
91 }
92 
93 template <>
94 struct type_casting_traits<float, numext::int16_t> {
95  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
96 };
97 template <>
98 EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
99  return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
100 }
101 template <>
102 EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
103  return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
104 }
105 
106 template <>
107 struct type_casting_traits<float, numext::uint16_t> {
108  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
109 };
110 template <>
111 EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
112  return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
113 }
114 template <>
115 EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
116  return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
117 }
118 
119 template <>
120 struct type_casting_traits<float, numext::int8_t> {
121  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
122 };
123 template <>
124 EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
125  const Packet4f& d) {
126  const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
127  const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
128  return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
129 }
130 template <>
131 EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
132  const Packet2f& d) {
133  const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
134  const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
135  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
136 }
137 
138 template <>
139 struct type_casting_traits<float, numext::uint8_t> {
140  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
141 };
142 template <>
143 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
144  const Packet4f& d) {
145  const uint16x8_t ab_u16 = pcast<Packet4f, Packet8us>(a, b);
146  const uint16x8_t cd_u16 = pcast<Packet4f, Packet8us>(c, d);
147  return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
148 }
149 template <>
150 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
151  const Packet2f& d) {
152  const uint16x4_t ab_u16 = pcast<Packet2f, Packet4us>(a, b);
153  const uint16x4_t cd_u16 = pcast<Packet2f, Packet4us>(c, d);
154  return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
155 }
156 
157 //==============================================================================
158 // pcast, SrcType = int8_t
159 //==============================================================================
160 template <>
161 struct type_casting_traits<numext::int8_t, float> {
162  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
163 };
164 template <>
165 EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
166  // Discard all but first 4 bytes.
167  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
168 }
169 template <>
170 EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
171  // Discard all but first 2 bytes.
172  return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
173 }
174 
175 template <>
176 struct type_casting_traits<numext::int8_t, numext::int64_t> {
177  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
178 };
179 template <>
180 EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
181  // Discard all but first two bytes.
182  return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
183 }
184 
185 template <>
186 struct type_casting_traits<numext::int8_t, numext::uint64_t> {
187  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
188 };
189 template <>
190 EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
191  return vreinterpretq_u64_s64(pcast<Packet16c, Packet2l>(a));
192 }
193 
194 template <>
195 struct type_casting_traits<numext::int8_t, numext::int32_t> {
196  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
197 };
198 template <>
199 EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
200  // Discard all but first 4 bytes.
201  return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
202 }
203 template <>
204 EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
205  // Discard all but first 2 bytes.
206  return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
207 }
208 
209 template <>
210 struct type_casting_traits<numext::int8_t, numext::uint32_t> {
211  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
212 };
213 template <>
214 EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
215  return vreinterpretq_u32_s32(pcast<Packet16c, Packet4i>(a));
216 }
217 template <>
218 EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
219  return vreinterpret_u32_s32(pcast<Packet8c, Packet2i>(a));
220 }
221 
222 template <>
223 struct type_casting_traits<numext::int8_t, numext::int16_t> {
224  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
225 };
226 template <>
227 EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
228  // Discard second half of input.
229  return vmovl_s8(vget_low_s8(a));
230 }
231 template <>
232 EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
233  // Discard second half of input.
234  return vget_low_s16(vmovl_s8(a));
235 }
236 
237 template <>
238 struct type_casting_traits<numext::int8_t, numext::uint16_t> {
239  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
240 };
241 template <>
242 EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
243  return vreinterpretq_u16_s16(pcast<Packet16c, Packet8s>(a));
244 }
245 template <>
246 EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
247  return vreinterpret_u16_s16(pcast<Packet8c, Packet4s>(a));
248 }
249 
250 template <>
251 struct type_casting_traits<numext::int8_t, numext::int8_t> {
252  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
253 };
254 template <>
255 EIGEN_STRONG_INLINE Packet16c pcast<Packet16c, Packet16c>(const Packet16c& a) {
256  return a;
257 }
258 template <>
259 EIGEN_STRONG_INLINE Packet8c pcast<Packet8c, Packet8c>(const Packet8c& a) {
260  return a;
261 }
262 template <>
263 EIGEN_STRONG_INLINE Packet4c pcast<Packet4c, Packet4c>(const Packet4c& a) {
264  return a;
265 }
266 
267 template <>
268 struct type_casting_traits<numext::int8_t, numext::uint8_t> {
269  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
270 };
271 template <>
272 EIGEN_STRONG_INLINE Packet16uc pcast<Packet16c, Packet16uc>(const Packet16c& a) {
273  return vreinterpretq_u8_s8(a);
274 }
275 template <>
276 EIGEN_STRONG_INLINE Packet8uc pcast<Packet8c, Packet8uc>(const Packet8c& a) {
277  return vreinterpret_u8_s8(a);
278 }
279 template <>
280 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4c, Packet4uc>(const Packet4c& a) {
281  return static_cast<Packet4uc>(a);
282 }
283 
284 //==============================================================================
285 // pcast, SrcType = uint8_t
286 //==============================================================================
287 template <>
288 struct type_casting_traits<numext::uint8_t, float> {
289  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
290 };
291 template <>
292 EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
293  // Discard all but first 4 bytes.
294  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
295 }
296 template <>
297 EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
298  // Discard all but first 2 bytes.
299  return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
300 }
301 
302 template <>
303 struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
304  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
305 };
306 template <>
307 EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
308  // Discard all but first two bytes.
309  return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
310 }
311 
312 template <>
313 struct type_casting_traits<numext::uint8_t, numext::int64_t> {
314  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
315 };
316 template <>
317 EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
318  return vreinterpretq_s64_u64(pcast<Packet16uc, Packet2ul>(a));
319 }
320 
321 template <>
322 struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
323  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
324 };
325 template <>
326 EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
327  // Discard all but first 4 bytes.
328  return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
329 }
330 template <>
331 EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
332  // Discard all but first 2 bytes.
333  return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
334 }
335 
336 template <>
337 struct type_casting_traits<numext::uint8_t, numext::int32_t> {
338  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
339 };
340 template <>
341 EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
342  return vreinterpretq_s32_u32(pcast<Packet16uc, Packet4ui>(a));
343 }
344 template <>
345 EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
346  return vreinterpret_s32_u32(pcast<Packet8uc, Packet2ui>(a));
347 }
348 
349 template <>
350 struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
351  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
352 };
353 template <>
354 EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
355  // Discard second half of input.
356  return vmovl_u8(vget_low_u8(a));
357 }
358 template <>
359 EIGEN_STRONG_INLINE Packet4us pcast<Packet8uc, Packet4us>(const Packet8uc& a) {
360  // Discard second half of input.
361  return vget_low_u16(vmovl_u8(a));
362 }
363 
364 template <>
365 struct type_casting_traits<numext::uint8_t, numext::int16_t> {
366  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
367 };
368 template <>
369 EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
370  return vreinterpretq_s16_u16(pcast<Packet16uc, Packet8us>(a));
371 }
372 template <>
373 EIGEN_STRONG_INLINE Packet4s pcast<Packet8uc, Packet4s>(const Packet8uc& a) {
374  return vreinterpret_s16_u16(pcast<Packet8uc, Packet4us>(a));
375 }
376 
377 template <>
378 struct type_casting_traits<numext::uint8_t, numext::uint8_t> {
379  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
380 };
381 template <>
382 EIGEN_STRONG_INLINE Packet16uc pcast<Packet16uc, Packet16uc>(const Packet16uc& a) {
383  return a;
384 }
385 template <>
386 EIGEN_STRONG_INLINE Packet8uc pcast<Packet8uc, Packet8uc>(const Packet8uc& a) {
387  return a;
388 }
389 template <>
390 EIGEN_STRONG_INLINE Packet4uc pcast<Packet4uc, Packet4uc>(const Packet4uc& a) {
391  return a;
392 }
393 
394 template <>
395 struct type_casting_traits<numext::uint8_t, numext::int8_t> {
396  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
397 };
398 template <>
399 EIGEN_STRONG_INLINE Packet16c pcast<Packet16uc, Packet16c>(const Packet16uc& a) {
400  return vreinterpretq_s8_u8(a);
401 }
402 template <>
403 EIGEN_STRONG_INLINE Packet8c pcast<Packet8uc, Packet8c>(const Packet8uc& a) {
404  return vreinterpret_s8_u8(a);
405 }
406 template <>
407 EIGEN_STRONG_INLINE Packet4c pcast<Packet4uc, Packet4c>(const Packet4uc& a) {
408  return static_cast<Packet4c>(a);
409 }
410 
411 //==============================================================================
412 // pcast, SrcType = int16_t
413 //==============================================================================
414 template <>
415 struct type_casting_traits<numext::int16_t, float> {
416  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
417 };
418 template <>
419 EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
420  // Discard second half of input.
421  return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
422 }
423 template <>
424 EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
425  // Discard second half of input.
426  return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
427 }
428 
429 template <>
430 struct type_casting_traits<numext::int16_t, numext::int64_t> {
431  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
432 };
433 template <>
434 EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
435  // Discard all but first two values.
436  return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
437 }
438 
439 template <>
440 struct type_casting_traits<numext::int16_t, numext::uint64_t> {
441  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
442 };
443 template <>
444 EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
445  return vreinterpretq_u64_s64(pcast<Packet8s, Packet2l>(a));
446 }
447 
448 template <>
449 struct type_casting_traits<numext::int16_t, numext::int32_t> {
450  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
451 };
452 template <>
453 EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
454  // Discard second half of input.
455  return vmovl_s16(vget_low_s16(a));
456 }
457 template <>
458 EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
459  // Discard second half of input.
460  return vget_low_s32(vmovl_s16(a));
461 }
462 
463 template <>
464 struct type_casting_traits<numext::int16_t, numext::uint32_t> {
465  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
466 };
467 template <>
468 EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
469  return vreinterpretq_u32_s32(pcast<Packet8s, Packet4i>(a));
470 }
471 template <>
472 EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
473  return vreinterpret_u32_s32(pcast<Packet4s, Packet2i>(a));
474 }
475 
476 template <>
477 struct type_casting_traits<numext::int16_t, numext::int16_t> {
478  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
479 };
480 template <>
481 EIGEN_STRONG_INLINE Packet8s pcast<Packet8s, Packet8s>(const Packet8s& a) {
482  return a;
483 }
484 template <>
485 EIGEN_STRONG_INLINE Packet4s pcast<Packet4s, Packet4s>(const Packet4s& a) {
486  return a;
487 }
488 
489 template <>
490 struct type_casting_traits<numext::int16_t, numext::uint16_t> {
491  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
492 };
493 template <>
494 EIGEN_STRONG_INLINE Packet8us pcast<Packet8s, Packet8us>(const Packet8s& a) {
495  return vreinterpretq_u16_s16(a);
496 }
497 template <>
498 EIGEN_STRONG_INLINE Packet4us pcast<Packet4s, Packet4us>(const Packet4s& a) {
499  return vreinterpret_u16_s16(a);
500 }
501 
502 template <>
503 struct type_casting_traits<numext::int16_t, numext::int8_t> {
504  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
505 };
506 template <>
507 EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
508  return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
509 }
510 template <>
511 EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
512  return vmovn_s16(vcombine_s16(a, b));
513 }
514 
515 template <>
516 struct type_casting_traits<numext::int16_t, numext::uint8_t> {
517  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
518 };
519 template <>
520 EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
521  return vcombine_u8(vmovn_u16(vreinterpretq_u16_s16(a)), vmovn_u16(vreinterpretq_u16_s16(b)));
522 }
523 template <>
524 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
525  return vmovn_u16(vcombine_u16(vreinterpret_u16_s16(a), vreinterpret_u16_s16(b)));
526 }
527 
528 //==============================================================================
529 // pcast, SrcType = uint16_t
530 //==============================================================================
531 template <>
532 struct type_casting_traits<numext::uint16_t, float> {
533  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
534 };
535 template <>
536 EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
537  // Discard second half of input.
538  return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
539 }
540 template <>
541 EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
542  // Discard second half of input.
543  return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
544 }
545 
546 template <>
547 struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
548  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
549 };
550 template <>
551 EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
552  // Discard all but first two values.
553  return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
554 }
555 
556 template <>
557 struct type_casting_traits<numext::uint16_t, numext::int64_t> {
558  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
559 };
560 template <>
561 EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
562  return vreinterpretq_s64_u64(pcast<Packet8us, Packet2ul>(a));
563 }
564 
565 template <>
566 struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
567  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
568 };
569 template <>
570 EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
571  // Discard second half of input.
572  return vmovl_u16(vget_low_u16(a));
573 }
574 template <>
575 EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
576  // Discard second half of input.
577  return vget_low_u32(vmovl_u16(a));
578 }
579 
580 template <>
581 struct type_casting_traits<numext::uint16_t, numext::int32_t> {
582  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
583 };
584 template <>
585 EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
586  return vreinterpretq_s32_u32(pcast<Packet8us, Packet4ui>(a));
587 }
588 template <>
589 EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
590  return vreinterpret_s32_u32(pcast<Packet4us, Packet2ui>(a));
591 }
592 
593 template <>
594 struct type_casting_traits<numext::uint16_t, numext::uint16_t> {
595  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
596 };
597 template <>
598 EIGEN_STRONG_INLINE Packet8us pcast<Packet8us, Packet8us>(const Packet8us& a) {
599  return a;
600 }
601 template <>
602 EIGEN_STRONG_INLINE Packet4us pcast<Packet4us, Packet4us>(const Packet4us& a) {
603  return a;
604 }
605 
606 template <>
607 struct type_casting_traits<numext::uint16_t, numext::int16_t> {
608  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
609 };
610 template <>
611 EIGEN_STRONG_INLINE Packet8s pcast<Packet8us, Packet8s>(const Packet8us& a) {
612  return vreinterpretq_s16_u16(a);
613 }
614 template <>
615 EIGEN_STRONG_INLINE Packet4s pcast<Packet4us, Packet4s>(const Packet4us& a) {
616  return vreinterpret_s16_u16(a);
617 }
618 
619 template <>
620 struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
621  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
622 };
623 template <>
624 EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
625  return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
626 }
627 template <>
628 EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
629  return vmovn_u16(vcombine_u16(a, b));
630 }
631 
632 template <>
633 struct type_casting_traits<numext::uint16_t, numext::int8_t> {
634  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
635 };
636 template <>
637 EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
638  return vreinterpretq_s8_u8(pcast<Packet8us, Packet16uc>(a, b));
639 }
640 template <>
641 EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
642  return vreinterpret_s8_u8(pcast<Packet4us, Packet8uc>(a, b));
643 }
644 
645 //==============================================================================
646 // pcast, SrcType = int32_t
647 //==============================================================================
648 template <>
649 struct type_casting_traits<numext::int32_t, float> {
650  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
651 };
652 template <>
653 EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
654  return vcvtq_f32_s32(a);
655 }
656 template <>
657 EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
658  return vcvt_f32_s32(a);
659 }
660 
661 template <>
662 struct type_casting_traits<numext::int32_t, numext::int64_t> {
663  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
664 };
665 template <>
666 EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
667  // Discard second half of input.
668  return vmovl_s32(vget_low_s32(a));
669 }
670 
671 template <>
672 struct type_casting_traits<numext::int32_t, numext::uint64_t> {
673  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
674 };
675 template <>
676 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
677  return vreinterpretq_u64_s64(pcast<Packet4i, Packet2l>(a));
678 }
679 
680 template <>
681 struct type_casting_traits<numext::int32_t, numext::int32_t> {
682  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
683 };
684 template <>
685 EIGEN_STRONG_INLINE Packet4i pcast<Packet4i, Packet4i>(const Packet4i& a) {
686  return a;
687 }
688 template <>
689 EIGEN_STRONG_INLINE Packet2i pcast<Packet2i, Packet2i>(const Packet2i& a) {
690  return a;
691 }
692 
693 template <>
694 struct type_casting_traits<numext::int32_t, numext::uint32_t> {
695  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
696 };
697 template <>
698 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4i, Packet4ui>(const Packet4i& a) {
699  return vreinterpretq_u32_s32(a);
700 }
701 template <>
702 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2i, Packet2ui>(const Packet2i& a) {
703  return vreinterpret_u32_s32(a);
704 }
705 
706 template <>
707 struct type_casting_traits<numext::int32_t, numext::int16_t> {
708  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
709 };
710 template <>
711 EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
712  return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
713 }
714 template <>
715 EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
716  return vmovn_s32(vcombine_s32(a, b));
717 }
718 
719 template <>
720 struct type_casting_traits<numext::int32_t, numext::uint16_t> {
721  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
722 };
723 template <>
724 EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
725  return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
726 }
727 template <>
728 EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
729  return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
730 }
731 
732 template <>
733 struct type_casting_traits<numext::int32_t, numext::int8_t> {
734  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
735 };
736 template <>
737 EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
738  const Packet4i& d) {
739  const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
740  const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
741  return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
742 }
743 template <>
744 EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
745  const Packet2i& d) {
746  const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
747  const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
748  return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
749 }
750 
751 template <>
752 struct type_casting_traits<numext::int32_t, numext::uint8_t> {
753  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
754 };
755 template <>
756 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
757  const Packet4i& d) {
758  const uint16x8_t ab_u16 = pcast<Packet4i, Packet8us>(a, b);
759  const uint16x8_t cd_u16 = pcast<Packet4i, Packet8us>(c, d);
760  return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
761 }
762 template <>
763 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
764  const Packet2i& d) {
765  const uint16x4_t ab_u16 = pcast<Packet2i, Packet4us>(a, b);
766  const uint16x4_t cd_u16 = pcast<Packet2i, Packet4us>(c, d);
767  return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
768 }
769 
770 //==============================================================================
771 // pcast, SrcType = uint32_t
772 //==============================================================================
773 template <>
774 struct type_casting_traits<numext::uint32_t, float> {
775  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
776 };
777 template <>
778 EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
779  return vcvtq_f32_u32(a);
780 }
781 template <>
782 EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
783  return vcvt_f32_u32(a);
784 }
785 
786 template <>
787 struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
788  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
789 };
790 template <>
791 EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
792  // Discard second half of input.
793  return vmovl_u32(vget_low_u32(a));
794 }
795 
796 template <>
797 struct type_casting_traits<numext::uint32_t, numext::int64_t> {
798  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
799 };
800 template <>
801 EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
802  return vreinterpretq_s64_u64(pcast<Packet4ui, Packet2ul>(a));
803 }
804 
805 template <>
806 struct type_casting_traits<numext::uint32_t, numext::uint32_t> {
807  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
808 };
809 template <>
810 EIGEN_STRONG_INLINE Packet4ui pcast<Packet4ui, Packet4ui>(const Packet4ui& a) {
811  return a;
812 }
813 template <>
814 EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ui, Packet2ui>(const Packet2ui& a) {
815  return a;
816 }
817 
818 template <>
819 struct type_casting_traits<numext::uint32_t, numext::int32_t> {
820  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
821 };
822 template <>
823 EIGEN_STRONG_INLINE Packet4i pcast<Packet4ui, Packet4i>(const Packet4ui& a) {
824  return vreinterpretq_s32_u32(a);
825 }
826 template <>
827 EIGEN_STRONG_INLINE Packet2i pcast<Packet2ui, Packet2i>(const Packet2ui& a) {
828  return vreinterpret_s32_u32(a);
829 }
830 
831 template <>
832 struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
833  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
834 };
835 template <>
836 EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
837  return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
838 }
839 template <>
840 EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
841  return vmovn_u32(vcombine_u32(a, b));
842 }
843 
844 template <>
845 struct type_casting_traits<numext::uint32_t, numext::int16_t> {
846  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
847 };
848 template <>
849 EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
850  return vreinterpretq_s16_u16(pcast<Packet4ui, Packet8us>(a, b));
851 }
852 template <>
853 EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
854  return vreinterpret_s16_u16(pcast<Packet2ui, Packet4us>(a, b));
855 }
856 
857 template <>
858 struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
859  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
860 };
861 template <>
862 EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
863  const Packet4ui& d) {
864  const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
865  const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
866  return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
867 }
868 template <>
869 EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
870  const Packet2ui& d) {
871  const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
872  const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
873  return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
874 }
875 
876 template <>
877 struct type_casting_traits<numext::uint32_t, numext::int8_t> {
878  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
879 };
880 template <>
881 EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
882  const Packet4ui& d) {
883  return vreinterpretq_s8_u8(pcast<Packet4ui, Packet16uc>(a, b, c, d));
884 }
885 template <>
886 EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
887  const Packet2ui& d) {
888  return vreinterpret_s8_u8(pcast<Packet2ui, Packet8uc>(a, b, c, d));
889 }
890 
891 //==============================================================================
892 // pcast, SrcType = int64_t
893 //==============================================================================
894 template <>
895 struct type_casting_traits<numext::int64_t, float> {
896  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
897 };
898 template <>
899 EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
900  return vcvtq_f32_s32(vcombine_s32(vmovn_s64(a), vmovn_s64(b)));
901 }
902 
903 template <>
904 struct type_casting_traits<numext::int64_t, numext::int64_t> {
905  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
906 };
907 template <>
908 EIGEN_STRONG_INLINE Packet2l pcast<Packet2l, Packet2l>(const Packet2l& a) {
909  return a;
910 }
911 
912 template <>
913 struct type_casting_traits<numext::int64_t, numext::uint64_t> {
914  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
915 };
916 template <>
917 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2l, Packet2ul>(const Packet2l& a) {
918  return vreinterpretq_u64_s64(a);
919 }
920 
921 template <>
922 struct type_casting_traits<numext::int64_t, numext::int32_t> {
923  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
924 };
925 template <>
926 EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
927  return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
928 }
929 
930 template <>
931 struct type_casting_traits<numext::int64_t, numext::uint32_t> {
932  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
933 };
934 template <>
935 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
936  return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
937 }
938 
939 template <>
940 struct type_casting_traits<numext::int64_t, numext::int16_t> {
941  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
942 };
943 template <>
944 EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
945  const Packet2l& d) {
946  const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
947  const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
948  return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
949 }
950 
951 template <>
952 struct type_casting_traits<numext::int64_t, numext::uint16_t> {
953  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
954 };
955 template <>
956 EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
957  const Packet2l& d) {
958  const uint32x4_t ab_u32 = pcast<Packet2l, Packet4ui>(a, b);
959  const uint32x4_t cd_u32 = pcast<Packet2l, Packet4ui>(c, d);
960  return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
961 }
962 
963 template <>
964 struct type_casting_traits<numext::int64_t, numext::int8_t> {
965  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
966 };
967 template <>
968 EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
969  const Packet2l& d, const Packet2l& e, const Packet2l& f,
970  const Packet2l& g, const Packet2l& h) {
971  const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
972  const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
973  return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
974 }
975 
976 template <>
977 struct type_casting_traits<numext::int64_t, numext::uint8_t> {
978  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
979 };
980 template <>
981 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
982  const Packet2l& d, const Packet2l& e, const Packet2l& f,
983  const Packet2l& g, const Packet2l& h) {
984  const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
985  const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
986  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
987 }
988 
989 //==============================================================================
990 // pcast, SrcType = uint64_t
991 //==============================================================================
992 template <>
993 struct type_casting_traits<numext::uint64_t, float> {
994  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
995 };
996 template <>
997 EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
998  return vcvtq_f32_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
999 }
1000 
1001 template <>
1002 struct type_casting_traits<numext::uint64_t, numext::uint64_t> {
1003  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1004 };
1005 template <>
1006 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ul, Packet2ul>(const Packet2ul& a) {
1007  return a;
1008 }
1009 
1010 template <>
1011 struct type_casting_traits<numext::uint64_t, numext::int64_t> {
1012  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1013 };
1014 template <>
1015 EIGEN_STRONG_INLINE Packet2l pcast<Packet2ul, Packet2l>(const Packet2ul& a) {
1016  return vreinterpretq_s64_u64(a);
1017 }
1018 
1019 template <>
1020 struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1021  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1022 };
1023 template <>
1024 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1025  return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1026 }
1027 
1028 template <>
1029 struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1030  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1031 };
1032 template <>
1033 EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1034  return vreinterpretq_s32_u32(pcast<Packet2ul, Packet4ui>(a, b));
1035 }
1036 
1037 template <>
1038 struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1039  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1040 };
1041 template <>
1042 EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1043  const Packet2ul& d) {
1044  const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1045  const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1046  return vcombine_u16(ab_u16, cd_u16);
1047 }
1048 
1049 template <>
1050 struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1051  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1052 };
1053 template <>
1054 EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1055  const Packet2ul& d) {
1056  return vreinterpretq_s16_u16(pcast<Packet2ul, Packet8us>(a, b, c, d));
1057 }
1058 
1059 template <>
1060 struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1061  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1062 };
1063 template <>
1064 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1065  const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1066  const Packet2ul& g, const Packet2ul& h) {
1067  const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1068  const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1069  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1070 }
1071 
1072 template <>
1073 struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1074  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1075 };
1076 template <>
1077 EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1078  const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1079  const Packet2ul& g, const Packet2ul& h) {
1080  return vreinterpretq_s8_u8(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1081 }
1082 
1083 //==============================================================================
1084 // preinterpret
1085 //==============================================================================
1086 template <>
1087 EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
1088  return vreinterpret_f32_s32(a);
1089 }
1090 template <>
1091 EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
1092  return vreinterpret_f32_u32(a);
1093 }
1094 template <>
1095 EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
1096  return vreinterpretq_f32_s32(a);
1097 }
1098 template <>
1099 EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
1100  return vreinterpretq_f32_u32(a);
1101 }
1102 
1103 template <>
1104 EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
1105  return static_cast<Packet4c>(a);
1106 }
1107 template <>
1108 EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
1109  return vreinterpret_s8_u8(a);
1110 }
1111 template <>
1112 EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
1113  return vreinterpretq_s8_u8(a);
1114 }
1115 
1116 template <>
1117 EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
1118  return static_cast<Packet4uc>(a);
1119 }
1120 template <>
1121 EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
1122  return vreinterpret_u8_s8(a);
1123 }
1124 template <>
1125 EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
1126  return vreinterpretq_u8_s8(a);
1127 }
1128 
1129 template <>
1130 EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
1131  return vreinterpret_s16_u16(a);
1132 }
1133 template <>
1134 EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
1135  return vreinterpretq_s16_u16(a);
1136 }
1137 
1138 template <>
1139 EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
1140  return vreinterpret_u16_s16(a);
1141 }
1142 template <>
1143 EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
1144  return vreinterpretq_u16_s16(a);
1145 }
1146 
1147 template <>
1148 EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
1149  return vreinterpret_s32_f32(a);
1150 }
1151 template <>
1152 EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
1153  return vreinterpret_s32_u32(a);
1154 }
1155 template <>
1156 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
1157  return vreinterpretq_s32_f32(a);
1158 }
1159 template <>
1160 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
1161  return vreinterpretq_s32_u32(a);
1162 }
1163 
1164 template <>
1165 EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
1166  return vreinterpret_u32_f32(a);
1167 }
1168 template <>
1169 EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
1170  return vreinterpret_u32_s32(a);
1171 }
1172 template <>
1173 EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
1174  return vreinterpretq_u32_f32(a);
1175 }
1176 template <>
1177 EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
1178  return vreinterpretq_u32_s32(a);
1179 }
1180 
1181 template <>
1182 EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
1183  return vreinterpretq_s64_u64(a);
1184 }
1185 template <>
1186 EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
1187  return vreinterpretq_u64_s64(a);
1188 }
1189 
1190 #if EIGEN_ARCH_ARM64
1191 
1192 //==============================================================================
1193 // pcast/preinterpret, Double
1194 //==============================================================================
1195 
1196 template <>
1197 struct type_casting_traits<double, double> {
1198  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1199 };
1200 template <>
1201 EIGEN_STRONG_INLINE Packet2d pcast<Packet2d, Packet2d>(const Packet2d& a) {
1202  return a;
1203 }
1204 
1205 template <>
1206 struct type_casting_traits<double, float> {
1207  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1208 };
1209 template <>
1210 EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1211  return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1212 }
1213 
1214 template <>
1215 struct type_casting_traits<double, numext::int64_t> {
1216  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1217 };
1218 template <>
1219 EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1220  return vcvtq_s64_f64(a);
1221 }
1222 
1223 template <>
1224 struct type_casting_traits<double, numext::uint64_t> {
1225  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1226 };
1227 template <>
1228 EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1229  return vcvtq_u64_f64(a);
1230 }
1231 
1232 template <>
1233 struct type_casting_traits<double, numext::int32_t> {
1234  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1235 };
1236 template <>
1237 EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1238  return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1239 }
1240 
1241 template <>
1242 struct type_casting_traits<double, numext::uint32_t> {
1243  enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1244 };
1245 template <>
1246 EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1247  return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1248 }
1249 
1250 template <>
1251 struct type_casting_traits<double, numext::int16_t> {
1252  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1253 };
1254 template <>
1255 EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1256  const Packet2d& d) {
1257  const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1258  const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1259  return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1260 }
1261 
1262 template <>
1263 struct type_casting_traits<double, numext::uint16_t> {
1264  enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1265 };
1266 template <>
1267 EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1268  const Packet2d& d) {
1269  const uint32x4_t ab_u32 = pcast<Packet2d, Packet4ui>(a, b);
1270  const uint32x4_t cd_u32 = pcast<Packet2d, Packet4ui>(c, d);
1271  return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
1272 }
1273 
1274 template <>
1275 struct type_casting_traits<double, numext::int8_t> {
1276  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1277 };
1278 template <>
1279 EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1280  const Packet2d& d, const Packet2d& e, const Packet2d& f,
1281  const Packet2d& g, const Packet2d& h) {
1282  const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1283  const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1284  return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1285 }
1286 
1287 template <>
1288 struct type_casting_traits<double, numext::uint8_t> {
1289  enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1290 };
1291 template <>
1292 EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1293  const Packet2d& d, const Packet2d& e, const Packet2d& f,
1294  const Packet2d& g, const Packet2d& h) {
1295  const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1296  const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1297  return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1298 }
1299 
1300 template <>
1301 struct type_casting_traits<float, double> {
1302  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1303 };
1304 template <>
1305 EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1306  // Discard second-half of input.
1307  return vcvt_f64_f32(vget_low_f32(a));
1308 }
1309 
1310 template <>
1311 struct type_casting_traits<numext::int8_t, double> {
1312  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1313 };
1314 template <>
1315 EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1316  // Discard all but first two values.
1317  return vcvt_f64_f32(pcast<Packet8c, Packet2f>(vget_low_s8(a)));
1318 }
1319 
1320 template <>
1321 struct type_casting_traits<numext::uint8_t, double> {
1322  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1323 };
1324 template <>
1325 EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1326  // Discard all but first two values.
1327  return vcvt_f64_f32(pcast<Packet8uc, Packet2f>(vget_low_u8(a)));
1328 }
1329 
1330 template <>
1331 struct type_casting_traits<numext::int16_t, double> {
1332  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1333 };
1334 template <>
1335 EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1336  // Discard all but first two values.
1337  return vcvt_f64_f32(pcast<Packet4s, Packet2f>(vget_low_s16(a)));
1338 }
1339 
1340 template <>
1341 struct type_casting_traits<numext::uint16_t, double> {
1342  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1343 };
1344 template <>
1345 EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1346  // Discard all but first two values.
1347  return vcvt_f64_f32(pcast<Packet4us, Packet2f>(vget_low_u16(a)));
1348 }
1349 
1350 template <>
1351 struct type_casting_traits<numext::int32_t, double> {
1352  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1353 };
1354 template <>
1355 EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1356  // Discard second half of input.
1357  return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1358 }
1359 
1360 template <>
1361 struct type_casting_traits<numext::uint32_t, double> {
1362  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1363 };
1364 template <>
1365 EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1366  // Discard second half of input.
1367  return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1368 }
1369 
1370 template <>
1371 struct type_casting_traits<numext::int64_t, double> {
1372  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1373 };
1374 template <>
1375 EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1376  return vcvtq_f64_s64(a);
1377 }
1378 
1379 template <>
1380 struct type_casting_traits<numext::uint64_t, double> {
1381  enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1382 };
1383 template <>
1384 EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1385  return vcvtq_f64_u64(a);
1386 }
1387 
1388 template <>
1389 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1390  return vreinterpretq_f64_s64(a);
1391 }
1392 template <>
1393 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1394  return vreinterpretq_f64_u64(a);
1395 }
1396 template <>
1397 EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1398  return vreinterpretq_s64_f64(a);
1399 }
1400 template <>
1401 EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1402  return vreinterpretq_u64_f64(a);
1403 }
1404 template <>
1405 EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1406  return vreinterpretq_f64_s32(a);
1407 }
1408 template <>
1409 EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1410  return vreinterpretq_s32_f64(a);
1411 }
1412 
1413 #endif // EIGEN_ARCH_ARM64
1414 
1415 } // end namespace internal
1416 
1417 } // end namespace Eigen
1418 
1419 #endif // EIGEN_TYPE_CASTING_NEON_H
Namespace containing all symbols from the Eigen library.
Definition: Core:141
Definition: Eigen_Colamd.h:50