11 #ifndef EIGEN_MATRIX_PRODUCT_ALTIVEC_H 12 #define EIGEN_MATRIX_PRODUCT_ALTIVEC_H 14 #ifndef EIGEN_ALTIVEC_USE_CUSTOM_PACK 15 #define EIGEN_ALTIVEC_USE_CUSTOM_PACK 1 18 #include "MatrixProductCommon.h" 22 #if !defined(EIGEN_ALTIVEC_DISABLE_MMA) && !defined(EIGEN_ALTIVEC_MMA_ONLY) 24 #define EIGEN_ALTIVEC_MMA_ONLY 26 #define EIGEN_ALTIVEC_DISABLE_MMA 32 #if __has_builtin(__builtin_mma_assemble_acc) 33 #define ALTIVEC_MMA_SUPPORT 37 #if defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 38 #include "MatrixProductMMA.h" 53 template<
typename Scalar>
56 typedef typename packet_traits<Scalar>::type vectortype;
57 typedef PacketBlock<vectortype,4> type;
58 typedef vectortype rhstype;
61 vectorsize = packet_traits<Scalar>::size,
68 struct quad_traits<double>
70 typedef Packet2d vectortype;
71 typedef PacketBlock<vectortype,4> type;
72 typedef PacketBlock<Packet2d,2> rhstype;
75 vectorsize = packet_traits<double>::size,
85 const static Packet16uc p16uc_GETREAL32 = { 0, 1, 2, 3,
90 const static Packet16uc p16uc_GETIMAG32 = { 4, 5, 6, 7,
94 const static Packet16uc p16uc_GETREAL64 = { 0, 1, 2, 3, 4, 5, 6, 7,
95 16, 17, 18, 19, 20, 21, 22, 23};
98 const static Packet16uc p16uc_GETIMAG64 = { 8, 9, 10, 11, 12, 13, 14, 15,
99 24, 25, 26, 27, 28, 29, 30, 31};
119 template<
typename Scalar,
typename Index,
int StorageOrder>
120 EIGEN_ALWAYS_INLINE std::complex<Scalar> getAdjointVal(
Index i,
Index j, const_blas_data_mapper<std::complex<Scalar>,
Index, StorageOrder>& dt)
122 std::complex<Scalar> v;
125 v.real( dt(j,i).
real());
126 v.imag(-dt(j,i).
imag());
129 v.real( dt(i,j).
real());
130 v.imag( dt(i,j).
imag());
132 v.real( dt(i,j).
real());
138 template<
typename Scalar,
typename Index,
int StorageOrder,
int N>
139 EIGEN_STRONG_INLINE
void symm_pack_complex_rhs_helper(std::complex<Scalar>* blockB,
const std::complex<Scalar>* _rhs,
Index rhsStride,
Index rows,
Index cols,
Index k2)
141 const Index depth = k2 + rows;
142 const_blas_data_mapper<std::complex<Scalar>,
Index, StorageOrder> rhs(_rhs, rhsStride);
143 const Index vectorSize = N*quad_traits<Scalar>::vectorsize;
144 const Index vectorDelta = vectorSize * rows;
145 Scalar* blockBf =
reinterpret_cast<Scalar *
>(blockB);
147 Index rir = 0, rii, j = 0;
148 for(; j + vectorSize <= cols; j+=vectorSize)
150 rii = rir + vectorDelta;
152 for(Index i = k2; i < depth; i++)
154 for(Index k = 0; k < vectorSize; k++)
156 std::complex<Scalar> v = getAdjointVal<Scalar, Index, StorageOrder>(i, j + k, rhs);
158 blockBf[rir + k] = v.real();
159 blockBf[rii + k] = v.imag();
169 rii = rir + ((cols - j) * rows);
171 for(Index i = k2; i < depth; i++)
176 std::complex<Scalar> v = getAdjointVal<Scalar, Index, StorageOrder>(i, k, rhs);
178 blockBf[rir] = v.real();
179 blockBf[rii] = v.imag();
188 template<
typename Scalar,
typename Index,
int StorageOrder>
189 EIGEN_STRONG_INLINE
void symm_pack_complex_lhs_helper(std::complex<Scalar>* blockA,
const std::complex<Scalar>* _lhs, Index lhsStride, Index cols, Index rows)
191 const Index depth = cols;
192 const_blas_data_mapper<std::complex<Scalar>,
Index, StorageOrder> lhs(_lhs, lhsStride);
193 const Index vectorSize = quad_traits<Scalar>::vectorsize;
194 const Index vectorDelta = vectorSize * depth;
195 Scalar* blockAf = (Scalar *)(blockA);
197 Index rir = 0, rii, j = 0;
198 for(; j + vectorSize <= rows; j+=vectorSize)
200 rii = rir + vectorDelta;
202 for(Index i = 0; i < depth; i++)
204 for(Index k = 0; k < vectorSize; k++)
206 std::complex<Scalar> v = getAdjointVal<Scalar, Index, StorageOrder>(j+k, i, lhs);
208 blockAf[rir + k] = v.real();
209 blockAf[rii + k] = v.imag();
220 rii = rir + ((rows - j) * depth);
222 for(Index i = 0; i < depth; i++)
227 std::complex<Scalar> v = getAdjointVal<Scalar, Index, StorageOrder>(k, i, lhs);
229 blockAf[rir] = v.real();
230 blockAf[rii] = v.imag();
239 template<
typename Scalar,
typename Index,
int StorageOrder,
int N>
240 EIGEN_STRONG_INLINE
void symm_pack_rhs_helper(Scalar* blockB,
const Scalar* _rhs, Index rhsStride, Index rows, Index cols, Index k2)
242 const Index depth = k2 + rows;
243 const_blas_data_mapper<Scalar, Index, StorageOrder> rhs(_rhs, rhsStride);
244 const Index vectorSize = quad_traits<Scalar>::vectorsize;
247 for(; j + N*vectorSize <= cols; j+=N*vectorSize)
250 for(; i < depth; i++)
252 for(Index k = 0; k < N*vectorSize; k++)
255 blockB[ri + k] = rhs(j+k, i);
257 blockB[ri + k] = rhs(i, j+k);
265 for(Index i = k2; i < depth; i++)
271 blockB[ri] = rhs(i, k);
273 blockB[ri] = rhs(k, i);
280 template<
typename Scalar,
typename Index,
int StorageOrder>
281 EIGEN_STRONG_INLINE
void symm_pack_lhs_helper(Scalar* blockA,
const Scalar* _lhs, Index lhsStride, Index cols, Index rows)
283 const Index depth = cols;
284 const_blas_data_mapper<Scalar, Index, StorageOrder> lhs(_lhs, lhsStride);
285 const Index vectorSize = quad_traits<Scalar>::vectorsize;
288 for(; j + vectorSize <= rows; j+=vectorSize)
292 for(; i < depth; i++)
294 for(Index k = 0; k < vectorSize; k++)
297 blockA[ri + k] = lhs(j+k, i);
299 blockA[ri + k] = lhs(i, j+k);
307 for(Index i = 0; i < depth; i++)
313 blockA[ri] = lhs(k, i);
315 blockA[ri] = lhs(i, k);
322 template<
typename Index,
int nr,
int StorageOrder>
323 struct symm_pack_rhs<
std::complex<float>, Index, nr, StorageOrder>
325 void operator()(std::complex<float>* blockB,
const std::complex<float>* _rhs, Index rhsStride, Index rows, Index cols, Index k2)
327 symm_pack_complex_rhs_helper<float, Index, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
331 template<
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
332 struct symm_pack_lhs<
std::complex<float>, Index, Pack1, Pack2_dummy, StorageOrder>
334 void operator()(std::complex<float>* blockA,
const std::complex<float>* _lhs, Index lhsStride, Index cols, Index rows)
336 symm_pack_complex_lhs_helper<float, Index, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
342 template<
typename Index,
int nr,
int StorageOrder>
343 struct symm_pack_rhs<
std::complex<double>, Index, nr, StorageOrder>
345 void operator()(std::complex<double>* blockB,
const std::complex<double>* _rhs, Index rhsStride, Index rows, Index cols, Index k2)
347 symm_pack_complex_rhs_helper<double, Index, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
351 template<
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
352 struct symm_pack_lhs<
std::complex<double>, Index, Pack1, Pack2_dummy, StorageOrder>
354 void operator()(std::complex<double>* blockA,
const std::complex<double>* _lhs, Index lhsStride, Index cols, Index rows)
356 symm_pack_complex_lhs_helper<double, Index, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
361 template<
typename Index,
int nr,
int StorageOrder>
362 struct symm_pack_rhs<float, Index, nr, StorageOrder>
364 void operator()(
float* blockB,
const float* _rhs, Index rhsStride, Index rows, Index cols, Index k2)
366 symm_pack_rhs_helper<float, Index, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
370 template<
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
371 struct symm_pack_lhs<float, Index, Pack1, Pack2_dummy, StorageOrder>
373 void operator()(
float* blockA,
const float* _lhs, Index lhsStride, Index cols, Index rows)
375 symm_pack_lhs_helper<float, Index, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
380 template<
typename Index,
int nr,
int StorageOrder>
381 struct symm_pack_rhs<double, Index, nr, StorageOrder>
383 void operator()(
double* blockB,
const double* _rhs, Index rhsStride, Index rows, Index cols, Index k2)
385 symm_pack_rhs_helper<double, Index, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
389 template<
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
390 struct symm_pack_lhs<double, Index, Pack1, Pack2_dummy, StorageOrder>
392 void operator()(
double* blockA,
const double* _lhs, Index lhsStride, Index cols, Index rows)
394 symm_pack_lhs_helper<double, Index, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
409 template<
typename Scalar,
typename Packet,
typename Index>
410 EIGEN_ALWAYS_INLINE
void storeBlock(Scalar* to, PacketBlock<Packet,4>& block)
412 const Index size = 16 /
sizeof(Scalar);
413 pstore<Scalar>(to + (0 * size), block.packet[0]);
414 pstore<Scalar>(to + (1 * size), block.packet[1]);
415 pstore<Scalar>(to + (2 * size), block.packet[2]);
416 pstore<Scalar>(to + (3 * size), block.packet[3]);
419 template<
typename Scalar,
typename Packet,
typename Index>
420 EIGEN_ALWAYS_INLINE
void storeBlock(Scalar* to, PacketBlock<Packet,2>& block)
422 const Index size = 16 /
sizeof(Scalar);
423 pstore<Scalar>(to + (0 * size), block.packet[0]);
424 pstore<Scalar>(to + (1 * size), block.packet[1]);
428 template<
typename Scalar,
typename Index,
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
bool PanelMode,
bool UseLhs>
430 EIGEN_STRONG_INLINE
void operator()(std::complex<Scalar>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
432 const Index vectorSize = quad_traits<Scalar>::vectorsize;
433 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
434 Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii;
435 Scalar* blockAt =
reinterpret_cast<Scalar *
>(blockA);
438 for(; j + vectorSize <= rows; j+=vectorSize)
442 rii = rir + vectorDelta;
444 for(; i + vectorSize <= depth; i+=vectorSize)
446 PacketBlock<Packet,4> blockr, blocki;
447 PacketBlock<PacketC,8> cblock;
450 bload<DataMapper, PacketC, Index, 2, 0, StorageOrder>(cblock, lhs, j, i);
452 bload<DataMapper, PacketC, Index, 2, 0, StorageOrder>(cblock, lhs, i, j);
455 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETREAL32);
456 blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETREAL32);
457 blockr.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETREAL32);
458 blockr.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETREAL32);
460 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, p16uc_GETIMAG32);
461 blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, p16uc_GETIMAG32);
462 blocki.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, p16uc_GETIMAG32);
463 blocki.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, p16uc_GETIMAG32);
467 blocki.packet[0] = -blocki.packet[0];
468 blocki.packet[1] = -blocki.packet[1];
469 blocki.packet[2] = -blocki.packet[2];
470 blocki.packet[3] = -blocki.packet[3];
473 if(((StorageOrder ==
RowMajor) && UseLhs) || (((StorageOrder ==
ColMajor) && !UseLhs)))
479 storeBlock<Scalar, Packet, Index>(blockAt + rir, blockr);
480 storeBlock<Scalar, Packet, Index>(blockAt + rii, blocki);
485 for(; i < depth; i++)
487 PacketBlock<Packet,1> blockr, blocki;
488 PacketBlock<PacketC,2> cblock;
490 if(((StorageOrder ==
ColMajor) && UseLhs) || (((StorageOrder ==
RowMajor) && !UseLhs)))
493 cblock.packet[0] = lhs.template loadPacket<PacketC>(j + 0, i);
494 cblock.packet[1] = lhs.template loadPacket<PacketC>(j + 2, i);
496 cblock.packet[0] = lhs.template loadPacket<PacketC>(i, j + 0);
497 cblock.packet[1] = lhs.template loadPacket<PacketC>(i, j + 2);
500 std::complex<Scalar> lhs0, lhs1;
502 lhs0 = lhs(j + 0, i);
503 lhs1 = lhs(j + 1, i);
504 cblock.packet[0] = pload2(&lhs0, &lhs1);
505 lhs0 = lhs(j + 2, i);
506 lhs1 = lhs(j + 3, i);
507 cblock.packet[1] = pload2(&lhs0, &lhs1);
509 lhs0 = lhs(i, j + 0);
510 lhs1 = lhs(i, j + 1);
511 cblock.packet[0] = pload2(&lhs0, &lhs1);
512 lhs0 = lhs(i, j + 2);
513 lhs1 = lhs(i, j + 3);
514 cblock.packet[1] = pload2(&lhs0, &lhs1);
518 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32);
519 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32);
523 blocki.packet[0] = -blocki.packet[0];
526 pstore<Scalar>(blockAt + rir, blockr.packet[0]);
527 pstore<Scalar>(blockAt + rii, blocki.packet[0]);
533 rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta);
538 if(PanelMode) rir += (offset*(rows - j - vectorSize));
539 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
541 for(Index i = 0; i < depth; i++)
547 blockAt[rir] = lhs(k, i).real();
550 blockAt[rii] = -lhs(k, i).imag();
552 blockAt[rii] = lhs(k, i).imag();
554 blockAt[rir] = lhs(i, k).real();
557 blockAt[rii] = -lhs(i, k).imag();
559 blockAt[rii] = lhs(i, k).imag();
571 template<
typename Scalar,
typename Index,
typename DataMapper,
typename Packet,
int StorageOrder,
bool PanelMode,
bool UseLhs>
573 EIGEN_STRONG_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
575 const Index vectorSize = quad_traits<Scalar>::vectorsize;
578 for(; j + vectorSize <= rows; j+=vectorSize)
582 if(PanelMode) ri += vectorSize*offset;
584 for(; i + vectorSize <= depth; i+=vectorSize)
586 PacketBlock<Packet,4> block;
589 bload<DataMapper, Packet, Index, 4, 0, StorageOrder>(block, lhs, j, i);
591 bload<DataMapper, Packet, Index, 4, 0, StorageOrder>(block, lhs, i, j);
593 if(((StorageOrder ==
RowMajor) && UseLhs) || ((StorageOrder ==
ColMajor) && !UseLhs))
598 storeBlock<Scalar, Packet, Index>(blockA + ri, block);
602 for(; i < depth; i++)
604 if(((StorageOrder ==
RowMajor) && UseLhs) || ((StorageOrder ==
ColMajor) && !UseLhs))
607 blockA[ri+0] = lhs(j+0, i);
608 blockA[ri+1] = lhs(j+1, i);
609 blockA[ri+2] = lhs(j+2, i);
610 blockA[ri+3] = lhs(j+3, i);
612 blockA[ri+0] = lhs(i, j+0);
613 blockA[ri+1] = lhs(i, j+1);
614 blockA[ri+2] = lhs(i, j+2);
615 blockA[ri+3] = lhs(i, j+3);
620 lhsV = lhs.template loadPacket<Packet>(j, i);
622 lhsV = lhs.template loadPacket<Packet>(i, j);
624 pstore<Scalar>(blockA + ri, lhsV);
630 if(PanelMode) ri += vectorSize*(stride - offset - depth);
635 if(PanelMode) ri += offset*(rows - j);
637 for(Index i = 0; i < depth; i++)
643 blockA[ri] = lhs(k, i);
645 blockA[ri] = lhs(i, k);
655 template<
typename Index,
typename DataMapper,
int StorageOrder,
bool PanelMode>
656 struct dhs_pack<double, Index, DataMapper, Packet2d, StorageOrder, PanelMode, true>
658 EIGEN_STRONG_INLINE
void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
660 const Index vectorSize = quad_traits<double>::vectorsize;
663 for(; j + vectorSize <= rows; j+=vectorSize)
667 if(PanelMode) ri += vectorSize*offset;
669 for(; i + vectorSize <= depth; i+=vectorSize)
671 PacketBlock<Packet2d,2> block;
674 block.packet[0] = lhs.template loadPacket<Packet2d>(j + 0, i);
675 block.packet[1] = lhs.template loadPacket<Packet2d>(j + 1, i);
679 block.packet[0] = lhs.template loadPacket<Packet2d>(j, i + 0);
680 block.packet[1] = lhs.template loadPacket<Packet2d>(j, i + 1);
683 storeBlock<double, Packet2d, Index>(blockA + ri, block);
687 for(; i < depth; i++)
691 blockA[ri+0] = lhs(j+0, i);
692 blockA[ri+1] = lhs(j+1, i);
694 Packet2d lhsV = lhs.template loadPacket<Packet2d>(j, i);
695 pstore<double>(blockA + ri, lhsV);
701 if(PanelMode) ri += vectorSize*(stride - offset - depth);
706 if(PanelMode) ri += offset*(rows - j);
708 for(Index i = 0; i < depth; i++)
713 blockA[ri] = lhs(k, i);
722 template<
typename Index,
typename DataMapper,
int StorageOrder,
bool PanelMode>
723 struct dhs_pack<double, Index, DataMapper, Packet2d, StorageOrder, PanelMode, false>
725 EIGEN_STRONG_INLINE
void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
727 const Index vectorSize = quad_traits<double>::vectorsize;
730 for(; j + 2*vectorSize <= cols; j+=2*vectorSize)
734 if(PanelMode) ri += offset*(2*vectorSize);
736 for(; i + vectorSize <= depth; i+=vectorSize)
738 PacketBlock<Packet2d,4> block;
741 PacketBlock<Packet2d,2> block1, block2;
742 block1.packet[0] = rhs.template loadPacket<Packet2d>(i, j + 0);
743 block1.packet[1] = rhs.template loadPacket<Packet2d>(i, j + 1);
744 block2.packet[0] = rhs.template loadPacket<Packet2d>(i, j + 2);
745 block2.packet[1] = rhs.template loadPacket<Packet2d>(i, j + 3);
750 pstore<double>(blockB + ri , block1.packet[0]);
751 pstore<double>(blockB + ri + 2, block2.packet[0]);
752 pstore<double>(blockB + ri + 4, block1.packet[1]);
753 pstore<double>(blockB + ri + 6, block2.packet[1]);
755 block.packet[0] = rhs.template loadPacket<Packet2d>(i + 0, j + 0);
756 block.packet[1] = rhs.template loadPacket<Packet2d>(i + 0, j + 2);
757 block.packet[2] = rhs.template loadPacket<Packet2d>(i + 1, j + 0);
758 block.packet[3] = rhs.template loadPacket<Packet2d>(i + 1, j + 2);
760 storeBlock<double, Packet2d, Index>(blockB + ri, block);
765 for(; i < depth; i++)
769 blockB[ri+0] = rhs(i, j+0);
770 blockB[ri+1] = rhs(i, j+1);
774 blockB[ri+0] = rhs(i, j+2);
775 blockB[ri+1] = rhs(i, j+3);
777 Packet2d rhsV = rhs.template loadPacket<Packet2d>(i, j);
778 pstore<double>(blockB + ri, rhsV);
782 rhsV = rhs.template loadPacket<Packet2d>(i, j + 2);
783 pstore<double>(blockB + ri, rhsV);
788 if(PanelMode) ri += (2*vectorSize)*(stride - offset - depth);
793 if(PanelMode) ri += offset*(cols - j);
795 for(Index i = 0; i < depth; i++)
800 blockB[ri] = rhs(i, k);
809 template<
typename Index,
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
bool PanelMode>
810 struct dhs_cpack<double, Index, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true>
812 EIGEN_STRONG_INLINE
void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
814 const Index vectorSize = quad_traits<double>::vectorsize;
815 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
816 Index rir = ((PanelMode) ? (vectorSize*offset) : 0), rii;
817 double* blockAt =
reinterpret_cast<double *
>(blockA);
820 for(; j + vectorSize <= rows; j+=vectorSize)
824 rii = rir + vectorDelta;
826 for(; i + vectorSize <= depth; i+=vectorSize)
828 PacketBlock<Packet,2> blockr, blocki;
829 PacketBlock<PacketC,4> cblock;
833 cblock.packet[0] = lhs.template loadPacket<PacketC>(j, i + 0);
834 cblock.packet[1] = lhs.template loadPacket<PacketC>(j, i + 1);
836 cblock.packet[2] = lhs.template loadPacket<PacketC>(j + 1, i + 0);
837 cblock.packet[3] = lhs.template loadPacket<PacketC>(j + 1, i + 1);
839 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETREAL64);
840 blockr.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETREAL64);
842 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[2].v, p16uc_GETIMAG64);
843 blocki.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[3].v, p16uc_GETIMAG64);
845 cblock.packet[0] = lhs.template loadPacket<PacketC>(j + 0, i);
846 cblock.packet[1] = lhs.template loadPacket<PacketC>(j + 1, i);
848 cblock.packet[2] = lhs.template loadPacket<PacketC>(j + 0, i + 1);
849 cblock.packet[3] = lhs.template loadPacket<PacketC>(j + 1, i + 1);
851 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64);
852 blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64);
854 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64);
855 blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64);
860 blocki.packet[0] = -blocki.packet[0];
861 blocki.packet[1] = -blocki.packet[1];
864 storeBlock<double, Packet, Index>(blockAt + rir, blockr);
865 storeBlock<double, Packet, Index>(blockAt + rii, blocki);
870 for(; i < depth; i++)
872 PacketBlock<Packet,1> blockr, blocki;
873 PacketBlock<PacketC,2> cblock;
875 cblock.packet[0] = lhs.template loadPacket<PacketC>(j + 0, i);
876 cblock.packet[1] = lhs.template loadPacket<PacketC>(j + 1, i);
878 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64);
879 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64);
883 blocki.packet[0] = -blocki.packet[0];
886 pstore<double>(blockAt + rir, blockr.packet[0]);
887 pstore<double>(blockAt + rii, blocki.packet[0]);
893 rir += ((PanelMode) ? (vectorSize*(2*stride - depth)) : vectorDelta);
898 if(PanelMode) rir += (offset*(rows - j - vectorSize));
899 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
901 for(Index i = 0; i < depth; i++)
906 blockAt[rir] = lhs(k, i).real();
909 blockAt[rii] = -lhs(k, i).imag();
911 blockAt[rii] = lhs(k, i).imag();
922 template<
typename Index,
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
bool PanelMode>
923 struct dhs_cpack<double, Index, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, false>
925 EIGEN_STRONG_INLINE
void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
927 const Index vectorSize = quad_traits<double>::vectorsize;
928 const Index vectorDelta = 2*vectorSize * ((PanelMode) ? stride : depth);
929 Index rir = ((PanelMode) ? (2*vectorSize*offset) : 0), rii;
930 double* blockBt =
reinterpret_cast<double *
>(blockB);
933 for(; j + 2*vectorSize <= cols; j+=2*vectorSize)
937 rii = rir + vectorDelta;
939 for(; i < depth; i++)
941 PacketBlock<PacketC,4> cblock;
942 PacketBlock<Packet,2> blockr, blocki;
944 bload<DataMapper, PacketC, Index, 2, 0, ColMajor>(cblock, rhs, i, j);
946 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL64);
947 blockr.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETREAL64);
949 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG64);
950 blocki.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, p16uc_GETIMAG64);
954 blocki.packet[0] = -blocki.packet[0];
955 blocki.packet[1] = -blocki.packet[1];
958 storeBlock<double, Packet, Index>(blockBt + rir, blockr);
959 storeBlock<double, Packet, Index>(blockBt + rii, blocki);
965 rir += ((PanelMode) ? (2*vectorSize*(2*stride - depth)) : vectorDelta);
970 if(PanelMode) rir += (offset*(cols - j - 2*vectorSize));
971 rii = rir + (((PanelMode) ? stride : depth) * (cols - j));
973 for(Index i = 0; i < depth; i++)
978 blockBt[rir] = rhs(i, k).real();
981 blockBt[rii] = -rhs(i, k).imag();
983 blockBt[rii] = rhs(i, k).imag();
998 template<
typename Packet,
bool NegativeAccumulate>
999 EIGEN_ALWAYS_INLINE
void pger_common(PacketBlock<Packet,4>* acc,
const Packet& lhsV,
const Packet* rhsV)
1001 if(NegativeAccumulate)
1003 acc->packet[0] = vec_nmsub(lhsV, rhsV[0], acc->packet[0]);
1004 acc->packet[1] = vec_nmsub(lhsV, rhsV[1], acc->packet[1]);
1005 acc->packet[2] = vec_nmsub(lhsV, rhsV[2], acc->packet[2]);
1006 acc->packet[3] = vec_nmsub(lhsV, rhsV[3], acc->packet[3]);
1008 acc->packet[0] = vec_madd(lhsV, rhsV[0], acc->packet[0]);
1009 acc->packet[1] = vec_madd(lhsV, rhsV[1], acc->packet[1]);
1010 acc->packet[2] = vec_madd(lhsV, rhsV[2], acc->packet[2]);
1011 acc->packet[3] = vec_madd(lhsV, rhsV[3], acc->packet[3]);
1015 template<
typename Packet,
bool NegativeAccumulate>
1016 EIGEN_ALWAYS_INLINE
void pger_common(PacketBlock<Packet,1>* acc,
const Packet& lhsV,
const Packet* rhsV)
1018 if(NegativeAccumulate)
1020 acc->packet[0] = vec_nmsub(lhsV, rhsV[0], acc->packet[0]);
1022 acc->packet[0] = vec_madd(lhsV, rhsV[0], acc->packet[0]);
1026 template<
int N,
typename Scalar,
typename Packet,
bool NegativeAccumulate>
1027 EIGEN_ALWAYS_INLINE
void pger(PacketBlock<Packet,N>* acc,
const Scalar* lhs,
const Packet* rhsV)
1029 Packet lhsV = pload<Packet>(lhs);
1031 pger_common<Packet, NegativeAccumulate>(acc, lhsV, rhsV);
1034 template<
typename Scalar,
typename Packet,
typename Index>
1035 EIGEN_ALWAYS_INLINE
void loadPacketRemaining(
const Scalar* lhs, Packet &lhsV, Index remaining_rows)
1038 lhsV = vec_xl_len((Scalar *)lhs, remaining_rows *
sizeof(Scalar));
1043 }
while (++i < remaining_rows);
1047 template<
int N,
typename Scalar,
typename Packet,
typename Index,
bool NegativeAccumulate>
1048 EIGEN_ALWAYS_INLINE
void pger(PacketBlock<Packet,N>* acc,
const Scalar* lhs,
const Packet* rhsV, Index remaining_rows)
1051 loadPacketRemaining<Scalar, Packet, Index>(lhs, lhsV, remaining_rows);
1053 pger_common<Packet, NegativeAccumulate>(acc, lhsV, rhsV);
1057 template<
int N,
typename Packet,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1058 EIGEN_ALWAYS_INLINE
void pgerc_common(PacketBlock<Packet,N>* accReal, PacketBlock<Packet,N>* accImag,
const Packet &lhsV,
const Packet &lhsVi,
const Packet* rhsV,
const Packet* rhsVi)
1060 pger_common<Packet, false>(accReal, lhsV, rhsV);
1063 pger_common<Packet, ConjugateRhs>(accImag, lhsV, rhsVi);
1064 EIGEN_UNUSED_VARIABLE(lhsVi);
1067 pger_common<Packet, ConjugateLhs == ConjugateRhs>(accReal, lhsVi, rhsVi);
1068 pger_common<Packet, ConjugateRhs>(accImag, lhsV, rhsVi);
1070 EIGEN_UNUSED_VARIABLE(rhsVi);
1072 pger_common<Packet, ConjugateLhs>(accImag, lhsVi, rhsV);
1076 template<
int N,
typename Scalar,
typename Packet,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1077 EIGEN_ALWAYS_INLINE
void pgerc(PacketBlock<Packet,N>* accReal, PacketBlock<Packet,N>* accImag,
const Scalar* lhs_ptr,
const Scalar* lhs_ptr_imag,
const Packet* rhsV,
const Packet* rhsVi)
1079 Packet lhsV = ploadLhs<Scalar, Packet>(lhs_ptr);
1081 if(!LhsIsReal) lhsVi = ploadLhs<Scalar, Packet>(lhs_ptr_imag);
1082 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1084 pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi);
1087 template<
typename Scalar,
typename Packet,
typename Index,
bool LhsIsReal>
1088 EIGEN_ALWAYS_INLINE
void loadPacketRemaining(
const Scalar* lhs_ptr,
const Scalar* lhs_ptr_imag, Packet &lhsV, Packet &lhsVi, Index remaining_rows)
1091 lhsV = vec_xl_len((Scalar *)lhs_ptr, remaining_rows *
sizeof(Scalar));
1092 if(!LhsIsReal) lhsVi = vec_xl_len((Scalar *)lhs_ptr_imag, remaining_rows *
sizeof(Scalar));
1093 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1097 lhsV[i] = lhs_ptr[i];
1098 if(!LhsIsReal) lhsVi[i] = lhs_ptr_imag[i];
1099 }
while (++i < remaining_rows);
1100 if(LhsIsReal) EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1104 template<
int N,
typename Scalar,
typename Packet,
typename Index,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1105 EIGEN_ALWAYS_INLINE
void pgerc(PacketBlock<Packet,N>* accReal, PacketBlock<Packet,N>* accImag,
const Scalar* lhs_ptr,
const Scalar* lhs_ptr_imag,
const Packet* rhsV,
const Packet* rhsVi, Index remaining_rows)
1108 loadPacketRemaining<Scalar, Packet, Index, LhsIsReal>(lhs_ptr, lhs_ptr_imag, lhsV, lhsVi, remaining_rows);
1110 pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi);
1113 template<
typename Scalar,
typename Packet>
1114 EIGEN_ALWAYS_INLINE Packet ploadLhs(
const Scalar* lhs)
1116 return ploadu<Packet>(lhs);
1120 template<
typename Scalar,
typename Packet>
1121 EIGEN_ALWAYS_INLINE
void bsetzero(PacketBlock<Packet,4>& acc)
1123 acc.packet[0] = pset1<Packet>((Scalar)0);
1124 acc.packet[1] = pset1<Packet>((Scalar)0);
1125 acc.packet[2] = pset1<Packet>((Scalar)0);
1126 acc.packet[3] = pset1<Packet>((Scalar)0);
1129 template<
typename Scalar,
typename Packet>
1130 EIGEN_ALWAYS_INLINE
void bsetzero(PacketBlock<Packet,1>& acc)
1132 acc.packet[0] = pset1<Packet>((Scalar)0);
1136 template<
typename Packet>
1137 EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet,4>& acc, PacketBlock<Packet,4>& accZ,
const Packet& pAlpha)
1139 acc.packet[0] = pmadd(pAlpha, accZ.packet[0], acc.packet[0]);
1140 acc.packet[1] = pmadd(pAlpha, accZ.packet[1], acc.packet[1]);
1141 acc.packet[2] = pmadd(pAlpha, accZ.packet[2], acc.packet[2]);
1142 acc.packet[3] = pmadd(pAlpha, accZ.packet[3], acc.packet[3]);
1145 template<
typename Packet>
1146 EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet,1>& acc, PacketBlock<Packet,1>& accZ,
const Packet& pAlpha)
1148 acc.packet[0] = pmadd(pAlpha, accZ.packet[0], acc.packet[0]);
1151 template<
typename Packet>
1152 EIGEN_ALWAYS_INLINE
void bscalec_common(PacketBlock<Packet,4>& acc, PacketBlock<Packet,4>& accZ,
const Packet& pAlpha)
1154 acc.packet[0] = pmul<Packet>(accZ.packet[0], pAlpha);
1155 acc.packet[1] = pmul<Packet>(accZ.packet[1], pAlpha);
1156 acc.packet[2] = pmul<Packet>(accZ.packet[2], pAlpha);
1157 acc.packet[3] = pmul<Packet>(accZ.packet[3], pAlpha);
1160 template<
typename Packet>
1161 EIGEN_ALWAYS_INLINE
void bscalec_common(PacketBlock<Packet,1>& acc, PacketBlock<Packet,1>& accZ,
const Packet& pAlpha)
1163 acc.packet[0] = pmul<Packet>(accZ.packet[0], pAlpha);
1167 template<
typename Packet,
int N>
1168 EIGEN_ALWAYS_INLINE
void bscalec(PacketBlock<Packet,N>& aReal, PacketBlock<Packet,N>& aImag,
const Packet& bReal,
const Packet& bImag, PacketBlock<Packet,N>& cReal, PacketBlock<Packet,N>& cImag)
1170 bscalec_common<Packet>(cReal, aReal, bReal);
1172 bscalec_common<Packet>(cImag, aImag, bReal);
1174 pger_common<Packet, true>(&cReal, bImag, aImag.packet);
1176 pger_common<Packet, false>(&cImag, bImag, aReal.packet);
1179 template<
typename Packet>
1180 EIGEN_ALWAYS_INLINE
void band(PacketBlock<Packet,4>& acc,
const Packet& pMask)
1182 acc.packet[0] = pand(acc.packet[0], pMask);
1183 acc.packet[1] = pand(acc.packet[1], pMask);
1184 acc.packet[2] = pand(acc.packet[2], pMask);
1185 acc.packet[3] = pand(acc.packet[3], pMask);
1188 template<
typename Packet>
1189 EIGEN_ALWAYS_INLINE
void bscalec(PacketBlock<Packet,4>& aReal, PacketBlock<Packet,4>& aImag,
const Packet& bReal,
const Packet& bImag, PacketBlock<Packet,4>& cReal, PacketBlock<Packet,4>& cImag,
const Packet& pMask)
1191 band<Packet>(aReal, pMask);
1192 band<Packet>(aImag, pMask);
1194 bscalec<Packet,4>(aReal, aImag, bReal, bImag, cReal, cImag);
1198 template<
typename DataMapper,
typename Packet,
typename Index, const Index accCols,
int N,
int StorageOrder>
1199 EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet,4>& acc,
const DataMapper& res, Index row, Index col)
1202 acc.packet[0] = res.template loadPacket<Packet>(row + 0, col + N*accCols);
1203 acc.packet[1] = res.template loadPacket<Packet>(row + 1, col + N*accCols);
1204 acc.packet[2] = res.template loadPacket<Packet>(row + 2, col + N*accCols);
1205 acc.packet[3] = res.template loadPacket<Packet>(row + 3, col + N*accCols);
1207 acc.packet[0] = res.template loadPacket<Packet>(row + N*accCols, col + 0);
1208 acc.packet[1] = res.template loadPacket<Packet>(row + N*accCols, col + 1);
1209 acc.packet[2] = res.template loadPacket<Packet>(row + N*accCols, col + 2);
1210 acc.packet[3] = res.template loadPacket<Packet>(row + N*accCols, col + 3);
1215 template<
typename DataMapper,
typename Packet,
typename Index, const Index accCols,
int N,
int StorageOrder>
1216 EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet,8>& acc,
const DataMapper& res, Index row, Index col)
1219 acc.packet[0] = res.template loadPacket<Packet>(row + 0, col + N*accCols);
1220 acc.packet[1] = res.template loadPacket<Packet>(row + 1, col + N*accCols);
1221 acc.packet[2] = res.template loadPacket<Packet>(row + 2, col + N*accCols);
1222 acc.packet[3] = res.template loadPacket<Packet>(row + 3, col + N*accCols);
1223 acc.packet[4] = res.template loadPacket<Packet>(row + 0, col + (N+1)*accCols);
1224 acc.packet[5] = res.template loadPacket<Packet>(row + 1, col + (N+1)*accCols);
1225 acc.packet[6] = res.template loadPacket<Packet>(row + 2, col + (N+1)*accCols);
1226 acc.packet[7] = res.template loadPacket<Packet>(row + 3, col + (N+1)*accCols);
1228 acc.packet[0] = res.template loadPacket<Packet>(row + N*accCols, col + 0);
1229 acc.packet[1] = res.template loadPacket<Packet>(row + N*accCols, col + 1);
1230 acc.packet[2] = res.template loadPacket<Packet>(row + N*accCols, col + 2);
1231 acc.packet[3] = res.template loadPacket<Packet>(row + N*accCols, col + 3);
1232 acc.packet[4] = res.template loadPacket<Packet>(row + (N+1)*accCols, col + 0);
1233 acc.packet[5] = res.template loadPacket<Packet>(row + (N+1)*accCols, col + 1);
1234 acc.packet[6] = res.template loadPacket<Packet>(row + (N+1)*accCols, col + 2);
1235 acc.packet[7] = res.template loadPacket<Packet>(row + (N+1)*accCols, col + 3);
1239 template<
typename DataMapper,
typename Packet,
typename Index, const Index accCols,
int N,
int StorageOrder>
1240 EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet,2>& acc,
const DataMapper& res, Index row, Index col)
1242 acc.packet[0] = res.template loadPacket<Packet>(row + N*accCols, col + 0);
1243 acc.packet[1] = res.template loadPacket<Packet>(row + (N+1)*accCols, col + 0);
1246 const static Packet4i mask41 = { -1, 0, 0, 0 };
1247 const static Packet4i mask42 = { -1, -1, 0, 0 };
1248 const static Packet4i mask43 = { -1, -1, -1, 0 };
1250 const static Packet2l mask21 = { -1, 0 };
1252 template<
typename Packet>
1253 EIGEN_ALWAYS_INLINE Packet bmask(
const int remaining_rows)
1255 if (remaining_rows == 0) {
1256 return pset1<Packet>(float(0.0));
1258 switch (remaining_rows) {
1259 case 1:
return Packet(mask41);
1260 case 2:
return Packet(mask42);
1261 default:
return Packet(mask43);
1267 EIGEN_ALWAYS_INLINE Packet2d bmask<Packet2d>(
const int remaining_rows)
1269 if (remaining_rows == 0) {
1270 return pset1<Packet2d>(double(0.0));
1272 return Packet2d(mask21);
1276 template<
typename Packet>
1277 EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet,4>& acc, PacketBlock<Packet,4>& accZ,
const Packet& pAlpha,
const Packet& pMask)
1279 band<Packet>(accZ, pMask);
1281 bscale<Packet>(acc, accZ, pAlpha);
1284 template<
typename Packet>
1285 EIGEN_ALWAYS_INLINE
void pbroadcast4_old(
const __UNPACK_TYPE__(Packet)* a, Packet& a0, Packet& a1, Packet& a2, Packet& a3)
1287 pbroadcast4<Packet>(a, a0, a1, a2, a3);
1291 EIGEN_ALWAYS_INLINE
void pbroadcast4_old<Packet2d>(
const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
1293 a1 = pload<Packet2d>(a);
1294 a3 = pload<Packet2d>(a + 2);
1295 a0 = vec_splat(a1, 0);
1296 a1 = vec_splat(a1, 1);
1297 a2 = vec_splat(a3, 0);
1298 a3 = vec_splat(a3, 1);
1304 template<
typename Scalar,
typename Packet,
typename Index>
1305 EIGEN_ALWAYS_INLINE
void MICRO_EXTRA_COL(
1306 const Scalar* &lhs_ptr,
1307 const Scalar* &rhs_ptr,
1308 PacketBlock<Packet,1> &accZero,
1309 Index remaining_rows,
1310 Index remaining_cols)
1313 rhsV[0] = pset1<Packet>(rhs_ptr[0]);
1314 pger<1,Scalar, Packet, false>(&accZero, lhs_ptr, rhsV);
1315 lhs_ptr += remaining_rows;
1316 rhs_ptr += remaining_cols;
1319 template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accRows>
1320 EIGEN_STRONG_INLINE
void gemm_extra_col(
1321 const DataMapper& res,
1322 const Scalar* lhs_base,
1323 const Scalar* rhs_base,
1329 Index remaining_rows,
1330 Index remaining_cols,
1331 const Packet& pAlpha)
1333 const Scalar* rhs_ptr = rhs_base;
1334 const Scalar* lhs_ptr = lhs_base + row*strideA + remaining_rows*offsetA;
1335 PacketBlock<Packet,1> accZero;
1337 bsetzero<Scalar, Packet>(accZero);
1339 Index remaining_depth = (depth & -accRows);
1341 for(; k + PEEL <= remaining_depth; k+= PEEL)
1343 EIGEN_POWER_PREFETCH(rhs_ptr);
1344 EIGEN_POWER_PREFETCH(lhs_ptr);
1345 for (
int l = 0; l < PEEL; l++) {
1346 MICRO_EXTRA_COL<Scalar, Packet, Index>(lhs_ptr, rhs_ptr, accZero, remaining_rows, remaining_cols);
1349 for(; k < remaining_depth; k++)
1351 MICRO_EXTRA_COL<Scalar, Packet, Index>(lhs_ptr, rhs_ptr, accZero, remaining_rows, remaining_cols);
1353 for(; k < depth; k++)
1356 rhsV[0] = pset1<Packet>(rhs_ptr[0]);
1357 pger<1, Scalar, Packet, Index, false>(&accZero, lhs_ptr, rhsV, remaining_rows);
1358 lhs_ptr += remaining_rows;
1359 rhs_ptr += remaining_cols;
1362 accZero.packet[0] = vec_mul(pAlpha, accZero.packet[0]);
1363 for(Index i = 0; i < remaining_rows; i++) {
1364 res(row + i, col) += accZero.packet[0][i];
1368 template<
typename Scalar,
typename Packet,
typename Index, const Index accRows>
1369 EIGEN_ALWAYS_INLINE
void MICRO_EXTRA_ROW(
1370 const Scalar* &lhs_ptr,
1371 const Scalar* &rhs_ptr,
1372 PacketBlock<Packet,4> &accZero,
1373 Index remaining_rows)
1376 pbroadcast4<Packet>(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]);
1377 pger<4, Scalar, Packet, false>(&accZero, lhs_ptr, rhsV);
1378 lhs_ptr += remaining_rows;
1382 template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accRows, const Index accCols>
1383 EIGEN_STRONG_INLINE
void gemm_extra_row(
1384 const DataMapper& res,
1385 const Scalar* lhs_base,
1386 const Scalar* rhs_base,
1394 Index remaining_rows,
1395 const Packet& pAlpha,
1396 const Packet& pMask)
1398 const Scalar* rhs_ptr = rhs_base;
1399 const Scalar* lhs_ptr = lhs_base + row*strideA + remaining_rows*offsetA;
1400 PacketBlock<Packet,4> accZero, acc;
1402 bsetzero<Scalar, Packet>(accZero);
1404 Index remaining_depth = (col + accRows < cols) ? depth : (depth & -accRows);
1406 for(; k + PEEL <= remaining_depth; k+= PEEL)
1408 EIGEN_POWER_PREFETCH(rhs_ptr);
1409 EIGEN_POWER_PREFETCH(lhs_ptr);
1410 for (
int l = 0; l < PEEL; l++) {
1411 MICRO_EXTRA_ROW<Scalar, Packet, Index, accRows>(lhs_ptr, rhs_ptr, accZero, remaining_rows);
1414 for(; k < remaining_depth; k++)
1416 MICRO_EXTRA_ROW<Scalar, Packet, Index, accRows>(lhs_ptr, rhs_ptr, accZero, remaining_rows);
1419 if ((remaining_depth == depth) && (rows >= accCols))
1421 for(Index j = 0; j < 4; j++) {
1422 acc.packet[j] = res.template loadPacket<Packet>(row, col + j);
1424 bscale<Packet>(acc, accZero, pAlpha, pMask);
1425 res.template storePacketBlock<Packet,4>(row, col, acc);
1427 for(; k < depth; k++)
1430 pbroadcast4<Packet>(rhs_ptr, rhsV[0], rhsV[1], rhsV[2], rhsV[3]);
1431 pger<4, Scalar, Packet, Index, false>(&accZero, lhs_ptr, rhsV, remaining_rows);
1432 lhs_ptr += remaining_rows;
1436 for(Index j = 0; j < 4; j++) {
1437 accZero.packet[j] = vec_mul(pAlpha, accZero.packet[j]);
1439 for(Index j = 0; j < 4; j++) {
1440 for(Index i = 0; i < remaining_rows; i++) {
1441 res(row + i, col + j) += accZero.packet[j][i];
1447 #define MICRO_UNROLL(func) \ 1448 func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7) 1450 #define MICRO_UNROLL_WORK(func, func2, peel) \ 1451 MICRO_UNROLL(func2); \ 1452 func(0,peel) func(1,peel) func(2,peel) func(3,peel) \ 1453 func(4,peel) func(5,peel) func(6,peel) func(7,peel) 1455 #define MICRO_LOAD_ONE(iter) \ 1456 if (unroll_factor > iter) { \ 1457 lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr##iter); \ 1458 lhs_ptr##iter += accCols; \ 1460 EIGEN_UNUSED_VARIABLE(lhsV##iter); \ 1463 #define MICRO_WORK_ONE(iter, peel) \ 1464 if (unroll_factor > iter) { \ 1465 pger_common<Packet, false>(&accZero##iter, lhsV##iter, rhsV##peel); \ 1468 #define MICRO_TYPE_PEEL4(func, func2, peel) \ 1469 if (PEEL > peel) { \ 1470 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ 1471 pbroadcast4<Packet>(rhs_ptr + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ 1472 MICRO_UNROLL_WORK(func, func2, peel) \ 1474 EIGEN_UNUSED_VARIABLE(rhsV##peel); \ 1477 #define MICRO_TYPE_PEEL1(func, func2, peel) \ 1478 if (PEEL > peel) { \ 1479 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \ 1480 rhsV##peel[0] = pset1<Packet>(rhs_ptr[remaining_cols * peel]); \ 1481 MICRO_UNROLL_WORK(func, func2, peel) \ 1483 EIGEN_UNUSED_VARIABLE(rhsV##peel); \ 1486 #define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2) \ 1487 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M], rhsV8[M], rhsV9[M]; \ 1488 func(func1,func2,0); func(func1,func2,1); \ 1489 func(func1,func2,2); func(func1,func2,3); \ 1490 func(func1,func2,4); func(func1,func2,5); \ 1491 func(func1,func2,6); func(func1,func2,7); \ 1492 func(func1,func2,8); func(func1,func2,9); 1494 #define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2) \ 1496 func(func1,func2,0); 1498 #define MICRO_ONE_PEEL4 \ 1499 MICRO_UNROLL_TYPE_PEEL(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ 1500 rhs_ptr += (accRows * PEEL); 1502 #define MICRO_ONE4 \ 1503 MICRO_UNROLL_TYPE_ONE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ 1506 #define MICRO_ONE_PEEL1 \ 1507 MICRO_UNROLL_TYPE_PEEL(1, MICRO_TYPE_PEEL1, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ 1508 rhs_ptr += (remaining_cols * PEEL); 1510 #define MICRO_ONE1 \ 1511 MICRO_UNROLL_TYPE_ONE(1, MICRO_TYPE_PEEL1, MICRO_WORK_ONE, MICRO_LOAD_ONE); \ 1512 rhs_ptr += remaining_cols; 1514 #define MICRO_DST_PTR_ONE(iter) \ 1515 if (unroll_factor > iter) { \ 1516 bsetzero<Scalar, Packet>(accZero##iter); \ 1518 EIGEN_UNUSED_VARIABLE(accZero##iter); \ 1521 #define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE) 1523 #define MICRO_SRC_PTR_ONE(iter) \ 1524 if (unroll_factor > iter) { \ 1525 lhs_ptr##iter = lhs_base + ( (row/accCols) + iter )*strideA*accCols + accCols*offsetA; \ 1527 EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \ 1530 #define MICRO_SRC_PTR MICRO_UNROLL(MICRO_SRC_PTR_ONE) 1532 #define MICRO_PREFETCH_ONE(iter) \ 1533 if (unroll_factor > iter) { \ 1534 EIGEN_POWER_PREFETCH(lhs_ptr##iter); \ 1537 #define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE) 1539 #define MICRO_STORE_ONE(iter) \ 1540 if (unroll_factor > iter) { \ 1541 acc.packet[0] = res.template loadPacket<Packet>(row + iter*accCols, col + 0); \ 1542 acc.packet[1] = res.template loadPacket<Packet>(row + iter*accCols, col + 1); \ 1543 acc.packet[2] = res.template loadPacket<Packet>(row + iter*accCols, col + 2); \ 1544 acc.packet[3] = res.template loadPacket<Packet>(row + iter*accCols, col + 3); \ 1545 bscale<Packet>(acc, accZero##iter, pAlpha); \ 1546 res.template storePacketBlock<Packet,4>(row + iter*accCols, col, acc); \ 1549 #define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE) 1551 #define MICRO_COL_STORE_ONE(iter) \ 1552 if (unroll_factor > iter) { \ 1553 acc.packet[0] = res.template loadPacket<Packet>(row + iter*accCols, col + 0); \ 1554 bscale<Packet>(acc, accZero##iter, pAlpha); \ 1555 res.template storePacketBlock<Packet,1>(row + iter*accCols, col, acc); \ 1558 #define MICRO_COL_STORE MICRO_UNROLL(MICRO_COL_STORE_ONE) 1560 template<
int unroll_factor,
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accRows, const Index accCols>
1561 EIGEN_STRONG_INLINE
void gemm_unrolled_iteration(
1562 const DataMapper& res,
1563 const Scalar* lhs_base,
1564 const Scalar* rhs_base,
1570 const Packet& pAlpha)
1572 const Scalar* rhs_ptr = rhs_base;
1573 const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, * lhs_ptr7 = NULL;
1574 PacketBlock<Packet,4> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
1575 PacketBlock<Packet,4> acc;
1581 for(; k + PEEL <= depth; k+= PEEL)
1583 EIGEN_POWER_PREFETCH(rhs_ptr);
1587 for(; k < depth; k++)
1593 row += unroll_factor*accCols;
1596 template<
int unroll_factor,
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accCols>
1597 EIGEN_STRONG_INLINE
void gemm_unrolled_col_iteration(
1598 const DataMapper& res,
1599 const Scalar* lhs_base,
1600 const Scalar* rhs_base,
1606 Index remaining_cols,
1607 const Packet& pAlpha)
1609 const Scalar* rhs_ptr = rhs_base;
1610 const Scalar* lhs_ptr0 = NULL, * lhs_ptr1 = NULL, * lhs_ptr2 = NULL, * lhs_ptr3 = NULL, * lhs_ptr4 = NULL, * lhs_ptr5 = NULL, * lhs_ptr6 = NULL, *lhs_ptr7 = NULL;
1611 PacketBlock<Packet,1> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
1612 PacketBlock<Packet,1> acc;
1618 for(; k + PEEL <= depth; k+= PEEL)
1620 EIGEN_POWER_PREFETCH(rhs_ptr);
1624 for(; k < depth; k++)
1630 row += unroll_factor*accCols;
1633 template<
typename Scalar,
typename Packet,
typename DataMapper,
typename Index, const Index accCols>
1634 EIGEN_STRONG_INLINE
void gemm_unrolled_col(
1635 const DataMapper& res,
1636 const Scalar* lhs_base,
1637 const Scalar* rhs_base,
1644 Index remaining_cols,
1645 const Packet& pAlpha)
1647 #define MAX_UNROLL 6 1648 while(row + MAX_UNROLL*accCols <= rows) {
1649 gemm_unrolled_col_iteration<MAX_UNROLL, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1651 switch( (rows-row)/accCols ) {
1654 gemm_unrolled_col_iteration<7, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1659 gemm_unrolled_col_iteration<6, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1664 gemm_unrolled_col_iteration<5, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1669 gemm_unrolled_col_iteration<4, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1674 gemm_unrolled_col_iteration<3, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1679 gemm_unrolled_col_iteration<2, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1684 gemm_unrolled_col_iteration<1, Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_cols, pAlpha);
1696 template<
typename Scalar,
typename Index,
typename Packet,
typename RhsPacket,
typename DataMapper, const Index accRows, const Index accCols>
1697 EIGEN_STRONG_INLINE
void gemm(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
1699 const Index remaining_rows = rows % accCols;
1700 const Index remaining_cols = cols % accRows;
1702 if( strideA == -1 ) strideA = depth;
1703 if( strideB == -1 ) strideB = depth;
1705 const Packet pAlpha = pset1<Packet>(alpha);
1706 const Packet pMask = bmask<Packet>((
const int)(remaining_rows));
1709 for(; col + accRows <= cols; col += accRows)
1711 const Scalar* rhs_base = blockB + col*strideB + accRows*offsetB;
1712 const Scalar* lhs_base = blockA;
1715 #define MAX_UNROLL 6 1716 while(row + MAX_UNROLL*accCols <= rows) {
1717 gemm_unrolled_iteration<MAX_UNROLL, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1719 switch( (rows-row)/accCols ) {
1722 gemm_unrolled_iteration<7, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1727 gemm_unrolled_iteration<6, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1732 gemm_unrolled_iteration<5, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1737 gemm_unrolled_iteration<4, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1742 gemm_unrolled_iteration<3, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1747 gemm_unrolled_iteration<2, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1752 gemm_unrolled_iteration<1, Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, pAlpha);
1760 if(remaining_rows > 0)
1762 gemm_extra_row<Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask);
1766 if(remaining_cols > 0)
1768 const Scalar* rhs_base = blockB + col*strideB + remaining_cols*offsetB;
1769 const Scalar* lhs_base = blockA;
1771 for(; col < cols; col++)
1775 gemm_unrolled_col<Scalar, Packet, DataMapper, Index, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, rows, col, remaining_cols, pAlpha);
1777 if (remaining_rows > 0)
1779 gemm_extra_col<Scalar, Packet, DataMapper, Index, accRows>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, remaining_rows, remaining_cols, pAlpha);
1786 #define accColsC (accCols / 2) 1787 #define advanceRows ((LhsIsReal) ? 1 : 2) 1788 #define advanceCols ((RhsIsReal) ? 1 : 2) 1791 #define PEEL_COMPLEX 3 1793 template<
typename Scalar,
typename Packet,
typename Index, const Index accRows,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1794 EIGEN_ALWAYS_INLINE
void MICRO_COMPLEX_EXTRA_COL(
1795 const Scalar* &lhs_ptr_real,
const Scalar* &lhs_ptr_imag,
1796 const Scalar* &rhs_ptr_real,
const Scalar* &rhs_ptr_imag,
1797 PacketBlock<Packet,1> &accReal, PacketBlock<Packet,1> &accImag,
1798 Index remaining_rows,
1799 Index remaining_cols)
1801 Packet rhsV[1], rhsVi[1];
1802 rhsV[0] = pset1<Packet>(rhs_ptr_real[0]);
1803 if(!RhsIsReal) rhsVi[0] = pset1<Packet>(rhs_ptr_imag[0]);
1804 pgerc<1, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi);
1805 lhs_ptr_real += remaining_rows;
1806 if(!LhsIsReal) lhs_ptr_imag += remaining_rows;
1807 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1808 rhs_ptr_real += remaining_cols;
1809 if(!RhsIsReal) rhs_ptr_imag += remaining_cols;
1810 else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
1813 template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1814 EIGEN_STRONG_INLINE
void gemm_complex_extra_col(
1815 const DataMapper& res,
1816 const Scalar* lhs_base,
1817 const Scalar* rhs_base,
1824 Index remaining_rows,
1825 Index remaining_cols,
1826 const Packet& pAlphaReal,
1827 const Packet& pAlphaImag)
1829 const Scalar* rhs_ptr_real = rhs_base;
1830 const Scalar* rhs_ptr_imag;
1831 if(!RhsIsReal) rhs_ptr_imag = rhs_base + remaining_cols*strideB;
1832 else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
1833 const Scalar* lhs_ptr_real = lhs_base + advanceRows*row*strideA + remaining_rows*offsetA;
1834 const Scalar* lhs_ptr_imag;
1835 if(!LhsIsReal) lhs_ptr_imag = lhs_ptr_real + remaining_rows*strideA;
1836 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1837 PacketBlock<Packet,1> accReal, accImag;
1838 PacketBlock<Packet,1> taccReal, taccImag;
1839 PacketBlock<Packetc,1> acc0, acc1;
1841 bsetzero<Scalar, Packet>(accReal);
1842 bsetzero<Scalar, Packet>(accImag);
1844 Index remaining_depth = (depth & -accRows);
1846 for(; k + PEEL_COMPLEX <= remaining_depth; k+= PEEL_COMPLEX)
1848 EIGEN_POWER_PREFETCH(rhs_ptr_real);
1850 EIGEN_POWER_PREFETCH(rhs_ptr_imag);
1852 EIGEN_POWER_PREFETCH(lhs_ptr_real);
1854 EIGEN_POWER_PREFETCH(lhs_ptr_imag);
1856 for (
int l = 0; l < PEEL_COMPLEX; l++) {
1857 MICRO_COMPLEX_EXTRA_COL<Scalar, Packet, Index, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows, remaining_cols);
1860 for(; k < remaining_depth; k++)
1862 MICRO_COMPLEX_EXTRA_COL<Scalar, Packet, Index, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows, remaining_cols);
1865 for(; k < depth; k++)
1867 Packet rhsV[1], rhsVi[1];
1868 rhsV[0] = pset1<Packet>(rhs_ptr_real[0]);
1869 if(!RhsIsReal) rhsVi[0] = pset1<Packet>(rhs_ptr_imag[0]);
1870 pgerc<1, Scalar, Packet, Index, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi, remaining_rows);
1871 lhs_ptr_real += remaining_rows;
1872 if(!LhsIsReal) lhs_ptr_imag += remaining_rows;
1873 rhs_ptr_real += remaining_cols;
1874 if(!RhsIsReal) rhs_ptr_imag += remaining_cols;
1877 bscalec<Packet,1>(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag);
1878 bcouple_common<Packet, Packetc>(taccReal, taccImag, acc0, acc1);
1880 if ((
sizeof(Scalar) ==
sizeof(float)) && (remaining_rows == 1))
1882 res(row + 0, col + 0) += pfirst<Packetc>(acc0.packet[0]);
1884 acc0.packet[0] += res.template loadPacket<Packetc>(row + 0, col + 0);
1885 res.template storePacketBlock<Packetc,1>(row + 0, col + 0, acc0);
1886 if(remaining_rows > accColsC) {
1887 res(row + accColsC, col + 0) += pfirst<Packetc>(acc1.packet[0]);
1892 template<
typename Scalar,
typename Packet,
typename Index, const Index accRows,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1893 EIGEN_ALWAYS_INLINE
void MICRO_COMPLEX_EXTRA_ROW(
1894 const Scalar* &lhs_ptr_real,
const Scalar* &lhs_ptr_imag,
1895 const Scalar* &rhs_ptr_real,
const Scalar* &rhs_ptr_imag,
1896 PacketBlock<Packet,4> &accReal, PacketBlock<Packet,4> &accImag,
1897 Index remaining_rows)
1899 Packet rhsV[4], rhsVi[4];
1900 pbroadcast4_old<Packet>(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]);
1901 if(!RhsIsReal) pbroadcast4_old<Packet>(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]);
1902 pgerc<4, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi);
1903 lhs_ptr_real += remaining_rows;
1904 if(!LhsIsReal) lhs_ptr_imag += remaining_rows;
1905 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1906 rhs_ptr_real += accRows;
1907 if(!RhsIsReal) rhs_ptr_imag += accRows;
1908 else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
1911 template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1912 EIGEN_STRONG_INLINE
void gemm_complex_extra_row(
1913 const DataMapper& res,
1914 const Scalar* lhs_base,
1915 const Scalar* rhs_base,
1924 Index remaining_rows,
1925 const Packet& pAlphaReal,
1926 const Packet& pAlphaImag,
1927 const Packet& pMask)
1929 const Scalar* rhs_ptr_real = rhs_base;
1930 const Scalar* rhs_ptr_imag;
1931 if(!RhsIsReal) rhs_ptr_imag = rhs_base + accRows*strideB;
1932 else EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
1933 const Scalar* lhs_ptr_real = lhs_base + advanceRows*row*strideA + remaining_rows*offsetA;
1934 const Scalar* lhs_ptr_imag;
1935 if(!LhsIsReal) lhs_ptr_imag = lhs_ptr_real + remaining_rows*strideA;
1936 else EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1937 PacketBlock<Packet,4> accReal, accImag;
1938 PacketBlock<Packet,4> taccReal, taccImag;
1939 PacketBlock<Packetc,4> acc0, acc1;
1940 PacketBlock<Packetc,8> tRes;
1942 bsetzero<Scalar, Packet>(accReal);
1943 bsetzero<Scalar, Packet>(accImag);
1945 Index remaining_depth = (col + accRows < cols) ? depth : (depth & -accRows);
1947 for(; k + PEEL_COMPLEX <= remaining_depth; k+= PEEL_COMPLEX)
1949 EIGEN_POWER_PREFETCH(rhs_ptr_real);
1951 EIGEN_POWER_PREFETCH(rhs_ptr_imag);
1953 EIGEN_POWER_PREFETCH(lhs_ptr_real);
1955 EIGEN_POWER_PREFETCH(lhs_ptr_imag);
1957 for (
int l = 0; l < PEEL_COMPLEX; l++) {
1958 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, Index, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows);
1961 for(; k < remaining_depth; k++)
1963 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, Index, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real, rhs_ptr_imag, accReal, accImag, remaining_rows);
1966 if ((remaining_depth == depth) && (rows >= accCols))
1968 bload<DataMapper, Packetc, Index, accColsC, 0, ColMajor>(tRes, res, row, col);
1969 bscalec<Packet>(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask);
1970 bcouple<Packet, Packetc>(taccReal, taccImag, tRes, acc0, acc1);
1971 res.template storePacketBlock<Packetc,4>(row + 0, col, acc0);
1972 res.template storePacketBlock<Packetc,4>(row + accColsC, col, acc1);
1974 for(; k < depth; k++)
1976 Packet rhsV[4], rhsVi[4];
1977 pbroadcast4_old<Packet>(rhs_ptr_real, rhsV[0], rhsV[1], rhsV[2], rhsV[3]);
1978 if(!RhsIsReal) pbroadcast4_old<Packet>(rhs_ptr_imag, rhsVi[0], rhsVi[1], rhsVi[2], rhsVi[3]);
1979 pgerc<4, Scalar, Packet, Index, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi, remaining_rows);
1980 lhs_ptr_real += remaining_rows;
1981 if(!LhsIsReal) lhs_ptr_imag += remaining_rows;
1982 rhs_ptr_real += accRows;
1983 if(!RhsIsReal) rhs_ptr_imag += accRows;
1986 bscalec<Packet,4>(accReal, accImag, pAlphaReal, pAlphaImag, taccReal, taccImag);
1987 bcouple_common<Packet, Packetc>(taccReal, taccImag, acc0, acc1);
1989 if ((
sizeof(Scalar) ==
sizeof(float)) && (remaining_rows == 1))
1991 for(Index j = 0; j < 4; j++) {
1992 res(row + 0, col + j) += pfirst<Packetc>(acc0.packet[j]);
1995 for(Index j = 0; j < 4; j++) {
1996 PacketBlock<Packetc,1> acc2;
1997 acc2.packet[0] = res.template loadPacket<Packetc>(row + 0, col + j) + acc0.packet[j];
1998 res.template storePacketBlock<Packetc,1>(row + 0, col + j, acc2);
1999 if(remaining_rows > accColsC) {
2000 res(row + accColsC, col + j) += pfirst<Packetc>(acc1.packet[j]);
2007 #define MICRO_COMPLEX_UNROLL(func) \ 2008 func(0) func(1) func(2) func(3) func(4) 2010 #define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ 2011 MICRO_COMPLEX_UNROLL(func2); \ 2012 func(0,peel) func(1,peel) func(2,peel) func(3,peel) func(4,peel) 2014 #define MICRO_COMPLEX_LOAD_ONE(iter) \ 2015 if (unroll_factor > iter) { \ 2016 lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr_real##iter); \ 2017 lhs_ptr_real##iter += accCols; \ 2019 lhsVi##iter = ploadLhs<Scalar, Packet>(lhs_ptr_imag##iter); \ 2020 lhs_ptr_imag##iter += accCols; \ 2022 EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ 2025 EIGEN_UNUSED_VARIABLE(lhsV##iter); \ 2026 EIGEN_UNUSED_VARIABLE(lhsVi##iter); \ 2029 #define MICRO_COMPLEX_WORK_ONE4(iter, peel) \ 2030 if (unroll_factor > iter) { \ 2031 pgerc_common<4, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ 2034 #define MICRO_COMPLEX_WORK_ONE1(iter, peel) \ 2035 if (unroll_factor > iter) { \ 2036 pgerc_common<1, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \ 2039 #define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \ 2040 if (PEEL_COMPLEX > peel) { \ 2041 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \ 2042 Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \ 2043 pbroadcast4_old<Packet>(rhs_ptr_real + (accRows * peel), rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \ 2045 pbroadcast4_old<Packet>(rhs_ptr_imag + (accRows * peel), rhsVi##peel[0], rhsVi##peel[1], rhsVi##peel[2], rhsVi##peel[3]); \ 2047 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ 2049 MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ 2051 EIGEN_UNUSED_VARIABLE(rhsV##peel); \ 2052 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ 2055 #define MICRO_COMPLEX_TYPE_PEEL1(func, func2, peel) \ 2056 if (PEEL_COMPLEX > peel) { \ 2057 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \ 2058 Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \ 2059 rhsV##peel[0] = pset1<Packet>(rhs_ptr_real[remaining_cols * peel]); \ 2061 rhsVi##peel[0] = pset1<Packet>(rhs_ptr_imag[remaining_cols * peel]); \ 2063 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ 2065 MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \ 2067 EIGEN_UNUSED_VARIABLE(rhsV##peel); \ 2068 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \ 2071 #define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2) \ 2072 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M], rhsV8[M], rhsV9[M]; \ 2073 Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M], rhsVi4[M], rhsVi5[M], rhsVi6[M], rhsVi7[M], rhsVi8[M], rhsVi9[M]; \ 2074 func(func1,func2,0); func(func1,func2,1); \ 2075 func(func1,func2,2); func(func1,func2,3); \ 2076 func(func1,func2,4); func(func1,func2,5); \ 2077 func(func1,func2,6); func(func1,func2,7); \ 2078 func(func1,func2,8); func(func1,func2,9); 2080 #define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2) \ 2081 Packet rhsV0[M], rhsVi0[M];\ 2082 func(func1,func2,0); 2084 #define MICRO_COMPLEX_ONE_PEEL4 \ 2085 MICRO_COMPLEX_UNROLL_TYPE_PEEL(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ 2086 rhs_ptr_real += (accRows * PEEL_COMPLEX); \ 2087 if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX); 2089 #define MICRO_COMPLEX_ONE4 \ 2090 MICRO_COMPLEX_UNROLL_TYPE_ONE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE); \ 2091 rhs_ptr_real += accRows; \ 2092 if(!RhsIsReal) rhs_ptr_imag += accRows; 2094 #define MICRO_COMPLEX_ONE_PEEL1 \ 2095 MICRO_COMPLEX_UNROLL_TYPE_PEEL(1, MICRO_COMPLEX_TYPE_PEEL1, MICRO_COMPLEX_WORK_ONE1, MICRO_COMPLEX_LOAD_ONE); \ 2096 rhs_ptr_real += (remaining_cols * PEEL_COMPLEX); \ 2097 if(!RhsIsReal) rhs_ptr_imag += (remaining_cols * PEEL_COMPLEX); 2099 #define MICRO_COMPLEX_ONE1 \ 2100 MICRO_COMPLEX_UNROLL_TYPE_ONE(1, MICRO_COMPLEX_TYPE_PEEL1, MICRO_COMPLEX_WORK_ONE1, MICRO_COMPLEX_LOAD_ONE); \ 2101 rhs_ptr_real += remaining_cols; \ 2102 if(!RhsIsReal) rhs_ptr_imag += remaining_cols; 2104 #define MICRO_COMPLEX_DST_PTR_ONE(iter) \ 2105 if (unroll_factor > iter) { \ 2106 bsetzero<Scalar, Packet>(accReal##iter); \ 2107 bsetzero<Scalar, Packet>(accImag##iter); \ 2109 EIGEN_UNUSED_VARIABLE(accReal##iter); \ 2110 EIGEN_UNUSED_VARIABLE(accImag##iter); \ 2113 #define MICRO_COMPLEX_DST_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE) 2115 #define MICRO_COMPLEX_SRC_PTR_ONE(iter) \ 2116 if (unroll_factor > iter) { \ 2117 lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \ 2119 lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \ 2121 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ 2124 EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \ 2125 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \ 2128 #define MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE) 2130 #define MICRO_COMPLEX_PREFETCH_ONE(iter) \ 2131 if (unroll_factor > iter) { \ 2132 EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \ 2134 EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \ 2138 #define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE) 2140 #define MICRO_COMPLEX_STORE_ONE(iter) \ 2141 if (unroll_factor > iter) { \ 2142 bload<DataMapper, Packetc, Index, accColsC, 0, ColMajor>(tRes, res, row + iter*accCols, col); \ 2143 bscalec<Packet,4>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag); \ 2144 bcouple<Packet, Packetc>(taccReal, taccImag, tRes, acc0, acc1); \ 2145 res.template storePacketBlock<Packetc,4>(row + iter*accCols + 0, col, acc0); \ 2146 res.template storePacketBlock<Packetc,4>(row + iter*accCols + accColsC, col, acc1); \ 2149 #define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE) 2151 #define MICRO_COMPLEX_COL_STORE_ONE(iter) \ 2152 if (unroll_factor > iter) { \ 2153 bload<DataMapper, Packetc, Index, accColsC, 0, ColMajor>(tRes, res, row + iter*accCols, col); \ 2154 bscalec<Packet,1>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, taccImag); \ 2155 bcouple<Packet, Packetc>(taccReal, taccImag, tRes, acc0, acc1); \ 2156 res.template storePacketBlock<Packetc,1>(row + iter*accCols + 0, col, acc0); \ 2157 res.template storePacketBlock<Packetc,1>(row + iter*accCols + accColsC, col, acc1); \ 2160 #define MICRO_COMPLEX_COL_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_COL_STORE_ONE) 2162 template<
int unroll_factor,
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2163 EIGEN_STRONG_INLINE
void gemm_complex_unrolled_iteration(
2164 const DataMapper& res,
2165 const Scalar* lhs_base,
2166 const Scalar* rhs_base,
2173 const Packet& pAlphaReal,
2174 const Packet& pAlphaImag)
2176 const Scalar* rhs_ptr_real = rhs_base;
2177 const Scalar* rhs_ptr_imag;
2179 rhs_ptr_imag = rhs_base + accRows*strideB;
2181 EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
2183 const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL;
2184 const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL;
2185 const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL;
2186 PacketBlock<Packet,4> accReal0, accImag0, accReal1, accImag1;
2187 PacketBlock<Packet,4> accReal2, accImag2, accReal3, accImag3;
2188 PacketBlock<Packet,4> accReal4, accImag4;
2189 PacketBlock<Packet,4> taccReal, taccImag;
2190 PacketBlock<Packetc,4> acc0, acc1;
2191 PacketBlock<Packetc,8> tRes;
2193 MICRO_COMPLEX_SRC_PTR
2194 MICRO_COMPLEX_DST_PTR
2197 for(; k + PEEL_COMPLEX <= depth; k+= PEEL_COMPLEX)
2199 EIGEN_POWER_PREFETCH(rhs_ptr_real);
2201 EIGEN_POWER_PREFETCH(rhs_ptr_imag);
2203 MICRO_COMPLEX_PREFETCH
2204 MICRO_COMPLEX_ONE_PEEL4
2206 for(; k < depth; k++)
2212 row += unroll_factor*accCols;
2215 template<
int unroll_factor,
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2216 EIGEN_STRONG_INLINE
void gemm_complex_unrolled_col_iteration(
2217 const DataMapper& res,
2218 const Scalar* lhs_base,
2219 const Scalar* rhs_base,
2226 Index remaining_cols,
2227 const Packet& pAlphaReal,
2228 const Packet& pAlphaImag)
2230 const Scalar* rhs_ptr_real = rhs_base;
2231 const Scalar* rhs_ptr_imag;
2233 rhs_ptr_imag = rhs_base + remaining_cols*strideB;
2235 EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
2237 const Scalar* lhs_ptr_real0 = NULL, * lhs_ptr_imag0 = NULL, * lhs_ptr_real1 = NULL, * lhs_ptr_imag1 = NULL;
2238 const Scalar* lhs_ptr_real2 = NULL, * lhs_ptr_imag2 = NULL, * lhs_ptr_real3 = NULL, * lhs_ptr_imag3 = NULL;
2239 const Scalar* lhs_ptr_real4 = NULL, * lhs_ptr_imag4 = NULL;
2240 PacketBlock<Packet,1> accReal0, accImag0, accReal1, accImag1;
2241 PacketBlock<Packet,1> accReal2, accImag2, accReal3, accImag3;
2242 PacketBlock<Packet,1> accReal4, accImag4;
2243 PacketBlock<Packet,1> taccReal, taccImag;
2244 PacketBlock<Packetc,1> acc0, acc1;
2245 PacketBlock<Packetc,2> tRes;
2247 MICRO_COMPLEX_SRC_PTR
2248 MICRO_COMPLEX_DST_PTR
2251 for(; k + PEEL_COMPLEX <= depth; k+= PEEL_COMPLEX)
2253 EIGEN_POWER_PREFETCH(rhs_ptr_real);
2255 EIGEN_POWER_PREFETCH(rhs_ptr_imag);
2257 MICRO_COMPLEX_PREFETCH
2258 MICRO_COMPLEX_ONE_PEEL1
2260 for(; k < depth; k++)
2264 MICRO_COMPLEX_COL_STORE
2266 row += unroll_factor*accCols;
2269 template<
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
typename Index, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2270 EIGEN_STRONG_INLINE
void gemm_complex_unrolled_col(
2271 const DataMapper& res,
2272 const Scalar* lhs_base,
2273 const Scalar* rhs_base,
2281 Index remaining_cols,
2282 const Packet& pAlphaReal,
2283 const Packet& pAlphaImag)
2285 #define MAX_COMPLEX_UNROLL 3 2286 while(row + MAX_COMPLEX_UNROLL*accCols <= rows) {
2287 gemm_complex_unrolled_col_iteration<MAX_COMPLEX_UNROLL, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag);
2289 switch( (rows-row)/accCols ) {
2290 #if MAX_COMPLEX_UNROLL > 4 2292 gemm_complex_unrolled_col_iteration<4, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag);
2295 #if MAX_COMPLEX_UNROLL > 3 2297 gemm_complex_unrolled_col_iteration<3, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag);
2300 #if MAX_COMPLEX_UNROLL > 2 2302 gemm_complex_unrolled_col_iteration<2, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag);
2305 #if MAX_COMPLEX_UNROLL > 1 2307 gemm_complex_unrolled_col_iteration<1, Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_cols, pAlphaReal, pAlphaImag);
2313 #undef MAX_COMPLEX_UNROLL 2316 template<
typename LhsScalar,
typename RhsScalar,
typename Scalarc,
typename Scalar,
typename Index,
typename Packet,
typename Packetc,
typename RhsPacket,
typename DataMapper, const Index accRows, const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2317 EIGEN_STRONG_INLINE
void gemm_complex(
const DataMapper& res,
const LhsScalar* blockAc,
const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
2319 const Index remaining_rows = rows % accCols;
2320 const Index remaining_cols = cols % accRows;
2322 if( strideA == -1 ) strideA = depth;
2323 if( strideB == -1 ) strideB = depth;
2325 const Packet pAlphaReal = pset1<Packet>(alpha.real());
2326 const Packet pAlphaImag = pset1<Packet>(alpha.imag());
2327 const Packet pMask = bmask<Packet>((
const int)(remaining_rows));
2329 const Scalar* blockA = (Scalar *) blockAc;
2330 const Scalar* blockB = (Scalar *) blockBc;
2333 for(; col + accRows <= cols; col += accRows)
2335 const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB;
2336 const Scalar* lhs_base = blockA;
2339 #define MAX_COMPLEX_UNROLL 3 2340 while(row + MAX_COMPLEX_UNROLL*accCols <= rows) {
2341 gemm_complex_unrolled_iteration<MAX_COMPLEX_UNROLL, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
2343 switch( (rows-row)/accCols ) {
2344 #if MAX_COMPLEX_UNROLL > 4 2346 gemm_complex_unrolled_iteration<4, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
2349 #if MAX_COMPLEX_UNROLL > 3 2351 gemm_complex_unrolled_iteration<3, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
2354 #if MAX_COMPLEX_UNROLL > 2 2356 gemm_complex_unrolled_iteration<2, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
2359 #if MAX_COMPLEX_UNROLL > 1 2361 gemm_complex_unrolled_iteration<1, Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
2367 #undef MAX_COMPLEX_UNROLL 2369 if(remaining_rows > 0)
2371 gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask);
2375 if(remaining_cols > 0)
2377 const Scalar* rhs_base = blockB + advanceCols*col*strideB + remaining_cols*offsetB;
2378 const Scalar* lhs_base = blockA;
2380 for(; col < cols; col++)
2384 gemm_complex_unrolled_col<Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, col, remaining_cols, pAlphaReal, pAlphaImag);
2386 if (remaining_rows > 0)
2388 gemm_complex_extra_col<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_rows, remaining_cols, pAlphaReal, pAlphaImag);
2402 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2403 struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode>
2405 void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2408 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2409 void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>
2410 ::operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2412 dhs_pack<double, Index, DataMapper, Packet2d, ColMajor, PanelMode, true> pack;
2413 pack(blockA, lhs, depth, rows, stride, offset);
2416 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2417 struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode>
2419 void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2422 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2423 void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>
2424 ::operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2426 dhs_pack<double, Index, DataMapper, Packet2d, RowMajor, PanelMode, true> pack;
2427 pack(blockA, lhs, depth, rows, stride, offset);
2430 #if EIGEN_ALTIVEC_USE_CUSTOM_PACK 2431 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2432 struct gemm_pack_rhs<double, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode>
2434 void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2437 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2438 void gemm_pack_rhs<double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>
2439 ::operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2441 dhs_pack<double, Index, DataMapper, Packet2d, ColMajor, PanelMode, false> pack;
2442 pack(blockB, rhs, depth, cols, stride, offset);
2445 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2446 struct gemm_pack_rhs<double, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode>
2448 void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2451 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2452 void gemm_pack_rhs<double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>
2453 ::operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2455 dhs_pack<double, Index, DataMapper, Packet2d, RowMajor, PanelMode, false> pack;
2456 pack(blockB, rhs, depth, cols, stride, offset);
2460 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2461 struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode>
2463 void operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2466 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2467 void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>
2468 ::operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2470 dhs_pack<float, Index, DataMapper, Packet4f, RowMajor, PanelMode, true> pack;
2471 pack(blockA, lhs, depth, rows, stride, offset);
2474 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2475 struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode>
2477 void operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2480 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2481 void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>
2482 ::operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2484 dhs_pack<float, Index, DataMapper, Packet4f, ColMajor, PanelMode, true> pack;
2485 pack(blockA, lhs, depth, rows, stride, offset);
2488 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2489 struct gemm_pack_lhs<
std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode>
2491 void operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2494 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2495 void gemm_pack_lhs<std::complex<float>,
Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode>
2496 ::operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2498 dhs_cpack<float, Index, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, true> pack;
2499 pack(blockA, lhs, depth, rows, stride, offset);
2502 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2503 struct gemm_pack_lhs<
std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode>
2505 void operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2508 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2509 void gemm_pack_lhs<std::complex<float>,
Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode>
2510 ::operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2512 dhs_cpack<float, Index, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, true> pack;
2513 pack(blockA, lhs, depth, rows, stride, offset);
2516 #if EIGEN_ALTIVEC_USE_CUSTOM_PACK 2517 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2518 struct gemm_pack_rhs<float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>
2520 void operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2523 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2524 void gemm_pack_rhs<float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>
2525 ::operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2527 dhs_pack<float, Index, DataMapper, Packet4f, ColMajor, PanelMode, false> pack;
2528 pack(blockB, rhs, depth, cols, stride, offset);
2531 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2532 struct gemm_pack_rhs<float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>
2534 void operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2537 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2538 void gemm_pack_rhs<float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>
2539 ::operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2541 dhs_pack<float, Index, DataMapper, Packet4f, RowMajor, PanelMode, false> pack;
2542 pack(blockB, rhs, depth, cols, stride, offset);
2546 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2547 struct gemm_pack_rhs<
std::complex<float>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>
2549 void operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2552 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2553 void gemm_pack_rhs<std::complex<float>,
Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode>
2554 ::operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2556 dhs_cpack<float, Index, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, false> pack;
2557 pack(blockB, rhs, depth, cols, stride, offset);
2560 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2561 struct gemm_pack_rhs<
std::complex<float>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>
2563 void operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2566 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2567 void gemm_pack_rhs<std::complex<float>,
Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode>
2568 ::operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2570 dhs_cpack<float, Index, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, false> pack;
2571 pack(blockB, rhs, depth, cols, stride, offset);
2574 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2575 struct gemm_pack_lhs<
std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>
2577 void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2580 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2581 void gemm_pack_lhs<std::complex<double>,
Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode>
2582 ::operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2584 dhs_cpack<double, Index, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, true> pack;
2585 pack(blockA, lhs, depth, rows, stride, offset);
2588 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2589 struct gemm_pack_lhs<
std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>
2591 void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride=0, Index offset=0);
2594 template<
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
2595 void gemm_pack_lhs<std::complex<double>,
Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode>
2596 ::operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset)
2598 dhs_cpack<double, Index, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, true> pack;
2599 pack(blockA, lhs, depth, rows, stride, offset);
2602 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2603 struct gemm_pack_rhs<
std::complex<double>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>
2605 void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2608 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2609 void gemm_pack_rhs<std::complex<double>,
Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode>
2610 ::operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2612 dhs_cpack<double, Index, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, false> pack;
2613 pack(blockB, rhs, depth, cols, stride, offset);
2616 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2617 struct gemm_pack_rhs<
std::complex<double>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>
2619 void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride=0, Index offset=0);
2622 template<
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
2623 void gemm_pack_rhs<std::complex<double>,
Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode>
2624 ::operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset)
2626 dhs_cpack<double, Index, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, false> pack;
2627 pack(blockB, rhs, depth, cols, stride, offset);
2631 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2632 struct gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2634 typedef typename quad_traits<float>::vectortype Packet;
2635 typedef typename quad_traits<float>::rhstype RhsPacket;
2637 void operator()(
const DataMapper& res,
const float* blockA,
const float* blockB,
2638 Index rows, Index depth, Index cols,
float alpha,
2639 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2642 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2643 void gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2644 ::operator()(
const DataMapper& res,
const float* blockA,
const float* blockB,
2645 Index rows, Index depth, Index cols,
float alpha,
2646 Index strideA, Index strideB, Index offsetA, Index offsetB)
2648 const Index accRows = quad_traits<float>::rows;
2649 const Index accCols = quad_traits<float>::size;
2650 void (*gemm_function)(
const DataMapper&,
const float*,
const float*,
Index,
Index,
Index, float,
Index,
Index,
Index,
Index);
2652 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2654 gemm_function = &Eigen::internal::gemmMMA<float, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2655 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2656 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2657 gemm_function = &Eigen::internal::gemmMMA<float, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2660 gemm_function = &Eigen::internal::gemm<float, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2663 gemm_function = &Eigen::internal::gemm<float, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2665 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2668 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2669 struct gebp_kernel<
std::complex<float>, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2671 typedef Packet4f Packet;
2672 typedef Packet2cf Packetc;
2673 typedef Packet4f RhsPacket;
2675 void operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const std::complex<float>* blockB,
2676 Index rows, Index depth, Index cols, std::complex<float> alpha,
2677 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2680 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2681 void gebp_kernel<std::complex<float>, std::complex<float>,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2682 ::operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const std::complex<float>* blockB,
2683 Index rows, Index depth, Index cols, std::complex<float> alpha,
2684 Index strideA, Index strideB, Index offsetA, Index offsetB)
2686 const Index accRows = quad_traits<float>::rows;
2687 const Index accCols = quad_traits<float>::size;
2688 void (*gemm_function)(
const DataMapper&,
const std::complex<float>*,
const std::complex<float>*,
2691 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2693 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<float>, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2694 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2695 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2696 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<float>, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2699 gemm_function = &Eigen::internal::gemm_complex<std::complex<float>, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2702 gemm_function = &Eigen::internal::gemm_complex<std::complex<float>, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2704 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2707 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2708 struct gebp_kernel<float,
std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2710 typedef Packet4f Packet;
2711 typedef Packet2cf Packetc;
2712 typedef Packet4f RhsPacket;
2714 void operator()(
const DataMapper& res,
const float* blockA,
const std::complex<float>* blockB,
2715 Index rows, Index depth, Index cols, std::complex<float> alpha,
2716 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2719 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2720 void gebp_kernel<float, std::complex<float>,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2721 ::operator()(
const DataMapper& res,
const float* blockA,
const std::complex<float>* blockB,
2722 Index rows, Index depth, Index cols, std::complex<float> alpha,
2723 Index strideA, Index strideB, Index offsetA, Index offsetB)
2725 const Index accRows = quad_traits<float>::rows;
2726 const Index accCols = quad_traits<float>::size;
2727 void (*gemm_function)(
const DataMapper&,
const float*,
const std::complex<float>*,
2729 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2731 gemm_function = &Eigen::internal::gemm_complexMMA<float, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2732 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2733 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2734 gemm_function = &Eigen::internal::gemm_complexMMA<float, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2737 gemm_function = &Eigen::internal::gemm_complex<float, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2740 gemm_function = &Eigen::internal::gemm_complex<float, std::complex<float>, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2742 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2745 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2746 struct gebp_kernel<
std::complex<float>, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2748 typedef Packet4f Packet;
2749 typedef Packet2cf Packetc;
2750 typedef Packet4f RhsPacket;
2752 void operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const float* blockB,
2753 Index rows, Index depth, Index cols, std::complex<float> alpha,
2754 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2757 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2758 void gebp_kernel<std::complex<float>, float,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2759 ::operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const float* blockB,
2760 Index rows, Index depth, Index cols, std::complex<float> alpha,
2761 Index strideA, Index strideB, Index offsetA, Index offsetB)
2763 const Index accRows = quad_traits<float>::rows;
2764 const Index accCols = quad_traits<float>::size;
2765 void (*gemm_function)(
const DataMapper&,
const std::complex<float>*,
const float*,
2767 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2769 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<float>, float, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2770 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2771 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2772 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<float>, float, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2775 gemm_function = &Eigen::internal::gemm_complex<std::complex<float>, float, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2778 gemm_function = &Eigen::internal::gemm_complex<std::complex<float>, float, std::complex<float>, float,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2780 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2783 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2784 struct gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2786 typedef typename quad_traits<double>::vectortype Packet;
2787 typedef typename quad_traits<double>::rhstype RhsPacket;
2789 void operator()(
const DataMapper& res,
const double* blockA,
const double* blockB,
2790 Index rows, Index depth, Index cols,
double alpha,
2791 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2794 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2795 void gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2796 ::operator()(
const DataMapper& res,
const double* blockA,
const double* blockB,
2797 Index rows, Index depth, Index cols,
double alpha,
2798 Index strideA, Index strideB, Index offsetA, Index offsetB)
2800 const Index accRows = quad_traits<double>::rows;
2801 const Index accCols = quad_traits<double>::size;
2802 void (*gemm_function)(
const DataMapper&,
const double*,
const double*,
Index,
Index,
Index, double,
Index,
Index,
Index,
Index);
2804 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2806 gemm_function = &Eigen::internal::gemmMMA<double, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2807 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2808 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2809 gemm_function = &Eigen::internal::gemmMMA<double, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2812 gemm_function = &Eigen::internal::gemm<double, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2815 gemm_function = &Eigen::internal::gemm<double, Index, Packet, RhsPacket, DataMapper, accRows, accCols>;
2817 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2820 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2821 struct gebp_kernel<
std::complex<double>, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2823 typedef quad_traits<double>::vectortype Packet;
2824 typedef Packet1cd Packetc;
2825 typedef quad_traits<double>::rhstype RhsPacket;
2827 void operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const std::complex<double>* blockB,
2828 Index rows, Index depth, Index cols, std::complex<double> alpha,
2829 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2832 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2833 void gebp_kernel<std::complex<double>, std::complex<double>,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2834 ::operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const std::complex<double>* blockB,
2835 Index rows, Index depth, Index cols, std::complex<double> alpha,
2836 Index strideA, Index strideB, Index offsetA, Index offsetB)
2838 const Index accRows = quad_traits<double>::rows;
2839 const Index accCols = quad_traits<double>::size;
2840 void (*gemm_function)(
const DataMapper&,
const std::complex<double>*,
const std::complex<double>*,
2842 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2844 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<double>, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2845 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2846 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2847 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<double>, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2850 gemm_function = &Eigen::internal::gemm_complex<std::complex<double>, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2853 gemm_function = &Eigen::internal::gemm_complex<std::complex<double>, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
false>;
2855 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2858 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2859 struct gebp_kernel<
std::complex<double>, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2861 typedef quad_traits<double>::vectortype Packet;
2862 typedef Packet1cd Packetc;
2863 typedef quad_traits<double>::rhstype RhsPacket;
2865 void operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const double* blockB,
2866 Index rows, Index depth, Index cols, std::complex<double> alpha,
2867 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2870 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2871 void gebp_kernel<std::complex<double>, double,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2872 ::operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const double* blockB,
2873 Index rows, Index depth, Index cols, std::complex<double> alpha,
2874 Index strideA, Index strideB, Index offsetA, Index offsetB)
2876 const Index accRows = quad_traits<double>::rows;
2877 const Index accCols = quad_traits<double>::size;
2878 void (*gemm_function)(
const DataMapper&,
const std::complex<double>*,
const double*,
2880 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2882 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<double>, double, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2883 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2884 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2885 gemm_function = &Eigen::internal::gemm_complexMMA<std::complex<double>, double, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2888 gemm_function = &Eigen::internal::gemm_complex<std::complex<double>, double, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2891 gemm_function = &Eigen::internal::gemm_complex<std::complex<double>, double, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
false,
true>;
2893 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2896 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2897 struct gebp_kernel<double,
std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2899 typedef quad_traits<double>::vectortype Packet;
2900 typedef Packet1cd Packetc;
2901 typedef quad_traits<double>::rhstype RhsPacket;
2903 void operator()(
const DataMapper& res,
const double* blockA,
const std::complex<double>* blockB,
2904 Index rows, Index depth, Index cols, std::complex<double> alpha,
2905 Index strideA=-1, Index strideB=-1, Index offsetA=0, Index offsetB=0);
2908 template<
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
2909 void gebp_kernel<double, std::complex<double>,
Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>
2910 ::operator()(
const DataMapper& res,
const double* blockA,
const std::complex<double>* blockB,
2911 Index rows, Index depth, Index cols, std::complex<double> alpha,
2912 Index strideA, Index strideB, Index offsetA, Index offsetB)
2914 const Index accRows = quad_traits<double>::rows;
2915 const Index accCols = quad_traits<double>::size;
2916 void (*gemm_function)(
const DataMapper&,
const double*,
const std::complex<double>*,
2918 #ifdef EIGEN_ALTIVEC_MMA_ONLY 2920 gemm_function = &Eigen::internal::gemm_complexMMA<double, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2921 #elif defined(ALTIVEC_MMA_SUPPORT) && !defined(EIGEN_ALTIVEC_DISABLE_MMA) 2922 if (__builtin_cpu_supports (
"arch_3_1") && __builtin_cpu_supports (
"mma")){
2923 gemm_function = &Eigen::internal::gemm_complexMMA<double, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2926 gemm_function = &Eigen::internal::gemm_complex<double, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2929 gemm_function = &Eigen::internal::gemm_complex<double, std::complex<double>, std::complex<double>, double,
Index, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs,
true,
false>;
2931 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
2937 #endif // EIGEN_MATRIX_PRODUCT_ALTIVEC_H Definition: Constants.h:319
Namespace containing all symbols from the Eigen library.
Definition: Core:141
Definition: BFloat16.h:88
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_imag_op< typename Derived::Scalar >, const Derived > imag(const Eigen::ArrayBase< Derived > &x)
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_real_op< typename Derived::Scalar >, const Derived > real(const Eigen::ArrayBase< Derived > &x)
Definition: Eigen_Colamd.h:50
Definition: Constants.h:321