12 #ifndef EIGEN_ASSIGN_EVALUATOR_H 13 #define EIGEN_ASSIGN_EVALUATOR_H 27 template <
typename DstEvaluator,
typename SrcEvaluator,
typename AssignFunc,
int MaxPacketSize = -1>
28 struct copy_using_evaluator_traits
30 typedef typename DstEvaluator::XprType Dst;
31 typedef typename Dst::Scalar DstScalar;
34 DstFlags = DstEvaluator::Flags,
35 SrcFlags = SrcEvaluator::Flags
40 DstAlignment = DstEvaluator::Alignment,
41 SrcAlignment = SrcEvaluator::Alignment,
43 JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment)
48 InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
49 : int(DstFlags)&
RowMajorBit ? int(Dst::ColsAtCompileTime)
50 : int(Dst::RowsAtCompileTime),
51 InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
52 : int(DstFlags)&
RowMajorBit ? int(Dst::MaxColsAtCompileTime)
53 : int(Dst::MaxRowsAtCompileTime),
54 RestrictedInnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(InnerSize,MaxPacketSize),
55 RestrictedLinearSize = EIGEN_SIZE_MIN_PREFER_FIXED(Dst::SizeAtCompileTime,MaxPacketSize),
56 OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
57 MaxSizeAtCompileTime = Dst::SizeAtCompileTime
61 typedef typename find_best_packet<DstScalar,RestrictedLinearSize>::type LinearPacketType;
62 typedef typename find_best_packet<DstScalar,RestrictedInnerSize>::type InnerPacketType;
65 LinearPacketSize = unpacket_traits<LinearPacketType>::size,
66 InnerPacketSize = unpacket_traits<InnerPacketType>::size
71 LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment,
72 InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment
79 StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
80 MightVectorize =
bool(StorageOrdersAgree)
82 &&
bool(functor_traits<AssignFunc>::PacketAccess),
83 MayInnerVectorize = MightVectorize
84 && int(InnerSize)!=
Dynamic && int(InnerSize)%int(InnerPacketSize)==0
85 && int(OuterStride)!=
Dynamic && int(OuterStride)%int(InnerPacketSize)==0
86 && (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment)>=int(InnerRequiredAlignment)),
87 MayLinearize =
bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) &
LinearAccessBit),
88 MayLinearVectorize =
bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess)
89 && (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime ==
Dynamic),
92 MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess)
93 && (int(InnerMaxSize)==
Dynamic || int(InnerMaxSize)>=(EIGEN_UNALIGNED_VECTORIZE?InnerPacketSize:(3*InnerPacketSize)))
102 Traversal = int(Dst::SizeAtCompileTime) == 0 ? int(AllAtOnceTraversal)
103 : (int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize)) ? int(LinearVectorizedTraversal)
104 : int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
105 : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
106 : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
107 : int(MayLinearize) ? int(LinearTraversal)
108 : int(DefaultTraversal),
109 Vectorized = int(Traversal) == InnerVectorizedTraversal
110 || int(Traversal) == LinearVectorizedTraversal
111 || int(Traversal) == SliceVectorizedTraversal
114 typedef typename conditional<int(Traversal)==LinearVectorizedTraversal, LinearPacketType, InnerPacketType>::type PacketType;
118 ActualPacketSize = int(Traversal)==LinearVectorizedTraversal ? LinearPacketSize
119 : Vectorized ? InnerPacketSize
121 UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize,
122 MayUnrollCompletely = int(Dst::SizeAtCompileTime) !=
Dynamic 123 && int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit),
124 MayUnrollInner = int(InnerSize) !=
Dynamic 125 && int(InnerSize) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <=
int(UnrollingLimit)
130 Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
132 int(MayUnrollCompletely) ? int(CompleteUnrolling)
133 : int(MayUnrollInner) ? int(InnerUnrolling)
136 : int(Traversal) == int(LinearVectorizedTraversal)
137 ? ( bool(MayUnrollCompletely) && ( EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)))
138 ? int(CompleteUnrolling)
140 : int(Traversal) == int(LinearTraversal)
141 ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling)
143 #if EIGEN_UNALIGNED_VECTORIZE
144 : int(Traversal) == int(SliceVectorizedTraversal)
145 ? ( bool(MayUnrollInner) ? int(InnerUnrolling)
151 #ifdef EIGEN_DEBUG_ASSIGN 154 std::cerr <<
"DstXpr: " <<
typeid(
typename DstEvaluator::XprType).name() << std::endl;
155 std::cerr <<
"SrcXpr: " <<
typeid(
typename SrcEvaluator::XprType).name() << std::endl;
156 std::cerr.setf(std::ios::hex, std::ios::basefield);
157 std::cerr <<
"DstFlags" <<
" = " << DstFlags <<
" (" << demangle_flags(DstFlags) <<
" )" << std::endl;
158 std::cerr <<
"SrcFlags" <<
" = " << SrcFlags <<
" (" << demangle_flags(SrcFlags) <<
" )" << std::endl;
159 std::cerr.unsetf(std::ios::hex);
160 EIGEN_DEBUG_VAR(DstAlignment)
161 EIGEN_DEBUG_VAR(SrcAlignment)
162 EIGEN_DEBUG_VAR(LinearRequiredAlignment)
163 EIGEN_DEBUG_VAR(InnerRequiredAlignment)
164 EIGEN_DEBUG_VAR(JointAlignment)
165 EIGEN_DEBUG_VAR(InnerSize)
166 EIGEN_DEBUG_VAR(InnerMaxSize)
167 EIGEN_DEBUG_VAR(LinearPacketSize)
168 EIGEN_DEBUG_VAR(InnerPacketSize)
169 EIGEN_DEBUG_VAR(ActualPacketSize)
170 EIGEN_DEBUG_VAR(StorageOrdersAgree)
171 EIGEN_DEBUG_VAR(MightVectorize)
172 EIGEN_DEBUG_VAR(MayLinearize)
173 EIGEN_DEBUG_VAR(MayInnerVectorize)
174 EIGEN_DEBUG_VAR(MayLinearVectorize)
175 EIGEN_DEBUG_VAR(MaySliceVectorize)
176 std::cerr <<
"Traversal" <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
177 EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
178 EIGEN_DEBUG_VAR(DstEvaluator::CoeffReadCost)
179 EIGEN_DEBUG_VAR(Dst::SizeAtCompileTime)
180 EIGEN_DEBUG_VAR(UnrollingLimit)
181 EIGEN_DEBUG_VAR(MayUnrollCompletely)
182 EIGEN_DEBUG_VAR(MayUnrollInner)
183 std::cerr <<
"Unrolling" <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
184 std::cerr << std::endl;
197 template<
typename Kernel,
int Index,
int Stop>
198 struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling
201 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
202 typedef typename DstEvaluatorType::XprType DstXprType;
205 outer =
Index / DstXprType::InnerSizeAtCompileTime,
206 inner =
Index % DstXprType::InnerSizeAtCompileTime
209 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
211 kernel.assignCoeffByOuterInner(outer, inner);
212 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
216 template<
typename Kernel,
int Stop>
217 struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop>
219 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
222 template<
typename Kernel,
int Index_,
int Stop>
223 struct copy_using_evaluator_DefaultTraversal_InnerUnrolling
225 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
Index outer)
227 kernel.assignCoeffByOuterInner(outer, Index_);
228 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_+1, Stop>::run(kernel, outer);
232 template<
typename Kernel,
int Stop>
233 struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop>
235 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&,
Index) { }
242 template<
typename Kernel,
int Index,
int Stop>
243 struct copy_using_evaluator_LinearTraversal_CompleteUnrolling
245 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel& kernel)
247 kernel.assignCoeff(
Index);
248 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
252 template<
typename Kernel,
int Stop>
253 struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop>
255 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
262 template<
typename Kernel,
int Index,
int Stop>
263 struct copy_using_evaluator_innervec_CompleteUnrolling
266 typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
267 typedef typename DstEvaluatorType::XprType DstXprType;
268 typedef typename Kernel::PacketType PacketType;
271 outer =
Index / DstXprType::InnerSizeAtCompileTime,
272 inner =
Index % DstXprType::InnerSizeAtCompileTime,
273 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
274 DstAlignment = Kernel::AssignmentTraits::DstAlignment
277 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
279 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
280 enum { NextIndex =
Index + unpacket_traits<PacketType>::size };
281 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
285 template<
typename Kernel,
int Stop>
286 struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop>
288 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&) { }
291 template<
typename Kernel,
int Index_,
int Stop,
int SrcAlignment,
int DstAlignment>
292 struct copy_using_evaluator_innervec_InnerUnrolling
294 typedef typename Kernel::PacketType PacketType;
295 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
Index outer)
297 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
298 enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
299 copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel, outer);
303 template<
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
304 struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment>
306 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &,
Index) { }
315 template<
typename Kernel,
316 int Traversal = Kernel::AssignmentTraits::Traversal,
317 int Unrolling = Kernel::AssignmentTraits::Unrolling>
318 struct dense_assignment_loop;
325 template<
typename Kernel,
int Unrolling>
326 struct dense_assignment_loop<Kernel, AllAtOnceTraversal, Unrolling>
328 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE run(Kernel& )
330 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
331 EIGEN_STATIC_ASSERT(
int(DstXprType::SizeAtCompileTime) == 0,
332 EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT)
340 template<
typename Kernel>
341 struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>
343 EIGEN_DEVICE_FUNC
static void EIGEN_STRONG_INLINE run(Kernel &kernel)
345 for(
Index outer = 0; outer < kernel.outerSize(); ++outer) {
346 for(
Index inner = 0; inner < kernel.innerSize(); ++inner) {
347 kernel.assignCoeffByOuterInner(outer, inner);
353 template<
typename Kernel>
354 struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling>
356 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
358 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
359 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
363 template<
typename Kernel>
364 struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling>
366 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
368 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
370 const Index outerSize = kernel.outerSize();
371 for(
Index outer = 0; outer < outerSize; ++outer)
372 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
384 template <
bool IsAligned = false>
385 struct unaligned_dense_assignment_loop
388 template <
typename Kernel>
389 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel&,
Index,
Index) {}
393 struct unaligned_dense_assignment_loop<false>
399 template <
typename Kernel>
400 static EIGEN_DONT_INLINE
void run(Kernel &kernel,
404 template <
typename Kernel>
405 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel,
410 for (
Index index = start; index < end; ++index)
411 kernel.assignCoeff(index);
415 template<
typename Kernel>
416 struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling>
418 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
420 const Index size = kernel.size();
421 typedef typename Kernel::Scalar Scalar;
422 typedef typename Kernel::PacketType PacketType;
424 requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
425 packetSize = unpacket_traits<PacketType>::size,
426 dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
427 dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
428 : int(Kernel::AssignmentTraits::DstAlignment),
429 srcAlignment = Kernel::AssignmentTraits::JointAlignment
431 const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size);
432 const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
434 unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart);
436 for(
Index index = alignedStart; index < alignedEnd; index += packetSize)
437 kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
439 unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
443 template<
typename Kernel>
444 struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling>
446 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
448 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
449 typedef typename Kernel::PacketType PacketType;
451 enum { size = DstXprType::SizeAtCompileTime,
452 packetSize =unpacket_traits<PacketType>::size,
453 alignedSize = (int(size)/packetSize)*packetSize };
455 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
456 copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
464 template<
typename Kernel>
465 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling>
467 typedef typename Kernel::PacketType PacketType;
469 SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
470 DstAlignment = Kernel::AssignmentTraits::DstAlignment
472 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
474 const Index innerSize = kernel.innerSize();
475 const Index outerSize = kernel.outerSize();
476 const Index packetSize = unpacket_traits<PacketType>::size;
477 for(
Index outer = 0; outer < outerSize; ++outer)
478 for(
Index inner = 0; inner < innerSize; inner+=packetSize)
479 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
483 template<
typename Kernel>
484 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling>
486 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
488 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
489 copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
493 template<
typename Kernel>
494 struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling>
496 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
498 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
499 typedef typename Kernel::AssignmentTraits Traits;
500 const Index outerSize = kernel.outerSize();
501 for(
Index outer = 0; outer < outerSize; ++outer)
502 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime,
503 Traits::SrcAlignment, Traits::DstAlignment>::run(kernel, outer);
511 template<
typename Kernel>
512 struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling>
514 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
516 const Index size = kernel.size();
517 for(
Index i = 0; i < size; ++i)
518 kernel.assignCoeff(i);
522 template<
typename Kernel>
523 struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling>
525 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
527 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
528 copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
536 template<
typename Kernel>
537 struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling>
539 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
541 typedef typename Kernel::Scalar Scalar;
542 typedef typename Kernel::PacketType PacketType;
544 packetSize = unpacket_traits<PacketType>::size,
545 requestedAlignment = int(Kernel::AssignmentTraits::InnerRequiredAlignment),
546 alignable = packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment)>=
sizeof(Scalar),
547 dstIsAligned =
int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
548 dstAlignment = alignable ? int(requestedAlignment)
549 : int(Kernel::AssignmentTraits::DstAlignment)
551 const Scalar *dst_ptr = kernel.dstDataPtr();
552 if((!
bool(dstIsAligned)) && (UIntPtr(dst_ptr) %
sizeof(Scalar))>0)
555 return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel);
557 const Index packetAlignedMask = packetSize - 1;
558 const Index innerSize = kernel.innerSize();
559 const Index outerSize = kernel.outerSize();
560 const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
561 Index alignedStart = ((!alignable) ||
bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
563 for(
Index outer = 0; outer < outerSize; ++outer)
565 const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
567 for(
Index inner = 0; inner<alignedStart ; ++inner)
568 kernel.assignCoeffByOuterInner(outer, inner);
571 for(
Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
572 kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
575 for(
Index inner = alignedEnd; inner<innerSize ; ++inner)
576 kernel.assignCoeffByOuterInner(outer, inner);
578 alignedStart = numext::mini((alignedStart+alignedStep)%packetSize, innerSize);
583 #if EIGEN_UNALIGNED_VECTORIZE 584 template<
typename Kernel>
585 struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling>
587 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
void run(Kernel &kernel)
589 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
590 typedef typename Kernel::PacketType PacketType;
592 enum { innerSize = DstXprType::InnerSizeAtCompileTime,
593 packetSize =unpacket_traits<PacketType>::size,
594 vectorizableSize = (int(innerSize) / int(packetSize)) *
int(packetSize),
595 size = DstXprType::SizeAtCompileTime };
597 for(
Index outer = 0; outer < kernel.outerSize(); ++outer)
599 copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer);
600 copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, innerSize>::run(kernel, outer);
617 template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor,
int Version = Specialized>
618 class generic_dense_assignment_kernel
621 typedef typename DstEvaluatorTypeT::XprType DstXprType;
622 typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
625 typedef DstEvaluatorTypeT DstEvaluatorType;
626 typedef SrcEvaluatorTypeT SrcEvaluatorType;
627 typedef typename DstEvaluatorType::Scalar Scalar;
628 typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
629 typedef typename AssignmentTraits::PacketType PacketType;
632 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
633 generic_dense_assignment_kernel(DstEvaluatorType &dst,
const SrcEvaluatorType &src,
const Functor &func, DstXprType& dstExpr)
634 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
636 #ifdef EIGEN_DEBUG_ASSIGN 637 AssignmentTraits::debug();
641 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index size() const EIGEN_NOEXCEPT {
return m_dstExpr.size(); }
642 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index innerSize() const EIGEN_NOEXCEPT {
return m_dstExpr.innerSize(); }
643 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index outerSize() const EIGEN_NOEXCEPT {
return m_dstExpr.outerSize(); }
644 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index rows() const EIGEN_NOEXCEPT {
return m_dstExpr.rows(); }
645 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index cols() const EIGEN_NOEXCEPT {
return m_dstExpr.cols(); }
646 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
Index outerStride() const EIGEN_NOEXCEPT {
return m_dstExpr.outerStride(); }
648 EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() EIGEN_NOEXCEPT {
return m_dst; }
649 EIGEN_DEVICE_FUNC
const SrcEvaluatorType& srcEvaluator() const EIGEN_NOEXCEPT {
return m_src; }
652 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(
Index row,
Index col)
654 m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
658 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeff(
Index index)
660 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
664 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignCoeffByOuterInner(
Index outer,
Index inner)
666 Index row = rowIndexByOuterInner(outer, inner);
667 Index col = colIndexByOuterInner(outer, inner);
668 assignCoeff(row, col);
672 template<
int StoreMode,
int LoadMode,
typename PacketType>
673 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(
Index row,
Index col)
675 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(row,col));
678 template<
int StoreMode,
int LoadMode,
typename PacketType>
679 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacket(
Index index)
681 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
684 template<
int StoreMode,
int LoadMode,
typename PacketType>
685 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void assignPacketByOuterInner(
Index outer,
Index inner)
687 Index row = rowIndexByOuterInner(outer, inner);
688 Index col = colIndexByOuterInner(outer, inner);
689 assignPacket<StoreMode,LoadMode,PacketType>(row, col);
692 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index rowIndexByOuterInner(
Index outer,
Index inner)
694 typedef typename DstEvaluatorType::ExpressionTraits Traits;
695 return int(Traits::RowsAtCompileTime) == 1 ? 0
696 : int(Traits::ColsAtCompileTime) == 1 ? inner
697 : int(DstEvaluatorType::Flags)&RowMajorBit ? outer
701 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
Index colIndexByOuterInner(
Index outer,
Index inner)
703 typedef typename DstEvaluatorType::ExpressionTraits Traits;
704 return int(Traits::ColsAtCompileTime) == 1 ? 0
705 : int(Traits::RowsAtCompileTime) == 1 ? inner
706 : int(DstEvaluatorType::Flags)&RowMajorBit ? inner
710 EIGEN_DEVICE_FUNC
const Scalar* dstDataPtr()
const 712 return m_dstExpr.data();
716 DstEvaluatorType& m_dst;
717 const SrcEvaluatorType& m_src;
718 const Functor &m_functor;
720 DstXprType& m_dstExpr;
727 template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor>
728 class restricted_packet_dense_assignment_kernel :
public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn>
731 typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, BuiltIn> Base;
733 typedef typename Base::Scalar Scalar;
734 typedef typename Base::DstXprType DstXprType;
735 typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, 4> AssignmentTraits;
736 typedef typename AssignmentTraits::PacketType PacketType;
738 EIGEN_DEVICE_FUNC restricted_packet_dense_assignment_kernel(DstEvaluatorTypeT &dst,
const SrcEvaluatorTypeT &src,
const Functor &func, DstXprType& dstExpr)
739 : Base(dst, src, func, dstExpr)
748 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
749 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
750 void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const Functor &)
752 EIGEN_ONLY_USED_FOR_DEBUG(dst);
753 EIGEN_ONLY_USED_FOR_DEBUG(src);
754 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
757 template<
typename DstXprType,
typename SrcXprType,
typename T1,
typename T2>
758 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
759 void resize_if_allowed(DstXprType &dst,
const SrcXprType& src,
const internal::assign_op<T1,T2> &)
761 Index dstRows = src.rows();
762 Index dstCols = src.cols();
763 if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols)))
764 dst.resize(dstRows, dstCols);
765 eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
768 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
769 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src,
const Functor &func)
771 typedef evaluator<DstXprType> DstEvaluatorType;
772 typedef evaluator<SrcXprType> SrcEvaluatorType;
774 SrcEvaluatorType srcEvaluator(src);
778 resize_if_allowed(dst, src, func);
780 DstEvaluatorType dstEvaluator(dst);
782 typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel;
783 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
785 dense_assignment_loop<Kernel>::run(kernel);
789 #ifndef EIGEN_GPU_COMPILE_PHASE 790 template<
typename DstXprType>
791 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<typename DstXprType::Scalar>, DstXprType>& src,
const internal::assign_op<typename DstXprType::Scalar,typename DstXprType::Scalar>& func)
793 resize_if_allowed(dst, src, func);
794 std::fill_n(dst.data(), dst.size(), src.functor()());
798 template<
typename DstXprType,
typename SrcXprType>
799 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_dense_assignment_loop(DstXprType& dst,
const SrcXprType& src)
801 call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>());
811 template<
typename DstShape,
typename SrcShape>
struct AssignmentKind;
814 struct Dense2Dense {};
815 struct EigenBase2EigenBase {};
817 template<
typename,
typename>
struct AssignmentKind {
typedef EigenBase2EigenBase Kind; };
818 template<>
struct AssignmentKind<DenseShape,DenseShape> {
typedef Dense2Dense Kind; };
821 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
822 typename Kind =
typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape ,
typename evaluator_traits<SrcXprType>::Shape >::Kind,
823 typename EnableIf =
void>
832 template<
typename Dst,
typename Src>
833 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
834 void call_assignment(Dst& dst,
const Src& src)
836 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
838 template<
typename Dst,
typename Src>
839 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
840 void call_assignment(
const Dst& dst,
const Src& src)
842 call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
846 template<
typename Dst,
typename Src,
typename Func>
847 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
848 void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if< evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
850 typename plain_matrix_type<Src>::type tmp(src);
851 call_assignment_no_alias(dst, tmp, func);
854 template<
typename Dst,
typename Src,
typename Func>
855 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
856 void call_assignment(Dst& dst,
const Src& src,
const Func& func,
typename enable_if<!evaluator_assume_aliasing<Src>::value,
void*>::type = 0)
858 call_assignment_no_alias(dst, src, func);
863 template<
typename Dst,
template <
typename>
class StorageBase,
typename Src,
typename Func>
864 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
865 void call_assignment(NoAlias<Dst,StorageBase>& dst,
const Src& src,
const Func& func)
867 call_assignment_no_alias(dst.expression(), src, func);
871 template<
typename Dst,
typename Src,
typename Func>
872 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
873 void call_assignment_no_alias(Dst& dst,
const Src& src,
const Func& func)
876 NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
877 || (
int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)
878 ) && int(Dst::SizeAtCompileTime) != 1
881 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned;
882 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType;
883 ActualDstType actualDst(dst);
886 EIGEN_STATIC_ASSERT_LVALUE(Dst)
887 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src)
888 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename ActualDstTypeCleaned::Scalar,
typename Src::Scalar);
890 Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
893 template<
typename Dst,
typename Src,
typename Func>
894 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
895 void call_restricted_packet_assignment_no_alias(Dst& dst,
const Src& src,
const Func& func)
897 typedef evaluator<Dst> DstEvaluatorType;
898 typedef evaluator<Src> SrcEvaluatorType;
899 typedef restricted_packet_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Func> Kernel;
901 EIGEN_STATIC_ASSERT_LVALUE(Dst)
902 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename Dst::Scalar,
typename Src::Scalar);
904 SrcEvaluatorType srcEvaluator(src);
905 resize_if_allowed(dst, src, func);
907 DstEvaluatorType dstEvaluator(dst);
908 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
910 dense_assignment_loop<Kernel>::run(kernel);
913 template<
typename Dst,
typename Src>
914 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
915 void call_assignment_no_alias(Dst& dst,
const Src& src)
917 call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
920 template<
typename Dst,
typename Src,
typename Func>
921 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
922 void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src,
const Func& func)
925 EIGEN_STATIC_ASSERT_LVALUE(Dst)
926 EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src)
927 EIGEN_CHECK_BINARY_COMPATIBILIY(Func,
typename Dst::Scalar,
typename Src::Scalar);
929 Assignment<Dst,Src,Func>::run(dst, src, func);
931 template<
typename Dst,
typename Src>
932 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
933 void call_assignment_no_alias_no_transpose(Dst& dst,
const Src& src)
935 call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
939 template<
typename Dst,
typename Src>
void check_for_aliasing(
const Dst &dst,
const Src &src);
944 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
945 struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak>
948 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const Functor &func)
950 #ifndef EIGEN_NO_DEBUG 951 internal::check_for_aliasing(dst, src);
954 call_dense_assignment_loop(dst, src, func);
962 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
963 struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak>
966 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &)
968 Index dstRows = src.rows();
969 Index dstCols = src.cols();
970 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
971 dst.resize(dstRows, dstCols);
973 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
979 template<
typename SrcScalarType>
981 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::add_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
983 Index dstRows = src.rows();
984 Index dstCols = src.cols();
985 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
986 dst.resize(dstRows, dstCols);
988 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
992 template<
typename SrcScalarType>
994 static EIGEN_STRONG_INLINE
void run(DstXprType &dst,
const SrcXprType &src,
const internal::sub_assign_op<typename DstXprType::Scalar,SrcScalarType> &)
996 Index dstRows = src.rows();
997 Index dstCols = src.cols();
998 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
999 dst.resize(dstRows, dstCols);
1001 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
1010 #endif // EIGEN_ASSIGN_EVALUATOR_H Generic expression of a matrix where all coefficients are defined by a functor.
Definition: CwiseNullaryOp.h:60
const unsigned int DirectAccessBit
Definition: Constants.h:155
Namespace containing all symbols from the Eigen library.
Definition: Core:141
const unsigned int RowMajorBit
Definition: Constants.h:66
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74
Definition: Eigen_Colamd.h:50
const int Dynamic
Definition: Constants.h:22
const unsigned int ActualPacketAccessBit
Definition: Constants.h:105
const unsigned int LinearAccessBit
Definition: Constants.h:130