26 template<
typename Func,
typename Evaluator>
30 typedef typename find_best_packet<typename Evaluator::Scalar,Evaluator::SizeAtCompileTime>::type PacketType;
32 PacketSize = unpacket_traits<PacketType>::size,
33 InnerMaxSize = int(Evaluator::IsRowMajor)
34 ? Evaluator::MaxColsAtCompileTime
35 : Evaluator::MaxRowsAtCompileTime,
36 OuterMaxSize = int(Evaluator::IsRowMajor)
37 ? Evaluator::MaxRowsAtCompileTime
38 : Evaluator::MaxColsAtCompileTime,
40 : int(OuterMaxSize)==
Dynamic ? (int(InnerMaxSize)>=int(PacketSize) ?
Dynamic : 0)
41 : (
int(InnerMaxSize)/int(PacketSize)) *
int(OuterMaxSize)
46 && (functor_traits<Func>::PacketAccess),
47 MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&
LinearAccessBit),
48 MaySliceVectorize =
bool(MightVectorize) && (int(SliceVectorizedWork)==
Dynamic || int(SliceVectorizedWork)>=3)
53 Traversal = int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
54 : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
55 : int(DefaultTraversal)
61 : Evaluator::SizeAtCompileTime * Evaluator::CoeffReadCost + (Evaluator::SizeAtCompileTime-1) * functor_traits<Func>::Cost,
62 UnrollingLimit = EIGEN_UNROLLING_LIMIT * (
int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize))
67 Unrolling = Cost <= UnrollingLimit ? CompleteUnrolling : NoUnrolling
70 #ifdef EIGEN_DEBUG_ASSIGN
73 std::cerr <<
"Xpr: " <<
typeid(
typename Evaluator::XprType).name() << std::endl;
74 std::cerr.setf(std::ios::hex, std::ios::basefield);
75 EIGEN_DEBUG_VAR(Evaluator::Flags)
76 std::cerr.unsetf(std::ios::hex);
77 EIGEN_DEBUG_VAR(InnerMaxSize)
78 EIGEN_DEBUG_VAR(OuterMaxSize)
79 EIGEN_DEBUG_VAR(SliceVectorizedWork)
80 EIGEN_DEBUG_VAR(PacketSize)
81 EIGEN_DEBUG_VAR(MightVectorize)
82 EIGEN_DEBUG_VAR(MayLinearVectorize)
83 EIGEN_DEBUG_VAR(MaySliceVectorize)
84 std::cerr <<
"Traversal" <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
85 EIGEN_DEBUG_VAR(UnrollingLimit)
86 std::cerr <<
"Unrolling" <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
87 std::cerr << std::endl;
98 template<
typename Func,
typename Evaluator,
int Start,
int Length>
99 struct redux_novec_unroller
102 HalfLength = Length/2
105 typedef typename Evaluator::Scalar Scalar;
108 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func& func)
110 return func(redux_novec_unroller<Func, Evaluator, Start, HalfLength>::run(eval,func),
111 redux_novec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::run(eval,func));
115 template<
typename Func,
typename Evaluator,
int Start>
116 struct redux_novec_unroller<Func, Evaluator, Start, 1>
119 outer = Start / Evaluator::InnerSizeAtCompileTime,
120 inner = Start % Evaluator::InnerSizeAtCompileTime
123 typedef typename Evaluator::Scalar Scalar;
126 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func&)
128 return eval.coeffByOuterInner(outer, inner);
135 template<
typename Func,
typename Evaluator,
int Start>
136 struct redux_novec_unroller<Func, Evaluator, Start, 0>
138 typedef typename Evaluator::Scalar Scalar;
140 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator&,
const Func&) {
return Scalar(); }
145 template<
typename Func,
typename Evaluator,
int Start,
int Length>
146 struct redux_vec_unroller
148 template<
typename PacketType>
150 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func& func)
153 PacketSize = unpacket_traits<PacketType>::size,
154 HalfLength = Length/2
157 return func.packetOp(
158 redux_vec_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval,func),
159 redux_vec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::template run<PacketType>(eval,func) );
163 template<
typename Func,
typename Evaluator,
int Start>
164 struct redux_vec_unroller<Func, Evaluator, Start, 1>
166 template<
typename PacketType>
168 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func&)
171 PacketSize = unpacket_traits<PacketType>::size,
172 index = Start * PacketSize,
173 outer = index / int(Evaluator::InnerSizeAtCompileTime),
174 inner = index % int(Evaluator::InnerSizeAtCompileTime),
175 alignment = Evaluator::Alignment
177 return eval.template packetByOuterInner<alignment,PacketType>(outer, inner);
185 template<
typename Func,
typename Evaluator,
186 int Traversal = redux_traits<Func, Evaluator>::Traversal,
187 int Unrolling = redux_traits<Func, Evaluator>::Unrolling
191 template<
typename Func,
typename Evaluator>
192 struct redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>
194 typedef typename Evaluator::Scalar Scalar;
196 template<
typename XprType>
197 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
198 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
200 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
202 res = eval.coeffByOuterInner(0, 0);
203 for(
Index i = 1; i < xpr.innerSize(); ++i)
204 res = func(res, eval.coeffByOuterInner(0, i));
205 for(
Index i = 1; i < xpr.outerSize(); ++i)
206 for(
Index j = 0; j < xpr.innerSize(); ++j)
207 res = func(res, eval.coeffByOuterInner(i, j));
212 template<
typename Func,
typename Evaluator>
213 struct redux_impl<Func,Evaluator, DefaultTraversal, CompleteUnrolling>
214 : redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime>
216 typedef redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
217 typedef typename Evaluator::Scalar Scalar;
218 template<
typename XprType>
219 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
220 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& )
222 return Base::run(eval,func);
226 template<
typename Func,
typename Evaluator>
227 struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, NoUnrolling>
229 typedef typename Evaluator::Scalar Scalar;
230 typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar;
232 template<
typename XprType>
233 static Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
235 const Index size = xpr.size();
237 const Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
238 const int packetAlignment = unpacket_traits<PacketScalar>::alignment;
240 alignment0 = (bool(Evaluator::Flags &
DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar)) ?
int(packetAlignment) : int(
Unaligned),
241 alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Evaluator::Alignment)
243 const Index alignedStart = internal::first_default_aligned(xpr);
244 const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize);
245 const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize);
246 const Index alignedEnd2 = alignedStart + alignedSize2;
247 const Index alignedEnd = alignedStart + alignedSize;
251 PacketScalar packet_res0 = eval.template packet<alignment,PacketScalar>(alignedStart);
252 if(alignedSize>packetSize)
254 PacketScalar packet_res1 = eval.template packet<alignment,PacketScalar>(alignedStart+packetSize);
255 for(
Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize)
257 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(index));
258 packet_res1 = func.packetOp(packet_res1, eval.template packet<alignment,PacketScalar>(index+packetSize));
261 packet_res0 = func.packetOp(packet_res0,packet_res1);
262 if(alignedEnd>alignedEnd2)
263 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(alignedEnd2));
265 res = func.predux(packet_res0);
267 for(
Index index = 0; index < alignedStart; ++index)
268 res = func(res,eval.coeff(index));
270 for(
Index index = alignedEnd; index < size; ++index)
271 res = func(res,eval.coeff(index));
277 for(
Index index = 1; index < size; ++index)
278 res = func(res,eval.coeff(index));
286 template<
typename Func,
typename Evaluator,
int Unrolling>
287 struct redux_impl<Func, Evaluator, SliceVectorizedTraversal, Unrolling>
289 typedef typename Evaluator::Scalar Scalar;
290 typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
292 template<
typename XprType>
293 EIGEN_DEVICE_FUNC
static Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
295 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
296 const Index innerSize = xpr.innerSize();
297 const Index outerSize = xpr.outerSize();
299 packetSize = redux_traits<Func, Evaluator>::PacketSize
301 const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize;
303 if(packetedInnerSize)
305 PacketType packet_res = eval.template packet<Unaligned,PacketType>(0,0);
306 for(
Index j=0; j<outerSize; ++j)
307 for(
Index i=(j==0?packetSize:0); i<packetedInnerSize; i+=
Index(packetSize))
308 packet_res = func.packetOp(packet_res, eval.template packetByOuterInner<Unaligned,PacketType>(j,i));
310 res = func.predux(packet_res);
311 for(
Index j=0; j<outerSize; ++j)
312 for(
Index i=packetedInnerSize; i<innerSize; ++i)
313 res = func(res, eval.coeffByOuterInner(j,i));
318 res = redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>::run(eval, func, xpr);
325 template<
typename Func,
typename Evaluator>
326 struct redux_impl<Func, Evaluator, LinearVectorizedTraversal, CompleteUnrolling>
328 typedef typename Evaluator::Scalar Scalar;
330 typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
332 PacketSize = redux_traits<Func, Evaluator>::PacketSize,
333 Size = Evaluator::SizeAtCompileTime,
334 VectorizedSize = (Size / PacketSize) * PacketSize
337 template<
typename XprType>
338 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
339 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType &xpr)
341 EIGEN_ONLY_USED_FOR_DEBUG(xpr)
342 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
343 if (VectorizedSize > 0) {
344 Scalar res = func.predux(redux_vec_unroller<Func, Evaluator, 0, Size / PacketSize>::template run<PacketType>(eval,func));
345 if (VectorizedSize != Size)
346 res = func(res,redux_novec_unroller<Func, Evaluator, VectorizedSize, Size-VectorizedSize>::run(eval,func));
350 return redux_novec_unroller<Func, Evaluator, 0, Size>::run(eval,func);
356 template<
typename _XprType>
357 class redux_evaluator :
public internal::evaluator<_XprType>
359 typedef internal::evaluator<_XprType> Base;
361 typedef _XprType XprType;
362 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
363 explicit redux_evaluator(
const XprType &xpr) : Base(xpr) {}
365 typedef typename XprType::Scalar Scalar;
366 typedef typename XprType::CoeffReturnType CoeffReturnType;
367 typedef typename XprType::PacketScalar PacketScalar;
370 MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime,
371 MaxColsAtCompileTime = XprType::MaxColsAtCompileTime,
374 IsRowMajor = XprType::IsRowMajor,
375 SizeAtCompileTime = XprType::SizeAtCompileTime,
376 InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime
379 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
380 CoeffReturnType coeffByOuterInner(
Index outer,
Index inner)
const
381 {
return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
383 template<
int LoadMode,
typename PacketType>
384 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
385 PacketType packetByOuterInner(
Index outer,
Index inner)
const
386 {
return Base::template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
406 template<
typename Derived>
407 template<
typename Func>
408 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
411 eigen_assert(this->rows()>0 && this->cols()>0 &&
"you are using an empty matrix");
413 typedef typename internal::redux_evaluator<Derived> ThisEvaluator;
414 ThisEvaluator thisEval(derived());
418 return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func, derived());
428 template<
typename Derived>
429 template<
int NaNPropagation>
430 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
433 return derived().redux(Eigen::internal::scalar_min_op<Scalar,Scalar, NaNPropagation>());
443 template<
typename Derived>
444 template<
int NaNPropagation>
445 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
448 return derived().redux(Eigen::internal::scalar_max_op<Scalar,Scalar, NaNPropagation>());
457 template<
typename Derived>
458 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
461 if(SizeAtCompileTime==0 || (SizeAtCompileTime==
Dynamic && size()==0))
463 return derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>());
470 template<
typename Derived>
471 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
474 #ifdef __INTEL_COMPILER
476 #pragma warning ( disable : 2259 )
478 return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>())) /
Scalar(this->size());
479 #ifdef __INTEL_COMPILER
491 template<
typename Derived>
492 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
495 if(SizeAtCompileTime==0 || (SizeAtCompileTime==
Dynamic && size()==0))
497 return derived().redux(Eigen::internal::scalar_product_op<Scalar>());
506 template<
typename Derived>
507 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
510 return derived().diagonal().sum();
Base class for all dense matrices, vectors, and arrays.
Definition: DenseBase.h:47
internal::traits< Derived >::Scalar minCoeff() const
Definition: Redux.h:431
Scalar mean() const
Definition: Redux.h:472
internal::traits< Derived >::Scalar Scalar
Definition: DenseBase.h:66
internal::traits< Derived >::Scalar maxCoeff() const
Definition: Redux.h:446
Scalar sum() const
Definition: Redux.h:459
Scalar prod() const
Definition: Redux.h:493
Scalar trace() const
Definition: Redux.h:508
@ Unaligned
Definition: Constants.h:233
const unsigned int ActualPacketAccessBit
Definition: Constants.h:105
const unsigned int LinearAccessBit
Definition: Constants.h:130
const unsigned int DirectAccessBit
Definition: Constants.h:155
Namespace containing all symbols from the Eigen library.
Definition: Core:134
const int HugeCost
Definition: Constants.h:44
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:74
const int Dynamic
Definition: Constants.h:22