Eigen  3.3.90 (mercurial changeset 94875feeeeb9)
SSE/PacketMath.h
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5 //
6 // This Source Code Form is subject to the terms of the Mozilla
7 // Public License v. 2.0. If a copy of the MPL was not distributed
8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 
10 #ifndef EIGEN_PACKET_MATH_SSE_H
11 #define EIGEN_PACKET_MATH_SSE_H
12 
13 namespace Eigen {
14 
15 namespace internal {
16 
17 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
19 #endif
20 
21 #if !defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS)
22 // 32 bits => 8 registers
23 // 64 bits => 16 registers
24 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
25 #endif
26 
27 #ifdef EIGEN_VECTORIZE_FMA
28 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
29 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 1
30 #endif
31 #endif
32 
33 #if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW) && (__GXX_ABI_VERSION < 1004)) || EIGEN_OS_QNX
34 // With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot
35 // have overloads for both types without linking error.
36 // One solution is to increase ABI version using -fabi-version=4 (or greater).
37 // Otherwise, we workaround this inconvenience by wrapping 128bit types into the following helper
38 // structure:
39 template<typename T>
40 struct eigen_packet_wrapper
41 {
42  EIGEN_ALWAYS_INLINE operator T&() { return m_val; }
43  EIGEN_ALWAYS_INLINE operator const T&() const { return m_val; }
44  EIGEN_ALWAYS_INLINE eigen_packet_wrapper() {}
45  EIGEN_ALWAYS_INLINE eigen_packet_wrapper(const T &v) : m_val(v) {}
46  EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(const T &v) {
47  m_val = v;
48  return *this;
49  }
50 
51  T m_val;
52 };
53 typedef eigen_packet_wrapper<__m128> Packet4f;
54 typedef eigen_packet_wrapper<__m128i> Packet4i;
55 typedef eigen_packet_wrapper<__m128d> Packet2d;
56 #else
57 typedef __m128 Packet4f;
58 typedef __m128i Packet4i;
59 typedef __m128d Packet2d;
60 #endif
61 
62 template<> struct is_arithmetic<__m128> { enum { value = true }; };
63 template<> struct is_arithmetic<__m128i> { enum { value = true }; };
64 template<> struct is_arithmetic<__m128d> { enum { value = true }; };
65 
66 #define EIGEN_SSE_SHUFFLE_MASK(p,q,r,s) ((s)<<6|(r)<<4|(q)<<2|(p))
67 
68 #define vec4f_swizzle1(v,p,q,r,s) \
69  (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s))))
70 
71 #define vec4i_swizzle1(v,p,q,r,s) \
72  (_mm_shuffle_epi32( v, EIGEN_SSE_SHUFFLE_MASK(p,q,r,s)))
73 
74 #define vec2d_swizzle1(v,p,q) \
75  (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), EIGEN_SSE_SHUFFLE_MASK(2*p,2*p+1,2*q,2*q+1))))
76 
77 #define vec4f_swizzle2(a,b,p,q,r,s) \
78  (_mm_shuffle_ps( (a), (b), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s)))
79 
80 #define vec4i_swizzle2(a,b,p,q,r,s) \
81  (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), EIGEN_SSE_SHUFFLE_MASK(p,q,r,s)))))
82 
83 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
84  const Packet4f p4f_##NAME = pset1<Packet4f>(X)
85 
86 #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
87  const Packet2d p2d_##NAME = pset1<Packet2d>(X)
88 
89 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
90  const Packet4f p4f_##NAME = pset1frombits<Packet4f>(X)
91 
92 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
93  const Packet4i p4i_##NAME = pset1<Packet4i>(X)
94 
95 
96 // Use the packet_traits defined in AVX/PacketMath.h instead if we're going
97 // to leverage AVX instructions.
98 #ifndef EIGEN_VECTORIZE_AVX
99 template <>
100 struct packet_traits<float> : default_packet_traits {
101  typedef Packet4f type;
102  typedef Packet4f half;
103  enum {
104  Vectorizable = 1,
105  AlignedOnScalar = 1,
106  size = 4,
107  HasHalfPacket = 0,
108 
109  HasDiv = 1,
110  HasSin = EIGEN_FAST_MATH,
111  HasCos = EIGEN_FAST_MATH,
112  HasLog = 1,
113  HasLog1p = 1,
114  HasExpm1 = 1,
115  HasNdtri = 1,
116  HasExp = 1,
117  HasBessel = 1,
118  HasSqrt = 1,
119  HasRsqrt = 1,
120  HasTanh = EIGEN_FAST_MATH,
121  HasErf = EIGEN_FAST_MATH,
122  HasBlend = 1,
123  HasFloor = 1
124 
125 #ifdef EIGEN_VECTORIZE_SSE4_1
126  ,
127  HasRound = 1,
128  HasCeil = 1
129 #endif
130  };
131 };
132 template <>
133 struct packet_traits<double> : default_packet_traits {
134  typedef Packet2d type;
135  typedef Packet2d half;
136  enum {
137  Vectorizable = 1,
138  AlignedOnScalar = 1,
139  size=2,
140  HasHalfPacket = 0,
141 
142  HasDiv = 1,
143  HasExp = 1,
144  HasSqrt = 1,
145  HasRsqrt = 1,
146  HasBlend = 1
147 
148 #ifdef EIGEN_VECTORIZE_SSE4_1
149  ,
150  HasRound = 1,
151  HasFloor = 1,
152  HasCeil = 1
153 #endif
154  };
155 };
156 #endif
157 template<> struct packet_traits<int> : default_packet_traits
158 {
159  typedef Packet4i type;
160  typedef Packet4i half;
161  enum {
162  Vectorizable = 1,
163  AlignedOnScalar = 1,
164  size=4,
165 
166  HasBlend = 1
167  };
168 };
169 
170 template<> struct unpacket_traits<Packet4f> {
171  typedef float type;
172  typedef Packet4f half;
173  typedef Packet4i integer_packet;
174  enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false};
175 };
176 template<> struct unpacket_traits<Packet2d> {
177  typedef double type;
178  typedef Packet2d half;
179  enum {size=2, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false};
180 };
181 template<> struct unpacket_traits<Packet4i> {
182  typedef int type;
183  typedef Packet4i half;
184  enum {size=4, alignment=Aligned16, vectorizable=false, masked_load_available=false, masked_store_available=false};
185 };
186 
187 #ifndef EIGEN_VECTORIZE_AVX
188 template<> struct scalar_div_cost<float,true> { enum { value = 7 }; };
189 template<> struct scalar_div_cost<double,true> { enum { value = 8 }; };
190 #endif
191 
192 #if EIGEN_COMP_MSVC==1500
193 // Workaround MSVC 9 internal compiler error.
194 // TODO: It has been detected with win64 builds (amd64), so let's check whether it also happens in 32bits+SSE mode
195 // TODO: let's check whether there does not exist a better fix, like adding a pset0() function. (it crashed on pset1(0)).
196 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set_ps(from,from,from,from); }
197 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set_pd(from,from); }
198 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set_epi32(from,from,from,from); }
199 #else
200 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set_ps1(from); }
201 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
202 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); }
203 #endif
204 
205 template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) { return _mm_castsi128_ps(pset1<Packet4i>(from)); }
206 
207 template<> EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) { return _mm_setzero_ps(); }
208 template<> EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /*a*/) { return _mm_setzero_pd(); }
209 template<> EIGEN_STRONG_INLINE Packet4i pzero(const Packet4i& /*a*/) { return _mm_setzero_si128(); }
210 
211 // GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction.
212 // However, using inrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203)
213 // Using inline assembly is also not an option because then gcc fails to reorder properly the instructions.
214 // Therefore, we introduced the pload1 functions to be used in product kernels for which bug 203 does not apply.
215 // Also note that with AVX, we want it to generate a vbroadcastss.
216 #if EIGEN_COMP_GNUC_STRICT && (!defined __AVX__)
217 template<> EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(const float *from) {
218  return vec4f_swizzle1(_mm_load_ss(from),0,0,0,0);
219 }
220 #endif
221 
222 template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
223 template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); }
224 template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
225 
226 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
227 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
228 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
229 
230 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
231 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
232 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
233 
234 template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
235 {
236  const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
237  return _mm_xor_ps(a,mask);
238 }
239 template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
240 {
241  const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000));
242  return _mm_xor_pd(a,mask);
243 }
244 template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a)
245 {
246  return psub(Packet4i(_mm_setr_epi32(0,0,0,0)), a);
247 }
248 
249 template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
250 template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
251 template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
252 
253 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
254 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
255 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
256 {
257 #ifdef EIGEN_VECTORIZE_SSE4_1
258  return _mm_mullo_epi32(a,b);
259 #else
260  // this version is slightly faster than 4 scalar products
261  return vec4i_swizzle1(
262  vec4i_swizzle2(
263  _mm_mul_epu32(a,b),
264  _mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2),
265  vec4i_swizzle1(b,1,0,3,2)),
266  0,2,0,2),
267  0,2,1,3);
268 #endif
269 }
270 
271 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
272 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
273 
274 // for some weird raisons, it has to be overloaded for packet of integers
275 template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); }
276 #ifdef EIGEN_VECTORIZE_FMA
277 template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return _mm_fmadd_ps(a,b,c); }
278 template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return _mm_fmadd_pd(a,b,c); }
279 #endif
280 
281 #ifdef EIGEN_VECTORIZE_SSE4_1
282 template<> EIGEN_DEVICE_FUNC inline Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { return _mm_blendv_ps(b,a,mask); }
283 
284 template<> EIGEN_DEVICE_FUNC inline Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) { return _mm_blendv_pd(b,a,mask); }
285 #endif
286 
287 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
288 #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63
289  // There appears to be a bug in GCC, by which the optimizer may
290  // flip the argument order in calls to _mm_min_ps, so we have to
291  // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
292  // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
293  #ifdef EIGEN_VECTORIZE_AVX
294  Packet4f res;
295  asm("vminps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b));
296  #else
297  Packet4f res = b;
298  asm("minps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a));
299  #endif
300  return res;
301 #else
302  // Arguments are reversed to match NaN propagation behavior of std::min.
303  return _mm_min_ps(b, a);
304 #endif
305 }
306 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
307 #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63
308  // There appears to be a bug in GCC, by which the optimizer may
309  // flip the argument order in calls to _mm_min_pd, so we have to
310  // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
311  // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
312  #ifdef EIGEN_VECTORIZE_AVX
313  Packet2d res;
314  asm("vminpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b));
315  #else
316  Packet2d res = b;
317  asm("minpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a));
318  #endif
319  return res;
320 #else
321  // Arguments are reversed to match NaN propagation behavior of std::min.
322  return _mm_min_pd(b, a);
323 #endif
324 }
325 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
326 {
327 #ifdef EIGEN_VECTORIZE_SSE4_1
328  return _mm_min_epi32(a,b);
329 #else
330  // after some bench, this version *is* faster than a scalar implementation
331  Packet4i mask = _mm_cmplt_epi32(a,b);
332  return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
333 #endif
334 }
335 
336 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
337 #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63
338  // There appears to be a bug in GCC, by which the optimizer may
339  // flip the argument order in calls to _mm_max_ps, so we have to
340  // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
341  // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
342  #ifdef EIGEN_VECTORIZE_AVX
343  Packet4f res;
344  asm("vmaxps %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b));
345  #else
346  Packet4f res = b;
347  asm("maxps %[a], %[res]" : [res] "+x" (res) : [a] "x" (a));
348  #endif
349  return res;
350 #else
351  // Arguments are reversed to match NaN propagation behavior of std::max.
352  return _mm_max_ps(b, a);
353 #endif
354 }
355 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
356 #if EIGEN_COMP_GNUC && EIGEN_COMP_GNUC < 63
357  // There appears to be a bug in GCC, by which the optimizer may
358  // flip the argument order in calls to _mm_max_pd, so we have to
359  // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
360  // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
361  #ifdef EIGEN_VECTORIZE_AVX
362  Packet2d res;
363  asm("vmaxpd %[a], %[b], %[res]" : [res] "=x" (res) : [a] "x" (a), [b] "x" (b));
364  #else
365  Packet2d res = b;
366  asm("maxpd %[a], %[res]" : [res] "+x" (res) : [a] "x" (a));
367  #endif
368  return res;
369 #else
370  // Arguments are reversed to match NaN propagation behavior of std::max.
371  return _mm_max_pd(b, a);
372 #endif
373 }
374 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
375 {
376 #ifdef EIGEN_VECTORIZE_SSE4_1
377  return _mm_max_epi32(a,b);
378 #else
379  // after some bench, this version *is* faster than a scalar implementation
380  Packet4i mask = _mm_cmpgt_epi32(a,b);
381  return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
382 #endif
383 }
384 
385 template<> EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { return _mm_cmple_ps(a,b); }
386 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { return _mm_cmplt_ps(a,b); }
387 template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { return _mm_cmpeq_ps(a,b); }
388 template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { return _mm_cmpeq_epi32(a,b); }
389 template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { return _mm_cmpeq_pd(a,b); }
390 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { return _mm_cmpnge_ps(a,b); }
391 
392 template<> EIGEN_STRONG_INLINE Packet4i ptrue<Packet4i>(const Packet4i& a) { return _mm_cmpeq_epi32(a, a); }
393 template<> EIGEN_STRONG_INLINE Packet4f
394 ptrue<Packet4f>(const Packet4f& a) {
395  Packet4i b = _mm_castps_si128(a);
396  return _mm_castsi128_ps(_mm_cmpeq_epi32(b, b));
397 }
398 template<> EIGEN_STRONG_INLINE Packet2d
399 ptrue<Packet2d>(const Packet2d& a) {
400  Packet4i b = _mm_castpd_si128(a);
401  return _mm_castsi128_pd(_mm_cmpeq_epi32(b, b));
402 }
403 
404 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
405 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
406 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
407 
408 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
409 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
410 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
411 
412 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
413 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
414 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
415 
416 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(b,a); }
417 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(b,a); }
418 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(b,a); }
419 
420 template<int N> EIGEN_STRONG_INLINE Packet4i pshiftright(Packet4i a) { return _mm_srli_epi32(a,N); }
421 template<int N> EIGEN_STRONG_INLINE Packet4i pshiftleft(Packet4i a) { return _mm_slli_epi32(a,N); }
422 
423 #ifdef EIGEN_VECTORIZE_SSE4_1
424 template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) { return _mm_round_ps(a, 0); }
425 template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return _mm_round_pd(a, 0); }
426 
427 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return _mm_ceil_ps(a); }
428 template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return _mm_ceil_pd(a); }
429 
430 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return _mm_floor_ps(a); }
431 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return _mm_floor_pd(a); }
432 #else
433 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
434 {
435  const Packet4f cst_1 = pset1<Packet4f>(1.0f);
436  Packet4i emm0 = _mm_cvttps_epi32(a);
437  Packet4f tmp = _mm_cvtepi32_ps(emm0);
438  /* if greater, substract 1 */
439  Packet4f mask = _mm_cmpgt_ps(tmp, a);
440  mask = pand(mask, cst_1);
441  return psub(tmp, mask);
442 }
443 
444 // WARNING: this pfloor implementation makes sense for small inputs only,
445 // It is currently only used by pexp and not exposed through HasFloor.
446 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a)
447 {
448  const Packet2d cst_1 = pset1<Packet2d>(1.0);
449  Packet4i emm0 = _mm_cvttpd_epi32(a);
450  Packet2d tmp = _mm_cvtepi32_pd(emm0);
451  /* if greater, substract 1 */
452  Packet2d mask = _mm_cmpgt_pd(tmp, a);
453  mask = pand(mask, cst_1);
454  return psub(tmp, mask);
455 }
456 #endif
457 
458 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
459 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
460 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from)); }
461 
462 #if EIGEN_COMP_MSVC
463  template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
464  EIGEN_DEBUG_UNALIGNED_LOAD
465  #if (EIGEN_COMP_MSVC==1600)
466  // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps
467  // (i.e., it does not generate an unaligned load!!
468  __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from));
469  res = _mm_loadh_pi(res, (const __m64*)(from+2));
470  return res;
471  #else
472  return _mm_loadu_ps(from);
473  #endif
474  }
475 #else
476 // NOTE: with the code below, MSVC's compiler crashes!
477 
478 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
479 {
480  EIGEN_DEBUG_UNALIGNED_LOAD
481  return _mm_loadu_ps(from);
482 }
483 #endif
484 
485 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
486 {
487  EIGEN_DEBUG_UNALIGNED_LOAD
488  return _mm_loadu_pd(from);
489 }
490 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
491 {
492  EIGEN_DEBUG_UNALIGNED_LOAD
493  return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
494 }
495 
496 
497 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
498 {
499  return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd(reinterpret_cast<const double*>(from))), 0, 0, 1, 1);
500 }
501 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
502 { return pset1<Packet2d>(from[0]); }
503 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
504 {
505  Packet4i tmp;
506  tmp = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(from));
507  return vec4i_swizzle1(tmp, 0, 0, 1, 1);
508 }
509 
510 template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
511 template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
512 template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from); }
513 
514 template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_pd(to, from); }
515 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ps(to, from); }
516 template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from); }
517 
518 template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride)
519 {
520  return _mm_set_ps(from[3*stride], from[2*stride], from[1*stride], from[0*stride]);
521 }
522 template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride)
523 {
524  return _mm_set_pd(from[1*stride], from[0*stride]);
525 }
526 template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride)
527 {
528  return _mm_set_epi32(from[3*stride], from[2*stride], from[1*stride], from[0*stride]);
529  }
530 
531 template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
532 {
533  to[stride*0] = _mm_cvtss_f32(from);
534  to[stride*1] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 1));
535  to[stride*2] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 2));
536  to[stride*3] = _mm_cvtss_f32(_mm_shuffle_ps(from, from, 3));
537 }
538 template<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
539 {
540  to[stride*0] = _mm_cvtsd_f64(from);
541  to[stride*1] = _mm_cvtsd_f64(_mm_shuffle_pd(from, from, 1));
542 }
543 template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride)
544 {
545  to[stride*0] = _mm_cvtsi128_si32(from);
546  to[stride*1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1));
547  to[stride*2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2));
548  to[stride*3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3));
549 }
550 
551 // some compilers might be tempted to perform multiple moves instead of using a vector path.
552 template<> EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a)
553 {
554  Packet4f pa = _mm_set_ss(a);
555  pstore(to, Packet4f(vec4f_swizzle1(pa,0,0,0,0)));
556 }
557 // some compilers might be tempted to perform multiple moves instead of using a vector path.
558 template<> EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a)
559 {
560  Packet2d pa = _mm_set_sd(a);
561  pstore(to, Packet2d(vec2d_swizzle1(pa,0,0)));
562 }
563 
564 #if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900
565 typedef const void * SsePrefetchPtrType;
566 #else
567 typedef const char * SsePrefetchPtrType;
568 #endif
569 
570 #ifndef EIGEN_VECTORIZE_AVX
571 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
572 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
573 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
574 #endif
575 
576 #if EIGEN_COMP_MSVC_STRICT && EIGEN_OS_WIN64
577 // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
578 // Direct of the struct members fixed bug #62.
579 template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
580 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
581 template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
582 #elif EIGEN_COMP_MSVC_STRICT
583 // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
584 template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
585 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
586 template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
587 #else
588 template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
589 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
590 template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
591 #endif
592 
593 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
594 { return _mm_shuffle_ps(a,a,0x1B); }
595 template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
596 { return _mm_shuffle_pd(a,a,0x1); }
597 template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
598 { return _mm_shuffle_epi32(a,0x1B); }
599 
600 template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a)
601 {
602  const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF));
603  return _mm_and_ps(a,mask);
604 }
605 template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a)
606 {
607  const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF));
608  return _mm_and_pd(a,mask);
609 }
610 template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a)
611 {
612  #ifdef EIGEN_VECTORIZE_SSSE3
613  return _mm_abs_epi32(a);
614  #else
615  Packet4i aux = _mm_srai_epi32(a,31);
616  return _mm_sub_epi32(_mm_xor_si128(a,aux),aux);
617  #endif
618 }
619 
620 template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
621  return pfrexp_float(a,exponent);
622 }
623 
624 template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
625  return pldexp_float(a,exponent);
626 }
627 
628 template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
629  const Packet4i cst_1023_0 = _mm_setr_epi32(1023, 1023, 0, 0);
630  Packet4i emm0 = _mm_cvttpd_epi32(exponent);
631  emm0 = padd(emm0, cst_1023_0);
632  emm0 = _mm_slli_epi32(emm0, 20);
633  emm0 = _mm_shuffle_epi32(emm0, _MM_SHUFFLE(1,2,0,3));
634  return pmul(a, Packet2d(_mm_castsi128_pd(emm0)));
635 }
636 
637 // with AVX, the default implementations based on pload1 are faster
638 #ifndef __AVX__
639 template<> EIGEN_STRONG_INLINE void
640 pbroadcast4<Packet4f>(const float *a,
641  Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
642 {
643  a3 = pload<Packet4f>(a);
644  a0 = vec4f_swizzle1(a3, 0,0,0,0);
645  a1 = vec4f_swizzle1(a3, 1,1,1,1);
646  a2 = vec4f_swizzle1(a3, 2,2,2,2);
647  a3 = vec4f_swizzle1(a3, 3,3,3,3);
648 }
649 template<> EIGEN_STRONG_INLINE void
650 pbroadcast4<Packet2d>(const double *a,
651  Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
652 {
653 #ifdef EIGEN_VECTORIZE_SSE3
654  a0 = _mm_loaddup_pd(a+0);
655  a1 = _mm_loaddup_pd(a+1);
656  a2 = _mm_loaddup_pd(a+2);
657  a3 = _mm_loaddup_pd(a+3);
658 #else
659  a1 = pload<Packet2d>(a);
660  a0 = vec2d_swizzle1(a1, 0,0);
661  a1 = vec2d_swizzle1(a1, 1,1);
662  a3 = pload<Packet2d>(a+2);
663  a2 = vec2d_swizzle1(a3, 0,0);
664  a3 = vec2d_swizzle1(a3, 1,1);
665 #endif
666 }
667 #endif
668 
669 EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs)
670 {
671  vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
672  vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
673  vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
674  vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
675 }
676 
677 #ifdef EIGEN_VECTORIZE_SSE3
678 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
679 {
680  return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3]));
681 }
682 
683 template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
684 {
685  return _mm_hadd_pd(vecs[0], vecs[1]);
686 }
687 
688 #else
689 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
690 {
691  Packet4f tmp0, tmp1, tmp2;
692  tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]);
693  tmp1 = _mm_unpackhi_ps(vecs[0], vecs[1]);
694  tmp2 = _mm_unpackhi_ps(vecs[2], vecs[3]);
695  tmp0 = _mm_add_ps(tmp0, tmp1);
696  tmp1 = _mm_unpacklo_ps(vecs[2], vecs[3]);
697  tmp1 = _mm_add_ps(tmp1, tmp2);
698  tmp2 = _mm_movehl_ps(tmp1, tmp0);
699  tmp0 = _mm_movelh_ps(tmp0, tmp1);
700  return _mm_add_ps(tmp0, tmp2);
701 }
702 
703 template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
704 {
705  return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1]));
706 }
707 #endif // SSE3
708 
709 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
710 {
711  // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures
712  // (from Nehalem to Haswell)
713 // #ifdef EIGEN_VECTORIZE_SSE3
714 // Packet4f tmp = _mm_add_ps(a, vec4f_swizzle1(a,2,3,2,3));
715 // return pfirst<Packet4f>(_mm_hadd_ps(tmp, tmp));
716 // #else
717  Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a));
718  return pfirst<Packet4f>(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
719 // #endif
720 }
721 
722 template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
723 {
724  // Disable SSE3 _mm_hadd_pd that is extremely slow on all existing Intel's architectures
725  // (from Nehalem to Haswell)
726 // #ifdef EIGEN_VECTORIZE_SSE3
727 // return pfirst<Packet2d>(_mm_hadd_pd(a, a));
728 // #else
729  return pfirst<Packet2d>(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
730 // #endif
731 }
732 
733 #ifdef EIGEN_VECTORIZE_SSSE3
734 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
735 {
736  return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3]));
737 }
738 template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
739 {
740  Packet4i tmp0 = _mm_hadd_epi32(a,a);
741  return pfirst<Packet4i>(_mm_hadd_epi32(tmp0,tmp0));
742 }
743 #else
744 template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
745 {
746  Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a));
747  return pfirst(tmp) + pfirst<Packet4i>(_mm_shuffle_epi32(tmp, 1));
748 }
749 
750 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
751 {
752  Packet4i tmp0, tmp1, tmp2;
753  tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
754  tmp1 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
755  tmp2 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
756  tmp0 = _mm_add_epi32(tmp0, tmp1);
757  tmp1 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
758  tmp1 = _mm_add_epi32(tmp1, tmp2);
759  tmp2 = _mm_unpacklo_epi64(tmp0, tmp1);
760  tmp0 = _mm_unpackhi_epi64(tmp0, tmp1);
761  return _mm_add_epi32(tmp0, tmp2);
762 }
763 #endif
764 // Other reduction functions:
765 
766 // mul
767 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
768 {
769  Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a));
770  return pfirst<Packet4f>(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
771 }
772 template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
773 {
774  return pfirst<Packet2d>(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
775 }
776 template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
777 {
778  // after some experiments, it is seems this is the fastest way to implement it
779  // for GCC (eg., reusing pmul is very slow !)
780  // TODO try to call _mm_mul_epu32 directly
781  EIGEN_ALIGN16 int aux[4];
782  pstore(aux, a);
783  return (aux[0] * aux[1]) * (aux[2] * aux[3]);
784 }
785 
786 // min
787 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
788 {
789  Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a));
790  return pfirst<Packet4f>(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
791 }
792 template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
793 {
794  return pfirst<Packet2d>(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
795 }
796 template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
797 {
798 #ifdef EIGEN_VECTORIZE_SSE4_1
799  Packet4i tmp = _mm_min_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2)));
800  return pfirst<Packet4i>(_mm_min_epi32(tmp,_mm_shuffle_epi32(tmp, 1)));
801 #else
802  // after some experiments, it is seems this is the fastest way to implement it
803  // for GCC (eg., it does not like using std::min after the pstore !!)
804  EIGEN_ALIGN16 int aux[4];
805  pstore(aux, a);
806  int aux0 = aux[0]<aux[1] ? aux[0] : aux[1];
807  int aux2 = aux[2]<aux[3] ? aux[2] : aux[3];
808  return aux0<aux2 ? aux0 : aux2;
809 #endif // EIGEN_VECTORIZE_SSE4_1
810 }
811 
812 // max
813 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
814 {
815  Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a));
816  return pfirst<Packet4f>(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
817 }
818 template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
819 {
820  return pfirst<Packet2d>(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
821 }
822 template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
823 {
824 #ifdef EIGEN_VECTORIZE_SSE4_1
825  Packet4i tmp = _mm_max_epi32(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0,0,3,2)));
826  return pfirst<Packet4i>(_mm_max_epi32(tmp,_mm_shuffle_epi32(tmp, 1)));
827 #else
828  // after some experiments, it is seems this is the fastest way to implement it
829  // for GCC (eg., it does not like using std::min after the pstore !!)
830  EIGEN_ALIGN16 int aux[4];
831  pstore(aux, a);
832  int aux0 = aux[0]>aux[1] ? aux[0] : aux[1];
833  int aux2 = aux[2]>aux[3] ? aux[2] : aux[3];
834  return aux0>aux2 ? aux0 : aux2;
835 #endif // EIGEN_VECTORIZE_SSE4_1
836 }
837 
838 // not needed yet
839 // template<> EIGEN_STRONG_INLINE bool predux_all(const Packet4f& x)
840 // {
841 // return _mm_movemask_ps(x) == 0xF;
842 // }
843 
844 template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x)
845 {
846  return _mm_movemask_ps(x) != 0x0;
847 }
848 
849 #if EIGEN_COMP_GNUC
850 // template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
851 // {
852 // Packet4f res = b;
853 // asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c));
854 // return res;
855 // }
856 // EIGEN_STRONG_INLINE Packet4i _mm_alignr_epi8(const Packet4i& a, const Packet4i& b, const int i)
857 // {
858 // Packet4i res = a;
859 // asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i));
860 // return res;
861 // }
862 #endif
863 
864 #ifdef EIGEN_VECTORIZE_SSSE3
865 // SSSE3 versions
866 template<int Offset>
867 struct palign_impl<Offset,Packet4f>
868 {
869  static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
870  {
871  if (Offset!=0)
872  first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4));
873  }
874 };
875 
876 template<int Offset>
877 struct palign_impl<Offset,Packet4i>
878 {
879  static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
880  {
881  if (Offset!=0)
882  first = _mm_alignr_epi8(second,first, Offset*4);
883  }
884 };
885 
886 template<int Offset>
887 struct palign_impl<Offset,Packet2d>
888 {
889  static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
890  {
891  if (Offset==1)
892  first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8));
893  }
894 };
895 #else
896 // SSE2 versions
897 template<int Offset>
898 struct palign_impl<Offset,Packet4f>
899 {
900  static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
901  {
902  if (Offset==1)
903  {
904  first = _mm_move_ss(first,second);
905  first = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(first),0x39));
906  }
907  else if (Offset==2)
908  {
909  first = _mm_movehl_ps(first,first);
910  first = _mm_movelh_ps(first,second);
911  }
912  else if (Offset==3)
913  {
914  first = _mm_move_ss(first,second);
915  first = _mm_shuffle_ps(first,second,0x93);
916  }
917  }
918 };
919 
920 template<int Offset>
921 struct palign_impl<Offset,Packet4i>
922 {
923  static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
924  {
925  if (Offset==1)
926  {
927  first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
928  first = _mm_shuffle_epi32(first,0x39);
929  }
930  else if (Offset==2)
931  {
932  first = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(first)));
933  first = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
934  }
935  else if (Offset==3)
936  {
937  first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
938  first = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second),0x93));
939  }
940  }
941 };
942 
943 template<int Offset>
944 struct palign_impl<Offset,Packet2d>
945 {
946  static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
947  {
948  if (Offset==1)
949  {
950  first = _mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(first),_mm_castpd_ps(first)));
951  first = _mm_castps_pd(_mm_movelh_ps(_mm_castpd_ps(first),_mm_castpd_ps(second)));
952  }
953  }
954 };
955 #endif
956 
957 EIGEN_DEVICE_FUNC inline void
958 ptranspose(PacketBlock<Packet4f,4>& kernel) {
959  _MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]);
960 }
961 
962 EIGEN_DEVICE_FUNC inline void
963 ptranspose(PacketBlock<Packet2d,2>& kernel) {
964  __m128d tmp = _mm_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
965  kernel.packet[0] = _mm_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
966  kernel.packet[1] = tmp;
967 }
968 
969 EIGEN_DEVICE_FUNC inline void
970 ptranspose(PacketBlock<Packet4i,4>& kernel) {
971  __m128i T0 = _mm_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
972  __m128i T1 = _mm_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
973  __m128i T2 = _mm_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
974  __m128i T3 = _mm_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
975 
976  kernel.packet[0] = _mm_unpacklo_epi64(T0, T1);
977  kernel.packet[1] = _mm_unpackhi_epi64(T0, T1);
978  kernel.packet[2] = _mm_unpacklo_epi64(T2, T3);
979  kernel.packet[3] = _mm_unpackhi_epi64(T2, T3);
980 }
981 
982 template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) {
983  const __m128i zero = _mm_setzero_si128();
984  const __m128i select = _mm_set_epi32(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]);
985  __m128i false_mask = _mm_cmpeq_epi32(select, zero);
986 #ifdef EIGEN_VECTORIZE_SSE4_1
987  return _mm_blendv_epi8(thenPacket, elsePacket, false_mask);
988 #else
989  return _mm_or_si128(_mm_andnot_si128(false_mask, thenPacket), _mm_and_si128(false_mask, elsePacket));
990 #endif
991 }
992 template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) {
993  const __m128 zero = _mm_setzero_ps();
994  const __m128 select = _mm_set_ps(ifPacket.select[3], ifPacket.select[2], ifPacket.select[1], ifPacket.select[0]);
995  __m128 false_mask = _mm_cmpeq_ps(select, zero);
996 #ifdef EIGEN_VECTORIZE_SSE4_1
997  return _mm_blendv_ps(thenPacket, elsePacket, false_mask);
998 #else
999  return _mm_or_ps(_mm_andnot_ps(false_mask, thenPacket), _mm_and_ps(false_mask, elsePacket));
1000 #endif
1001 }
1002 template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) {
1003  const __m128d zero = _mm_setzero_pd();
1004  const __m128d select = _mm_set_pd(ifPacket.select[1], ifPacket.select[0]);
1005  __m128d false_mask = _mm_cmpeq_pd(select, zero);
1006 #ifdef EIGEN_VECTORIZE_SSE4_1
1007  return _mm_blendv_pd(thenPacket, elsePacket, false_mask);
1008 #else
1009  return _mm_or_pd(_mm_andnot_pd(false_mask, thenPacket), _mm_and_pd(false_mask, elsePacket));
1010 #endif
1011 }
1012 
1013 template<> EIGEN_STRONG_INLINE Packet4f pinsertfirst(const Packet4f& a, float b)
1014 {
1015 #ifdef EIGEN_VECTORIZE_SSE4_1
1016  return _mm_blend_ps(a,pset1<Packet4f>(b),1);
1017 #else
1018  return _mm_move_ss(a, _mm_load_ss(&b));
1019 #endif
1020 }
1021 
1022 template<> EIGEN_STRONG_INLINE Packet2d pinsertfirst(const Packet2d& a, double b)
1023 {
1024 #ifdef EIGEN_VECTORIZE_SSE4_1
1025  return _mm_blend_pd(a,pset1<Packet2d>(b),1);
1026 #else
1027  return _mm_move_sd(a, _mm_load_sd(&b));
1028 #endif
1029 }
1030 
1031 template<> EIGEN_STRONG_INLINE Packet4f pinsertlast(const Packet4f& a, float b)
1032 {
1033 #ifdef EIGEN_VECTORIZE_SSE4_1
1034  return _mm_blend_ps(a,pset1<Packet4f>(b),(1<<3));
1035 #else
1036  const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x0,0x0,0x0,0xFFFFFFFF));
1037  return _mm_or_ps(_mm_andnot_ps(mask, a), _mm_and_ps(mask, pset1<Packet4f>(b)));
1038 #endif
1039 }
1040 
1041 template<> EIGEN_STRONG_INLINE Packet2d pinsertlast(const Packet2d& a, double b)
1042 {
1043 #ifdef EIGEN_VECTORIZE_SSE4_1
1044  return _mm_blend_pd(a,pset1<Packet2d>(b),(1<<1));
1045 #else
1046  const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x0,0xFFFFFFFF,0xFFFFFFFF));
1047  return _mm_or_pd(_mm_andnot_pd(mask, a), _mm_and_pd(mask, pset1<Packet2d>(b)));
1048 #endif
1049 }
1050 
1051 // Scalar path for pmadd with FMA to ensure consistency with vectorized path.
1052 #ifdef EIGEN_VECTORIZE_FMA
1053 template<> EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const float& c) {
1054  return ::fmaf(a,b,c);
1055 }
1056 template<> EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) {
1057  return ::fma(a,b,c);
1058 }
1059 #endif
1060 
1061 
1062 // Packet math for Eigen::half
1063 // Disable the following code since it's broken on too many platforms / compilers.
1064 //#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC)
1065 #if 0
1066 
1067 typedef struct {
1068  __m64 x;
1069 } Packet4h;
1070 
1071 
1072 template<> struct is_arithmetic<Packet4h> { enum { value = true }; };
1073 
1074 template <>
1075 struct packet_traits<Eigen::half> : default_packet_traits {
1076  typedef Packet4h type;
1077  // There is no half-size packet for Packet4h.
1078  typedef Packet4h half;
1079  enum {
1080  Vectorizable = 1,
1081  AlignedOnScalar = 1,
1082  size = 4,
1083  HasHalfPacket = 0,
1084  HasAdd = 1,
1085  HasSub = 1,
1086  HasMul = 1,
1087  HasDiv = 1,
1088  HasNegate = 0,
1089  HasAbs = 0,
1090  HasAbs2 = 0,
1091  HasMin = 0,
1092  HasMax = 0,
1093  HasConj = 0,
1094  HasSetLinear = 0,
1095  HasSqrt = 0,
1096  HasRsqrt = 0,
1097  HasExp = 0,
1098  HasLog = 0,
1099  HasBlend = 0
1100  };
1101 };
1102 
1103 
1104 template<> struct unpacket_traits<Packet4h> { typedef Eigen::half type; enum {size=4, alignment=Aligned16, vectorizable=true, masked_load_available=false, masked_store_available=false}; typedef Packet4h half; };
1105 
1106 template<> EIGEN_STRONG_INLINE Packet4h pset1<Packet4h>(const Eigen::half& from) {
1107  Packet4h result;
1108  result.x = _mm_set1_pi16(from.x);
1109  return result;
1110 }
1111 
1112 template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h>(const Packet4h& from) {
1113  return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_cvtsi64_si32(from.x)));
1114 }
1115 
1116 template<> EIGEN_STRONG_INLINE Packet4h pconj(const Packet4h& a) { return a; }
1117 
1118 template<> EIGEN_STRONG_INLINE Packet4h padd<Packet4h>(const Packet4h& a, const Packet4h& b) {
1119  __int64_t a64 = _mm_cvtm64_si64(a.x);
1120  __int64_t b64 = _mm_cvtm64_si64(b.x);
1121 
1122  Eigen::half h[4];
1123 
1124  Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1125  Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1126  h[0] = ha + hb;
1127  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1128  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1129  h[1] = ha + hb;
1130  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1131  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1132  h[2] = ha + hb;
1133  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1134  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1135  h[3] = ha + hb;
1136  Packet4h result;
1137  result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1138  return result;
1139 }
1140 
1141 template<> EIGEN_STRONG_INLINE Packet4h psub<Packet4h>(const Packet4h& a, const Packet4h& b) {
1142  __int64_t a64 = _mm_cvtm64_si64(a.x);
1143  __int64_t b64 = _mm_cvtm64_si64(b.x);
1144 
1145  Eigen::half h[4];
1146 
1147  Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1148  Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1149  h[0] = ha - hb;
1150  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1151  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1152  h[1] = ha - hb;
1153  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1154  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1155  h[2] = ha - hb;
1156  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1157  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1158  h[3] = ha - hb;
1159  Packet4h result;
1160  result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1161  return result;
1162 }
1163 
1164 template<> EIGEN_STRONG_INLINE Packet4h pmul<Packet4h>(const Packet4h& a, const Packet4h& b) {
1165  __int64_t a64 = _mm_cvtm64_si64(a.x);
1166  __int64_t b64 = _mm_cvtm64_si64(b.x);
1167 
1168  Eigen::half h[4];
1169 
1170  Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1171  Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1172  h[0] = ha * hb;
1173  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1174  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1175  h[1] = ha * hb;
1176  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1177  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1178  h[2] = ha * hb;
1179  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1180  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1181  h[3] = ha * hb;
1182  Packet4h result;
1183  result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1184  return result;
1185 }
1186 
1187 template<> EIGEN_STRONG_INLINE Packet4h pdiv<Packet4h>(const Packet4h& a, const Packet4h& b) {
1188  __int64_t a64 = _mm_cvtm64_si64(a.x);
1189  __int64_t b64 = _mm_cvtm64_si64(b.x);
1190 
1191  Eigen::half h[4];
1192 
1193  Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1194  Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1195  h[0] = ha / hb;
1196  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1197  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1198  h[1] = ha / hb;
1199  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1200  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1201  h[2] = ha / hb;
1202  ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1203  hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1204  h[3] = ha / hb;
1205  Packet4h result;
1206  result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1207  return result;
1208 }
1209 
1210 template<> EIGEN_STRONG_INLINE Packet4h pload<Packet4h>(const Eigen::half* from) {
1211  Packet4h result;
1212  result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from));
1213  return result;
1214 }
1215 
1216 template<> EIGEN_STRONG_INLINE Packet4h ploadu<Packet4h>(const Eigen::half* from) {
1217  Packet4h result;
1218  result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from));
1219  return result;
1220 }
1221 
1222 template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4h& from) {
1223  __int64_t r = _mm_cvtm64_si64(from.x);
1224  *(reinterpret_cast<__int64_t*>(to)) = r;
1225 }
1226 
1227 template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4h& from) {
1228  __int64_t r = _mm_cvtm64_si64(from.x);
1229  *(reinterpret_cast<__int64_t*>(to)) = r;
1230 }
1231 
1232 template<> EIGEN_STRONG_INLINE Packet4h
1233 ploadquad<Packet4h>(const Eigen::half* from) {
1234  return pset1<Packet4h>(*from);
1235 }
1236 
1237 template<> EIGEN_STRONG_INLINE Packet4h pgather<Eigen::half, Packet4h>(const Eigen::half* from, Index stride)
1238 {
1239  Packet4h result;
1240  result.x = _mm_set_pi16(from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
1241  return result;
1242 }
1243 
1244 template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4h>(Eigen::half* to, const Packet4h& from, Index stride)
1245 {
1246  __int64_t a = _mm_cvtm64_si64(from.x);
1247  to[stride*0].x = static_cast<unsigned short>(a);
1248  to[stride*1].x = static_cast<unsigned short>(a >> 16);
1249  to[stride*2].x = static_cast<unsigned short>(a >> 32);
1250  to[stride*3].x = static_cast<unsigned short>(a >> 48);
1251 }
1252 
1253 EIGEN_STRONG_INLINE void
1254 ptranspose(PacketBlock<Packet4h,4>& kernel) {
1255  __m64 T0 = _mm_unpacklo_pi16(kernel.packet[0].x, kernel.packet[1].x);
1256  __m64 T1 = _mm_unpacklo_pi16(kernel.packet[2].x, kernel.packet[3].x);
1257  __m64 T2 = _mm_unpackhi_pi16(kernel.packet[0].x, kernel.packet[1].x);
1258  __m64 T3 = _mm_unpackhi_pi16(kernel.packet[2].x, kernel.packet[3].x);
1259 
1260  kernel.packet[0].x = _mm_unpacklo_pi32(T0, T1);
1261  kernel.packet[1].x = _mm_unpackhi_pi32(T0, T1);
1262  kernel.packet[2].x = _mm_unpacklo_pi32(T2, T3);
1263  kernel.packet[3].x = _mm_unpackhi_pi32(T2, T3);
1264 }
1265 
1266 #endif
1267 
1268 
1269 } // end namespace internal
1270 
1271 } // end namespace Eigen
1272 
1273 #if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900
1274 // PGI++ does not define the following intrinsics in C++ mode.
1275 static inline __m128 _mm_castpd_ps (__m128d x) { return reinterpret_cast<__m128&>(x); }
1276 static inline __m128i _mm_castpd_si128(__m128d x) { return reinterpret_cast<__m128i&>(x); }
1277 static inline __m128d _mm_castps_pd (__m128 x) { return reinterpret_cast<__m128d&>(x); }
1278 static inline __m128i _mm_castps_si128(__m128 x) { return reinterpret_cast<__m128i&>(x); }
1279 static inline __m128 _mm_castsi128_ps(__m128i x) { return reinterpret_cast<__m128&>(x); }
1280 static inline __m128d _mm_castsi128_pd(__m128i x) { return reinterpret_cast<__m128d&>(x); }
1281 #endif
1282 
1283 #endif // EIGEN_PACKET_MATH_SSE_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition: Core:129
Eigen::Aligned16
Definition: Constants.h:234
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:42