23#ifndef EIGEN_MALLOC_ALREADY_ALIGNED
34#if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
35 && defined(__LP64__) && ! defined( __SANITIZE_ADDRESS__ ) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
36 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
38 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
45#if defined(__FreeBSD__) && !(EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
46 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
48 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
51#if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
52 || (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
53 || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
54 || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
55 #define EIGEN_MALLOC_ALREADY_ALIGNED 1
57 #define EIGEN_MALLOC_ALREADY_ALIGNED 0
62#include "../InternalHeaderCheck.h"
69inline void throw_std_bad_alloc()
71 #ifdef EIGEN_EXCEPTIONS
72 throw std::bad_alloc();
74 std::size_t huge =
static_cast<std::size_t
>(-1);
75 #if defined(EIGEN_HIPCC)
87 void* unused = ::operator
new(huge);
88 EIGEN_UNUSED_VARIABLE(unused);
102EIGEN_DEVICE_FUNC
inline void* handmade_aligned_malloc(std::size_t size, std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES)
104 eigen_assert(alignment >=
sizeof(
void*) && (alignment & (alignment-1)) == 0 &&
"Alignment must be at least sizeof(void*) and a power of 2");
106 EIGEN_USING_STD(malloc)
107 void *original = malloc(size+alignment);
109 if (original == 0)
return 0;
110 void *aligned =
reinterpret_cast<void*
>((
reinterpret_cast<std::size_t
>(original) & ~(std::size_t(alignment-1))) + alignment);
111 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
116EIGEN_DEVICE_FUNC
inline void handmade_aligned_free(
void *ptr)
119 EIGEN_USING_STD(free)
120 free(*(
reinterpret_cast<void**
>(ptr) - 1));
129inline void* handmade_aligned_realloc(
void* ptr, std::size_t size, std::size_t = 0)
131 if (ptr == 0)
return handmade_aligned_malloc(size);
132 void *original = *(
reinterpret_cast<void**
>(ptr) - 1);
133 std::ptrdiff_t previous_offset =
static_cast<char *
>(ptr)-
static_cast<char *
>(original);
134 original = std::realloc(original,size+EIGEN_DEFAULT_ALIGN_BYTES);
135 if (original == 0)
return 0;
136 void *aligned =
reinterpret_cast<void*
>((
reinterpret_cast<std::size_t
>(original) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1))) + EIGEN_DEFAULT_ALIGN_BYTES);
137 void *previous_aligned =
static_cast<char *
>(original)+previous_offset;
138 if(aligned!=previous_aligned)
139 std::memmove(aligned, previous_aligned, size);
141 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
149#ifdef EIGEN_NO_MALLOC
150EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed()
152 eigen_assert(
false &&
"heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
154#elif defined EIGEN_RUNTIME_NO_MALLOC
155EIGEN_DEVICE_FUNC
inline bool is_malloc_allowed_impl(
bool update,
bool new_value =
false)
157 static bool value =
true;
162EIGEN_DEVICE_FUNC
inline bool is_malloc_allowed() {
return is_malloc_allowed_impl(
false); }
163EIGEN_DEVICE_FUNC
inline bool set_is_malloc_allowed(
bool new_value) {
return is_malloc_allowed_impl(
true, new_value); }
164EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed()
166 eigen_assert(is_malloc_allowed() &&
"heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)");
169EIGEN_DEVICE_FUNC
inline void check_that_malloc_is_allowed()
176EIGEN_DEVICE_FUNC
inline void* aligned_malloc(std::size_t size)
178 check_that_malloc_is_allowed();
181 #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
183 EIGEN_USING_STD(malloc)
184 result = malloc(size);
186 #if EIGEN_DEFAULT_ALIGN_BYTES==16
187 eigen_assert((size<16 || (std::size_t(result)%16)==0) &&
"System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback to handmade aligned memory allocator.");
190 result = handmade_aligned_malloc(size);
194 throw_std_bad_alloc();
200EIGEN_DEVICE_FUNC
inline void aligned_free(
void *ptr)
202 #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
204 EIGEN_USING_STD(free)
208 handmade_aligned_free(ptr);
217inline void* aligned_realloc(
void *ptr, std::size_t new_size, std::size_t old_size)
219 EIGEN_UNUSED_VARIABLE(old_size)
222#if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
223 result = std::realloc(ptr,new_size);
225 result = handmade_aligned_realloc(ptr,new_size,old_size);
228 if (!result && new_size)
229 throw_std_bad_alloc();
241template<
bool Align> EIGEN_DEVICE_FUNC
inline void* conditional_aligned_malloc(std::size_t size)
243 return aligned_malloc(size);
246template<> EIGEN_DEVICE_FUNC
inline void* conditional_aligned_malloc<false>(std::size_t size)
248 check_that_malloc_is_allowed();
250 EIGEN_USING_STD(malloc)
251 void *result = malloc(size);
254 throw_std_bad_alloc();
259template<
bool Align> EIGEN_DEVICE_FUNC
inline void conditional_aligned_free(
void *ptr)
264template<> EIGEN_DEVICE_FUNC
inline void conditional_aligned_free<false>(
void *ptr)
266 EIGEN_USING_STD(free)
270template<
bool Align>
inline void* conditional_aligned_realloc(
void* ptr, std::size_t new_size, std::size_t old_size)
272 return aligned_realloc(ptr, new_size, old_size);
275template<>
inline void* conditional_aligned_realloc<false>(
void* ptr, std::size_t new_size, std::size_t)
277 return std::realloc(ptr, new_size);
287template<
typename T> EIGEN_DEVICE_FUNC
inline void destruct_elements_of_array(T *ptr, std::size_t size)
291 while(size) ptr[--size].~T();
297template<
typename T> EIGEN_DEVICE_FUNC
inline T* construct_elements_of_array(T *ptr, std::size_t size)
302 for (i = 0; i < size; ++i) ::
new (ptr + i) T;
307 destruct_elements_of_array(ptr, i);
318EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void check_size_for_overflow(std::size_t size)
320 if(size > std::size_t(-1) /
sizeof(T))
321 throw_std_bad_alloc();
328template<
typename T> EIGEN_DEVICE_FUNC
inline T* aligned_new(std::size_t size)
330 check_size_for_overflow<T>(size);
331 T *result =
reinterpret_cast<T*
>(aligned_malloc(
sizeof(T)*size));
334 return construct_elements_of_array(result, size);
338 aligned_free(result);
344template<
typename T,
bool Align> EIGEN_DEVICE_FUNC
inline T* conditional_aligned_new(std::size_t size)
346 check_size_for_overflow<T>(size);
347 T *result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T)*size));
350 return construct_elements_of_array(result, size);
354 conditional_aligned_free<Align>(result);
363template<
typename T> EIGEN_DEVICE_FUNC
inline void aligned_delete(T *ptr, std::size_t size)
365 destruct_elements_of_array<T>(ptr, size);
366 Eigen::internal::aligned_free(ptr);
372template<
typename T,
bool Align> EIGEN_DEVICE_FUNC
inline void conditional_aligned_delete(T *ptr, std::size_t size)
374 destruct_elements_of_array<T>(ptr, size);
375 conditional_aligned_free<Align>(ptr);
378template<
typename T,
bool Align> EIGEN_DEVICE_FUNC
inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size)
380 check_size_for_overflow<T>(new_size);
381 check_size_for_overflow<T>(old_size);
382 if(new_size < old_size)
383 destruct_elements_of_array(pts+new_size, old_size-new_size);
384 T *result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(T)*new_size,
sizeof(T)*old_size));
385 if(new_size > old_size)
389 construct_elements_of_array(result+old_size, new_size-old_size);
393 conditional_aligned_free<Align>(result);
401template<
typename T,
bool Align> EIGEN_DEVICE_FUNC
inline T* conditional_aligned_new_auto(std::size_t size)
405 check_size_for_overflow<T>(size);
406 T *result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(T)*size));
407 if(NumTraits<T>::RequireInitialization)
411 construct_elements_of_array(result, size);
415 conditional_aligned_free<Align>(result);
422template<
typename T,
bool Align>
inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size)
424 check_size_for_overflow<T>(new_size);
425 check_size_for_overflow<T>(old_size);
426 if(NumTraits<T>::RequireInitialization && (new_size < old_size))
427 destruct_elements_of_array(pts+new_size, old_size-new_size);
428 T *result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(T)*new_size,
sizeof(T)*old_size));
429 if(NumTraits<T>::RequireInitialization && (new_size > old_size))
433 construct_elements_of_array(result+old_size, new_size-old_size);
437 conditional_aligned_free<Align>(result);
444template<
typename T,
bool Align> EIGEN_DEVICE_FUNC
inline void conditional_aligned_delete_auto(T *ptr, std::size_t size)
446 if(NumTraits<T>::RequireInitialization)
447 destruct_elements_of_array<T>(ptr, size);
448 conditional_aligned_free<Align>(ptr);
470template<
int Alignment,
typename Scalar,
typename Index>
471EIGEN_DEVICE_FUNC
inline Index first_aligned(
const Scalar* array,
Index size)
473 const Index ScalarSize =
sizeof(Scalar);
474 const Index AlignmentSize = Alignment / ScalarSize;
475 const Index AlignmentMask = AlignmentSize-1;
483 else if( (UIntPtr(array) & (
sizeof(Scalar)-1)) || (Alignment%ScalarSize)!=0)
491 Index first = (AlignmentSize - (
Index((UIntPtr(array)/
sizeof(Scalar))) & AlignmentMask)) & AlignmentMask;
492 return (first < size) ? first : size;
498template<
typename Scalar,
typename Index>
499EIGEN_DEVICE_FUNC
inline Index first_default_aligned(
const Scalar* array,
Index size)
501 typedef typename packet_traits<Scalar>::type DefaultPacketType;
502 return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(array, size);
507template<
typename Index>
510 return ((size+base-1)/base)*base;
515template<
typename T,
bool UseMemcpy>
struct smart_copy_helper;
517template<
typename T> EIGEN_DEVICE_FUNC
void smart_copy(
const T* start,
const T*
end, T* target)
519 smart_copy_helper<T,!NumTraits<T>::RequireInitialization>::run(start,
end, target);
522template<
typename T>
struct smart_copy_helper<T,true> {
523 EIGEN_DEVICE_FUNC
static inline void run(
const T* start,
const T*
end, T* target)
525 IntPtr size = IntPtr(
end)-IntPtr(start);
527 eigen_internal_assert(start!=0 &&
end!=0 && target!=0);
528 EIGEN_USING_STD(memcpy)
529 memcpy(target, start, size);
533template<
typename T>
struct smart_copy_helper<T,false> {
534 EIGEN_DEVICE_FUNC
static inline void run(
const T* start,
const T*
end, T* target)
535 { std::copy(start,
end, target); }
539template<
typename T,
bool UseMemmove>
struct smart_memmove_helper;
541template<
typename T>
void smart_memmove(
const T* start,
const T*
end, T* target)
543 smart_memmove_helper<T,!NumTraits<T>::RequireInitialization>::run(start,
end, target);
546template<
typename T>
struct smart_memmove_helper<T,true> {
547 static inline void run(
const T* start,
const T*
end, T* target)
549 IntPtr size = IntPtr(
end)-IntPtr(start);
551 eigen_internal_assert(start!=0 &&
end!=0 && target!=0);
552 std::memmove(target, start, size);
556template<
typename T>
struct smart_memmove_helper<T,false> {
557 static inline void run(
const T* start,
const T*
end, T* target)
559 if (UIntPtr(target) < UIntPtr(start))
561 std::copy(start,
end, target);
565 std::ptrdiff_t count = (std::ptrdiff_t(
end)-std::ptrdiff_t(start)) /
sizeof(T);
566 std::copy_backward(start,
end, target + count);
571template<
typename T> EIGEN_DEVICE_FUNC T* smart_move(T* start, T*
end, T* target)
573 return std::move(start,
end, target);
582#if ! defined EIGEN_ALLOCA && ! defined EIGEN_GPU_COMPILE_PHASE
583 #if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca)
584 #define EIGEN_ALLOCA alloca
585 #elif EIGEN_COMP_MSVC
586 #define EIGEN_ALLOCA _alloca
595#if defined(__clang__) && defined(__thumb__)
601template<
typename T>
class aligned_stack_memory_handler : noncopyable
611 aligned_stack_memory_handler(T* ptr, std::size_t size,
bool dealloc)
612 : m_ptr(ptr), m_size(size), m_deallocate(dealloc)
614 if(NumTraits<T>::RequireInitialization && m_ptr)
615 Eigen::internal::construct_elements_of_array(m_ptr, size);
618 ~aligned_stack_memory_handler()
620 if(NumTraits<T>::RequireInitialization && m_ptr)
621 Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
623 Eigen::internal::aligned_free(m_ptr);
633template<
typename Xpr,
int NbEvaluations,
634 bool MapExternalBuffer = nested_eval<Xpr,NbEvaluations>::Evaluate && Xpr::MaxSizeAtCompileTime==
Dynamic
636struct local_nested_eval_wrapper
638 static const bool NeedExternalBuffer =
false;
639 typedef typename Xpr::Scalar Scalar;
640 typedef typename nested_eval<Xpr,NbEvaluations>::type ObjectType;
644 local_nested_eval_wrapper(
const Xpr& xpr, Scalar* ptr) : object(xpr)
646 EIGEN_UNUSED_VARIABLE(ptr);
647 eigen_internal_assert(ptr==0);
651template<
typename Xpr,
int NbEvaluations>
652struct local_nested_eval_wrapper<Xpr,NbEvaluations,true>
654 static const bool NeedExternalBuffer =
true;
655 typedef typename Xpr::Scalar Scalar;
656 typedef typename plain_object_eval<Xpr>::type PlainObject;
657 typedef Map<PlainObject,EIGEN_DEFAULT_ALIGN_BYTES> ObjectType;
661 local_nested_eval_wrapper(
const Xpr& xpr, Scalar* ptr)
662 : object(ptr==0 ? reinterpret_cast<Scalar*>(
Eigen::internal::aligned_malloc(sizeof(Scalar)*xpr.size())) : ptr, xpr.rows(), xpr.cols()),
665 if(NumTraits<Scalar>::RequireInitialization &&
object.data())
666 Eigen::internal::construct_elements_of_array(
object.data(),
object.size());
671 ~local_nested_eval_wrapper()
673 if(NumTraits<Scalar>::RequireInitialization &&
object.data())
674 Eigen::internal::destruct_elements_of_array(
object.data(),
object.size());
676 Eigen::internal::aligned_free(
object.data());
685template<
typename T>
class scoped_array : noncopyable
689 explicit scoped_array(std::ptrdiff_t size)
697 T& operator[](std::ptrdiff_t i) {
return m_ptr[i]; }
698 const T& operator[](std::ptrdiff_t i)
const {
return m_ptr[i]; }
699 T* &ptr() {
return m_ptr; }
700 const T* ptr()
const {
return m_ptr; }
701 operator const T*()
const {
return m_ptr; }
704template<
typename T>
void swap(scoped_array<T> &a,scoped_array<T> &b)
706 std::swap(a.ptr(),b.ptr());
738 #if EIGEN_DEFAULT_ALIGN_BYTES>0
741 #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((internal::UIntPtr(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)))
743 #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE)
746 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
747 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
748 TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \
749 : reinterpret_cast<TYPE*>( \
750 (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \
751 : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \
752 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT)
755 #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) \
756 Eigen::internal::local_nested_eval_wrapper<XPR_T,N> EIGEN_CAT(NAME,_wrapper)(XPR, reinterpret_cast<typename XPR_T::Scalar*>( \
757 ( (Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::NeedExternalBuffer) && ((sizeof(typename XPR_T::Scalar)*XPR.size())<=EIGEN_STACK_ALLOCATION_LIMIT) ) \
758 ? EIGEN_ALIGNED_ALLOCA( sizeof(typename XPR_T::Scalar)*XPR.size() ) : 0 ) ) ; \
759 typename Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::ObjectType NAME(EIGEN_CAT(NAME,_wrapper).object)
763 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
764 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
765 TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \
766 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true)
769#define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) typename Eigen::internal::nested_eval<XPR_T,N>::type NAME(XPR)
778#if EIGEN_HAS_CXX17_OVERALIGN
782#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign)
783#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
784#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW
785#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size)
790#if EIGEN_MAX_ALIGN_BYTES!=0 && !defined(EIGEN_HIP_DEVICE_COMPILE)
791 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
793 void* operator new(std::size_t size, const std::nothrow_t&) EIGEN_NO_THROW { \
794 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
795 EIGEN_CATCH (...) { return 0; } \
797 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
799 void *operator new(std::size_t size) { \
800 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
803 void *operator new[](std::size_t size) { \
804 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
807 void operator delete(void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
809 void operator delete[](void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
811 void operator delete(void * ptr, std::size_t ) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
813 void operator delete[](void * ptr, std::size_t ) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
818 static void *operator new(std::size_t size, void *ptr) { return ::operator new(size,ptr); } \
820 static void *operator new[](std::size_t size, void* ptr) { return ::operator new[](size,ptr); } \
822 void operator delete(void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete(memory,ptr); } \
824 void operator delete[](void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete[](memory,ptr); } \
826 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
828 void operator delete(void *ptr, const std::nothrow_t&) EIGEN_NO_THROW { \
829 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
831 typedef void eigen_aligned_operator_new_marker_type;
833 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
836#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
837#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \
838 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool( \
839 ((Size)!=Eigen::Dynamic) && \
840 (((EIGEN_MAX_ALIGN_BYTES>=16) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES )==0)) || \
841 ((EIGEN_MAX_ALIGN_BYTES>=32) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/2)==0)) || \
842 ((EIGEN_MAX_ALIGN_BYTES>=64) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/4)==0)) )))
876 typedef std::size_t size_type;
877 typedef std::ptrdiff_t difference_type;
879 typedef const T* const_pointer;
880 typedef T& reference;
881 typedef const T& const_reference;
882 typedef T value_type;
899 #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_LEAST(7,0)
903 size_type max_size()
const {
904 return (std::numeric_limits<std::ptrdiff_t>::max)()/
sizeof(T);
908 pointer allocate(size_type num,
const void* = 0)
910 internal::check_size_for_overflow<T>(num);
911 return static_cast<pointer
>( internal::aligned_malloc(num *
sizeof(T)) );
914 void deallocate(pointer p, size_type )
916 internal::aligned_free(p);
922#if !defined(EIGEN_NO_CPUID)
923# if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64
924# if defined(__PIC__) && EIGEN_ARCH_i386
926# define EIGEN_CPUID(abcd,func,id) \
927 __asm__ __volatile__ ("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id));
928# elif defined(__PIC__) && EIGEN_ARCH_x86_64
931# define EIGEN_CPUID(abcd,func,id) \
932 __asm__ __volatile__ ("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id));
935# define EIGEN_CPUID(abcd,func,id) \
936 __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id) );
938# elif EIGEN_COMP_MSVC
939# if EIGEN_ARCH_i386_OR_x86_64
940# define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id)
949inline bool cpuid_is_vendor(
int abcd[4],
const int vendor[3])
951 return abcd[1]==vendor[0] && abcd[3]==vendor[1] && abcd[2]==vendor[2];
954inline void queryCacheSizes_intel_direct(
int& l1,
int& l2,
int& l3)
961 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
962 EIGEN_CPUID(abcd,0x4,cache_id);
963 cache_type = (abcd[0] & 0x0F) >> 0;
964 if(cache_type==1||cache_type==3)
966 int cache_level = (abcd[0] & 0xE0) >> 5;
967 int ways = (abcd[1] & 0xFFC00000) >> 22;
968 int partitions = (abcd[1] & 0x003FF000) >> 12;
969 int line_size = (abcd[1] & 0x00000FFF) >> 0;
970 int sets = (abcd[2]);
972 int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1);
976 case 1: l1 = cache_size;
break;
977 case 2: l2 = cache_size;
break;
978 case 3: l3 = cache_size;
break;
983 }
while(cache_type>0 && cache_id<16);
986inline void queryCacheSizes_intel_codes(
int& l1,
int& l2,
int& l3)
989 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
991 EIGEN_CPUID(abcd,0x00000002,0);
992 unsigned char * bytes =
reinterpret_cast<unsigned char *
>(abcd)+2;
993 bool check_for_p2_core2 =
false;
994 for(
int i=0; i<14; ++i)
998 case 0x0A: l1 = 8;
break;
999 case 0x0C: l1 = 16;
break;
1000 case 0x0E: l1 = 24;
break;
1001 case 0x10: l1 = 16;
break;
1002 case 0x15: l1 = 16;
break;
1003 case 0x2C: l1 = 32;
break;
1004 case 0x30: l1 = 32;
break;
1005 case 0x60: l1 = 16;
break;
1006 case 0x66: l1 = 8;
break;
1007 case 0x67: l1 = 16;
break;
1008 case 0x68: l1 = 32;
break;
1009 case 0x1A: l2 = 96;
break;
1010 case 0x22: l3 = 512;
break;
1011 case 0x23: l3 = 1024;
break;
1012 case 0x25: l3 = 2048;
break;
1013 case 0x29: l3 = 4096;
break;
1014 case 0x39: l2 = 128;
break;
1015 case 0x3A: l2 = 192;
break;
1016 case 0x3B: l2 = 128;
break;
1017 case 0x3C: l2 = 256;
break;
1018 case 0x3D: l2 = 384;
break;
1019 case 0x3E: l2 = 512;
break;
1020 case 0x40: l2 = 0;
break;
1021 case 0x41: l2 = 128;
break;
1022 case 0x42: l2 = 256;
break;
1023 case 0x43: l2 = 512;
break;
1024 case 0x44: l2 = 1024;
break;
1025 case 0x45: l2 = 2048;
break;
1026 case 0x46: l3 = 4096;
break;
1027 case 0x47: l3 = 8192;
break;
1028 case 0x48: l2 = 3072;
break;
1029 case 0x49:
if(l2!=0) l3 = 4096;
else {check_for_p2_core2=
true; l3 = l2 = 4096;}
break;
1030 case 0x4A: l3 = 6144;
break;
1031 case 0x4B: l3 = 8192;
break;
1032 case 0x4C: l3 = 12288;
break;
1033 case 0x4D: l3 = 16384;
break;
1034 case 0x4E: l2 = 6144;
break;
1035 case 0x78: l2 = 1024;
break;
1036 case 0x79: l2 = 128;
break;
1037 case 0x7A: l2 = 256;
break;
1038 case 0x7B: l2 = 512;
break;
1039 case 0x7C: l2 = 1024;
break;
1040 case 0x7D: l2 = 2048;
break;
1041 case 0x7E: l2 = 256;
break;
1042 case 0x7F: l2 = 512;
break;
1043 case 0x80: l2 = 512;
break;
1044 case 0x81: l2 = 128;
break;
1045 case 0x82: l2 = 256;
break;
1046 case 0x83: l2 = 512;
break;
1047 case 0x84: l2 = 1024;
break;
1048 case 0x85: l2 = 2048;
break;
1049 case 0x86: l2 = 512;
break;
1050 case 0x87: l2 = 1024;
break;
1051 case 0x88: l3 = 2048;
break;
1052 case 0x89: l3 = 4096;
break;
1053 case 0x8A: l3 = 8192;
break;
1054 case 0x8D: l3 = 3072;
break;
1059 if(check_for_p2_core2 && l2 == l3)
1066inline void queryCacheSizes_intel(
int& l1,
int& l2,
int& l3,
int max_std_funcs)
1068 if(max_std_funcs>=4)
1069 queryCacheSizes_intel_direct(l1,l2,l3);
1070 else if(max_std_funcs>=2)
1071 queryCacheSizes_intel_codes(l1,l2,l3);
1076inline void queryCacheSizes_amd(
int& l1,
int& l2,
int& l3)
1079 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1082 EIGEN_CPUID(abcd,0x80000000,0);
1083 if(
static_cast<numext::uint32_t
>(abcd[0]) >=
static_cast<numext::uint32_t
>(0x80000006))
1085 EIGEN_CPUID(abcd,0x80000005,0);
1086 l1 = (abcd[2] >> 24) * 1024;
1087 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1088 EIGEN_CPUID(abcd,0x80000006,0);
1089 l2 = (abcd[2] >> 16) * 1024;
1090 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024;
1101inline void queryCacheSizes(
int& l1,
int& l2,
int& l3)
1105 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
1106 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
1107 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574};
1110 EIGEN_CPUID(abcd,0x0,0);
1111 int max_std_funcs = abcd[0];
1112 if(cpuid_is_vendor(abcd,GenuineIntel))
1113 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
1114 else if(cpuid_is_vendor(abcd,AuthenticAMD) || cpuid_is_vendor(abcd,AMDisbetter_))
1115 queryCacheSizes_amd(l1,l2,l3);
1118 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
1138inline int queryL1CacheSize()
1141 queryCacheSizes(l1,l2,l3);
1147inline int queryTopLevelCacheSize()
1149 int l1, l2(-1), l3(-1);
1150 queryCacheSizes(l1,l2,l3);
1151 return (std::max)(l2,l3);
STL compatible allocator to use with types requiring a non-standard alignment.
Definition: Memory.h:874
static const lastp1_t end
Definition: IndexedViewHelper.h:183
Namespace containing all symbols from the Eigen library.
Definition: B01_Experimental.dox:1
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:59
const int Dynamic
Definition: Constants.h:24