Eigen  5.0.1-dev
Loading...
Searching...
No Matches
Memory.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2015 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2008-2009 Benoit Jacob <jacob.benoit.1@gmail.com>
6// Copyright (C) 2009 Kenneth Riddile <kfriddile@yahoo.com>
7// Copyright (C) 2010 Hauke Heibel <hauke.heibel@gmail.com>
8// Copyright (C) 2010 Thomas Capricelli <orzel@freehackers.org>
9// Copyright (C) 2013 Pavel Holoborodko <pavel@holoborodko.com>
10//
11// This Source Code Form is subject to the terms of the Mozilla
12// Public License v. 2.0. If a copy of the MPL was not distributed
13// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
14
15/*****************************************************************************
16*** Platform checks for aligned malloc functions ***
17*****************************************************************************/
18
19#ifndef EIGEN_MEMORY_H
20#define EIGEN_MEMORY_H
21
22// Debian: aligned_malloc/free/realloc below are inline, so every translation
23// unit emits a weak copy of which the linker keeps one. Neither the allocator
24// they use nor the alignment they return may depend on the build flags, or
25// objects built with different -m flags crash once linked together.
26// See Debian bug #1064320.
27#undef EIGEN_MALLOC_ALREADY_ALIGNED
28#define EIGEN_MALLOC_ALREADY_ALIGNED 0
29
30// Fixed per-architecture alignment, wide enough for any SIMD flags.
31#if defined(__x86_64__) || defined(__i386__)
32#define EIGEN_HEAP_ALIGN_BYTES 64 // AVX-512
33#elif defined(__HVX__) && (__HVX_LENGTH__ == 128)
34#define EIGEN_HEAP_ALIGN_BYTES 128
35#else
36#define EIGEN_HEAP_ALIGN_BYTES 16
37#endif
38
39#ifndef EIGEN_MALLOC_CHECK_THREAD_LOCAL
40
41// Check whether we can use the thread_local keyword to allow or disallow
42// allocating memory with per-thread granularity, by means of the
43// set_is_malloc_allowed() function.
44#ifndef EIGEN_AVOID_THREAD_LOCAL
45
46#if ((EIGEN_COMP_GNUC) || __has_feature(cxx_thread_local) || EIGEN_COMP_MSVC >= 1900) && \
47 !defined(EIGEN_GPU_COMPILE_PHASE)
48#define EIGEN_MALLOC_CHECK_THREAD_LOCAL thread_local
49#else
50#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
51#endif
52
53#else // EIGEN_AVOID_THREAD_LOCAL
54#define EIGEN_MALLOC_CHECK_THREAD_LOCAL
55#endif // EIGEN_AVOID_THREAD_LOCAL
56
57#endif
58
59// IWYU pragma: private
60#include "../InternalHeaderCheck.h"
61
62namespace Eigen {
63
64namespace internal {
65
66/*****************************************************************************
67*** Implementation of portable aligned versions of malloc/free/realloc ***
68*****************************************************************************/
69
70#ifdef EIGEN_NO_MALLOC
71EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {
72 eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
73}
74EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {
75 eigen_assert(false && "heap deallocation is forbidden (EIGEN_NO_MALLOC is defined)");
76}
77#elif defined EIGEN_RUNTIME_NO_MALLOC
78EIGEN_DEVICE_FUNC inline bool is_malloc_allowed_impl(bool update, bool new_value = false) {
79 EIGEN_MALLOC_CHECK_THREAD_LOCAL static bool value = true;
80 if (update == 1) value = new_value;
81 return value;
82}
83EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); }
84EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); }
85EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {
86 eigen_assert(is_malloc_allowed() &&
87 "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_malloc_allowed is false)");
88}
89EIGEN_DEVICE_FUNC inline bool is_free_allowed_impl(bool update, bool new_value = false) {
90 EIGEN_MALLOC_CHECK_THREAD_LOCAL static bool value = true;
91 if (update == 1) value = new_value;
92 return value;
93}
94EIGEN_DEVICE_FUNC inline bool is_free_allowed() { return is_free_allowed_impl(false); }
95EIGEN_DEVICE_FUNC inline bool set_is_free_allowed(bool new_value) { return is_free_allowed_impl(true, new_value); }
96EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {
97 eigen_assert(is_malloc_allowed() &&
98 "heap deallocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and set_is_free_allowed is false)");
99}
100#else
101EIGEN_DEVICE_FUNC inline void check_that_malloc_is_allowed() {}
102EIGEN_DEVICE_FUNC inline void check_that_free_is_allowed() {}
103#endif
104
105EIGEN_DEVICE_FUNC inline void throw_std_bad_alloc() {
106#ifdef EIGEN_EXCEPTIONS
107 throw std::bad_alloc();
108#else
109 std::size_t huge = static_cast<std::size_t>(-1);
110#if defined(EIGEN_HIPCC)
111 //
112 // calls to "::operator new" are to be treated as opaque function calls (i.e no inlining),
113 // and as a consequence the code in the #else block triggers the hipcc warning :
114 // "no overloaded function has restriction specifiers that are compatible with the ambient context"
115 //
116 // "throw_std_bad_alloc" has the EIGEN_DEVICE_FUNC attribute, so it seems that hipcc expects
117 // the same on "operator new"
118 // Reverting code back to the old version in this #if block for the hipcc compiler
119 //
120 new int[huge];
121#else
122 void* unused = ::operator new(huge);
123 EIGEN_UNUSED_VARIABLE(unused);
124#endif
125#endif
126}
127
128/*****************************************************************************
129*** Implementation of handmade aligned functions ***
130*****************************************************************************/
131
132/* ----- Hand made implementations of aligned malloc/free and realloc ----- */
133
137EIGEN_DEVICE_FUNC inline void* handmade_aligned_malloc(std::size_t size,
138 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
139 eigen_assert(alignment >= sizeof(void*) && alignment <= 256 && (alignment & (alignment - 1)) == 0 &&
140 "Alignment must be at least sizeof(void*), less than or equal to 256, and a power of 2");
141
142 check_that_malloc_is_allowed();
143 EIGEN_USING_STD(malloc)
144 void* original = malloc(size + alignment);
145 if (original == nullptr) return nullptr;
146 std::size_t offset = alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1));
147 void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
148 // Store offset - 1, since it is guaranteed to be at least 1.
149 *(static_cast<uint8_t*>(aligned) - 1) = static_cast<uint8_t>(offset - 1);
150 return aligned;
151}
152
154EIGEN_DEVICE_FUNC inline void handmade_aligned_free(void* ptr) {
155 if (ptr != nullptr) {
156 std::size_t offset = static_cast<std::size_t>(*(static_cast<uint8_t*>(ptr) - 1)) + 1;
157 void* original = static_cast<void*>(static_cast<uint8_t*>(ptr) - offset);
158
159 check_that_free_is_allowed();
160 EIGEN_USING_STD(free)
161 free(original);
162 }
163}
164
170EIGEN_DEVICE_FUNC inline void* handmade_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size,
171 std::size_t alignment = EIGEN_DEFAULT_ALIGN_BYTES) {
172 if (ptr == nullptr) return handmade_aligned_malloc(new_size, alignment);
173 std::size_t old_offset = static_cast<std::size_t>(*(static_cast<uint8_t*>(ptr) - 1)) + 1;
174 void* old_original = static_cast<uint8_t*>(ptr) - old_offset;
175
176 check_that_malloc_is_allowed();
177 EIGEN_USING_STD(realloc)
178 void* original = realloc(old_original, new_size + alignment);
179 if (original == nullptr) return nullptr;
180 if (original == old_original) return ptr;
181 std::size_t offset = alignment - (reinterpret_cast<std::size_t>(original) & (alignment - 1));
182 void* aligned = static_cast<void*>(static_cast<uint8_t*>(original) + offset);
183 if (offset != old_offset) {
184 const void* src = static_cast<const void*>(static_cast<uint8_t*>(original) + old_offset);
185 std::size_t count = (std::min)(new_size, old_size);
186 std::memmove(aligned, src, count);
187 }
188 // Store offset - 1, since it is guaranteed to be at least 1.
189 *(static_cast<uint8_t*>(aligned) - 1) = static_cast<uint8_t>(offset - 1);
190 return aligned;
191}
192
196EIGEN_DEVICE_FUNC inline void* aligned_malloc(std::size_t size) {
197 if (size == 0) return nullptr;
198
199 void* result = handmade_aligned_malloc(size, EIGEN_HEAP_ALIGN_BYTES);
200
201 if (!result && size) throw_std_bad_alloc();
202
203 return result;
204}
205
207EIGEN_DEVICE_FUNC inline void aligned_free(void* ptr) { handmade_aligned_free(ptr); }
208
214EIGEN_DEVICE_FUNC inline void* aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) {
215 if (ptr == nullptr) return aligned_malloc(new_size);
216 if (old_size == new_size) return ptr;
217 if (new_size == 0) {
218 aligned_free(ptr);
219 return nullptr;
220 }
221
222 void* result = handmade_aligned_realloc(ptr, new_size, old_size, EIGEN_HEAP_ALIGN_BYTES);
223
224 if (!result && new_size) throw_std_bad_alloc();
225
226 return result;
227}
228
229/*****************************************************************************
230*** Implementation of conditionally aligned functions ***
231*****************************************************************************/
232
236template <bool Align>
237EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc(std::size_t size) {
238 return aligned_malloc(size);
239}
240
241template <>
242EIGEN_DEVICE_FUNC inline void* conditional_aligned_malloc<false>(std::size_t size) {
243 if (size == 0) return nullptr;
244
245 check_that_malloc_is_allowed();
246 EIGEN_USING_STD(malloc)
247 void* result = malloc(size);
248
249 if (!result && size) throw_std_bad_alloc();
250 return result;
251}
252
254template <bool Align>
255EIGEN_DEVICE_FUNC inline void conditional_aligned_free(void* ptr) {
256 aligned_free(ptr);
257}
258
259template <>
260EIGEN_DEVICE_FUNC inline void conditional_aligned_free<false>(void* ptr) {
261 if (ptr != nullptr) {
262 check_that_free_is_allowed();
263 EIGEN_USING_STD(free)
264 free(ptr);
265 }
266}
267
268template <bool Align>
269EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc(void* ptr, std::size_t new_size, std::size_t old_size) {
270 return aligned_realloc(ptr, new_size, old_size);
271}
272
273template <>
274EIGEN_DEVICE_FUNC inline void* conditional_aligned_realloc<false>(void* ptr, std::size_t new_size,
275 std::size_t old_size) {
276 if (ptr == nullptr) return conditional_aligned_malloc<false>(new_size);
277 if (old_size == new_size) return ptr;
278 if (new_size == 0) {
279 conditional_aligned_free<false>(ptr);
280 return nullptr;
281 }
282
283 check_that_malloc_is_allowed();
284 EIGEN_USING_STD(realloc)
285 return realloc(ptr, new_size);
286}
287
288/*****************************************************************************
289*** Construction/destruction of array elements ***
290*****************************************************************************/
291
295template <typename T>
296EIGEN_DEVICE_FUNC inline void destruct_elements_of_array(T* ptr, std::size_t size) {
297 // always destruct an array starting from the end.
298 if (ptr)
299 while (size) ptr[--size].~T();
300}
301
305template <typename T>
306EIGEN_DEVICE_FUNC inline T* default_construct_elements_of_array(T* ptr, std::size_t size) {
307 std::size_t i = 0;
308 EIGEN_TRY {
309 for (i = 0; i < size; ++i) ::new (ptr + i) T;
310 }
311 EIGEN_CATCH(...) {
312 destruct_elements_of_array(ptr, i);
313 EIGEN_THROW;
314 }
315 return ptr;
316}
317
321template <typename T>
322EIGEN_DEVICE_FUNC inline T* copy_construct_elements_of_array(T* ptr, const T* src, std::size_t size) {
323 std::size_t i = 0;
324 EIGEN_TRY {
325 for (i = 0; i < size; ++i) ::new (ptr + i) T(*(src + i));
326 }
327 EIGEN_CATCH(...) {
328 destruct_elements_of_array(ptr, i);
329 EIGEN_THROW;
330 }
331 return ptr;
332}
333
337template <typename T>
338EIGEN_DEVICE_FUNC inline T* move_construct_elements_of_array(T* ptr, T* src, std::size_t size) {
339 std::size_t i = 0;
340 EIGEN_TRY {
341 for (i = 0; i < size; ++i) ::new (ptr + i) T(std::move(*(src + i)));
342 }
343 EIGEN_CATCH(...) {
344 destruct_elements_of_array(ptr, i);
345 EIGEN_THROW;
346 }
347 return ptr;
348}
349
350/*****************************************************************************
351*** Implementation of aligned new/delete-like functions ***
352*****************************************************************************/
353
354template <typename T>
355EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void check_size_for_overflow(std::size_t size) {
356 constexpr std::size_t max_elements = (std::numeric_limits<std::ptrdiff_t>::max)() / sizeof(T);
357 if (size > max_elements) throw_std_bad_alloc();
358}
359
364template <typename T>
365EIGEN_DEVICE_FUNC inline T* aligned_new(std::size_t size) {
366 check_size_for_overflow<T>(size);
367 T* result = static_cast<T*>(aligned_malloc(sizeof(T) * size));
368 EIGEN_TRY { return default_construct_elements_of_array(result, size); }
369 EIGEN_CATCH(...) {
370 aligned_free(result);
371 EIGEN_THROW;
372 }
373 return result;
374}
375
376template <typename T, bool Align>
377EIGEN_DEVICE_FUNC inline T* conditional_aligned_new(std::size_t size) {
378 check_size_for_overflow<T>(size);
379 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
380 EIGEN_TRY { return default_construct_elements_of_array(result, size); }
381 EIGEN_CATCH(...) {
382 conditional_aligned_free<Align>(result);
383 EIGEN_THROW;
384 }
385 return result;
386}
387
391template <typename T>
392EIGEN_DEVICE_FUNC inline void aligned_delete(T* ptr, std::size_t size) {
393 destruct_elements_of_array<T>(ptr, size);
394 aligned_free(ptr);
395}
396
400template <typename T, bool Align>
401EIGEN_DEVICE_FUNC inline void conditional_aligned_delete(T* ptr, std::size_t size) {
402 destruct_elements_of_array<T>(ptr, size);
403 conditional_aligned_free<Align>(ptr);
404}
405
406template <typename T, bool Align>
407EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new(T* pts, std::size_t new_size, std::size_t old_size) {
408 check_size_for_overflow<T>(new_size);
409 check_size_for_overflow<T>(old_size);
410
411 // If elements need to be explicitly initialized, we cannot simply realloc
412 // (or memcpy) the memory block - each element needs to be reconstructed.
413 // Otherwise, objects that contain internal pointers like mpfr or
414 // AnnoyingScalar can be pointing to the wrong thing.
415 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * new_size));
416 EIGEN_TRY {
417 // Move-construct initial elements.
418 std::size_t copy_size = (std::min)(old_size, new_size);
419 move_construct_elements_of_array(result, pts, copy_size);
420
421 // Default-construct remaining elements.
422 if (new_size > old_size) {
423 default_construct_elements_of_array(result + copy_size, new_size - old_size);
424 }
425
426 // Delete old elements.
427 conditional_aligned_delete<T, Align>(pts, old_size);
428 }
429 EIGEN_CATCH(...) {
430 conditional_aligned_free<Align>(result);
431 EIGEN_THROW;
432 }
433
434 return result;
435}
436
437template <typename T, bool Align>
438EIGEN_DEVICE_FUNC inline T* conditional_aligned_new_auto(std::size_t size) {
439 if (size == 0) return nullptr; // short-cut. Also fixes Bug 884
440 check_size_for_overflow<T>(size);
441 T* result = static_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T) * size));
442 if (NumTraits<T>::RequireInitialization) {
443 EIGEN_TRY { default_construct_elements_of_array(result, size); }
444 EIGEN_CATCH(...) {
445 conditional_aligned_free<Align>(result);
446 EIGEN_THROW;
447 }
448 }
449 return result;
450}
451
452template <typename T, bool Align>
453EIGEN_DEVICE_FUNC inline T* conditional_aligned_realloc_new_auto(T* pts, std::size_t new_size, std::size_t old_size) {
454 if (NumTraits<T>::RequireInitialization) {
455 return conditional_aligned_realloc_new<T, Align>(pts, new_size, old_size);
456 }
457
458 check_size_for_overflow<T>(new_size);
459 check_size_for_overflow<T>(old_size);
460 return static_cast<T*>(
461 conditional_aligned_realloc<Align>(static_cast<void*>(pts), sizeof(T) * new_size, sizeof(T) * old_size));
462}
463
464template <typename T, bool Align>
465EIGEN_DEVICE_FUNC inline void conditional_aligned_delete_auto(T* ptr, std::size_t size) {
466 if (NumTraits<T>::RequireInitialization) destruct_elements_of_array<T>(ptr, size);
467 conditional_aligned_free<Align>(ptr);
468}
469
470/****************************************************************************/
471
490template <int Alignment, typename Scalar, typename Index>
491EIGEN_DEVICE_FUNC inline Index first_aligned(const Scalar* array, Index size) {
492 const Index ScalarSize = sizeof(Scalar);
493 const Index AlignmentSize = Alignment / ScalarSize;
494 const Index AlignmentMask = AlignmentSize - 1;
495
496 if (AlignmentSize <= 1) {
497 // Either the requested alignment if smaller than a scalar, or it exactly match a 1 scalar
498 // so that all elements of the array have the same alignment.
499 return 0;
500 } else if ((std::uintptr_t(array) & (sizeof(Scalar) - 1)) || (Alignment % ScalarSize) != 0) {
501 // The array is not aligned to the size of a single scalar, or the requested alignment is not a multiple of the
502 // scalar size. Consequently, no element of the array is well aligned.
503 return size;
504 } else {
505 Index first = (AlignmentSize - (Index((std::uintptr_t(array) / sizeof(Scalar))) & AlignmentMask)) & AlignmentMask;
506 return (first < size) ? first : size;
507 }
508}
509
512template <typename Scalar, typename Index>
513EIGEN_DEVICE_FUNC inline Index first_default_aligned(const Scalar* array, Index size) {
514 typedef typename packet_traits<Scalar>::type DefaultPacketType;
515 return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(array, size);
516}
517
520template <typename Index>
521inline Index first_multiple(Index size, Index base) {
522 return ((size + base - 1) / base) * base;
523}
524
525// std::copy is much slower than memcpy, so let's introduce a smart_copy which
526// use memcpy on trivial types, i.e., on types that does not require an initialization ctor.
527template <typename T, bool UseMemcpy>
528struct smart_copy_helper;
529
530template <typename T>
531EIGEN_DEVICE_FUNC void smart_copy(const T* start, const T* end, T* target) {
532 smart_copy_helper<T, !NumTraits<T>::RequireInitialization>::run(start, end, target);
533}
534
535template <typename T>
536struct smart_copy_helper<T, true> {
537 EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) {
538 std::intptr_t size = std::intptr_t(end) - std::intptr_t(start);
539 if (size == 0) return;
540 eigen_internal_assert(start != 0 && end != 0 && target != 0);
541 EIGEN_USING_STD(memcpy)
542 memcpy(target, start, size);
543 }
544};
545
546template <typename T>
547struct smart_copy_helper<T, false> {
548 EIGEN_DEVICE_FUNC static inline void run(const T* start, const T* end, T* target) { std::copy(start, end, target); }
549};
550
551// intelligent memmove. falls back to std::memmove for POD types, uses std::copy otherwise.
552template <typename T, bool UseMemmove>
553struct smart_memmove_helper;
554
555template <typename T>
556void smart_memmove(const T* start, const T* end, T* target) {
557 smart_memmove_helper<T, !NumTraits<T>::RequireInitialization>::run(start, end, target);
558}
559
560template <typename T>
561struct smart_memmove_helper<T, true> {
562 static inline void run(const T* start, const T* end, T* target) {
563 std::intptr_t size = std::intptr_t(end) - std::intptr_t(start);
564 if (size == 0) return;
565 eigen_internal_assert(start != 0 && end != 0 && target != 0);
566 std::memmove(target, start, size);
567 }
568};
569
570template <typename T>
571struct smart_memmove_helper<T, false> {
572 static inline void run(const T* start, const T* end, T* target) {
573 if (std::uintptr_t(target) < std::uintptr_t(start)) {
574 std::copy(start, end, target);
575 } else {
576 std::ptrdiff_t count = (std::ptrdiff_t(end) - std::ptrdiff_t(start)) / sizeof(T);
577 std::copy_backward(start, end, target + count);
578 }
579 }
580};
581
582template <typename T>
583EIGEN_DEVICE_FUNC T* smart_move(T* start, T* end, T* target) {
584 return std::move(start, end, target);
585}
586
587/*****************************************************************************
588*** Implementation of runtime stack allocation (falling back to malloc) ***
589*****************************************************************************/
590
591// you can overwrite Eigen's default behavior regarding alloca by defining EIGEN_ALLOCA
592// to the appropriate stack allocation function
593#if !defined EIGEN_ALLOCA && !defined EIGEN_GPU_COMPILE_PHASE
594#if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca)
595#define EIGEN_ALLOCA alloca
596#elif EIGEN_COMP_MSVC
597#define EIGEN_ALLOCA _alloca
598#endif
599#endif
600
601// With clang -Oz -mthumb, alloca changes the stack pointer in a way that is
602// not allowed in Thumb2. -DEIGEN_STACK_ALLOCATION_LIMIT=0 doesn't work because
603// the compiler still emits bad code because stack allocation checks use "<=".
604// TODO: Eliminate after https://bugs.llvm.org/show_bug.cgi?id=23772
605// is fixed.
606#if defined(__clang__) && defined(__thumb__)
607#undef EIGEN_ALLOCA
608#endif
609
610// This helper class construct the allocated memory, and takes care of destructing and freeing the handled data
611// at destruction time. In practice this helper class is mainly useful to avoid memory leak in case of exceptions.
612template <typename T>
613class aligned_stack_memory_handler : noncopyable {
614 public:
615 /* Creates a stack_memory_handler responsible for the buffer \a ptr of size \a size.
616 * Note that \a ptr can be 0 regardless of the other parameters.
617 * This constructor takes care of constructing/initializing the elements of the buffer if required by the scalar type
618 *T (see NumTraits<T>::RequireInitialization). In this case, the buffer elements will also be destructed when this
619 *handler will be destructed. Finally, if \a dealloc is true, then the pointer \a ptr is freed.
620 **/
621 EIGEN_DEVICE_FUNC aligned_stack_memory_handler(T* ptr, std::size_t size, bool dealloc)
622 : m_ptr(ptr), m_size(size), m_deallocate(dealloc) {
623 if (NumTraits<T>::RequireInitialization && m_ptr) Eigen::internal::default_construct_elements_of_array(m_ptr, size);
624 }
625 EIGEN_DEVICE_FUNC ~aligned_stack_memory_handler() {
626 if (NumTraits<T>::RequireInitialization && m_ptr) Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
627 if (m_deallocate) Eigen::internal::aligned_free(m_ptr);
628 }
629
630 protected:
631 T* m_ptr;
632 std::size_t m_size;
633 bool m_deallocate;
634};
635
636#ifdef EIGEN_ALLOCA
637
638template <typename Xpr, int NbEvaluations,
639 bool MapExternalBuffer = nested_eval<Xpr, NbEvaluations>::Evaluate && Xpr::MaxSizeAtCompileTime == Dynamic>
640struct local_nested_eval_wrapper {
641 static constexpr bool NeedExternalBuffer = false;
642 typedef typename Xpr::Scalar Scalar;
643 typedef typename nested_eval<Xpr, NbEvaluations>::type ObjectType;
644 ObjectType object;
645
646 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr) : object(xpr) {
647 EIGEN_UNUSED_VARIABLE(ptr);
648 eigen_internal_assert(ptr == 0);
649 }
650};
651
652template <typename Xpr, int NbEvaluations>
653struct local_nested_eval_wrapper<Xpr, NbEvaluations, true> {
654 static constexpr bool NeedExternalBuffer = true;
655 typedef typename Xpr::Scalar Scalar;
656 typedef typename plain_object_eval<Xpr>::type PlainObject;
657 typedef Map<PlainObject, EIGEN_DEFAULT_ALIGN_BYTES> ObjectType;
658 ObjectType object;
659
660 EIGEN_DEVICE_FUNC local_nested_eval_wrapper(const Xpr& xpr, Scalar* ptr)
661 : object(ptr == 0 ? reinterpret_cast<Scalar*>(Eigen::internal::aligned_malloc(sizeof(Scalar) * xpr.size())) : ptr,
662 xpr.rows(), xpr.cols()),
663 m_deallocate(ptr == 0) {
664 if (NumTraits<Scalar>::RequireInitialization && object.data())
665 Eigen::internal::default_construct_elements_of_array(object.data(), object.size());
666 object = xpr;
667 }
668
669 EIGEN_DEVICE_FUNC ~local_nested_eval_wrapper() {
670 if (NumTraits<Scalar>::RequireInitialization && object.data())
671 Eigen::internal::destruct_elements_of_array(object.data(), object.size());
672 if (m_deallocate) Eigen::internal::aligned_free(object.data());
673 }
674
675 private:
676 bool m_deallocate;
677};
678
679#endif // EIGEN_ALLOCA
680
681template <typename T>
682class scoped_array : noncopyable {
683 T* m_ptr;
684
685 public:
686 explicit scoped_array(std::ptrdiff_t size) { m_ptr = new T[size]; }
687 ~scoped_array() { delete[] m_ptr; }
688 T& operator[](std::ptrdiff_t i) { return m_ptr[i]; }
689 const T& operator[](std::ptrdiff_t i) const { return m_ptr[i]; }
690 T*& ptr() { return m_ptr; }
691 const T* ptr() const { return m_ptr; }
692 operator const T*() const { return m_ptr; }
693};
694
695template <typename T>
696void swap(scoped_array<T>& a, scoped_array<T>& b) {
697 std::swap(a.ptr(), b.ptr());
698}
699
700} // end namespace internal
701
725#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
726
727#if EIGEN_DEFAULT_ALIGN_BYTES > 0
728// We always manually re-align the result of EIGEN_ALLOCA.
729// If alloca is already aligned, the compiler should be smart enough to optimize away the re-alignment.
730
731#if ((EIGEN_COMP_GNUC || EIGEN_COMP_CLANG) && !EIGEN_COMP_NVHPC)
732#define EIGEN_ALIGNED_ALLOCA(SIZE) __builtin_alloca_with_align(SIZE, CHAR_BIT* EIGEN_DEFAULT_ALIGN_BYTES)
733#else
734EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void* eigen_aligned_alloca_helper(void* ptr) {
735 constexpr std::uintptr_t mask = EIGEN_DEFAULT_ALIGN_BYTES - 1;
736 std::uintptr_t ptr_int = std::uintptr_t(ptr);
737 std::uintptr_t aligned_ptr_int = (ptr_int + mask) & ~mask;
738 std::uintptr_t offset = aligned_ptr_int - ptr_int;
739 return static_cast<void*>(static_cast<uint8_t*>(ptr) + offset);
740}
741#define EIGEN_ALIGNED_ALLOCA(SIZE) eigen_aligned_alloca_helper(EIGEN_ALLOCA(SIZE + EIGEN_DEFAULT_ALIGN_BYTES - 1))
742#endif
743
744#else
745#define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE)
746#endif
747
748#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
749 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
750 TYPE* NAME = (BUFFER) != 0 ? (BUFFER) \
751 : reinterpret_cast<TYPE*>((sizeof(TYPE) * (SIZE) <= EIGEN_STACK_ALLOCATION_LIMIT) \
752 ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE) * (SIZE)) \
753 : Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \
754 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
755 (BUFFER) == 0 ? NAME : 0, SIZE, sizeof(TYPE) * (SIZE) > EIGEN_STACK_ALLOCATION_LIMIT)
756
757#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
758 Eigen::internal::local_nested_eval_wrapper<XPR_T, N> EIGEN_CAT(NAME, _wrapper)( \
759 XPR, reinterpret_cast<typename XPR_T::Scalar*>( \
760 ((Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::NeedExternalBuffer) && \
761 ((sizeof(typename XPR_T::Scalar) * XPR.size()) <= EIGEN_STACK_ALLOCATION_LIMIT)) \
762 ? EIGEN_ALIGNED_ALLOCA(sizeof(typename XPR_T::Scalar) * XPR.size()) \
763 : 0)); \
764 typename Eigen::internal::local_nested_eval_wrapper<XPR_T, N>::ObjectType NAME(EIGEN_CAT(NAME, _wrapper).object)
765
766#else
767
768#define ei_declare_aligned_stack_constructed_variable(TYPE, NAME, SIZE, BUFFER) \
769 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
770 TYPE* NAME = \
771 (BUFFER) != 0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE) * (SIZE))); \
772 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME, _stack_memory_destructor)( \
773 (BUFFER) == 0 ? NAME : 0, SIZE, true)
774
775#define ei_declare_local_nested_eval(XPR_T, XPR, N, NAME) \
776 typename Eigen::internal::nested_eval<XPR_T, N>::type NAME(XPR)
777
778#endif
779
780/*****************************************************************************
781*** Implementation of EIGEN_MAKE_ALIGNED_OPERATOR_NEW [_IF] ***
782*****************************************************************************/
783
784#if EIGEN_HAS_CXX17_OVERALIGN
785
786// C++17 -> no need to bother about alignment anymore :)
787
788#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign)
789#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
790#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW
791#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size)
792
793#else
794
795// HIP does not support new/delete on device.
796#if EIGEN_MAX_ALIGN_BYTES != 0 && !defined(EIGEN_HIP_DEVICE_COMPILE)
797#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
798 EIGEN_DEVICE_FUNC void* operator new(std::size_t size, const std::nothrow_t&) noexcept { \
799 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
800 EIGEN_CATCH(...) { return 0; } \
801 }
802#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
803 EIGEN_DEVICE_FUNC void* operator new(std::size_t size) { \
804 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
805 } \
806 EIGEN_DEVICE_FUNC void* operator new[](std::size_t size) { \
807 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
808 } \
809 EIGEN_DEVICE_FUNC void operator delete(void* ptr) noexcept { \
810 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
811 } \
812 EIGEN_DEVICE_FUNC void operator delete[](void* ptr) noexcept { \
813 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
814 } \
815 EIGEN_DEVICE_FUNC void operator delete(void* ptr, std::size_t /* sz */) noexcept { \
816 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
817 } \
818 EIGEN_DEVICE_FUNC void operator delete[](void* ptr, std::size_t /* sz */) noexcept { \
819 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
820 } \
821 /* in-place new and delete. since (at least afaik) there is no actual */ \
822 /* memory allocated we can safely let the default implementation handle */ \
823 /* this particular case. */ \
824 EIGEN_DEVICE_FUNC static void* operator new(std::size_t size, void* ptr) { return ::operator new(size, ptr); } \
825 EIGEN_DEVICE_FUNC static void* operator new[](std::size_t size, void* ptr) { return ::operator new[](size, ptr); } \
826 EIGEN_DEVICE_FUNC void operator delete(void* memory, void* ptr) noexcept { return ::operator delete(memory, ptr); } \
827 EIGEN_DEVICE_FUNC void operator delete[](void* memory, void* ptr) noexcept { \
828 return ::operator delete[](memory, ptr); \
829 } \
830 /* nothrow-new (returns zero instead of std::bad_alloc) */ \
831 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
832 EIGEN_DEVICE_FUNC void operator delete(void* ptr, const std::nothrow_t&) noexcept { \
833 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
834 } \
835 typedef void eigen_aligned_operator_new_marker_type;
836#else
837#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
838#endif
839
840#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
841#define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar, Size) \
842 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF( \
843 bool(((Size) != Eigen::Dynamic) && \
844 (((EIGEN_MAX_ALIGN_BYTES >= 16) && ((sizeof(Scalar) * (Size)) % (EIGEN_MAX_ALIGN_BYTES) == 0)) || \
845 ((EIGEN_MAX_ALIGN_BYTES >= 32) && ((sizeof(Scalar) * (Size)) % (EIGEN_MAX_ALIGN_BYTES / 2) == 0)) || \
846 ((EIGEN_MAX_ALIGN_BYTES >= 64) && ((sizeof(Scalar) * (Size)) % (EIGEN_MAX_ALIGN_BYTES / 4) == 0)))))
847
848#endif
849
850/****************************************************************************/
851
876template <class T>
877class aligned_allocator {
878 public:
879 typedef std::size_t size_type;
880 typedef std::ptrdiff_t difference_type;
881 typedef T* pointer;
882 typedef const T* const_pointer;
883 typedef T& reference;
884 typedef const T& const_reference;
885 typedef T value_type;
886
887 template <class U>
888 struct rebind {
889 typedef aligned_allocator<U> other;
890 };
891
892 aligned_allocator() = default;
893
894 aligned_allocator(const aligned_allocator&) = default;
895
896 template <class U>
897 aligned_allocator(const aligned_allocator<U>&) {}
898
899 template <class U>
900 constexpr bool operator==(const aligned_allocator<U>&) const noexcept {
901 return true;
902 }
903 template <class U>
904 constexpr bool operator!=(const aligned_allocator<U>&) const noexcept {
905 return false;
906 }
907
908#if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_STRICT_AT_LEAST(7, 0, 0)
909 // In gcc std::allocator::max_size() is bugged making gcc triggers a warning:
910 // eigen/Eigen/src/Core/util/Memory.h:189:12: warning: argument 1 value '18446744073709551612' exceeds maximum object
911 // size 9223372036854775807 See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87544
912 size_type max_size() const { return (std::numeric_limits<std::ptrdiff_t>::max)() / sizeof(T); }
913#endif
914
915 pointer allocate(size_type num, const void* /*hint*/ = 0) {
916 internal::check_size_for_overflow<T>(num);
917 return static_cast<pointer>(internal::aligned_malloc(num * sizeof(T)));
918 }
919
920 void deallocate(pointer p, size_type /*num*/) { internal::aligned_free(p); }
921};
922
923//---------- Cache sizes ----------
924
925#if !defined(EIGEN_NO_CPUID)
926#if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64
927#if defined(__PIC__) && EIGEN_ARCH_i386
928// Case for x86 with PIC
929#define EIGEN_CPUID(abcd, func, id) \
930 __asm__ __volatile__("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1" \
931 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
932 : "a"(func), "c"(id));
933#elif defined(__PIC__) && EIGEN_ARCH_x86_64
934// Case for x64 with PIC. In theory this is only a problem with recent gcc and with medium or large code model, not with
935// the default small code model. However, we cannot detect which code model is used, and the xchg overhead is negligible
936// anyway.
937#define EIGEN_CPUID(abcd, func, id) \
938 __asm__ __volatile__("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1" \
939 : "=a"(abcd[0]), "=&r"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) \
940 : "0"(func), "2"(id));
941#else
942// Case for x86_64 or x86 w/o PIC
943#define EIGEN_CPUID(abcd, func, id) \
944 __asm__ __volatile__("cpuid" : "=a"(abcd[0]), "=b"(abcd[1]), "=c"(abcd[2]), "=d"(abcd[3]) : "0"(func), "2"(id));
945#endif
946#elif EIGEN_COMP_MSVC
947#if EIGEN_ARCH_i386_OR_x86_64
948#define EIGEN_CPUID(abcd, func, id) __cpuidex((int*)abcd, func, id)
949#endif
950#endif
951#endif
952
953namespace internal {
954
955#ifdef EIGEN_CPUID
956
957inline bool cpuid_is_vendor(int abcd[4], const int vendor[3]) {
958 return abcd[1] == vendor[0] && abcd[3] == vendor[1] && abcd[2] == vendor[2];
959}
960
961inline void queryCacheSizes_intel_direct(int& l1, int& l2, int& l3) {
962 int abcd[4];
963 l1 = l2 = l3 = 0;
964 int cache_id = 0;
965 int cache_type = 0;
966 do {
967 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
968 EIGEN_CPUID(abcd, 0x4, cache_id);
969 cache_type = (abcd[0] & 0x0F) >> 0;
970 if (cache_type == 1 || cache_type == 3) // data or unified cache
971 {
972 int cache_level = (abcd[0] & 0xE0) >> 5; // A[7:5]
973 int ways = (abcd[1] & 0xFFC00000) >> 22; // B[31:22]
974 int partitions = (abcd[1] & 0x003FF000) >> 12; // B[21:12]
975 int line_size = (abcd[1] & 0x00000FFF) >> 0; // B[11:0]
976 int sets = (abcd[2]); // C[31:0]
977
978 int cache_size = (ways + 1) * (partitions + 1) * (line_size + 1) * (sets + 1);
979
980 switch (cache_level) {
981 case 1:
982 l1 = cache_size;
983 break;
984 case 2:
985 l2 = cache_size;
986 break;
987 case 3:
988 l3 = cache_size;
989 break;
990 default:
991 break;
992 }
993 }
994 cache_id++;
995 } while (cache_type > 0 && cache_id < 16);
996}
997
998inline void queryCacheSizes_intel_codes(int& l1, int& l2, int& l3) {
999 int abcd[4];
1000 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1001 l1 = l2 = l3 = 0;
1002 EIGEN_CPUID(abcd, 0x00000002, 0);
1003 unsigned char* bytes = reinterpret_cast<unsigned char*>(abcd) + 2;
1004 bool check_for_p2_core2 = false;
1005 for (int i = 0; i < 14; ++i) {
1006 switch (bytes[i]) {
1007 case 0x0A:
1008 l1 = 8;
1009 break; // 0Ah data L1 cache, 8 KB, 2 ways, 32 byte lines
1010 case 0x0C:
1011 l1 = 16;
1012 break; // 0Ch data L1 cache, 16 KB, 4 ways, 32 byte lines
1013 case 0x0E:
1014 l1 = 24;
1015 break; // 0Eh data L1 cache, 24 KB, 6 ways, 64 byte lines
1016 case 0x10:
1017 l1 = 16;
1018 break; // 10h data L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
1019 case 0x15:
1020 l1 = 16;
1021 break; // 15h code L1 cache, 16 KB, 4 ways, 32 byte lines (IA-64)
1022 case 0x2C:
1023 l1 = 32;
1024 break; // 2Ch data L1 cache, 32 KB, 8 ways, 64 byte lines
1025 case 0x30:
1026 l1 = 32;
1027 break; // 30h code L1 cache, 32 KB, 8 ways, 64 byte lines
1028 case 0x60:
1029 l1 = 16;
1030 break; // 60h data L1 cache, 16 KB, 8 ways, 64 byte lines, sectored
1031 case 0x66:
1032 l1 = 8;
1033 break; // 66h data L1 cache, 8 KB, 4 ways, 64 byte lines, sectored
1034 case 0x67:
1035 l1 = 16;
1036 break; // 67h data L1 cache, 16 KB, 4 ways, 64 byte lines, sectored
1037 case 0x68:
1038 l1 = 32;
1039 break; // 68h data L1 cache, 32 KB, 4 ways, 64 byte lines, sectored
1040 case 0x1A:
1041 l2 = 96;
1042 break; // code and data L2 cache, 96 KB, 6 ways, 64 byte lines (IA-64)
1043 case 0x22:
1044 l3 = 512;
1045 break; // code and data L3 cache, 512 KB, 4 ways (!), 64 byte lines, dual-sectored
1046 case 0x23:
1047 l3 = 1024;
1048 break; // code and data L3 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
1049 case 0x25:
1050 l3 = 2048;
1051 break; // code and data L3 cache, 2048 KB, 8 ways, 64 byte lines, dual-sectored
1052 case 0x29:
1053 l3 = 4096;
1054 break; // code and data L3 cache, 4096 KB, 8 ways, 64 byte lines, dual-sectored
1055 case 0x39:
1056 l2 = 128;
1057 break; // code and data L2 cache, 128 KB, 4 ways, 64 byte lines, sectored
1058 case 0x3A:
1059 l2 = 192;
1060 break; // code and data L2 cache, 192 KB, 6 ways, 64 byte lines, sectored
1061 case 0x3B:
1062 l2 = 128;
1063 break; // code and data L2 cache, 128 KB, 2 ways, 64 byte lines, sectored
1064 case 0x3C:
1065 l2 = 256;
1066 break; // code and data L2 cache, 256 KB, 4 ways, 64 byte lines, sectored
1067 case 0x3D:
1068 l2 = 384;
1069 break; // code and data L2 cache, 384 KB, 6 ways, 64 byte lines, sectored
1070 case 0x3E:
1071 l2 = 512;
1072 break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines, sectored
1073 case 0x40:
1074 l2 = 0;
1075 break; // no integrated L2 cache (P6 core) or L3 cache (P4 core)
1076 case 0x41:
1077 l2 = 128;
1078 break; // code and data L2 cache, 128 KB, 4 ways, 32 byte lines
1079 case 0x42:
1080 l2 = 256;
1081 break; // code and data L2 cache, 256 KB, 4 ways, 32 byte lines
1082 case 0x43:
1083 l2 = 512;
1084 break; // code and data L2 cache, 512 KB, 4 ways, 32 byte lines
1085 case 0x44:
1086 l2 = 1024;
1087 break; // code and data L2 cache, 1024 KB, 4 ways, 32 byte lines
1088 case 0x45:
1089 l2 = 2048;
1090 break; // code and data L2 cache, 2048 KB, 4 ways, 32 byte lines
1091 case 0x46:
1092 l3 = 4096;
1093 break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines
1094 case 0x47:
1095 l3 = 8192;
1096 break; // code and data L3 cache, 8192 KB, 8 ways, 64 byte lines
1097 case 0x48:
1098 l2 = 3072;
1099 break; // code and data L2 cache, 3072 KB, 12 ways, 64 byte lines
1100 case 0x49:
1101 if (l2 != 0)
1102 l3 = 4096;
1103 else {
1104 check_for_p2_core2 = true;
1105 l3 = l2 = 4096;
1106 }
1107 break; // code and data L3 cache, 4096 KB, 16 ways, 64 byte lines (P4) or L2 for core2
1108 case 0x4A:
1109 l3 = 6144;
1110 break; // code and data L3 cache, 6144 KB, 12 ways, 64 byte lines
1111 case 0x4B:
1112 l3 = 8192;
1113 break; // code and data L3 cache, 8192 KB, 16 ways, 64 byte lines
1114 case 0x4C:
1115 l3 = 12288;
1116 break; // code and data L3 cache, 12288 KB, 12 ways, 64 byte lines
1117 case 0x4D:
1118 l3 = 16384;
1119 break; // code and data L3 cache, 16384 KB, 16 ways, 64 byte lines
1120 case 0x4E:
1121 l2 = 6144;
1122 break; // code and data L2 cache, 6144 KB, 24 ways, 64 byte lines
1123 case 0x78:
1124 l2 = 1024;
1125 break; // code and data L2 cache, 1024 KB, 4 ways, 64 byte lines
1126 case 0x79:
1127 l2 = 128;
1128 break; // code and data L2 cache, 128 KB, 8 ways, 64 byte lines, dual-sectored
1129 case 0x7A:
1130 l2 = 256;
1131 break; // code and data L2 cache, 256 KB, 8 ways, 64 byte lines, dual-sectored
1132 case 0x7B:
1133 l2 = 512;
1134 break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines, dual-sectored
1135 case 0x7C:
1136 l2 = 1024;
1137 break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines, dual-sectored
1138 case 0x7D:
1139 l2 = 2048;
1140 break; // code and data L2 cache, 2048 KB, 8 ways, 64 byte lines
1141 case 0x7E:
1142 l2 = 256;
1143 break; // code and data L2 cache, 256 KB, 8 ways, 128 byte lines, sect. (IA-64)
1144 case 0x7F:
1145 l2 = 512;
1146 break; // code and data L2 cache, 512 KB, 2 ways, 64 byte lines
1147 case 0x80:
1148 l2 = 512;
1149 break; // code and data L2 cache, 512 KB, 8 ways, 64 byte lines
1150 case 0x81:
1151 l2 = 128;
1152 break; // code and data L2 cache, 128 KB, 8 ways, 32 byte lines
1153 case 0x82:
1154 l2 = 256;
1155 break; // code and data L2 cache, 256 KB, 8 ways, 32 byte lines
1156 case 0x83:
1157 l2 = 512;
1158 break; // code and data L2 cache, 512 KB, 8 ways, 32 byte lines
1159 case 0x84:
1160 l2 = 1024;
1161 break; // code and data L2 cache, 1024 KB, 8 ways, 32 byte lines
1162 case 0x85:
1163 l2 = 2048;
1164 break; // code and data L2 cache, 2048 KB, 8 ways, 32 byte lines
1165 case 0x86:
1166 l2 = 512;
1167 break; // code and data L2 cache, 512 KB, 4 ways, 64 byte lines
1168 case 0x87:
1169 l2 = 1024;
1170 break; // code and data L2 cache, 1024 KB, 8 ways, 64 byte lines
1171 case 0x88:
1172 l3 = 2048;
1173 break; // code and data L3 cache, 2048 KB, 4 ways, 64 byte lines (IA-64)
1174 case 0x89:
1175 l3 = 4096;
1176 break; // code and data L3 cache, 4096 KB, 4 ways, 64 byte lines (IA-64)
1177 case 0x8A:
1178 l3 = 8192;
1179 break; // code and data L3 cache, 8192 KB, 4 ways, 64 byte lines (IA-64)
1180 case 0x8D:
1181 l3 = 3072;
1182 break; // code and data L3 cache, 3072 KB, 12 ways, 128 byte lines (IA-64)
1183
1184 default:
1185 break;
1186 }
1187 }
1188 if (check_for_p2_core2 && l2 == l3) l3 = 0;
1189 l1 *= 1024;
1190 l2 *= 1024;
1191 l3 *= 1024;
1192}
1193
1194inline void queryCacheSizes_intel(int& l1, int& l2, int& l3, int max_std_funcs) {
1195 if (max_std_funcs >= 4)
1196 queryCacheSizes_intel_direct(l1, l2, l3);
1197 else if (max_std_funcs >= 2)
1198 queryCacheSizes_intel_codes(l1, l2, l3);
1199 else
1200 l1 = l2 = l3 = 0;
1201}
1202
1203inline void queryCacheSizes_amd(int& l1, int& l2, int& l3) {
1204 int abcd[4];
1205 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1206
1207 // First query the max supported function.
1208 EIGEN_CPUID(abcd, 0x80000000, 0);
1209 if (static_cast<numext::uint32_t>(abcd[0]) >= static_cast<numext::uint32_t>(0x80000006)) {
1210 EIGEN_CPUID(abcd, 0x80000005, 0);
1211 l1 = (abcd[2] >> 24) * 1024; // C[31:24] = L1 size in KB
1212 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1213 EIGEN_CPUID(abcd, 0x80000006, 0);
1214 l2 = (abcd[2] >> 16) * 1024; // C[31;16] = l2 cache size in KB
1215 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024; // D[31;18] = l3 cache size in 512KB
1216 } else {
1217 l1 = l2 = l3 = 0;
1218 }
1219}
1220#endif
1221
1224inline void queryCacheSizes(int& l1, int& l2, int& l3) {
1225#ifdef EIGEN_CPUID
1226 int abcd[4];
1227 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
1228 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
1229 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574}; // "AMDisbetter!"
1230
1231 // identify the CPU vendor
1232 EIGEN_CPUID(abcd, 0x0, 0);
1233 int max_std_funcs = abcd[0];
1234 if (cpuid_is_vendor(abcd, GenuineIntel))
1235 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1236 else if (cpuid_is_vendor(abcd, AuthenticAMD) || cpuid_is_vendor(abcd, AMDisbetter_))
1237 queryCacheSizes_amd(l1, l2, l3);
1238 else
1239 // by default let's use Intel's API
1240 queryCacheSizes_intel(l1, l2, l3, max_std_funcs);
1241
1242 // here is the list of other vendors:
1243 // ||cpuid_is_vendor(abcd,"VIA VIA VIA ")
1244 // ||cpuid_is_vendor(abcd,"CyrixInstead")
1245 // ||cpuid_is_vendor(abcd,"CentaurHauls")
1246 // ||cpuid_is_vendor(abcd,"GenuineTMx86")
1247 // ||cpuid_is_vendor(abcd,"TransmetaCPU")
1248 // ||cpuid_is_vendor(abcd,"RiseRiseRise")
1249 // ||cpuid_is_vendor(abcd,"Geode by NSC")
1250 // ||cpuid_is_vendor(abcd,"SiS SiS SiS ")
1251 // ||cpuid_is_vendor(abcd,"UMC UMC UMC ")
1252 // ||cpuid_is_vendor(abcd,"NexGenDriven")
1253#else
1254 l1 = l2 = l3 = -1;
1255#endif
1256}
1257
1260inline int queryL1CacheSize() {
1261 int l1(-1), l2, l3;
1262 queryCacheSizes(l1, l2, l3);
1263 return l1;
1264}
1265
1268inline int queryTopLevelCacheSize() {
1269 int l1, l2(-1), l3(-1);
1270 queryCacheSizes(l1, l2, l3);
1271 return (std::max)(l2, l3);
1272}
1273
1277
1278#if EIGEN_COMP_CXXVER >= 20 && defined(__cpp_lib_constexpr_dynamic_alloc) && \
1279 __cpp_lib_constexpr_dynamic_alloc >= 201907L
1280using std::construct_at;
1281#else
1282template <class T, class... Args>
1283EIGEN_DEVICE_FUNC T* construct_at(T* p, Args&&... args) {
1284 return ::new (const_cast<void*>(static_cast<const volatile void*>(p))) T(std::forward<Args>(args)...);
1285}
1286#endif
1287
1293#if EIGEN_COMP_CXXVER >= 17
1294using std::destroy_at;
1295#else
1296template <class T>
1297EIGEN_DEVICE_FUNC void destroy_at(T* p) {
1298 p->~T();
1299}
1300#endif
1301
1302// FIXME(rmlarsen): Work around missing linker symbol with msan on ARM.
1303#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && __has_feature(memory_sanitizer) && \
1304 (EIGEN_ARCH_ARM || EIGEN_ARCH_ARM64)
1305#define EIGEN_DONT_ASSUME_ALIGNED
1306#endif
1307
1308
1309#if !defined(EIGEN_DONT_ASSUME_ALIGNED) && defined(__cpp_lib_assume_aligned) && (__cpp_lib_assume_aligned >= 201811L)
1310template <std::size_t N, typename T>
1311EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr T* assume_aligned(T* ptr) {
1312 return std::assume_aligned<N, T>(ptr);
1313}
1314#elif !defined(EIGEN_DONT_ASSUME_ALIGNED) && EIGEN_HAS_BUILTIN(__builtin_assume_aligned)
1315template <std::size_t N, typename T>
1316EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC T* assume_aligned(T* ptr) {
1317 return static_cast<T*>(__builtin_assume_aligned(ptr, N));
1318}
1319#else
1320template <std::size_t N, typename T>
1321EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr T* assume_aligned(T* ptr) {
1322 return ptr;
1323}
1324#endif
1325
1326} // end namespace internal
1327
1328} // end namespace Eigen
1329
1330#endif // EIGEN_MEMORY_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition B01_Experimental.dox:1
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:82
Eigen::Dynamic
const int Dynamic
Definition Constants.h:25