@@ -16,13 +16,11 @@ terms of the MIT license. A copy of the license can be found in the file
1616// ------------------------------------------------------
1717
1818static bool mi_malloc_is_naturally_aligned ( size_t size , size_t alignment ) {
19- // certain blocks are always allocated at a certain natural alignment.
20- // (see also `arena.c:mi_arenas_page_alloc_fresh`).
19+ // objects up to `MI_PAGE_MIN_BLOCK_ALIGN` are always allocated aligned to their size
2120 mi_assert_internal (_mi_is_power_of_two (alignment ) && (alignment > 0 ));
2221 if (alignment > size ) return false;
2322 const size_t bsize = mi_good_size (size );
24- const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two (bsize )) || // power-of-two under N
25- (alignment == MI_PAGE_OSPAGE_BLOCK_ALIGN2 && (bsize % MI_PAGE_OSPAGE_BLOCK_ALIGN2 )== 0 ); // or multiple of N
23+ const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two (bsize ));
2624 if (ok ) { mi_assert_internal ((bsize & (alignment - 1 )) == 0 ); } // since both power of 2 and alignment <= size
2725 return ok ;
2826}
@@ -40,22 +38,22 @@ static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, si
4038 return p ;
4139}
4240
43- static void * mi_heap_malloc_zero_no_guarded (mi_heap_t * heap , size_t size , bool zero , size_t * usable ) {
41+ static void * mi_heap_malloc_zero_no_guarded (mi_heap_t * heap , size_t size , bool zero ) {
4442 const size_t rate = heap -> guarded_sample_rate ;
4543 // only write if `rate!=0` so we don't write to the constant `_mi_heap_empty`
4644 if (rate != 0 ) { heap -> guarded_sample_rate = 0 ; }
47- void * p = _mi_heap_malloc_zero_ex (heap , size , zero , 0 , usable );
45+ void * p = _mi_heap_malloc_zero (heap , size , zero );
4846 if (rate != 0 ) { heap -> guarded_sample_rate = rate ; }
4947 return p ;
5048}
5149#else
52- static void * mi_heap_malloc_zero_no_guarded (mi_heap_t * heap , size_t size , bool zero , size_t * usable ) {
53- return _mi_heap_malloc_zero_ex (heap , size , zero , 0 , usable );
50+ static void * mi_heap_malloc_zero_no_guarded (mi_heap_t * heap , size_t size , bool zero ) {
51+ return _mi_heap_malloc_zero (heap , size , zero );
5452}
5553#endif
5654
5755// Fallback aligned allocation that over-allocates -- split out for better codegen
58- static mi_decl_noinline void * mi_heap_malloc_zero_aligned_at_overalloc (mi_heap_t * const heap , const size_t size , const size_t alignment , const size_t offset , const bool zero , size_t * usable ) mi_attr_noexcept
56+ static mi_decl_noinline void * mi_heap_malloc_zero_aligned_at_overalloc (mi_heap_t * const heap , const size_t size , const size_t alignment , const size_t offset , const bool zero ) mi_attr_noexcept
5957{
6058 mi_assert_internal (size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE ));
6159 mi_assert_internal (alignment != 0 && _mi_is_power_of_two (alignment ));
@@ -75,30 +73,25 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
7573 }
7674 oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size );
7775 // note: no guarded as alignment > 0
78- p = _mi_heap_malloc_zero_ex (heap , oversize , zero , alignment , usable ); // the page block size should be large enough to align in the single huge page block
76+ p = _mi_heap_malloc_zero_ex (heap , oversize , zero , alignment ); // the page block size should be large enough to align in the single huge page block
7977 if (p == NULL ) return NULL ;
8078 }
8179 else {
8280 // otherwise over-allocate
8381 oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size ) + alignment - 1 ; // adjust for size <= 16; with size 0 and aligment 64k, we would allocate a 64k block and pointing just beyond that.
84- p = mi_heap_malloc_zero_no_guarded (heap , oversize , zero , usable );
82+ p = mi_heap_malloc_zero_no_guarded (heap , oversize , zero );
8583 if (p == NULL ) return NULL ;
8684 }
87-
85+ mi_page_t * page = _mi_ptr_page (p );
86+
8887 // .. and align within the allocation
8988 const uintptr_t align_mask = alignment - 1 ; // for any x, `(x & align_mask) == (x % alignment)`
9089 const uintptr_t poffset = ((uintptr_t )p + offset ) & align_mask ;
9190 const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset );
9291 mi_assert_internal (adjust < alignment );
9392 void * aligned_p = (void * )((uintptr_t )p + adjust );
94-
95- // note: after the above allocation, the page may be abandoned now (as it became full, see `page.c:_mi_malloc_generic`)
96- // and we no longer own it. We should be careful to only read constant fields in the page,
97- // or use safe atomic access as in `mi_page_set_has_interior_pointers`.
98- // (we can access the page though since the just allocated pointer keeps it alive)
99- mi_page_t * page = _mi_ptr_page (p );
10093 if (aligned_p != p ) {
101- mi_page_set_has_interior_pointers (page , true);
94+ mi_page_set_has_aligned (page , true);
10295 #if MI_GUARDED
10396 // set tag to aligned so mi_usable_size works with guard pages
10497 if (adjust >= sizeof (mi_block_t )) {
@@ -139,7 +132,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
139132}
140133
141134// Generic primitive aligned allocation -- split out for better codegen
142- static mi_decl_noinline void * mi_heap_malloc_zero_aligned_at_generic (mi_heap_t * const heap , const size_t size , const size_t alignment , const size_t offset , const bool zero , size_t * usable ) mi_attr_noexcept
135+ static mi_decl_noinline void * mi_heap_malloc_zero_aligned_at_generic (mi_heap_t * const heap , const size_t size , const size_t alignment , const size_t offset , const bool zero ) mi_attr_noexcept
143136{
144137 mi_assert_internal (alignment != 0 && _mi_is_power_of_two (alignment ));
145138 // we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
@@ -154,7 +147,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t*
154147 // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist
155148 // a page with the right block size, and if we always use the over-alloc fallback that would never happen.
156149 if (offset == 0 && mi_malloc_is_naturally_aligned (size ,alignment )) {
157- void * p = mi_heap_malloc_zero_no_guarded (heap , size , zero , usable );
150+ void * p = mi_heap_malloc_zero_no_guarded (heap , size , zero );
158151 mi_assert_internal (p == NULL || ((uintptr_t )p % alignment ) == 0 );
159152 const bool is_aligned_or_null = (((uintptr_t )p ) & (alignment - 1 ))== 0 ;
160153 if mi_likely (is_aligned_or_null ) {
@@ -168,14 +161,12 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t*
168161 }
169162
170163 // fall back to over-allocation
171- return mi_heap_malloc_zero_aligned_at_overalloc (heap ,size ,alignment ,offset ,zero , usable );
164+ return mi_heap_malloc_zero_aligned_at_overalloc (heap ,size ,alignment ,offset ,zero );
172165}
173166
174167
175168// Primitive aligned allocation
176- static void * mi_heap_malloc_zero_aligned_at (mi_heap_t * const heap , const size_t size ,
177- const size_t alignment , const size_t offset , const bool zero ,
178- size_t * usable ) mi_attr_noexcept
169+ static void * mi_heap_malloc_zero_aligned_at (mi_heap_t * const heap , const size_t size , const size_t alignment , const size_t offset , const bool zero ) mi_attr_noexcept
179170{
180171 // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
181172 if mi_unlikely (alignment == 0 || !_mi_is_power_of_two (alignment )) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
@@ -202,7 +193,6 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
202193 const bool is_aligned = (((uintptr_t )page -> free + offset ) & align_mask )== 0 ;
203194 if mi_likely (is_aligned )
204195 {
205- if (usable != NULL ) { * usable = mi_page_usable_block_size (page ); }
206196 void * p = (zero ? _mi_page_malloc_zeroed (heap ,page ,padsize ) : _mi_page_malloc (heap ,page ,padsize )); // call specific page malloc for better codegen
207197 mi_assert_internal (p != NULL );
208198 mi_assert_internal (((uintptr_t )p + offset ) % alignment == 0 );
@@ -213,7 +203,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
213203 }
214204
215205 // fallback to generic aligned allocation
216- return mi_heap_malloc_zero_aligned_at_generic (heap , size , alignment , offset , zero , usable );
206+ return mi_heap_malloc_zero_aligned_at_generic (heap , size , alignment , offset , zero );
217207}
218208
219209
@@ -222,7 +212,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
222212// ------------------------------------------------------
223213
224214mi_decl_nodiscard mi_decl_restrict void * mi_heap_malloc_aligned_at (mi_heap_t * heap , size_t size , size_t alignment , size_t offset ) mi_attr_noexcept {
225- return mi_heap_malloc_zero_aligned_at (heap , size , alignment , offset , false, NULL );
215+ return mi_heap_malloc_zero_aligned_at (heap , size , alignment , offset , false);
226216}
227217
228218mi_decl_nodiscard mi_decl_restrict void * mi_heap_malloc_aligned (mi_heap_t * heap , size_t size , size_t alignment ) mi_attr_noexcept {
@@ -234,7 +224,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap,
234224// ------------------------------------------------------
235225
236226mi_decl_nodiscard mi_decl_restrict void * mi_heap_zalloc_aligned_at (mi_heap_t * heap , size_t size , size_t alignment , size_t offset ) mi_attr_noexcept {
237- return mi_heap_malloc_zero_aligned_at (heap , size , alignment , offset , true, NULL );
227+ return mi_heap_malloc_zero_aligned_at (heap , size , alignment , offset , true);
238228}
239229
240230mi_decl_nodiscard mi_decl_restrict void * mi_heap_zalloc_aligned (mi_heap_t * heap , size_t size , size_t alignment ) mi_attr_noexcept {
@@ -259,10 +249,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a
259249 return mi_heap_malloc_aligned (mi_prim_get_default_heap (), size , alignment );
260250}
261251
262- mi_decl_nodiscard mi_decl_restrict void * mi_umalloc_aligned (size_t size , size_t alignment , size_t * block_size ) mi_attr_noexcept {
263- return mi_heap_malloc_zero_aligned_at (mi_prim_get_default_heap (), size , alignment , 0 , false, block_size );
264- }
265-
266252mi_decl_nodiscard mi_decl_restrict void * mi_zalloc_aligned_at (size_t size , size_t alignment , size_t offset ) mi_attr_noexcept {
267253 return mi_heap_zalloc_aligned_at (mi_prim_get_default_heap (), size , alignment , offset );
268254}
@@ -271,10 +257,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a
271257 return mi_heap_zalloc_aligned (mi_prim_get_default_heap (), size , alignment );
272258}
273259
274- mi_decl_nodiscard mi_decl_restrict void * mi_uzalloc_aligned (size_t size , size_t alignment , size_t * block_size ) mi_attr_noexcept {
275- return mi_heap_malloc_zero_aligned_at (mi_prim_get_default_heap (), size , alignment , 0 , true, block_size );
276- }
277-
278260mi_decl_nodiscard mi_decl_restrict void * mi_calloc_aligned_at (size_t count , size_t size , size_t alignment , size_t offset ) mi_attr_noexcept {
279261 return mi_heap_calloc_aligned_at (mi_prim_get_default_heap (), count , size , alignment , offset );
280262}
@@ -290,8 +272,8 @@ mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t
290272
291273static void * mi_heap_realloc_zero_aligned_at (mi_heap_t * heap , void * p , size_t newsize , size_t alignment , size_t offset , bool zero ) mi_attr_noexcept {
292274 mi_assert (alignment > 0 );
293- if (alignment <= sizeof (uintptr_t )) return _mi_heap_realloc_zero (heap ,p ,newsize ,zero , NULL , NULL );
294- if (p == NULL ) return mi_heap_malloc_zero_aligned_at (heap ,newsize ,alignment ,offset ,zero , NULL );
275+ if (alignment <= sizeof (uintptr_t )) return _mi_heap_realloc_zero (heap ,p ,newsize ,zero );
276+ if (p == NULL ) return mi_heap_malloc_zero_aligned_at (heap ,newsize ,alignment ,offset ,zero );
295277 size_t size = mi_usable_size (p );
296278 if (newsize <= size && newsize >= (size - (size / 2 ))
297279 && (((uintptr_t )p + offset ) % alignment ) == 0 ) {
@@ -315,7 +297,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne
315297
316298static void * mi_heap_realloc_zero_aligned (mi_heap_t * heap , void * p , size_t newsize , size_t alignment , bool zero ) mi_attr_noexcept {
317299 mi_assert (alignment > 0 );
318- if (alignment <= sizeof (uintptr_t )) return _mi_heap_realloc_zero (heap ,p ,newsize ,zero , NULL , NULL );
300+ if (alignment <= sizeof (uintptr_t )) return _mi_heap_realloc_zero (heap ,p ,newsize ,zero );
319301 size_t offset = ((uintptr_t )p % alignment ); // use offset of previous allocation (p can be NULL)
320302 return mi_heap_realloc_zero_aligned_at (heap ,p ,newsize ,alignment ,offset ,zero );
321303}
0 commit comments