diff --git a/Makefile b/Makefile index 6790209..6636251 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ PERM_FREE_REALLOC = -DPERM_FREE_REALLOC=0 ## 1 byte tag per addressable chunk of memory. These tags can ## be retrieved and verified. This feature will likely interfere ## with ARM MTE and PAC. See MEMORY_TAGGING.md for more information -MEMORY_TAGGING = -DMEMORY_TAGGING=0 +MEMORY_TAGGING = -DMEMORY_TAGGING=1 ## Enable abort() when isoalloc can't gather enough entropy. ABORT_NO_ENTROPY = -DABORT_NO_ENTROPY=1 @@ -90,8 +90,12 @@ THP_PAGES = -DTHP_PAGES=1 ## program. This file encodes the heap usage patterns of ## the target. This file can be consumed by the profiler ## CLI utility. See PROFILER.md for the format of this file -#HEAP_PROFILER = -DHEAP_PROFILER=1 -fno-omit-frame-pointer \ -# -fno-optimize-sibling-calls -ldl +#HEAP_PROFILER = $(HEAP_PROFILER_FLAGS) + +## Flags required to build with the heap profiler. Backtraces +## are collected by walking frame pointers so they must be kept +HEAP_PROFILER_FLAGS = -DHEAP_PROFILER=1 -fno-omit-frame-pointer \ + -fno-optimize-sibling-calls ## Enable CPU pinning support on a per-zone basis. This is ## a minor security feature which introduces an allocation @@ -425,6 +429,7 @@ tests: clean library_debug_unit_tests $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tests $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) $(UNIT_TESTING) tests/uaf.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/uaf $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/interfaces_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/interfaces_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/aligned_alloc_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/aligned_alloc_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/thread_tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/thread_tests $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) $(UNIT_TESTING) tests/big_canary_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/big_canary_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/big_tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/big_tests $(LDFLAGS) @@ -463,9 +468,19 @@ tagging_tests: clean cpp_library_debug $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/uaf_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/uaf_tag_ptr_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/bad_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/bad_tag_ptr_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/verify_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/verify_tag_ptr_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/valid_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/valid_tag_ptr_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/tag_refresh_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tag_refresh_test $(LDFLAGS) $(CXX) -DMEMORY_TAGGING=1 $(CXXFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(EXE_CFLAGS) $(OS_FLAGS) tests/tagged_ptr_test.cpp $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tagged_ptr_test_cpp $(LDFLAGS) utils/run_tagging_tests.sh +## Build the library with the heap profiler and run its test. +## The profiler data file is checked by the runner script +profiler_tests: clean + @echo "make profiler_tests" + $(CC) $(CFLAGS) $(HEAP_PROFILER_FLAGS) $(LIBRARY) $(OS_FLAGS) $(UNIT_TESTING) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(C_SRCS) -o $(BUILD_DIR)/$(LIBNAME) -ldl + $(CC) $(CFLAGS) $(HEAP_PROFILER_FLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/profiler_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/profiler_test $(LDFLAGS) -ldl + utils/run_profiler_tests.sh + init_test: clean library_debug_unit_tests @echo "make init_test" $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/init_destroy.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/init_destroy $(LDFLAGS) @@ -530,6 +545,6 @@ format-ci: clean: rm -rf build/* tests_perf_analysis.txt big_tests_perf_analysis.txt gmon.out \ - tagging_test_output.txt test_output.txt *.dSYM core* iso_alloc_profiler.data + tagging_test_output.txt test_output.txt profiler_test_output.txt *.dSYM core* iso_alloc_profiler.data rm -rf android/libs android/obj mkdir -p build/ diff --git a/include/iso_alloc_internal.h b/include/iso_alloc_internal.h index b8a036f..89bc094 100644 --- a/include/iso_alloc_internal.h +++ b/include/iso_alloc_internal.h @@ -367,6 +367,10 @@ extern pthread_mutex_t root_busy_mutex; /* The global root */ extern iso_alloc_root *_root; +/* These are C functions and may be called from iso_alloc.cpp */ +#ifdef __cplusplus +extern "C" { +#endif INTERNAL_HIDDEN INLINE void check_big_canary(iso_alloc_big_zone_t *big); INTERNAL_HIDDEN INLINE void check_canary(iso_alloc_zone_t *zone, const void *p); INTERNAL_HIDDEN INLINE void iso_clear_user_chunk(uint8_t *p, size_t size); @@ -377,6 +381,7 @@ INTERNAL_HIDDEN INLINE void flush_chunk_quarantine(void); INTERNAL_HIDDEN INLINE void clear_zone_cache(void); INTERNAL_HIDDEN iso_alloc_big_zone_t *iso_find_big_zone(void *p, bool remove); INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size); +INTERNAL_HIDDEN iso_alloc_zone_t *find_exact_fit_zone(size_t size); INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size); INTERNAL_HIDDEN iso_alloc_zone_t *iso_new_zone(size_t size, bool internal); INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal, int32_t index); @@ -389,6 +394,7 @@ INTERNAL_HIDDEN INLINE bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone) INTERNAL_HIDDEN iso_alloc_root *iso_alloc_new_root(void); INTERNAL_HIDDEN bool is_pow2(uint64_t sz); INTERNAL_HIDDEN bool _is_zone_retired(iso_alloc_zone_t *zone); +INTERNAL_HIDDEN void _iso_alloc_refresh_chunk_tag(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN bool _refresh_zone_mem_tags(iso_alloc_zone_t *zone); INTERNAL_HIDDEN iso_alloc_zone_t *_iso_free_internal_unlocked(void *p, bool permanent, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void fill_free_bit_slots(iso_alloc_zone_t *zone); @@ -414,8 +420,9 @@ INTERNAL_HIDDEN INLINE void dont_need_pages(void *p, size_t size); INTERNAL_HIDDEN void *_tag_ptr(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void _free_big_zone_list(iso_alloc_big_zone_t *head); -INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size); +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size, size_t alignment); INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t size); +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_aligned(size_t size, size_t alignment); INTERNAL_HIDDEN INLINE ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone); INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_calloc(size_t nmemb, size_t size); #if UAF_PTR_PAGE @@ -430,6 +437,9 @@ INTERNAL_HIDDEN size_t _iso_chunk_size(void *p); INTERNAL_HIDDEN int64_t check_canary_no_abort(iso_alloc_zone_t *zone, const void *p); INTERNAL_HIDDEN void _iso_alloc_initialize(void); INTERNAL_HIDDEN void _iso_alloc_destroy(void); +#ifdef __cplusplus +} +#endif #if ARM_MTE INLINE void *iso_mte_untag_ptr(void *p); diff --git a/include/iso_alloc_profiler.h b/include/iso_alloc_profiler.h index b122318..8093024 100644 --- a/include/iso_alloc_profiler.h +++ b/include/iso_alloc_profiler.h @@ -18,29 +18,33 @@ /* The IsoAlloc profiler is not thread local but these * globals should only ever be touched by internal * allocator functions when the root is locked */ -uint64_t _alloc_count; -uint64_t _free_count; -uint64_t _alloc_sampled_count; -uint64_t _free_sampled_count; - -int32_t profiler_fd; +extern uint64_t _alloc_count; +extern uint64_t _free_count; +extern uint64_t _alloc_sampled_count; +extern uint64_t _free_sampled_count; typedef struct { uint64_t total; uint64_t count; } zone_profiler_map_t; -zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX]; +/* These are defined once in iso_alloc_profiler.c. This + * header is included by several translation units so + * defining them here fails to link under -fno-common */ +extern int32_t profiler_fd; + +/* Indexed by chunk size, which can be exactly SMALL_SIZE_MAX */ +extern zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX + 1]; /* iso_alloc_traces_t is a public structure, and * is defined in the public header iso_alloc.h */ -iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; -size_t _alloc_bts_count; +extern iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; +extern size_t _alloc_bts_count; /* iso_free_traces_t is a public structure, and * is defined in the public header iso_alloc.h */ -iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; -size_t _free_bts_count; +extern iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; +extern size_t _free_bts_count; INTERNAL_HIDDEN INLINE uint64_t _get_backtrace_hash(void); INTERNAL_HIDDEN INLINE void _save_backtrace(iso_alloc_traces_t *abts); diff --git a/src/iso_alloc.c b/src/iso_alloc.c index 6bfecdf..bd804ac 100644 --- a/src/iso_alloc.c +++ b/src/iso_alloc.c @@ -867,6 +867,44 @@ INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, } } +/* Walks the lookup table chain for internal zones whose + * chunk size is exactly size and returns the first usable + * one, or NULL if none of them can hold another chunk */ +INTERNAL_HIDDEN iso_alloc_zone_t *find_exact_fit_zone(size_t size) { + int32_t i = _root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)]; + + if(i == 0) { + return NULL; + } + + const size_t zones_used = _root->zones_used; + + for(; i < zones_used;) { + iso_alloc_zone_t *zone = &_root->zones[i]; + + if(zone->chunk_size != size) { + LOG_AND_ABORT("Zone lookup table failed to match sizes for zone[%d](%d) for chunk size (%d)", zone->index, zone->chunk_size, size); + } + + if(zone->internal == false) { + LOG_AND_ABORT("Lookup table should never contain private zones"); + } + + if(is_zone_usable(zone, size) != NULL) { + return zone; + } + + if(zone->next_sz_index != 0) { + i = zone->next_sz_index; + } else { + /* We have reached the end of our linked zones */ + break; + } + } + + return NULL; +} + /* Finds a zone that can fit this allocation request */ INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size) { iso_alloc_zone_t *zone = NULL; @@ -891,37 +929,14 @@ INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size) { } #endif - /* Fast path via lookup table */ - if(_root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)] != 0) { - i = _root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)]; + /* Fast path via lookup table. If this fails to find a + * usable zone we fall through and iterate through all + * zones, including ones we may have skipped over, to + * find a suitable candidate instead of creating a new one */ + zone = find_exact_fit_zone(size); - for(; i < zones_used;) { - iso_alloc_zone_t *zone = &_root->zones[i]; - - if(zone->chunk_size != size) { - LOG_AND_ABORT("Zone lookup table failed to match sizes for zone[%d](%d) for chunk size (%d)", zone->index, zone->chunk_size, size); - } - - if(zone->internal == false) { - LOG_AND_ABORT("Lookup table should never contain private zones"); - } - - if(is_zone_usable(zone, size) != NULL) { - return zone; - } - - if(zone->next_sz_index != 0) { - i = zone->next_sz_index; - } else { - /* We have reached the end of our linked zones. The - * lookup table failed to find us a usable zone. - * Instead of creating a new one we will break out - * of this loop and try iterating through all zones, - * including ones we may have skipped over, to find - * a suitable candidate. */ - break; - } - } + if(zone != NULL) { + return zone; } #if SMALL_MEM_STARTUP @@ -1235,10 +1250,102 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t s LOG_AND_ABORT("Allocation size of %d is > %d and cannot use a private zone (%d)", size, SMALL_SIZE_MAX, zone->chunk_size); } - return _iso_big_alloc(size); + return _iso_big_alloc(size, g_page_size); } } +/* Allocates a chunk aligned to alignment bytes, which must be + * a power of 2. Every chunk is at least SZ_ALIGNMENT aligned. + * Stronger alignments are satisfied by rounding the size up to + * a multiple of the alignment and allocating from an internal + * zone whose chunk size is exactly that. Chunks are laid out + * at multiples of the chunk size from a page aligned base so + * this guarantees the alignment for anything up to a page. + * Larger alignments and sizes use the big zone path */ +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_aligned(size_t size, size_t alignment) { + if(UNLIKELY(is_pow2(alignment) == false)) { + return NULL; + } + + /* Every chunk is already SZ_ALIGNMENT aligned, and any + * smaller power of 2 divides SZ_ALIGNMENT */ + if(alignment <= SZ_ALIGNMENT) { + return _iso_alloc(NULL, size); + } + + if(size == 0) { + size = alignment; + } + + const size_t s = ((size + (alignment - 1)) & ~(alignment - 1)); + + if(UNLIKELY(s < size)) { + return NULL; + } + + LOCK_ROOT(); + + if(UNLIKELY(_root == NULL)) { +#if AUTO_CTOR_DTOR + g_page_size = sysconf(_SC_PAGESIZE); + g_page_size_shift = _log2(g_page_size); + iso_alloc_initialize_global_root(); +#else + LOG_AND_ABORT("Root never initialized!"); +#endif + } + +#if HEAP_PROFILER + _iso_alloc_profile(s); +#endif + + if(alignment > g_page_size || s > SMALL_SIZE_MAX) { + /* The big zone path uses a different lock */ + UNLOCK_ROOT(); + + /* Big zones must be at least SMALL_SIZE_MAX bytes. That + * is a power of 2 so rounding up to it keeps the size a + * multiple of any smaller alignment */ + return _iso_big_alloc((s < SMALL_SIZE_MAX) ? SMALL_SIZE_MAX : s, alignment); + } + + iso_alloc_zone_t *zone = find_exact_fit_zone(s); + + if(zone == NULL) { + zone = _iso_new_zone(s, true, -1); + + if(UNLIKELY(zone == NULL)) { + LOG_AND_ABORT("Failed to create a zone for aligned allocation of %zu bytes", s); + } + } + + bit_slot_t free_bit_slot = zone->next_free_bit_slot; + + if(UNLIKELY(free_bit_slot == BAD_BIT_SLOT)) { + UNLOCK_ROOT(); +#if ABORT_ON_NULL + LOG_AND_ABORT("isoalloc configured to abort on NULL"); +#endif + return NULL; + } + + UNMASK_ZONE_PTRS(zone); + + zone->next_free_bit_slot = BAD_BIT_SLOT; + void *p = _iso_alloc_bitslot_from_zone(free_bit_slot, zone); + + MASK_ZONE_PTRS(zone); + UNLOCK_ROOT(); + populate_zone_cache(zone); + +#if ARM_MTE + if(_root->arm_mte_enabled == true) { + return iso_mte_set_tag_range(p, zone->chunk_size); + } +#endif + return p; +} + INTERNAL_HIDDEN iso_alloc_zone_t *search_chunk_lookup_table(const void *restrict p) { /* The chunk lookup table is the fastest way to find a * zone given a pointer to a chunk so we check it first */ @@ -1597,7 +1704,13 @@ INTERNAL_HIDDEN void _iso_free(void *p, bool permanent) { #endif #if HEAP_PROFILER + /* The alloc side profiles with the root locked. The + * free profiler updates the same globals so it needs + * the same protection or racing frees can push the + * backtrace count past the end of the array */ + LOCK_ROOT(); _iso_free_profile(); + UNLOCK_ROOT(); #endif #if ARM_MTE @@ -1747,14 +1860,10 @@ INTERNAL_HIDDEN iso_alloc_zone_t *_iso_free_internal_unlocked(void *p, bool perm * be refreshed then just generate a new tag for this chunk */ if(zone->tagged == true) { if(_refresh_zone_mem_tags(zone) == false) { - /* We only need to refresh this single tag */ - void *user_pages_start = UNMASK_USER_PTR(zone); - uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); - uint64_t chunk_offset = (uint64_t) (p - user_pages_start); - _mtp += (chunk_offset / zone->chunk_size); - - /* Generate and write a new tag for this chunk */ - *_mtp = (uint8_t) us_rand_uint64(&_root->seed); + /* We only need to refresh this single tag. The + * new tag is guaranteed to differ from the old one + * so a stale tagged pointer to p can never verify */ + _iso_alloc_refresh_chunk_tag(p, zone); } } #endif @@ -1909,8 +2018,12 @@ INTERNAL_HIDDEN void iso_free_big_zone(iso_alloc_big_zone_t *big_zone, bool perm } } -INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { +/* alignment must be a power of 2. Every big zone mapping is + * page aligned so only alignments larger than a page need + * any special handling */ +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size, size_t alignment) { const size_t new_size = ROUND_UP_PAGE(size); + const bool aligned = (alignment > g_page_size); if(new_size < size || new_size > BIG_SZ_MAX || size < SMALL_SIZE_MAX) { #if ABORT_ON_NULL @@ -1950,7 +2063,8 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { check_big_canary(big); /* We found a suitable big zone we can reuse */ - if(big->size >= size && (big->size - size) <= BIG_ZONE_WASTE * 2) { + if(big->size >= size && (big->size - size) <= BIG_ZONE_WASTE * 2 && + (aligned == false || ((uintptr_t) big->user_pages_start & (alignment - 1)) == 0)) { big->free = false; _root->big_zone_free_count--; UNPOISON_BIG_ZONE(big); @@ -2000,21 +2114,33 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { UNLOCK_BIG_ZONE_FREE(); /* The free list contained no usable entries - * so we need to create a new one */ + * so we need to create a new one. An alignment + * larger than a page is satisfied by mapping an + * extra alignment bytes and trimming both ends */ + size_t map_size = size; + + if(aligned == true) { + map_size += alignment; + + if(UNLIKELY(map_size < size)) { + return NULL; + } + } + void *user_pages = NULL; #if ARM_MTE #if BIG_ZONE_GUARD - user_pages = mmap_guarded_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_guarded_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #else if(_root->arm_mte_enabled == true) { - user_pages = mmap_rw_mte_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_rw_mte_pages(map_size, false, BIG_ZONE_UD_NAME); } #endif #else #if BIG_ZONE_GUARD - user_pages = mmap_guarded_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_guarded_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #else - user_pages = mmap_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #endif #endif @@ -2025,6 +2151,31 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { return NULL; } + if(aligned == true) { + void *aligned_pages = (void *) (((uintptr_t) user_pages + (alignment - 1)) & ~(alignment - 1)); + const size_t head = (size_t) (aligned_pages - user_pages); + const size_t tail = (alignment - head); +#if BIG_ZONE_GUARD + /* Move the guard pages so they sit directly below and + * above the aligned range, then release everything else. + * The upper munmap also covers the original upper guard */ + if(head != 0) { + munmap(user_pages - g_page_size, head); + mprotect_pages(aligned_pages - g_page_size, g_page_size, PROT_NONE); + } + + mprotect_pages(aligned_pages + size, g_page_size, PROT_NONE); + munmap(aligned_pages + size + g_page_size, tail); +#else + if(head != 0) { + munmap(user_pages, head); + } + + munmap(aligned_pages + size, tail); +#endif + user_pages = aligned_pages; + } + /* We only need a single page for big zone meta data */ static_assert(BIG_ZONE_META_DATA_PAGE_COUNT == 1, "Big zone meta data only needs a single page"); #if BIG_ZONE_META_DATA_GUARD diff --git a/src/iso_alloc.cpp b/src/iso_alloc.cpp index 90fce59..61d987d 100644 --- a/src/iso_alloc.cpp +++ b/src/iso_alloc.cpp @@ -23,11 +23,54 @@ extern "C" __attribute__((constructor(FIRST_CTOR + 1))) void iso_cpp(void) { #define NEW_EXCEPT #endif +// A throwing operator new may never return NULL. The +// standard requires that we retry through the installed +// new handler and throw bad_alloc if there is none +static inline void *iso_cpp_new(size_t size) { + void *p = iso_alloc(size); + + while(UNLIKELY(p == NULL)) { + std::new_handler h = std::get_new_handler(); + + if(h == nullptr) { + throw std::bad_alloc(); + } + + h(); + p = iso_alloc(size); + } + + return p; +} + +static inline void *iso_cpp_new_aligned(size_t size, std::align_val_t val) { + const size_t alignment = static_cast(val); + + if(is_pow2(alignment) == false) { + throw std::bad_alloc(); + } + + void *p = _iso_alloc_aligned(size, alignment); + + while(UNLIKELY(p == NULL)) { + std::new_handler h = std::get_new_handler(); + + if(h == nullptr) { + throw std::bad_alloc(); + } + + h(); + p = _iso_alloc_aligned(size, alignment); + } + + return p; +} + // These hooks override the basic new/delete // operators to use the iso_alloc API EXTERNAL_API FLATTEN void *operator new(size_t size) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new(size); } EXTERNAL_API FLATTEN void operator delete(void *p) noexcept { @@ -35,7 +78,7 @@ EXTERNAL_API FLATTEN void operator delete(void *p) noexcept { } EXTERNAL_API FLATTEN void *operator new[](size_t size) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new(size); } EXTERNAL_API FLATTEN void operator delete[](void *p) noexcept { @@ -67,11 +110,11 @@ EXTERNAL_API FLATTEN void operator delete[](void *ptr, const std::nothrow_t &) n } EXTERNAL_API FLATTEN void *operator new(size_t size, std::align_val_t val) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new_aligned(size, val); } EXTERNAL_API FLATTEN void *operator new[](size_t size, std::align_val_t val) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new_aligned(size, val); } EXTERNAL_API FLATTEN void operator delete(void *p, std::align_val_t val) noexcept { @@ -83,11 +126,11 @@ EXTERNAL_API FLATTEN void operator delete[](void *p, std::align_val_t val) noexc } EXTERNAL_API FLATTEN void *operator new(size_t size, std::align_val_t val, std::nothrow_t &) noexcept { - return iso_alloc(size); + return _iso_alloc_aligned(size, static_cast(val)); } EXTERNAL_API FLATTEN void *operator new[](size_t size, std::align_val_t val, std::nothrow_t &) noexcept { - return iso_alloc(size); + return _iso_alloc_aligned(size, static_cast(val)); } EXTERNAL_API FLATTEN void operator delete(void *p, std::align_val_t val, std::nothrow_t &) noexcept { diff --git a/src/iso_alloc_interfaces.c b/src/iso_alloc_interfaces.c index 8dbd2ae..4febfb0 100644 --- a/src/iso_alloc_interfaces.c +++ b/src/iso_alloc_interfaces.c @@ -53,6 +53,10 @@ EXTERNAL_API FLATTEN size_t iso_chunksz(void *p) { } EXTERNAL_API FLATTEN NO_DISCARD size_t iso_zone_chunk_count(iso_alloc_zone_handle *zone) { + if(zone == NULL) { + return 0; + } + UNMASK_ZONE_HANDLE(zone); iso_alloc_zone_t *_zone = (iso_alloc_zone_t *) zone; size_t canaries = 0; @@ -236,6 +240,13 @@ EXTERNAL_API FLATTEN void iso_alloc_destroy_zone(iso_alloc_zone_handle *zone) { EXTERNAL_API FLATTEN NO_DISCARD iso_alloc_zone_handle *iso_alloc_new_zone(size_t size) { iso_alloc_zone_handle *zone = (iso_alloc_zone_handle *) iso_new_zone(size, false); + + /* A NULL must not be masked or the caller gets back + * a non-NULL handle that is really just the mask */ + if(zone == NULL) { + return NULL; + } + UNMASK_ZONE_HANDLE(zone); return zone; } diff --git a/src/iso_alloc_mem_tags.c b/src/iso_alloc_mem_tags.c index 2bb3713..f90b496 100644 --- a/src/iso_alloc_mem_tags.c +++ b/src/iso_alloc_mem_tags.c @@ -11,15 +11,16 @@ INTERNAL_HIDDEN uint8_t _iso_alloc_get_mem_tag(void *p, iso_alloc_zone_t *zone) #if MEMORY_TAGGING void *user_pages_start = UNMASK_USER_PTR(zone); - if(user_pages_start > p || (user_pages_start + ZONE_USER_SIZE) < p) { + if(user_pages_start > p || (user_pages_start + ZONE_USER_SIZE) <= p) { LOG_AND_ABORT("Cannot get tag for pointer %p with wrong zone %d %p - %p", p, zone->index, user_pages_start, user_pages_start + ZONE_USER_SIZE); } uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); const uint64_t chunk_offset = (uint64_t) (p - user_pages_start); - /* Ensure the pointer is a multiple of chunk size */ - if(UNLIKELY((chunk_offset & (zone->chunk_size - 1)) != 0)) { + /* Ensure the pointer is a multiple of chunk size. Chunk + * sizes are not always a power of 2 so we can't use a mask */ + if(UNLIKELY((chunk_offset % zone->chunk_size) != 0)) { LOG_AND_ABORT("Chunk offset %d not an alignment of %d", chunk_offset, zone->chunk_size); } @@ -40,8 +41,10 @@ INTERNAL_HIDDEN void _iso_alloc_verify_tag(void *p, iso_alloc_zone_t *zone) { void *untagged_p = (void *) ((uintptr_t) p & TAGGED_PTR_MASK); const uint64_t tag = _iso_alloc_get_mem_tag(untagged_p, zone); - if(tag != ((uintptr_t) p & IS_TAGGED_PTR_MASK)) { - LOG_AND_ABORT("Pointer %p has wrong tag %x, expected %x", p, ((uintptr_t) p & IS_TAGGED_PTR_MASK), tag); + const uint64_t ptr_tag = (((uintptr_t) p & IS_TAGGED_PTR_MASK) >> UNTAGGED_BITS); + + if(tag != ptr_tag) { + LOG_AND_ABORT("Pointer %p has wrong tag %x, expected %x", p, ptr_tag, tag); } #endif return; @@ -74,18 +77,48 @@ INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone) { #endif } +#if MEMORY_TAGGING +/* Returns a random tag that is guaranteed to differ from + * old_tag. A stale tagged pointer must never verify against + * the tag written when its chunk was freed, so we draw the + * new tag uniformly from the 255 values that are not old_tag */ +static INLINE uint8_t _new_mem_tag(uint8_t old_tag, uint8_t rnd) { + return (uint8_t) (old_tag + 1 + (rnd % 255)); +} +#endif + +/* Replaces the tag for the chunk at p, which must be an + * untagged pointer to the start of a chunk in zone, with + * a new random tag that differs from the current one */ +INTERNAL_HIDDEN void _iso_alloc_refresh_chunk_tag(void *p, iso_alloc_zone_t *zone) { +#if MEMORY_TAGGING + void *user_pages_start = UNMASK_USER_PTR(zone); + uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); + const uint64_t chunk_offset = (uint64_t) (p - user_pages_start); + _mtp += (chunk_offset / zone->chunk_size); + *_mtp = _new_mem_tag(*_mtp, (uint8_t) us_rand_uint64(&_root->seed)); +#endif +} + INTERNAL_HIDDEN bool _refresh_zone_mem_tags(iso_alloc_zone_t *zone) { #if MEMORY_TAGGING /* This implements a similar policy to zone retirement. * The only difference is that we refresh all tags at * %25 of the configured zone retirement age */ - if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count << _root->zone_retirement_shf)) >> 2) { + if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > ((zone->chunk_count << _root->zone_retirement_shf) >> 2))) { size_t s = ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE); - uint64_t *_mtp = (zone->user_pages_start - g_page_size - s); - size_t tms = s / sizeof(uint64_t); - - for(uint64_t o = 0; o < tms; o++) { - _mtp[o] = us_rand_uint64(&_root->seed); + uint8_t *_mtp = (UNMASK_USER_PTR(zone) - g_page_size - s); + + /* Every tag must change so that no stale tagged + * pointer created before the refresh can verify + * against the new tags. One random qword provides + * the entropy for 8 consecutive tags */ + for(size_t o = 0; o < s; o += sizeof(uint64_t)) { + uint64_t r = us_rand_uint64(&_root->seed); + + for(size_t b = 0; b < sizeof(uint64_t); b++) { + _mtp[o + b] = _new_mem_tag(_mtp[o + b], (uint8_t) (r >> (b * 8))); + } } return true; diff --git a/src/iso_alloc_profiler.c b/src/iso_alloc_profiler.c index 3fea22b..0f28ad8 100644 --- a/src/iso_alloc_profiler.c +++ b/src/iso_alloc_profiler.c @@ -13,6 +13,19 @@ #include +#if HEAP_PROFILER +uint64_t _alloc_count; +uint64_t _free_count; +uint64_t _alloc_sampled_count; +uint64_t _free_sampled_count; +int32_t profiler_fd; +zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX + 1]; +iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; +size_t _alloc_bts_count; +iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; +size_t _free_bts_count; +#endif + INTERNAL_HIDDEN uint64_t _iso_alloc_detect_leaks_in_zone(iso_alloc_zone_t *zone) { LOCK_ROOT(); uint64_t leaks = _iso_alloc_zone_leak_detector(zone, false); @@ -173,7 +186,7 @@ INTERNAL_HIDDEN uint64_t _iso_alloc_zone_leak_detector(iso_alloc_zone_t *zone, b if(profile == false) { LOG("Zone[%d] Total number of %d byte chunks(%d) used and free'd (%d) (%d percent), in use = %d", zone->index, zone->chunk_size, zone->chunk_count, - was_used, (int32_t) ((float) was_used / zone->chunk_count) * 100, in_use); + was_used, (int32_t) (((float) was_used / zone->chunk_count) * 100), in_use); } MASK_ZONE_PTRS(zone); @@ -184,8 +197,7 @@ INTERNAL_HIDDEN uint64_t _iso_alloc_zone_leak_detector(iso_alloc_zone_t *zone, b * the total number of allocations both currently * in use and previously used by this zone */ if(profile == true) { - uint64_t total = (in_use + was_used); - return (uint64_t) ((float) (total / zone->chunk_count) * 100.0); + return (uint64_t) (in_use + was_used); } #endif return in_use; @@ -253,16 +265,16 @@ INTERNAL_HIDDEN uint64_t __iso_alloc_big_zone_mem_usage(iso_alloc_big_zone_t *he * be used to interpret allocation patterns */ INTERNAL_HIDDEN size_t _iso_get_alloc_traces(iso_alloc_traces_t *traces_out) { LOCK_ROOT(); - __iso_memcpy(traces_out, _alloc_bts, sizeof(iso_alloc_traces_t)); size_t sz = _alloc_bts_count; + __iso_memcpy(traces_out, _alloc_bts, sz * sizeof(iso_alloc_traces_t)); UNLOCK_ROOT(); return sz; } INTERNAL_HIDDEN size_t _iso_get_free_traces(iso_free_traces_t *traces_out) { LOCK_ROOT(); - __iso_memcpy(traces_out, _free_bts, sizeof(iso_free_traces_t)); size_t sz = _free_bts_count; + __iso_memcpy(traces_out, _free_bts, sz * sizeof(iso_free_traces_t)); UNLOCK_ROOT(); return sz; } @@ -276,61 +288,60 @@ INTERNAL_HIDDEN void _iso_alloc_reset_traces(void) { UNLOCK_ROOT(); } -#define UPDATE_BT_HASH(frame, hash) \ - if(__builtin_frame_address(frame)) { \ - hash ^= (uint64_t) __builtin_return_address(frame); \ - } else { \ - return hash; \ +/* Walks the frame pointer chain. Both x86_64 and aarch64 + * save the previous frame pointer at fp[0] and the return + * address at fp[1] when built with frame pointers. We walk + * the chain ourselves because __builtin_return_address(n) + * for n > 0 is a constant 0 on aarch64 gcc, which left the + * profiler without any backtraces on that target. The walk + * stops at the first frame that is NULL, misaligned, or does + * not sit above the current one on the stack */ +static INLINE uintptr_t *_next_frame(uintptr_t *fp) { + uintptr_t *next = (uintptr_t *) fp[0]; + + if(next <= fp || ((uintptr_t) next & (sizeof(uintptr_t) * 2 - 1)) != 0) { + return NULL; } + return next; +} + INTERNAL_HIDDEN INLINE uint64_t _get_backtrace_hash(void) { uint64_t hash = 0; - UPDATE_BT_HASH(1, hash); - UPDATE_BT_HASH(2, hash); - UPDATE_BT_HASH(3, hash); - UPDATE_BT_HASH(4, hash); - UPDATE_BT_HASH(5, hash); - UPDATE_BT_HASH(6, hash); - UPDATE_BT_HASH(7, hash); - UPDATE_BT_HASH(8, hash); + uintptr_t *fp = (uintptr_t *) __builtin_frame_address(0); + + for(size_t i = 0; i < BACKTRACE_DEPTH && fp != NULL; i++) { + hash ^= (uint64_t) fp[1]; + fp = _next_frame(fp); + } + return hash; } -#define SAVE_BACKTRACE_FRAME(frame, bts) \ - if(__builtin_frame_address(frame)) { \ - uint64_t r = (uint64_t) __builtin_return_address(frame); \ - if(r > 0x1000) { \ - bts->callers[frame - 1] = (uint64_t) __builtin_return_address(frame); \ - } \ - } else { \ - return; \ +/* Stores up to BACKTRACE_DEPTH return addresses in callers */ +static INLINE void _save_backtrace_frames(uint64_t *callers) { + uintptr_t *fp = (uintptr_t *) __builtin_frame_address(0); + + for(size_t i = 0; i < BACKTRACE_DEPTH && fp != NULL; i++) { + uint64_t r = (uint64_t) fp[1]; + + if(r > 0x1000) { + callers[i] = r; + } + + fp = _next_frame(fp); } +} INTERNAL_HIDDEN INLINE void _save_alloc_backtrace(iso_alloc_traces_t *abts) { - SAVE_BACKTRACE_FRAME(1, abts); - SAVE_BACKTRACE_FRAME(2, abts); - SAVE_BACKTRACE_FRAME(3, abts); - SAVE_BACKTRACE_FRAME(4, abts); - SAVE_BACKTRACE_FRAME(5, abts); - SAVE_BACKTRACE_FRAME(6, abts); - SAVE_BACKTRACE_FRAME(7, abts); - SAVE_BACKTRACE_FRAME(8, abts); - return; + _save_backtrace_frames(abts->callers); } INTERNAL_HIDDEN INLINE void _save_free_backtrace(iso_free_traces_t *fbts) { - SAVE_BACKTRACE_FRAME(1, fbts); - SAVE_BACKTRACE_FRAME(2, fbts); - SAVE_BACKTRACE_FRAME(3, fbts); - SAVE_BACKTRACE_FRAME(4, fbts); - SAVE_BACKTRACE_FRAME(5, fbts); - SAVE_BACKTRACE_FRAME(6, fbts); - SAVE_BACKTRACE_FRAME(7, fbts); - SAVE_BACKTRACE_FRAME(8, fbts); - return; + _save_backtrace_frames(fbts->callers); } -INTERNAL_HIDDEN void _iso_output_profile() { +INTERNAL_HIDDEN void _iso_output_profile(void) { _iso_alloc_printf(profiler_fd, "allocated=%d\n", _alloc_count); _iso_alloc_printf(profiler_fd, "alloc_sampled=%d\n", _alloc_sampled_count); _iso_alloc_printf(profiler_fd, "freed=%d\n", _free_count); @@ -351,10 +362,9 @@ INTERNAL_HIDDEN void _iso_output_profile() { continue; } - Dl_info dl; - dladdr((void *) abts->callers[j], &dl); + Dl_info dl = {0}; - if(dl.dli_sname != NULL) { + if(dladdr((void *) abts->callers[j], &dl) != 0 && dl.dli_sname != NULL) { _iso_alloc_printf(profiler_fd, "\t0x%x -> %s %s\n", abts->callers[j], dl.dli_sname, dl.dli_fname); } else { _iso_alloc_printf(profiler_fd, "\t0x%x -> [?]\n", abts->callers[j]); @@ -372,10 +382,9 @@ INTERNAL_HIDDEN void _iso_output_profile() { continue; } - Dl_info dl; - dladdr((void *) fbts->callers[j], &dl); + Dl_info dl = {0}; - if(dl.dli_sname != NULL) { + if(dladdr((void *) fbts->callers[j], &dl) != 0 && dl.dli_sname != NULL) { _iso_alloc_printf(profiler_fd, "\t0x%x -> %s %s\n", fbts->callers[j], dl.dli_sname, dl.dli_fname); } else { _iso_alloc_printf(profiler_fd, "\t0x%x -> [?]\n", fbts->callers[j]); @@ -383,7 +392,7 @@ INTERNAL_HIDDEN void _iso_output_profile() { } } - for(int i = 0; i < SMALL_SIZE_MAX; i++) { + for(int i = 0; i <= SMALL_SIZE_MAX; i++) { if(_zone_profiler_map[i].count != 0) { _iso_alloc_printf(profiler_fd, "%d,%d,%d\n", i, _zone_profiler_map[i].total, _zone_profiler_map[i].count); } @@ -418,7 +427,8 @@ INTERNAL_HIDDEN void _iso_alloc_profile(size_t size) { used = _iso_alloc_zone_leak_detector(zone, true); } - used = (int32_t) ((float) (used / zone->chunk_count) * 100.0); + /* Convert the chunk count to a percentage of the zone */ + used = (uint32_t) (((float) used / zone->chunk_count) * 100.0f); if(used > CHUNK_USAGE_THRESHOLD) { _zone_profiler_map[zone->chunk_size].count++; @@ -458,6 +468,8 @@ INTERNAL_HIDDEN void _iso_alloc_profile(size_t size) { } } +/* Must be called with the root locked, it touches + * the profiler globals and the root seed */ INTERNAL_HIDDEN void _iso_free_profile(void) { _free_count++; diff --git a/src/iso_alloc_sanity.c b/src/iso_alloc_sanity.c index 1e601fd..6b60b71 100644 --- a/src/iso_alloc_sanity.c +++ b/src/iso_alloc_sanity.c @@ -392,7 +392,11 @@ INTERNAL_HIDDEN int32_t _iso_alloc_free_sane_sample(void *p) { _sane_allocation_t *sane_alloc = _get_sane_alloc(p); if(sane_alloc != NULL) { +#if !UNINIT_READ_SANITY + /* The canary is only written when we aren't using + * userfaultfd to detect uninitialized reads */ check_sanity_canary(sane_alloc); +#endif unmap_guarded_pages(sane_alloc->address, g_page_size); memset(sane_alloc, 0x0, sizeof(_sane_allocation_t)); _sane_cache[SANE_CACHE_IDX(p)]--; diff --git a/src/iso_alloc_util.c b/src/iso_alloc_util.c index fadd5a3..d968513 100644 --- a/src/iso_alloc_util.c +++ b/src/iso_alloc_util.c @@ -214,7 +214,7 @@ int32_t name_mapping(void *p, size_t sz, const char *name) { } bool is_pow2(uint64_t sz) { - return (sz & (sz - 1)) == 0; + return sz != 0 && (sz & (sz - 1)) == 0; } size_t next_pow2(size_t sz) { diff --git a/src/malloc_hook.c b/src/malloc_hook.c index 6110481..6331709 100644 --- a/src/malloc_hook.c +++ b/src/malloc_hook.c @@ -48,16 +48,17 @@ EXTERNAL_API void *__libc_reallocarray(void *p, size_t n, size_t s) ISO_FORWARD3 EXTERNAL_API void *reallocarray(void *p, size_t n, size_t s) ISO_FORWARD3(iso_reallocarray, p, n, s) EXTERNAL_API int __posix_memalign(void **r, size_t a, size_t s) { - if(is_pow2(a) == false) { - *r = NULL; + if(r == NULL) { return EINVAL; } - if(s < a) { - s = a; + /* POSIX requires a power of 2 that is a multiple of sizeof(void *) */ + if(a == 0 || (a % sizeof(void *)) != 0 || is_pow2(a) == false) { + *r = NULL; + return EINVAL; } - *r = iso_alloc(s); + *r = _iso_alloc_aligned(s, a); if(*r != NULL) { return 0; @@ -70,19 +71,21 @@ EXTERNAL_API int posix_memalign(void **r, size_t alignment, size_t s) { return __posix_memalign(r, alignment, s); } -EXTERNAL_API void *__libc_memalign(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *memalign(size_t alignment, size_t s) { + if(alignment == 0 || is_pow2(alignment) == false) { + errno = EINVAL; + return NULL; + } + + return _iso_alloc_aligned(s, alignment); } -EXTERNAL_API void *aligned_alloc(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *__libc_memalign(size_t alignment, size_t s) { + return memalign(alignment, s); } -EXTERNAL_API void *memalign(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *aligned_alloc(size_t alignment, size_t s) { + return memalign(alignment, s); } #if __ANDROID__ || __FreeBSD__ diff --git a/tests/aligned_alloc_test.c b/tests/aligned_alloc_test.c new file mode 100644 index 0000000..0835a49 --- /dev/null +++ b/tests/aligned_alloc_test.c @@ -0,0 +1,216 @@ +/* iso_alloc aligned_alloc_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for the aligned allocation interfaces. + * posix_memalign, memalign and aligned_alloc previously + * ignored the requested alignment and returned chunks with + * only SZ_ALIGNMENT bytes of alignment. They also accepted + * invalid alignments instead of returning EINVAL. This test + * also covers NULL handling in iso_alloc_new_zone and + * iso_zone_chunk_count */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if __linux__ +#include +#else +/* macOS does not declare memalign but IsoAlloc exports it */ +void *memalign(size_t alignment, size_t size); +#endif + +static const size_t alignments[] = {8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, + 8192, 16384, 32768, 65536, (1 << 20)}; + +static const size_t sizes[] = {1, 24, 100, 1000, 4097, 70000, 300000}; + +static void check_chunk(void *p, size_t alignment, size_t size, const char *api) { + if(p == NULL) { + LOG_AND_ABORT("%s(%d, %d) returned NULL", api, alignment, size); + } + + if(((uintptr_t) p & (alignment - 1)) != 0) { + LOG_AND_ABORT("%s(%d, %d) returned %p which is not aligned", api, alignment, size, p); + } + + size_t cs = iso_chunksz(p); + + if(cs < size) { + LOG_AND_ABORT("%s(%d, %d) returned a chunk of only %d bytes", api, alignment, size, cs); + } + + /* The entire requested size must be writable */ + memset(p, 0x41, size); +} + +int main(int argc, char *argv[]) { + const size_t page_size = (size_t) sysconf(_SC_PAGESIZE); + const size_t na = sizeof(alignments) / sizeof(alignments[0]); + const size_t ns = sizeof(sizes) / sizeof(sizes[0]); + void *ptrs[sizeof(alignments) / sizeof(alignments[0]) * sizeof(sizes) / sizeof(sizes[0]) * 3]; + size_t count = 0; + + /* Every combination through each interface. Chunks are + * held until the end so later allocations can't reuse + * an earlier aligned chunk by accident */ + for(size_t i = 0; i < na; i++) { + for(size_t j = 0; j < ns; j++) { + void *p = NULL; + int r = posix_memalign(&p, alignments[i], sizes[j]); + + if(r != 0) { + LOG_AND_ABORT("posix_memalign(%d, %d) returned %d", alignments[i], sizes[j], r); + } + + check_chunk(p, alignments[i], sizes[j], "posix_memalign"); + ptrs[count++] = p; + + p = memalign(alignments[i], sizes[j]); + check_chunk(p, alignments[i], sizes[j], "memalign"); + ptrs[count++] = p; + + /* C11 requires size to be a multiple of alignment */ + size_t s = (sizes[j] + (alignments[i] - 1)) & ~(alignments[i] - 1); + p = aligned_alloc(alignments[i], s); + check_chunk(p, alignments[i], s, "aligned_alloc"); + ptrs[count++] = p; + } + } + + for(size_t i = 0; i < count; i++) { + free(ptrs[i]); + } + + /* A big zone with an alignment larger than a page is + * trimmed from a larger mapping. Reusing it from the + * free list must preserve that alignment */ + const size_t big_alignment = (page_size << 2); + void *first = NULL; + int r = posix_memalign(&first, big_alignment, SMALL_SIZE_MAX * 2); + + if(r != 0 || first == NULL) { + LOG_AND_ABORT("posix_memalign(%d, %d) failed", big_alignment, SMALL_SIZE_MAX * 2); + } + + check_chunk(first, big_alignment, SMALL_SIZE_MAX * 2, "posix_memalign"); + free(first); + + for(size_t i = 0; i < 8; i++) { + void *p = NULL; + r = posix_memalign(&p, big_alignment, SMALL_SIZE_MAX * 2); + + if(r != 0) { + LOG_AND_ABORT("posix_memalign(%d, %d) returned %d on reuse", big_alignment, SMALL_SIZE_MAX * 2, r); + } + + check_chunk(p, big_alignment, SMALL_SIZE_MAX * 2, "posix_memalign"); + free(p); + } + + /* Invalid alignments must fail with EINVAL and never + * hand back memory with the wrong alignment. The values + * are volatile so the compiler can't reject them itself */ + static volatile size_t zero_alignment = 0; + static volatile size_t odd_alignment = 3; + static volatile size_t non_pow2_alignment = 24; + static volatile size_t small_alignment = 4; + static volatile size_t bad_c11_alignment = 48; + void *bad = (void *) 0x1; + + if(posix_memalign(&bad, zero_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 0"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, odd_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 3"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, non_pow2_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 24"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, small_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment smaller than sizeof(void *)"); + } + + /* The result pointer is volatile so the compiler can't + * see that it is NULL and reject the call itself */ + void **volatile null_result = NULL; + + if(posix_memalign(null_result, 64, 64) != EINVAL) { + LOG_AND_ABORT("posix_memalign accepted a NULL result pointer"); + } + + errno = 0; + + if(memalign(zero_alignment, 64) != NULL || errno != EINVAL) { + LOG_AND_ABORT("memalign accepted an alignment of 0"); + } + + errno = 0; + + if(memalign(non_pow2_alignment, 64) != NULL || errno != EINVAL) { + LOG_AND_ABORT("memalign accepted an alignment of 24"); + } + + errno = 0; + + if(aligned_alloc(bad_c11_alignment, 96) != NULL || errno != EINVAL) { + LOG_AND_ABORT("aligned_alloc accepted an alignment of 48"); + } + + /* Overflow in the size rounding must fail cleanly */ + bad = (void *) 0x1; + r = posix_memalign(&bad, 4096, SIZE_MAX - 100); + + if(r != ENOMEM || bad != NULL) { + LOG_AND_ABORT("posix_memalign did not fail on size overflow (%d, %p)", r, bad); + } + + /* A zero size with a valid alignment still returns + * an aligned chunk that can be freed */ + void *z = NULL; + r = posix_memalign(&z, 512, 0); + + if(r != 0 || z == NULL || ((uintptr_t) z & 511) != 0) { + LOG_AND_ABORT("posix_memalign(512, 0) failed (%d, %p)", r, z); + } + + free(z); + + /* iso_alloc_new_zone must return NULL for a size the + * zone allocator cannot handle, not a masked NULL */ + iso_alloc_zone_handle *zone = iso_alloc_new_zone(SMALL_SIZE_MAX + 1); + + if(zone != NULL) { + LOG_AND_ABORT("iso_alloc_new_zone returned %p for an oversized chunk", zone); + } + + if(iso_zone_chunk_count(NULL) != 0) { + LOG_AND_ABORT("iso_zone_chunk_count(NULL) is not 0"); + } + + zone = iso_alloc_new_zone(256); + + if(zone == NULL || iso_zone_chunk_count(zone) == 0) { + LOG_AND_ABORT("iso_alloc_new_zone(256) failed"); + } + + iso_alloc_destroy_zone(zone); + + iso_flush_caches(); + iso_verify_zones(); + + return 0; +} diff --git a/tests/profiler_test.c b/tests/profiler_test.c new file mode 100644 index 0000000..e30025e --- /dev/null +++ b/tests/profiler_test.c @@ -0,0 +1,174 @@ +/* iso_alloc profiler_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for the heap profiler. The traces API + * previously copied a single entry regardless of how many + * had been recorded, the per chunk size map was one entry + * too small for SMALL_SIZE_MAX byte zones, and the free + * profiler ran without the root lock. The profiler data + * file written at exit is checked by run_profiler_tests.sh */ +#include +#include +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" +#include "iso_alloc_profiler.h" + +#if !HEAP_PROFILER +#error "This test intended to be run with -DHEAP_PROFILER=1" +#endif + +/* Enough allocations that at least one is sampled at + * PROFILER_ODDS with overwhelming probability */ +#define CYCLES (PROFILER_ODDS * 50) +#define THREADS 4 +#define THREAD_CYCLES (PROFILER_ODDS * 10) +#define HELD 50 +#define FILL 0xab + +/* Backtraces are keyed on the call path so each of these + * is a distinct allocation or free site. They must not be + * inlined or they collapse into a single trace */ +#define ALLOC_SITE(name) \ + __attribute__((noinline)) static void *name(size_t size) { \ + void *p = iso_alloc(size); \ + if(p == NULL) { \ + LOG_AND_ABORT("iso_alloc failed in " #name); \ + } \ + return p; \ + } + +#define FREE_SITE(name) \ + __attribute__((noinline)) static void name(void *p) { \ + iso_free(p); \ + } + +ALLOC_SITE(alloc_site_a) +ALLOC_SITE(alloc_site_b) +ALLOC_SITE(alloc_site_c) +ALLOC_SITE(alloc_site_d) +FREE_SITE(free_site_a) +FREE_SITE(free_site_b) +FREE_SITE(free_site_c) +FREE_SITE(free_site_d) + +static void *churn(void *arg) { + for(size_t i = 0; i < THREAD_CYCLES; i++) { + void *p = iso_alloc((i % 4096) + 1); + + if(p == NULL) { + LOG_AND_ABORT("iso_alloc failed in thread"); + } + + iso_free(p); + } + + return NULL; +} + +int main(int argc, char *argv[]) { + /* Hold most of a SMALL_SIZE_MAX zone so the profiler + * records usage for the largest possible chunk size. + * That index lands one past the end of the old map */ + void *held[HELD]; + + for(size_t i = 0; i < HELD; i++) { + held[i] = iso_alloc(SMALL_SIZE_MAX); + + if(held[i] == NULL) { + LOG_AND_ABORT("Failed to allocate %d byte chunk", SMALL_SIZE_MAX); + } + } + + for(size_t i = 0; i < CYCLES; i++) { + const size_t size = (i % 2048) + 1; + free_site_a(alloc_site_a(size)); + free_site_b(alloc_site_b(size)); + free_site_c(alloc_site_c(size)); + free_site_d(alloc_site_d(size)); + } + + /* The traces API must copy every recorded entry. Fill + * the output with a pattern so untouched entries show */ + iso_alloc_traces_t at[BACKTRACE_DEPTH_SZ]; + memset(at, FILL, sizeof(at)); + size_t alloc_trace_count = iso_get_alloc_traces(at); + + /* Four sites each sampled many times over must yield + * more than one distinct trace */ + if(alloc_trace_count < 2 || alloc_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad alloc trace count %d", alloc_trace_count); + } + + iso_alloc_traces_t pattern; + memset(&pattern, FILL, sizeof(pattern)); + + for(size_t i = 0; i < alloc_trace_count; i++) { + if(memcmp(&at[i], &pattern, sizeof(pattern)) == 0) { + LOG_AND_ABORT("Alloc trace %d of %d was not copied", i, alloc_trace_count); + } + + if(at[i].call_count == 0) { + LOG_AND_ABORT("Alloc trace %d has no calls", i); + } + } + + iso_free_traces_t ft[BACKTRACE_DEPTH_SZ]; + memset(ft, FILL, sizeof(ft)); + size_t free_trace_count = iso_get_free_traces(ft); + + if(free_trace_count < 2 || free_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad free trace count %d", free_trace_count); + } + + iso_free_traces_t fpattern; + memset(&fpattern, FILL, sizeof(fpattern)); + + for(size_t i = 0; i < free_trace_count; i++) { + if(memcmp(&ft[i], &fpattern, sizeof(fpattern)) == 0) { + LOG_AND_ABORT("Free trace %d of %d was not copied", i, free_trace_count); + } + + if(ft[i].call_count == 0) { + LOG_AND_ABORT("Free trace %d has no calls", i); + } + } + + iso_alloc_reset_traces(); + + if(iso_get_alloc_traces(at) != 0 || iso_get_free_traces(ft) != 0) { + LOG_AND_ABORT("Traces were not reset"); + } + +#if THREAD_SUPPORT + /* Concurrent frees exercise the lock around the free + * profiler. A race here corrupts the backtrace arrays */ + pthread_t threads[THREADS]; + + for(size_t i = 0; i < THREADS; i++) { + if(pthread_create(&threads[i], NULL, churn, NULL) != 0) { + LOG_AND_ABORT("pthread_create failed"); + } + } + + for(size_t i = 0; i < THREADS; i++) { + pthread_join(threads[i], NULL); + } + + free_trace_count = iso_get_free_traces(ft); + + if(free_trace_count == 0 || free_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad free trace count %d after threads", free_trace_count); + } +#endif + + for(size_t i = 0; i < HELD; i++) { + iso_free(held[i]); + } + + iso_flush_caches(); + iso_verify_zones(); + + return 0; +} diff --git a/tests/tag_refresh_test.c b/tests/tag_refresh_test.c new file mode 100644 index 0000000..2cff353 --- /dev/null +++ b/tests/tag_refresh_test.c @@ -0,0 +1,125 @@ +/* iso_alloc tag_refresh_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for _refresh_zone_mem_tags. Once a zone + * has no live allocations and has seen more than a quarter + * of its retirement threshold in allocations every free + * regenerates every tag in the zone. The condition for this + * was previously miscomputed so it never fired, and when it + * did it wrote through the masked user pages pointer. This + * test fills a small zone, drives it past the threshold, and + * then checks that a single free changed the tag of every + * other chunk in the zone */ +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if !MEMORY_TAGGING +#error "This test intended to be run with -DMEMORY_TAGGING=1" +#endif + +/* Large chunks keep the zone small so the refresh + * threshold is reached in a few hundred cycles */ +#define SIZE SMALL_SIZE_MAX +#define CHUNKS (ZONE_USER_SIZE / SIZE) + +int main(int argc, char *argv[]) { + iso_alloc_zone_handle *_zone_handle = iso_alloc_new_zone(SIZE); + + if(_zone_handle == NULL) { + abort(); + } + + /* Allocate every chunk in the zone so we know their + * addresses, then free them all */ + void *chunks[CHUNKS]; + size_t count = 0; + + for(size_t i = 0; i < CHUNKS; i++) { + void *p = iso_alloc_from_zone(_zone_handle); + + if(p == NULL) { + break; + } + + chunks[count++] = p; + } + + if(count < (CHUNKS / 2)) { + LOG_AND_ABORT("Only allocated %d of %d chunks", count, CHUNKS); + } + + for(size_t i = 0; i < count; i++) { + iso_free_from_zone(chunks[i], _zone_handle); + } + + /* The refresh fires on a free once the zone has no live + * chunks and alloc_count exceeds a quarter of the + * retirement threshold. Overshoot it to be safe */ + const size_t threshold = (CHUNKS * ZONE_ALLOC_RETIRE) / 4; + + for(size_t i = 0; i < threshold + 64; i++) { + void *p = iso_alloc_from_zone_tagged(_zone_handle); + + if(p == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk in cycle %d", i); + } + + iso_alloc_verify_ptr_tag(p, _zone_handle); + iso_free_from_zone(p, _zone_handle); + } + + /* Allocate one chunk and snapshot the tags of every other + * chunk. The free below must refresh all of them */ + void *q = iso_alloc_from_zone_tagged(_zone_handle); + + if(q == NULL) { + LOG_AND_ABORT("Failed to allocate the final tagged chunk"); + } + + void *uq = iso_alloc_untag_ptr(q, _zone_handle); + uint8_t before[CHUNKS]; + + for(size_t i = 0; i < count; i++) { + before[i] = iso_alloc_get_mem_tag(chunks[i], _zone_handle); + } + + iso_free_from_zone(uq, _zone_handle); + + size_t unchanged = 0; + + for(size_t i = 0; i < count; i++) { + if(chunks[i] == uq) { + continue; + } + + if(iso_alloc_get_mem_tag(chunks[i], _zone_handle) == before[i]) { + unchanged++; + } + } + + if(unchanged != 0) { + LOG_AND_ABORT("Zone tag refresh left %d of %d chunk tags unchanged", unchanged, count); + } + + /* The stale pointer must no longer untag to its chunk + * and a fresh pointer must still verify after a refresh */ + if(iso_alloc_untag_ptr(q, _zone_handle) == uq) { + LOG_AND_ABORT("Stale pointer %p untagged to its original address after refresh", q); + } + + void *fresh = iso_alloc_from_zone_tagged(_zone_handle); + + if(fresh == NULL) { + LOG_AND_ABORT("Failed to allocate after refresh"); + } + + iso_alloc_verify_ptr_tag(fresh, _zone_handle); + iso_free_from_zone(fresh, _zone_handle); + + iso_flush_caches(); + iso_alloc_destroy_zone(_zone_handle); + + return 0; +} diff --git a/tests/tests.cpp b/tests/tests.cpp index c166fe6..dffd547 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #if THREAD_SUPPORT #include #endif @@ -20,6 +23,124 @@ static const uint32_t array_sizes[] = {16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}; int32_t alloc_count; +int32_t new_handler_calls; + +struct alignas(256) Aligned256 { + uint8_t data[300]; +}; + +struct alignas(4096) AlignedPage { + uint8_t data[100]; +}; + +static void check_alignment(void *p, size_t alignment) { + if(p == NULL) { + LOG_AND_ABORT("Aligned new returned NULL for alignment %d", alignment); + } + + if((reinterpret_cast(p) & (alignment - 1)) != 0) { + LOG_AND_ABORT("Pointer %p is not %d byte aligned", p, alignment); + } +} + +/* The aligned new operators previously ignored the requested + * alignment, and the throwing operators could return NULL + * instead of calling the new handler and throwing bad_alloc */ +static void aligned_new_tests(void) { + auto *a = new Aligned256(); + check_alignment(a, alignof(Aligned256)); + memset(a->data, 0x41, sizeof(a->data)); + delete a; + + auto *aa = new Aligned256[8]; + check_alignment(aa, alignof(Aligned256)); + + for(size_t i = 0; i < 8; i++) { + check_alignment(&aa[i], alignof(Aligned256)); + memset(aa[i].data, 0x41, sizeof(aa[i].data)); + } + + delete[] aa; + + auto *pg = new AlignedPage(); + check_alignment(pg, alignof(AlignedPage)); + delete pg; + + /* Alignments larger than a page go through the big zone path */ + const size_t big_alignment = (1 << 20); + void *b = ::operator new(1000, std::align_val_t(big_alignment)); + check_alignment(b, big_alignment); + memset(b, 0x41, 1000); + ::operator delete(b, std::align_val_t(big_alignment)); + + void *n = ::operator new(100, std::align_val_t(512), std::nothrow); + check_alignment(n, 512); + ::operator delete(n, std::align_val_t(512), std::nothrow); + + /* A throwing new must throw bad_alloc rather than return NULL. + * These impossible sizes make the allocator log a failed big + * zone allocation for each attempt, which is expected */ + LOG("The following big zone allocation failures are expected"); + static volatile size_t huge = (SIZE_MAX >> 1); + bool caught = false; + + try { + auto *h = new uint8_t[huge]; + delete[] h; + } catch(const std::bad_alloc &) { + caught = true; + } + + if(caught == false) { + LOG_AND_ABORT("new did not throw bad_alloc for an impossible size"); + } + + caught = false; + + try { + void *h = ::operator new(huge, std::align_val_t(4096)); + ::operator delete(h, std::align_val_t(4096)); + } catch(const std::bad_alloc &) { + caught = true; + } + + if(caught == false) { + LOG_AND_ABORT("aligned new did not throw bad_alloc for an impossible size"); + } + + /* The nothrow variants return NULL instead */ + if(new(std::nothrow) uint8_t[huge] != NULL) { + LOG_AND_ABORT("nothrow new returned memory for an impossible size"); + } + + if(::operator new(huge, std::align_val_t(4096), std::nothrow) != NULL) { + LOG_AND_ABORT("nothrow aligned new returned memory for an impossible size"); + } + + /* An installed new handler must be called once before + * bad_alloc is thrown. The handler uninstalls itself so + * the retry fails and the exception follows */ + new_handler_calls = 0; + std::set_new_handler([]() { + new_handler_calls++; + std::set_new_handler(nullptr); + }); + + caught = false; + + try { + auto *h = new uint8_t[huge]; + delete[] h; + } catch(const std::bad_alloc &) { + caught = true; + } + + std::set_new_handler(nullptr); + + if(caught == false || new_handler_calls != 1) { + LOG_AND_ABORT("new handler was called %d times, bad_alloc caught = %d", new_handler_calls, caught); + } +} class Base { public: @@ -132,6 +253,8 @@ int main(int argc, char *argv[]) { } #endif + aligned_new_tests(); + iso_verify_zones(); return 0; diff --git a/tests/valid_tag_ptr_test.c b/tests/valid_tag_ptr_test.c new file mode 100644 index 0000000..3a4b77e --- /dev/null +++ b/tests/valid_tag_ptr_test.c @@ -0,0 +1,85 @@ +/* iso_alloc valid_tag_ptr_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for iso_alloc_verify_ptr_tag. The tag + * comparison previously compared the 8 bit tag against the + * unshifted top byte of the pointer so every valid tagged + * pointer failed verification. The other tagging tests only + * check that verification aborts on a stale pointer, this + * one checks that it succeeds on a valid pointer. It also + * checks that freeing a chunk always changes its tag so a + * stale tagged pointer can never untag to the same address */ +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if !MEMORY_TAGGING +#error "This test intended to be run with -DMEMORY_TAGGING=1" +#endif + +#define SIZE 256 +#define HELD 64 +#define CYCLES 4096 + +int main(int argc, char *argv[]) { + iso_alloc_zone_handle *_zone_handle = iso_alloc_new_zone(SIZE); + + if(_zone_handle == NULL) { + abort(); + } + + void *held[HELD]; + + /* Every live tagged pointer must verify */ + for(size_t i = 0; i < HELD; i++) { + held[i] = iso_alloc_from_zone_tagged(_zone_handle); + + if(held[i] == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk %d", i); + } + + iso_alloc_verify_ptr_tag(held[i], _zone_handle); + + void *up = iso_alloc_untag_ptr(held[i], _zone_handle); + memset(up, 0x41, SIZE); + } + + for(size_t i = 0; i < HELD; i++) { + iso_alloc_verify_ptr_tag(held[i], _zone_handle); + iso_free_from_zone(held[i], _zone_handle); + } + + /* A tag must change on every free. If it did not then + * untagging the stale pointer yields the original chunk + * address and a use after free would go undetected */ + for(size_t i = 0; i < CYCLES; i++) { + void *p = iso_alloc_from_zone_tagged(_zone_handle); + + if(p == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk in cycle %d", i); + } + + iso_alloc_verify_ptr_tag(p, _zone_handle); + + void *up = iso_alloc_untag_ptr(p, _zone_handle); + const uint8_t old_tag = iso_alloc_get_mem_tag(up, _zone_handle); + + iso_free_from_zone(up, _zone_handle); + + const uint8_t new_tag = iso_alloc_get_mem_tag(up, _zone_handle); + + if(old_tag == new_tag) { + LOG_AND_ABORT("Tag %x for chunk %p did not change on free in cycle %d", old_tag, up, i); + } + + if(iso_alloc_untag_ptr(p, _zone_handle) == up) { + LOG_AND_ABORT("Stale pointer %p untagged to its original address in cycle %d", p, i); + } + } + + iso_flush_caches(); + iso_alloc_destroy_zone(_zone_handle); + + return 0; +} diff --git a/utils/run_profiler_tests.sh b/utils/run_profiler_tests.sh new file mode 100755 index 0000000..10c8780 --- /dev/null +++ b/utils/run_profiler_tests.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Runs the heap profiler test and checks the data file +# the profiler writes when the process exits +$(echo '' > profiler_test_output.txt) + +$(ulimit -c 0) + +export LD_LIBRARY_PATH=build/ +export ISO_ALLOC_PROFILER_FILE_PATH=build/iso_alloc_profiler.data +rm -f $ISO_ALLOC_PROFILER_FILE_PATH + +failure=0 + +echo -n "Running profiler_test test" +echo "Running profiler_test test" >> profiler_test_output.txt 2>&1 +$(build/profiler_test >> profiler_test_output.txt 2>&1) +ret=$? + +if [ $ret -ne 0 ]; then + echo "... Failed" + failure=$((failure+1)) +else + echo "... Succeeded" +fi + +# The profile must record sampled allocations and frees +# and a row for the largest chunk size. That row lands +# one entry past the end of an undersized map +echo -n "Checking profiler data file" + +if [ ! -s $ISO_ALLOC_PROFILER_FILE_PATH ]; then + echo "... Failed (no data file)" + failure=$((failure+1)) +elif ! grep -qE '^alloc_sampled=[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^free_sampled=[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^alloc_backtrace=' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^65536,[0-9]+,[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH; then + echo "... Failed (unexpected contents)" + cat $ISO_ALLOC_PROFILER_FILE_PATH + failure=$((failure+1)) +else + echo "... Succeeded" +fi + +unset LD_LIBRARY_PATH +unset LD_PRELOAD +unset ISO_ALLOC_PROFILER_FILE_PATH + +if [ $failure -ne 0 ]; then + cat profiler_test_output.txt + exit -1 +else + exit 0 +fi diff --git a/utils/run_tagging_tests.sh b/utils/run_tagging_tests.sh index 1327583..b78a4ba 100755 --- a/utils/run_tagging_tests.sh +++ b/utils/run_tagging_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash $(echo '' > tagging_test_output.txt) -tests=("tagged_ptr_test" "tagged_ptr_test_cpp") +tests=("tagged_ptr_test" "tagged_ptr_test_cpp" "valid_tag_ptr_test" "tag_refresh_test") failure=0 succeeded=0 diff --git a/utils/run_tests.sh b/utils/run_tests.sh index 4f2c529..107232e 100755 --- a/utils/run_tests.sh +++ b/utils/run_tests.sh @@ -4,7 +4,7 @@ $(echo '' > test_output.txt) tests=("tests" "big_tests" "big_zone_chunksz" "zone_retirement" "strdup_test" "interfaces_test" "thread_tests" "pool_test" - "rand_freelist") + "rand_freelist" "aligned_alloc_test") failure=0 succeeded=0 start=$SECONDS