From e66df637acfd8cd0b4f2ff987050dde90c6abf86 Mon Sep 17 00:00:00 2001 From: struct Date: Sat, 7 Nov 2026 15:00:31 -0500 Subject: [PATCH] Fix memory tagging, heap profiler, aligned allocation, and interface bugs Memory tagging: fix the tag verification comparison which never matched a valid tag, fix the whole zone tag refresh condition and write through the unmasked user pages pointer, reject one-past-the-end pointers, and use modulo for the chunk offset check since chunk sizes are not always a power of 2. Freed chunks now always receive a tag that differs from the previous one so a stale tagged pointer can never verify. Heap profiler: size the zone profiler map for SMALL_SIZE_MAX inclusive, copy the full trace arrays, compute usage percentages with float math, check dladdr results, and call the free profiler under the root lock. Walk frame pointers directly for backtraces since gcc returns 0 for __builtin_return_address(n > 0) on aarch64. Declare profiler globals extern in the header so the profiler links under -fno-common. Aligned allocation: posix_memalign, memalign, aligned_alloc, and the aligned C++ new operators now honor the requested alignment through _iso_alloc_aligned. Alignments up to a page come from an exact fit zone, larger ones from an over-mapped and trimmed big zone. Hooks validate alignment per POSIX. Throwing operator new retries via the new handler and throws bad_alloc rather than returning NULL. Interfaces: iso_alloc_new_zone returns NULL instead of a masked NULL, iso_zone_chunk_count handles a NULL handle. Sanity: only check the sampled chunk canary when it was written. Tests: add aligned_alloc_test to the test suite, aligned new coverage to the C++ tests, valid_tag_ptr_test and tag_refresh_test to the tagging suite, and a profiler_tests target with profiler_test. Co-Authored-By: Claude Fable 5.1 --- Makefile | 23 +++- include/iso_alloc_internal.h | 12 +- include/iso_alloc_profiler.h | 26 ++-- src/iso_alloc.c | 243 ++++++++++++++++++++++++++++------- src/iso_alloc.cpp | 55 +++++++- src/iso_alloc_interfaces.c | 11 ++ src/iso_alloc_mem_tags.c | 55 ++++++-- src/iso_alloc_profiler.c | 118 +++++++++-------- src/iso_alloc_sanity.c | 4 + src/iso_alloc_util.c | 2 +- src/malloc_hook.c | 31 +++-- tests/aligned_alloc_test.c | 216 +++++++++++++++++++++++++++++++ tests/profiler_test.c | 174 +++++++++++++++++++++++++ tests/tag_refresh_test.c | 125 ++++++++++++++++++ tests/tests.cpp | 123 ++++++++++++++++++ tests/valid_tag_ptr_test.c | 85 ++++++++++++ utils/run_profiler_tests.sh | 54 ++++++++ utils/run_tagging_tests.sh | 2 +- utils/run_tests.sh | 2 +- 19 files changed, 1212 insertions(+), 149 deletions(-) create mode 100644 tests/aligned_alloc_test.c create mode 100644 tests/profiler_test.c create mode 100644 tests/tag_refresh_test.c create mode 100644 tests/valid_tag_ptr_test.c create mode 100755 utils/run_profiler_tests.sh diff --git a/Makefile b/Makefile index 6790209..6636251 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ PERM_FREE_REALLOC = -DPERM_FREE_REALLOC=0 ## 1 byte tag per addressable chunk of memory. These tags can ## be retrieved and verified. This feature will likely interfere ## with ARM MTE and PAC. See MEMORY_TAGGING.md for more information -MEMORY_TAGGING = -DMEMORY_TAGGING=0 +MEMORY_TAGGING = -DMEMORY_TAGGING=1 ## Enable abort() when isoalloc can't gather enough entropy. ABORT_NO_ENTROPY = -DABORT_NO_ENTROPY=1 @@ -90,8 +90,12 @@ THP_PAGES = -DTHP_PAGES=1 ## program. This file encodes the heap usage patterns of ## the target. This file can be consumed by the profiler ## CLI utility. See PROFILER.md for the format of this file -#HEAP_PROFILER = -DHEAP_PROFILER=1 -fno-omit-frame-pointer \ -# -fno-optimize-sibling-calls -ldl +#HEAP_PROFILER = $(HEAP_PROFILER_FLAGS) + +## Flags required to build with the heap profiler. Backtraces +## are collected by walking frame pointers so they must be kept +HEAP_PROFILER_FLAGS = -DHEAP_PROFILER=1 -fno-omit-frame-pointer \ + -fno-optimize-sibling-calls ## Enable CPU pinning support on a per-zone basis. This is ## a minor security feature which introduces an allocation @@ -425,6 +429,7 @@ tests: clean library_debug_unit_tests $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tests $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) $(UNIT_TESTING) tests/uaf.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/uaf $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/interfaces_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/interfaces_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/aligned_alloc_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/aligned_alloc_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/thread_tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/thread_tests $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) $(UNIT_TESTING) tests/big_canary_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/big_canary_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/big_tests.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/big_tests $(LDFLAGS) @@ -463,9 +468,19 @@ tagging_tests: clean cpp_library_debug $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/uaf_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/uaf_tag_ptr_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/bad_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/bad_tag_ptr_test $(LDFLAGS) $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/verify_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/verify_tag_ptr_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/valid_tag_ptr_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/valid_tag_ptr_test $(LDFLAGS) + $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/tag_refresh_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tag_refresh_test $(LDFLAGS) $(CXX) -DMEMORY_TAGGING=1 $(CXXFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(EXE_CFLAGS) $(OS_FLAGS) tests/tagged_ptr_test.cpp $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/tagged_ptr_test_cpp $(LDFLAGS) utils/run_tagging_tests.sh +## Build the library with the heap profiler and run its test. +## The profiler data file is checked by the runner script +profiler_tests: clean + @echo "make profiler_tests" + $(CC) $(CFLAGS) $(HEAP_PROFILER_FLAGS) $(LIBRARY) $(OS_FLAGS) $(UNIT_TESTING) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(C_SRCS) -o $(BUILD_DIR)/$(LIBNAME) -ldl + $(CC) $(CFLAGS) $(HEAP_PROFILER_FLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/profiler_test.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/profiler_test $(LDFLAGS) -ldl + utils/run_profiler_tests.sh + init_test: clean library_debug_unit_tests @echo "make init_test" $(CC) $(CFLAGS) $(EXE_CFLAGS) $(DEBUG_LOG_FLAGS) $(GDB_FLAGS) $(OS_FLAGS) tests/init_destroy.c $(ISO_ALLOC_PRINTF_SRC) -o $(BUILD_DIR)/init_destroy $(LDFLAGS) @@ -530,6 +545,6 @@ format-ci: clean: rm -rf build/* tests_perf_analysis.txt big_tests_perf_analysis.txt gmon.out \ - tagging_test_output.txt test_output.txt *.dSYM core* iso_alloc_profiler.data + tagging_test_output.txt test_output.txt profiler_test_output.txt *.dSYM core* iso_alloc_profiler.data rm -rf android/libs android/obj mkdir -p build/ diff --git a/include/iso_alloc_internal.h b/include/iso_alloc_internal.h index b8a036f..89bc094 100644 --- a/include/iso_alloc_internal.h +++ b/include/iso_alloc_internal.h @@ -367,6 +367,10 @@ extern pthread_mutex_t root_busy_mutex; /* The global root */ extern iso_alloc_root *_root; +/* These are C functions and may be called from iso_alloc.cpp */ +#ifdef __cplusplus +extern "C" { +#endif INTERNAL_HIDDEN INLINE void check_big_canary(iso_alloc_big_zone_t *big); INTERNAL_HIDDEN INLINE void check_canary(iso_alloc_zone_t *zone, const void *p); INTERNAL_HIDDEN INLINE void iso_clear_user_chunk(uint8_t *p, size_t size); @@ -377,6 +381,7 @@ INTERNAL_HIDDEN INLINE void flush_chunk_quarantine(void); INTERNAL_HIDDEN INLINE void clear_zone_cache(void); INTERNAL_HIDDEN iso_alloc_big_zone_t *iso_find_big_zone(void *p, bool remove); INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, size_t size); +INTERNAL_HIDDEN iso_alloc_zone_t *find_exact_fit_zone(size_t size); INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size); INTERNAL_HIDDEN iso_alloc_zone_t *iso_new_zone(size_t size, bool internal); INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal, int32_t index); @@ -389,6 +394,7 @@ INTERNAL_HIDDEN INLINE bit_slot_t get_next_free_bit_slot(iso_alloc_zone_t *zone) INTERNAL_HIDDEN iso_alloc_root *iso_alloc_new_root(void); INTERNAL_HIDDEN bool is_pow2(uint64_t sz); INTERNAL_HIDDEN bool _is_zone_retired(iso_alloc_zone_t *zone); +INTERNAL_HIDDEN void _iso_alloc_refresh_chunk_tag(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN bool _refresh_zone_mem_tags(iso_alloc_zone_t *zone); INTERNAL_HIDDEN iso_alloc_zone_t *_iso_free_internal_unlocked(void *p, bool permanent, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void fill_free_bit_slots(iso_alloc_zone_t *zone); @@ -414,8 +420,9 @@ INTERNAL_HIDDEN INLINE void dont_need_pages(void *p, size_t size); INTERNAL_HIDDEN void *_tag_ptr(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone); INTERNAL_HIDDEN void _free_big_zone_list(iso_alloc_big_zone_t *head); -INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size); +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size, size_t alignment); INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t size); +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_aligned(size_t size, size_t alignment); INTERNAL_HIDDEN INLINE ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bitslot, iso_alloc_zone_t *zone); INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_calloc(size_t nmemb, size_t size); #if UAF_PTR_PAGE @@ -430,6 +437,9 @@ INTERNAL_HIDDEN size_t _iso_chunk_size(void *p); INTERNAL_HIDDEN int64_t check_canary_no_abort(iso_alloc_zone_t *zone, const void *p); INTERNAL_HIDDEN void _iso_alloc_initialize(void); INTERNAL_HIDDEN void _iso_alloc_destroy(void); +#ifdef __cplusplus +} +#endif #if ARM_MTE INLINE void *iso_mte_untag_ptr(void *p); diff --git a/include/iso_alloc_profiler.h b/include/iso_alloc_profiler.h index b122318..8093024 100644 --- a/include/iso_alloc_profiler.h +++ b/include/iso_alloc_profiler.h @@ -18,29 +18,33 @@ /* The IsoAlloc profiler is not thread local but these * globals should only ever be touched by internal * allocator functions when the root is locked */ -uint64_t _alloc_count; -uint64_t _free_count; -uint64_t _alloc_sampled_count; -uint64_t _free_sampled_count; - -int32_t profiler_fd; +extern uint64_t _alloc_count; +extern uint64_t _free_count; +extern uint64_t _alloc_sampled_count; +extern uint64_t _free_sampled_count; typedef struct { uint64_t total; uint64_t count; } zone_profiler_map_t; -zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX]; +/* These are defined once in iso_alloc_profiler.c. This + * header is included by several translation units so + * defining them here fails to link under -fno-common */ +extern int32_t profiler_fd; + +/* Indexed by chunk size, which can be exactly SMALL_SIZE_MAX */ +extern zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX + 1]; /* iso_alloc_traces_t is a public structure, and * is defined in the public header iso_alloc.h */ -iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; -size_t _alloc_bts_count; +extern iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; +extern size_t _alloc_bts_count; /* iso_free_traces_t is a public structure, and * is defined in the public header iso_alloc.h */ -iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; -size_t _free_bts_count; +extern iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; +extern size_t _free_bts_count; INTERNAL_HIDDEN INLINE uint64_t _get_backtrace_hash(void); INTERNAL_HIDDEN INLINE void _save_backtrace(iso_alloc_traces_t *abts); diff --git a/src/iso_alloc.c b/src/iso_alloc.c index 6bfecdf..bd804ac 100644 --- a/src/iso_alloc.c +++ b/src/iso_alloc.c @@ -867,6 +867,44 @@ INTERNAL_HIDDEN FLATTEN iso_alloc_zone_t *is_zone_usable(iso_alloc_zone_t *zone, } } +/* Walks the lookup table chain for internal zones whose + * chunk size is exactly size and returns the first usable + * one, or NULL if none of them can hold another chunk */ +INTERNAL_HIDDEN iso_alloc_zone_t *find_exact_fit_zone(size_t size) { + int32_t i = _root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)]; + + if(i == 0) { + return NULL; + } + + const size_t zones_used = _root->zones_used; + + for(; i < zones_used;) { + iso_alloc_zone_t *zone = &_root->zones[i]; + + if(zone->chunk_size != size) { + LOG_AND_ABORT("Zone lookup table failed to match sizes for zone[%d](%d) for chunk size (%d)", zone->index, zone->chunk_size, size); + } + + if(zone->internal == false) { + LOG_AND_ABORT("Lookup table should never contain private zones"); + } + + if(is_zone_usable(zone, size) != NULL) { + return zone; + } + + if(zone->next_sz_index != 0) { + i = zone->next_sz_index; + } else { + /* We have reached the end of our linked zones */ + break; + } + } + + return NULL; +} + /* Finds a zone that can fit this allocation request */ INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size) { iso_alloc_zone_t *zone = NULL; @@ -891,37 +929,14 @@ INTERNAL_HIDDEN iso_alloc_zone_t *find_suitable_zone(size_t size) { } #endif - /* Fast path via lookup table */ - if(_root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)] != 0) { - i = _root->zone_lookup_table[SZ_TO_ZONE_LOOKUP_IDX(size)]; + /* Fast path via lookup table. If this fails to find a + * usable zone we fall through and iterate through all + * zones, including ones we may have skipped over, to + * find a suitable candidate instead of creating a new one */ + zone = find_exact_fit_zone(size); - for(; i < zones_used;) { - iso_alloc_zone_t *zone = &_root->zones[i]; - - if(zone->chunk_size != size) { - LOG_AND_ABORT("Zone lookup table failed to match sizes for zone[%d](%d) for chunk size (%d)", zone->index, zone->chunk_size, size); - } - - if(zone->internal == false) { - LOG_AND_ABORT("Lookup table should never contain private zones"); - } - - if(is_zone_usable(zone, size) != NULL) { - return zone; - } - - if(zone->next_sz_index != 0) { - i = zone->next_sz_index; - } else { - /* We have reached the end of our linked zones. The - * lookup table failed to find us a usable zone. - * Instead of creating a new one we will break out - * of this loop and try iterating through all zones, - * including ones we may have skipped over, to find - * a suitable candidate. */ - break; - } - } + if(zone != NULL) { + return zone; } #if SMALL_MEM_STARTUP @@ -1235,10 +1250,102 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t s LOG_AND_ABORT("Allocation size of %d is > %d and cannot use a private zone (%d)", size, SMALL_SIZE_MAX, zone->chunk_size); } - return _iso_big_alloc(size); + return _iso_big_alloc(size, g_page_size); } } +/* Allocates a chunk aligned to alignment bytes, which must be + * a power of 2. Every chunk is at least SZ_ALIGNMENT aligned. + * Stronger alignments are satisfied by rounding the size up to + * a multiple of the alignment and allocating from an internal + * zone whose chunk size is exactly that. Chunks are laid out + * at multiples of the chunk size from a page aligned base so + * this guarantees the alignment for anything up to a page. + * Larger alignments and sizes use the big zone path */ +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_aligned(size_t size, size_t alignment) { + if(UNLIKELY(is_pow2(alignment) == false)) { + return NULL; + } + + /* Every chunk is already SZ_ALIGNMENT aligned, and any + * smaller power of 2 divides SZ_ALIGNMENT */ + if(alignment <= SZ_ALIGNMENT) { + return _iso_alloc(NULL, size); + } + + if(size == 0) { + size = alignment; + } + + const size_t s = ((size + (alignment - 1)) & ~(alignment - 1)); + + if(UNLIKELY(s < size)) { + return NULL; + } + + LOCK_ROOT(); + + if(UNLIKELY(_root == NULL)) { +#if AUTO_CTOR_DTOR + g_page_size = sysconf(_SC_PAGESIZE); + g_page_size_shift = _log2(g_page_size); + iso_alloc_initialize_global_root(); +#else + LOG_AND_ABORT("Root never initialized!"); +#endif + } + +#if HEAP_PROFILER + _iso_alloc_profile(s); +#endif + + if(alignment > g_page_size || s > SMALL_SIZE_MAX) { + /* The big zone path uses a different lock */ + UNLOCK_ROOT(); + + /* Big zones must be at least SMALL_SIZE_MAX bytes. That + * is a power of 2 so rounding up to it keeps the size a + * multiple of any smaller alignment */ + return _iso_big_alloc((s < SMALL_SIZE_MAX) ? SMALL_SIZE_MAX : s, alignment); + } + + iso_alloc_zone_t *zone = find_exact_fit_zone(s); + + if(zone == NULL) { + zone = _iso_new_zone(s, true, -1); + + if(UNLIKELY(zone == NULL)) { + LOG_AND_ABORT("Failed to create a zone for aligned allocation of %zu bytes", s); + } + } + + bit_slot_t free_bit_slot = zone->next_free_bit_slot; + + if(UNLIKELY(free_bit_slot == BAD_BIT_SLOT)) { + UNLOCK_ROOT(); +#if ABORT_ON_NULL + LOG_AND_ABORT("isoalloc configured to abort on NULL"); +#endif + return NULL; + } + + UNMASK_ZONE_PTRS(zone); + + zone->next_free_bit_slot = BAD_BIT_SLOT; + void *p = _iso_alloc_bitslot_from_zone(free_bit_slot, zone); + + MASK_ZONE_PTRS(zone); + UNLOCK_ROOT(); + populate_zone_cache(zone); + +#if ARM_MTE + if(_root->arm_mte_enabled == true) { + return iso_mte_set_tag_range(p, zone->chunk_size); + } +#endif + return p; +} + INTERNAL_HIDDEN iso_alloc_zone_t *search_chunk_lookup_table(const void *restrict p) { /* The chunk lookup table is the fastest way to find a * zone given a pointer to a chunk so we check it first */ @@ -1597,7 +1704,13 @@ INTERNAL_HIDDEN void _iso_free(void *p, bool permanent) { #endif #if HEAP_PROFILER + /* The alloc side profiles with the root locked. The + * free profiler updates the same globals so it needs + * the same protection or racing frees can push the + * backtrace count past the end of the array */ + LOCK_ROOT(); _iso_free_profile(); + UNLOCK_ROOT(); #endif #if ARM_MTE @@ -1747,14 +1860,10 @@ INTERNAL_HIDDEN iso_alloc_zone_t *_iso_free_internal_unlocked(void *p, bool perm * be refreshed then just generate a new tag for this chunk */ if(zone->tagged == true) { if(_refresh_zone_mem_tags(zone) == false) { - /* We only need to refresh this single tag */ - void *user_pages_start = UNMASK_USER_PTR(zone); - uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); - uint64_t chunk_offset = (uint64_t) (p - user_pages_start); - _mtp += (chunk_offset / zone->chunk_size); - - /* Generate and write a new tag for this chunk */ - *_mtp = (uint8_t) us_rand_uint64(&_root->seed); + /* We only need to refresh this single tag. The + * new tag is guaranteed to differ from the old one + * so a stale tagged pointer to p can never verify */ + _iso_alloc_refresh_chunk_tag(p, zone); } } #endif @@ -1909,8 +2018,12 @@ INTERNAL_HIDDEN void iso_free_big_zone(iso_alloc_big_zone_t *big_zone, bool perm } } -INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { +/* alignment must be a power of 2. Every big zone mapping is + * page aligned so only alignments larger than a page need + * any special handling */ +INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size, size_t alignment) { const size_t new_size = ROUND_UP_PAGE(size); + const bool aligned = (alignment > g_page_size); if(new_size < size || new_size > BIG_SZ_MAX || size < SMALL_SIZE_MAX) { #if ABORT_ON_NULL @@ -1950,7 +2063,8 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { check_big_canary(big); /* We found a suitable big zone we can reuse */ - if(big->size >= size && (big->size - size) <= BIG_ZONE_WASTE * 2) { + if(big->size >= size && (big->size - size) <= BIG_ZONE_WASTE * 2 && + (aligned == false || ((uintptr_t) big->user_pages_start & (alignment - 1)) == 0)) { big->free = false; _root->big_zone_free_count--; UNPOISON_BIG_ZONE(big); @@ -2000,21 +2114,33 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { UNLOCK_BIG_ZONE_FREE(); /* The free list contained no usable entries - * so we need to create a new one */ + * so we need to create a new one. An alignment + * larger than a page is satisfied by mapping an + * extra alignment bytes and trimming both ends */ + size_t map_size = size; + + if(aligned == true) { + map_size += alignment; + + if(UNLIKELY(map_size < size)) { + return NULL; + } + } + void *user_pages = NULL; #if ARM_MTE #if BIG_ZONE_GUARD - user_pages = mmap_guarded_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_guarded_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #else if(_root->arm_mte_enabled == true) { - user_pages = mmap_rw_mte_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_rw_mte_pages(map_size, false, BIG_ZONE_UD_NAME); } #endif #else #if BIG_ZONE_GUARD - user_pages = mmap_guarded_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_guarded_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #else - user_pages = mmap_rw_pages(size, false, BIG_ZONE_UD_NAME); + user_pages = mmap_rw_pages(map_size, false, BIG_ZONE_UD_NAME); #endif #endif @@ -2025,6 +2151,31 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_big_alloc(size_t size) { return NULL; } + if(aligned == true) { + void *aligned_pages = (void *) (((uintptr_t) user_pages + (alignment - 1)) & ~(alignment - 1)); + const size_t head = (size_t) (aligned_pages - user_pages); + const size_t tail = (alignment - head); +#if BIG_ZONE_GUARD + /* Move the guard pages so they sit directly below and + * above the aligned range, then release everything else. + * The upper munmap also covers the original upper guard */ + if(head != 0) { + munmap(user_pages - g_page_size, head); + mprotect_pages(aligned_pages - g_page_size, g_page_size, PROT_NONE); + } + + mprotect_pages(aligned_pages + size, g_page_size, PROT_NONE); + munmap(aligned_pages + size + g_page_size, tail); +#else + if(head != 0) { + munmap(user_pages, head); + } + + munmap(aligned_pages + size, tail); +#endif + user_pages = aligned_pages; + } + /* We only need a single page for big zone meta data */ static_assert(BIG_ZONE_META_DATA_PAGE_COUNT == 1, "Big zone meta data only needs a single page"); #if BIG_ZONE_META_DATA_GUARD diff --git a/src/iso_alloc.cpp b/src/iso_alloc.cpp index 90fce59..61d987d 100644 --- a/src/iso_alloc.cpp +++ b/src/iso_alloc.cpp @@ -23,11 +23,54 @@ extern "C" __attribute__((constructor(FIRST_CTOR + 1))) void iso_cpp(void) { #define NEW_EXCEPT #endif +// A throwing operator new may never return NULL. The +// standard requires that we retry through the installed +// new handler and throw bad_alloc if there is none +static inline void *iso_cpp_new(size_t size) { + void *p = iso_alloc(size); + + while(UNLIKELY(p == NULL)) { + std::new_handler h = std::get_new_handler(); + + if(h == nullptr) { + throw std::bad_alloc(); + } + + h(); + p = iso_alloc(size); + } + + return p; +} + +static inline void *iso_cpp_new_aligned(size_t size, std::align_val_t val) { + const size_t alignment = static_cast(val); + + if(is_pow2(alignment) == false) { + throw std::bad_alloc(); + } + + void *p = _iso_alloc_aligned(size, alignment); + + while(UNLIKELY(p == NULL)) { + std::new_handler h = std::get_new_handler(); + + if(h == nullptr) { + throw std::bad_alloc(); + } + + h(); + p = _iso_alloc_aligned(size, alignment); + } + + return p; +} + // These hooks override the basic new/delete // operators to use the iso_alloc API EXTERNAL_API FLATTEN void *operator new(size_t size) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new(size); } EXTERNAL_API FLATTEN void operator delete(void *p) noexcept { @@ -35,7 +78,7 @@ EXTERNAL_API FLATTEN void operator delete(void *p) noexcept { } EXTERNAL_API FLATTEN void *operator new[](size_t size) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new(size); } EXTERNAL_API FLATTEN void operator delete[](void *p) noexcept { @@ -67,11 +110,11 @@ EXTERNAL_API FLATTEN void operator delete[](void *ptr, const std::nothrow_t &) n } EXTERNAL_API FLATTEN void *operator new(size_t size, std::align_val_t val) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new_aligned(size, val); } EXTERNAL_API FLATTEN void *operator new[](size_t size, std::align_val_t val) NEW_EXCEPT { - return iso_alloc(size); + return iso_cpp_new_aligned(size, val); } EXTERNAL_API FLATTEN void operator delete(void *p, std::align_val_t val) noexcept { @@ -83,11 +126,11 @@ EXTERNAL_API FLATTEN void operator delete[](void *p, std::align_val_t val) noexc } EXTERNAL_API FLATTEN void *operator new(size_t size, std::align_val_t val, std::nothrow_t &) noexcept { - return iso_alloc(size); + return _iso_alloc_aligned(size, static_cast(val)); } EXTERNAL_API FLATTEN void *operator new[](size_t size, std::align_val_t val, std::nothrow_t &) noexcept { - return iso_alloc(size); + return _iso_alloc_aligned(size, static_cast(val)); } EXTERNAL_API FLATTEN void operator delete(void *p, std::align_val_t val, std::nothrow_t &) noexcept { diff --git a/src/iso_alloc_interfaces.c b/src/iso_alloc_interfaces.c index 8dbd2ae..4febfb0 100644 --- a/src/iso_alloc_interfaces.c +++ b/src/iso_alloc_interfaces.c @@ -53,6 +53,10 @@ EXTERNAL_API FLATTEN size_t iso_chunksz(void *p) { } EXTERNAL_API FLATTEN NO_DISCARD size_t iso_zone_chunk_count(iso_alloc_zone_handle *zone) { + if(zone == NULL) { + return 0; + } + UNMASK_ZONE_HANDLE(zone); iso_alloc_zone_t *_zone = (iso_alloc_zone_t *) zone; size_t canaries = 0; @@ -236,6 +240,13 @@ EXTERNAL_API FLATTEN void iso_alloc_destroy_zone(iso_alloc_zone_handle *zone) { EXTERNAL_API FLATTEN NO_DISCARD iso_alloc_zone_handle *iso_alloc_new_zone(size_t size) { iso_alloc_zone_handle *zone = (iso_alloc_zone_handle *) iso_new_zone(size, false); + + /* A NULL must not be masked or the caller gets back + * a non-NULL handle that is really just the mask */ + if(zone == NULL) { + return NULL; + } + UNMASK_ZONE_HANDLE(zone); return zone; } diff --git a/src/iso_alloc_mem_tags.c b/src/iso_alloc_mem_tags.c index 2bb3713..f90b496 100644 --- a/src/iso_alloc_mem_tags.c +++ b/src/iso_alloc_mem_tags.c @@ -11,15 +11,16 @@ INTERNAL_HIDDEN uint8_t _iso_alloc_get_mem_tag(void *p, iso_alloc_zone_t *zone) #if MEMORY_TAGGING void *user_pages_start = UNMASK_USER_PTR(zone); - if(user_pages_start > p || (user_pages_start + ZONE_USER_SIZE) < p) { + if(user_pages_start > p || (user_pages_start + ZONE_USER_SIZE) <= p) { LOG_AND_ABORT("Cannot get tag for pointer %p with wrong zone %d %p - %p", p, zone->index, user_pages_start, user_pages_start + ZONE_USER_SIZE); } uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); const uint64_t chunk_offset = (uint64_t) (p - user_pages_start); - /* Ensure the pointer is a multiple of chunk size */ - if(UNLIKELY((chunk_offset & (zone->chunk_size - 1)) != 0)) { + /* Ensure the pointer is a multiple of chunk size. Chunk + * sizes are not always a power of 2 so we can't use a mask */ + if(UNLIKELY((chunk_offset % zone->chunk_size) != 0)) { LOG_AND_ABORT("Chunk offset %d not an alignment of %d", chunk_offset, zone->chunk_size); } @@ -40,8 +41,10 @@ INTERNAL_HIDDEN void _iso_alloc_verify_tag(void *p, iso_alloc_zone_t *zone) { void *untagged_p = (void *) ((uintptr_t) p & TAGGED_PTR_MASK); const uint64_t tag = _iso_alloc_get_mem_tag(untagged_p, zone); - if(tag != ((uintptr_t) p & IS_TAGGED_PTR_MASK)) { - LOG_AND_ABORT("Pointer %p has wrong tag %x, expected %x", p, ((uintptr_t) p & IS_TAGGED_PTR_MASK), tag); + const uint64_t ptr_tag = (((uintptr_t) p & IS_TAGGED_PTR_MASK) >> UNTAGGED_BITS); + + if(tag != ptr_tag) { + LOG_AND_ABORT("Pointer %p has wrong tag %x, expected %x", p, ptr_tag, tag); } #endif return; @@ -74,18 +77,48 @@ INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone) { #endif } +#if MEMORY_TAGGING +/* Returns a random tag that is guaranteed to differ from + * old_tag. A stale tagged pointer must never verify against + * the tag written when its chunk was freed, so we draw the + * new tag uniformly from the 255 values that are not old_tag */ +static INLINE uint8_t _new_mem_tag(uint8_t old_tag, uint8_t rnd) { + return (uint8_t) (old_tag + 1 + (rnd % 255)); +} +#endif + +/* Replaces the tag for the chunk at p, which must be an + * untagged pointer to the start of a chunk in zone, with + * a new random tag that differs from the current one */ +INTERNAL_HIDDEN void _iso_alloc_refresh_chunk_tag(void *p, iso_alloc_zone_t *zone) { +#if MEMORY_TAGGING + void *user_pages_start = UNMASK_USER_PTR(zone); + uint8_t *_mtp = (user_pages_start - g_page_size - ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE)); + const uint64_t chunk_offset = (uint64_t) (p - user_pages_start); + _mtp += (chunk_offset / zone->chunk_size); + *_mtp = _new_mem_tag(*_mtp, (uint8_t) us_rand_uint64(&_root->seed)); +#endif +} + INTERNAL_HIDDEN bool _refresh_zone_mem_tags(iso_alloc_zone_t *zone) { #if MEMORY_TAGGING /* This implements a similar policy to zone retirement. * The only difference is that we refresh all tags at * %25 of the configured zone retirement age */ - if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count << _root->zone_retirement_shf)) >> 2) { + if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > ((zone->chunk_count << _root->zone_retirement_shf) >> 2))) { size_t s = ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE); - uint64_t *_mtp = (zone->user_pages_start - g_page_size - s); - size_t tms = s / sizeof(uint64_t); - - for(uint64_t o = 0; o < tms; o++) { - _mtp[o] = us_rand_uint64(&_root->seed); + uint8_t *_mtp = (UNMASK_USER_PTR(zone) - g_page_size - s); + + /* Every tag must change so that no stale tagged + * pointer created before the refresh can verify + * against the new tags. One random qword provides + * the entropy for 8 consecutive tags */ + for(size_t o = 0; o < s; o += sizeof(uint64_t)) { + uint64_t r = us_rand_uint64(&_root->seed); + + for(size_t b = 0; b < sizeof(uint64_t); b++) { + _mtp[o + b] = _new_mem_tag(_mtp[o + b], (uint8_t) (r >> (b * 8))); + } } return true; diff --git a/src/iso_alloc_profiler.c b/src/iso_alloc_profiler.c index 3fea22b..0f28ad8 100644 --- a/src/iso_alloc_profiler.c +++ b/src/iso_alloc_profiler.c @@ -13,6 +13,19 @@ #include +#if HEAP_PROFILER +uint64_t _alloc_count; +uint64_t _free_count; +uint64_t _alloc_sampled_count; +uint64_t _free_sampled_count; +int32_t profiler_fd; +zone_profiler_map_t _zone_profiler_map[SMALL_SIZE_MAX + 1]; +iso_alloc_traces_t _alloc_bts[BACKTRACE_DEPTH_SZ]; +size_t _alloc_bts_count; +iso_free_traces_t _free_bts[BACKTRACE_DEPTH_SZ]; +size_t _free_bts_count; +#endif + INTERNAL_HIDDEN uint64_t _iso_alloc_detect_leaks_in_zone(iso_alloc_zone_t *zone) { LOCK_ROOT(); uint64_t leaks = _iso_alloc_zone_leak_detector(zone, false); @@ -173,7 +186,7 @@ INTERNAL_HIDDEN uint64_t _iso_alloc_zone_leak_detector(iso_alloc_zone_t *zone, b if(profile == false) { LOG("Zone[%d] Total number of %d byte chunks(%d) used and free'd (%d) (%d percent), in use = %d", zone->index, zone->chunk_size, zone->chunk_count, - was_used, (int32_t) ((float) was_used / zone->chunk_count) * 100, in_use); + was_used, (int32_t) (((float) was_used / zone->chunk_count) * 100), in_use); } MASK_ZONE_PTRS(zone); @@ -184,8 +197,7 @@ INTERNAL_HIDDEN uint64_t _iso_alloc_zone_leak_detector(iso_alloc_zone_t *zone, b * the total number of allocations both currently * in use and previously used by this zone */ if(profile == true) { - uint64_t total = (in_use + was_used); - return (uint64_t) ((float) (total / zone->chunk_count) * 100.0); + return (uint64_t) (in_use + was_used); } #endif return in_use; @@ -253,16 +265,16 @@ INTERNAL_HIDDEN uint64_t __iso_alloc_big_zone_mem_usage(iso_alloc_big_zone_t *he * be used to interpret allocation patterns */ INTERNAL_HIDDEN size_t _iso_get_alloc_traces(iso_alloc_traces_t *traces_out) { LOCK_ROOT(); - __iso_memcpy(traces_out, _alloc_bts, sizeof(iso_alloc_traces_t)); size_t sz = _alloc_bts_count; + __iso_memcpy(traces_out, _alloc_bts, sz * sizeof(iso_alloc_traces_t)); UNLOCK_ROOT(); return sz; } INTERNAL_HIDDEN size_t _iso_get_free_traces(iso_free_traces_t *traces_out) { LOCK_ROOT(); - __iso_memcpy(traces_out, _free_bts, sizeof(iso_free_traces_t)); size_t sz = _free_bts_count; + __iso_memcpy(traces_out, _free_bts, sz * sizeof(iso_free_traces_t)); UNLOCK_ROOT(); return sz; } @@ -276,61 +288,60 @@ INTERNAL_HIDDEN void _iso_alloc_reset_traces(void) { UNLOCK_ROOT(); } -#define UPDATE_BT_HASH(frame, hash) \ - if(__builtin_frame_address(frame)) { \ - hash ^= (uint64_t) __builtin_return_address(frame); \ - } else { \ - return hash; \ +/* Walks the frame pointer chain. Both x86_64 and aarch64 + * save the previous frame pointer at fp[0] and the return + * address at fp[1] when built with frame pointers. We walk + * the chain ourselves because __builtin_return_address(n) + * for n > 0 is a constant 0 on aarch64 gcc, which left the + * profiler without any backtraces on that target. The walk + * stops at the first frame that is NULL, misaligned, or does + * not sit above the current one on the stack */ +static INLINE uintptr_t *_next_frame(uintptr_t *fp) { + uintptr_t *next = (uintptr_t *) fp[0]; + + if(next <= fp || ((uintptr_t) next & (sizeof(uintptr_t) * 2 - 1)) != 0) { + return NULL; } + return next; +} + INTERNAL_HIDDEN INLINE uint64_t _get_backtrace_hash(void) { uint64_t hash = 0; - UPDATE_BT_HASH(1, hash); - UPDATE_BT_HASH(2, hash); - UPDATE_BT_HASH(3, hash); - UPDATE_BT_HASH(4, hash); - UPDATE_BT_HASH(5, hash); - UPDATE_BT_HASH(6, hash); - UPDATE_BT_HASH(7, hash); - UPDATE_BT_HASH(8, hash); + uintptr_t *fp = (uintptr_t *) __builtin_frame_address(0); + + for(size_t i = 0; i < BACKTRACE_DEPTH && fp != NULL; i++) { + hash ^= (uint64_t) fp[1]; + fp = _next_frame(fp); + } + return hash; } -#define SAVE_BACKTRACE_FRAME(frame, bts) \ - if(__builtin_frame_address(frame)) { \ - uint64_t r = (uint64_t) __builtin_return_address(frame); \ - if(r > 0x1000) { \ - bts->callers[frame - 1] = (uint64_t) __builtin_return_address(frame); \ - } \ - } else { \ - return; \ +/* Stores up to BACKTRACE_DEPTH return addresses in callers */ +static INLINE void _save_backtrace_frames(uint64_t *callers) { + uintptr_t *fp = (uintptr_t *) __builtin_frame_address(0); + + for(size_t i = 0; i < BACKTRACE_DEPTH && fp != NULL; i++) { + uint64_t r = (uint64_t) fp[1]; + + if(r > 0x1000) { + callers[i] = r; + } + + fp = _next_frame(fp); } +} INTERNAL_HIDDEN INLINE void _save_alloc_backtrace(iso_alloc_traces_t *abts) { - SAVE_BACKTRACE_FRAME(1, abts); - SAVE_BACKTRACE_FRAME(2, abts); - SAVE_BACKTRACE_FRAME(3, abts); - SAVE_BACKTRACE_FRAME(4, abts); - SAVE_BACKTRACE_FRAME(5, abts); - SAVE_BACKTRACE_FRAME(6, abts); - SAVE_BACKTRACE_FRAME(7, abts); - SAVE_BACKTRACE_FRAME(8, abts); - return; + _save_backtrace_frames(abts->callers); } INTERNAL_HIDDEN INLINE void _save_free_backtrace(iso_free_traces_t *fbts) { - SAVE_BACKTRACE_FRAME(1, fbts); - SAVE_BACKTRACE_FRAME(2, fbts); - SAVE_BACKTRACE_FRAME(3, fbts); - SAVE_BACKTRACE_FRAME(4, fbts); - SAVE_BACKTRACE_FRAME(5, fbts); - SAVE_BACKTRACE_FRAME(6, fbts); - SAVE_BACKTRACE_FRAME(7, fbts); - SAVE_BACKTRACE_FRAME(8, fbts); - return; + _save_backtrace_frames(fbts->callers); } -INTERNAL_HIDDEN void _iso_output_profile() { +INTERNAL_HIDDEN void _iso_output_profile(void) { _iso_alloc_printf(profiler_fd, "allocated=%d\n", _alloc_count); _iso_alloc_printf(profiler_fd, "alloc_sampled=%d\n", _alloc_sampled_count); _iso_alloc_printf(profiler_fd, "freed=%d\n", _free_count); @@ -351,10 +362,9 @@ INTERNAL_HIDDEN void _iso_output_profile() { continue; } - Dl_info dl; - dladdr((void *) abts->callers[j], &dl); + Dl_info dl = {0}; - if(dl.dli_sname != NULL) { + if(dladdr((void *) abts->callers[j], &dl) != 0 && dl.dli_sname != NULL) { _iso_alloc_printf(profiler_fd, "\t0x%x -> %s %s\n", abts->callers[j], dl.dli_sname, dl.dli_fname); } else { _iso_alloc_printf(profiler_fd, "\t0x%x -> [?]\n", abts->callers[j]); @@ -372,10 +382,9 @@ INTERNAL_HIDDEN void _iso_output_profile() { continue; } - Dl_info dl; - dladdr((void *) fbts->callers[j], &dl); + Dl_info dl = {0}; - if(dl.dli_sname != NULL) { + if(dladdr((void *) fbts->callers[j], &dl) != 0 && dl.dli_sname != NULL) { _iso_alloc_printf(profiler_fd, "\t0x%x -> %s %s\n", fbts->callers[j], dl.dli_sname, dl.dli_fname); } else { _iso_alloc_printf(profiler_fd, "\t0x%x -> [?]\n", fbts->callers[j]); @@ -383,7 +392,7 @@ INTERNAL_HIDDEN void _iso_output_profile() { } } - for(int i = 0; i < SMALL_SIZE_MAX; i++) { + for(int i = 0; i <= SMALL_SIZE_MAX; i++) { if(_zone_profiler_map[i].count != 0) { _iso_alloc_printf(profiler_fd, "%d,%d,%d\n", i, _zone_profiler_map[i].total, _zone_profiler_map[i].count); } @@ -418,7 +427,8 @@ INTERNAL_HIDDEN void _iso_alloc_profile(size_t size) { used = _iso_alloc_zone_leak_detector(zone, true); } - used = (int32_t) ((float) (used / zone->chunk_count) * 100.0); + /* Convert the chunk count to a percentage of the zone */ + used = (uint32_t) (((float) used / zone->chunk_count) * 100.0f); if(used > CHUNK_USAGE_THRESHOLD) { _zone_profiler_map[zone->chunk_size].count++; @@ -458,6 +468,8 @@ INTERNAL_HIDDEN void _iso_alloc_profile(size_t size) { } } +/* Must be called with the root locked, it touches + * the profiler globals and the root seed */ INTERNAL_HIDDEN void _iso_free_profile(void) { _free_count++; diff --git a/src/iso_alloc_sanity.c b/src/iso_alloc_sanity.c index 1e601fd..6b60b71 100644 --- a/src/iso_alloc_sanity.c +++ b/src/iso_alloc_sanity.c @@ -392,7 +392,11 @@ INTERNAL_HIDDEN int32_t _iso_alloc_free_sane_sample(void *p) { _sane_allocation_t *sane_alloc = _get_sane_alloc(p); if(sane_alloc != NULL) { +#if !UNINIT_READ_SANITY + /* The canary is only written when we aren't using + * userfaultfd to detect uninitialized reads */ check_sanity_canary(sane_alloc); +#endif unmap_guarded_pages(sane_alloc->address, g_page_size); memset(sane_alloc, 0x0, sizeof(_sane_allocation_t)); _sane_cache[SANE_CACHE_IDX(p)]--; diff --git a/src/iso_alloc_util.c b/src/iso_alloc_util.c index fadd5a3..d968513 100644 --- a/src/iso_alloc_util.c +++ b/src/iso_alloc_util.c @@ -214,7 +214,7 @@ int32_t name_mapping(void *p, size_t sz, const char *name) { } bool is_pow2(uint64_t sz) { - return (sz & (sz - 1)) == 0; + return sz != 0 && (sz & (sz - 1)) == 0; } size_t next_pow2(size_t sz) { diff --git a/src/malloc_hook.c b/src/malloc_hook.c index 6110481..6331709 100644 --- a/src/malloc_hook.c +++ b/src/malloc_hook.c @@ -48,16 +48,17 @@ EXTERNAL_API void *__libc_reallocarray(void *p, size_t n, size_t s) ISO_FORWARD3 EXTERNAL_API void *reallocarray(void *p, size_t n, size_t s) ISO_FORWARD3(iso_reallocarray, p, n, s) EXTERNAL_API int __posix_memalign(void **r, size_t a, size_t s) { - if(is_pow2(a) == false) { - *r = NULL; + if(r == NULL) { return EINVAL; } - if(s < a) { - s = a; + /* POSIX requires a power of 2 that is a multiple of sizeof(void *) */ + if(a == 0 || (a % sizeof(void *)) != 0 || is_pow2(a) == false) { + *r = NULL; + return EINVAL; } - *r = iso_alloc(s); + *r = _iso_alloc_aligned(s, a); if(*r != NULL) { return 0; @@ -70,19 +71,21 @@ EXTERNAL_API int posix_memalign(void **r, size_t alignment, size_t s) { return __posix_memalign(r, alignment, s); } -EXTERNAL_API void *__libc_memalign(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *memalign(size_t alignment, size_t s) { + if(alignment == 0 || is_pow2(alignment) == false) { + errno = EINVAL; + return NULL; + } + + return _iso_alloc_aligned(s, alignment); } -EXTERNAL_API void *aligned_alloc(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *__libc_memalign(size_t alignment, size_t s) { + return memalign(alignment, s); } -EXTERNAL_API void *memalign(size_t alignment, size_t s) { - /* All iso_alloc allocations are 8 byte aligned */ - return iso_alloc(s); +EXTERNAL_API void *aligned_alloc(size_t alignment, size_t s) { + return memalign(alignment, s); } #if __ANDROID__ || __FreeBSD__ diff --git a/tests/aligned_alloc_test.c b/tests/aligned_alloc_test.c new file mode 100644 index 0000000..0835a49 --- /dev/null +++ b/tests/aligned_alloc_test.c @@ -0,0 +1,216 @@ +/* iso_alloc aligned_alloc_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for the aligned allocation interfaces. + * posix_memalign, memalign and aligned_alloc previously + * ignored the requested alignment and returned chunks with + * only SZ_ALIGNMENT bytes of alignment. They also accepted + * invalid alignments instead of returning EINVAL. This test + * also covers NULL handling in iso_alloc_new_zone and + * iso_zone_chunk_count */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if __linux__ +#include +#else +/* macOS does not declare memalign but IsoAlloc exports it */ +void *memalign(size_t alignment, size_t size); +#endif + +static const size_t alignments[] = {8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, + 8192, 16384, 32768, 65536, (1 << 20)}; + +static const size_t sizes[] = {1, 24, 100, 1000, 4097, 70000, 300000}; + +static void check_chunk(void *p, size_t alignment, size_t size, const char *api) { + if(p == NULL) { + LOG_AND_ABORT("%s(%d, %d) returned NULL", api, alignment, size); + } + + if(((uintptr_t) p & (alignment - 1)) != 0) { + LOG_AND_ABORT("%s(%d, %d) returned %p which is not aligned", api, alignment, size, p); + } + + size_t cs = iso_chunksz(p); + + if(cs < size) { + LOG_AND_ABORT("%s(%d, %d) returned a chunk of only %d bytes", api, alignment, size, cs); + } + + /* The entire requested size must be writable */ + memset(p, 0x41, size); +} + +int main(int argc, char *argv[]) { + const size_t page_size = (size_t) sysconf(_SC_PAGESIZE); + const size_t na = sizeof(alignments) / sizeof(alignments[0]); + const size_t ns = sizeof(sizes) / sizeof(sizes[0]); + void *ptrs[sizeof(alignments) / sizeof(alignments[0]) * sizeof(sizes) / sizeof(sizes[0]) * 3]; + size_t count = 0; + + /* Every combination through each interface. Chunks are + * held until the end so later allocations can't reuse + * an earlier aligned chunk by accident */ + for(size_t i = 0; i < na; i++) { + for(size_t j = 0; j < ns; j++) { + void *p = NULL; + int r = posix_memalign(&p, alignments[i], sizes[j]); + + if(r != 0) { + LOG_AND_ABORT("posix_memalign(%d, %d) returned %d", alignments[i], sizes[j], r); + } + + check_chunk(p, alignments[i], sizes[j], "posix_memalign"); + ptrs[count++] = p; + + p = memalign(alignments[i], sizes[j]); + check_chunk(p, alignments[i], sizes[j], "memalign"); + ptrs[count++] = p; + + /* C11 requires size to be a multiple of alignment */ + size_t s = (sizes[j] + (alignments[i] - 1)) & ~(alignments[i] - 1); + p = aligned_alloc(alignments[i], s); + check_chunk(p, alignments[i], s, "aligned_alloc"); + ptrs[count++] = p; + } + } + + for(size_t i = 0; i < count; i++) { + free(ptrs[i]); + } + + /* A big zone with an alignment larger than a page is + * trimmed from a larger mapping. Reusing it from the + * free list must preserve that alignment */ + const size_t big_alignment = (page_size << 2); + void *first = NULL; + int r = posix_memalign(&first, big_alignment, SMALL_SIZE_MAX * 2); + + if(r != 0 || first == NULL) { + LOG_AND_ABORT("posix_memalign(%d, %d) failed", big_alignment, SMALL_SIZE_MAX * 2); + } + + check_chunk(first, big_alignment, SMALL_SIZE_MAX * 2, "posix_memalign"); + free(first); + + for(size_t i = 0; i < 8; i++) { + void *p = NULL; + r = posix_memalign(&p, big_alignment, SMALL_SIZE_MAX * 2); + + if(r != 0) { + LOG_AND_ABORT("posix_memalign(%d, %d) returned %d on reuse", big_alignment, SMALL_SIZE_MAX * 2, r); + } + + check_chunk(p, big_alignment, SMALL_SIZE_MAX * 2, "posix_memalign"); + free(p); + } + + /* Invalid alignments must fail with EINVAL and never + * hand back memory with the wrong alignment. The values + * are volatile so the compiler can't reject them itself */ + static volatile size_t zero_alignment = 0; + static volatile size_t odd_alignment = 3; + static volatile size_t non_pow2_alignment = 24; + static volatile size_t small_alignment = 4; + static volatile size_t bad_c11_alignment = 48; + void *bad = (void *) 0x1; + + if(posix_memalign(&bad, zero_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 0"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, odd_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 3"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, non_pow2_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment of 24"); + } + + bad = (void *) 0x1; + + if(posix_memalign(&bad, small_alignment, 64) != EINVAL || bad != NULL) { + LOG_AND_ABORT("posix_memalign accepted an alignment smaller than sizeof(void *)"); + } + + /* The result pointer is volatile so the compiler can't + * see that it is NULL and reject the call itself */ + void **volatile null_result = NULL; + + if(posix_memalign(null_result, 64, 64) != EINVAL) { + LOG_AND_ABORT("posix_memalign accepted a NULL result pointer"); + } + + errno = 0; + + if(memalign(zero_alignment, 64) != NULL || errno != EINVAL) { + LOG_AND_ABORT("memalign accepted an alignment of 0"); + } + + errno = 0; + + if(memalign(non_pow2_alignment, 64) != NULL || errno != EINVAL) { + LOG_AND_ABORT("memalign accepted an alignment of 24"); + } + + errno = 0; + + if(aligned_alloc(bad_c11_alignment, 96) != NULL || errno != EINVAL) { + LOG_AND_ABORT("aligned_alloc accepted an alignment of 48"); + } + + /* Overflow in the size rounding must fail cleanly */ + bad = (void *) 0x1; + r = posix_memalign(&bad, 4096, SIZE_MAX - 100); + + if(r != ENOMEM || bad != NULL) { + LOG_AND_ABORT("posix_memalign did not fail on size overflow (%d, %p)", r, bad); + } + + /* A zero size with a valid alignment still returns + * an aligned chunk that can be freed */ + void *z = NULL; + r = posix_memalign(&z, 512, 0); + + if(r != 0 || z == NULL || ((uintptr_t) z & 511) != 0) { + LOG_AND_ABORT("posix_memalign(512, 0) failed (%d, %p)", r, z); + } + + free(z); + + /* iso_alloc_new_zone must return NULL for a size the + * zone allocator cannot handle, not a masked NULL */ + iso_alloc_zone_handle *zone = iso_alloc_new_zone(SMALL_SIZE_MAX + 1); + + if(zone != NULL) { + LOG_AND_ABORT("iso_alloc_new_zone returned %p for an oversized chunk", zone); + } + + if(iso_zone_chunk_count(NULL) != 0) { + LOG_AND_ABORT("iso_zone_chunk_count(NULL) is not 0"); + } + + zone = iso_alloc_new_zone(256); + + if(zone == NULL || iso_zone_chunk_count(zone) == 0) { + LOG_AND_ABORT("iso_alloc_new_zone(256) failed"); + } + + iso_alloc_destroy_zone(zone); + + iso_flush_caches(); + iso_verify_zones(); + + return 0; +} diff --git a/tests/profiler_test.c b/tests/profiler_test.c new file mode 100644 index 0000000..e30025e --- /dev/null +++ b/tests/profiler_test.c @@ -0,0 +1,174 @@ +/* iso_alloc profiler_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for the heap profiler. The traces API + * previously copied a single entry regardless of how many + * had been recorded, the per chunk size map was one entry + * too small for SMALL_SIZE_MAX byte zones, and the free + * profiler ran without the root lock. The profiler data + * file written at exit is checked by run_profiler_tests.sh */ +#include +#include +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" +#include "iso_alloc_profiler.h" + +#if !HEAP_PROFILER +#error "This test intended to be run with -DHEAP_PROFILER=1" +#endif + +/* Enough allocations that at least one is sampled at + * PROFILER_ODDS with overwhelming probability */ +#define CYCLES (PROFILER_ODDS * 50) +#define THREADS 4 +#define THREAD_CYCLES (PROFILER_ODDS * 10) +#define HELD 50 +#define FILL 0xab + +/* Backtraces are keyed on the call path so each of these + * is a distinct allocation or free site. They must not be + * inlined or they collapse into a single trace */ +#define ALLOC_SITE(name) \ + __attribute__((noinline)) static void *name(size_t size) { \ + void *p = iso_alloc(size); \ + if(p == NULL) { \ + LOG_AND_ABORT("iso_alloc failed in " #name); \ + } \ + return p; \ + } + +#define FREE_SITE(name) \ + __attribute__((noinline)) static void name(void *p) { \ + iso_free(p); \ + } + +ALLOC_SITE(alloc_site_a) +ALLOC_SITE(alloc_site_b) +ALLOC_SITE(alloc_site_c) +ALLOC_SITE(alloc_site_d) +FREE_SITE(free_site_a) +FREE_SITE(free_site_b) +FREE_SITE(free_site_c) +FREE_SITE(free_site_d) + +static void *churn(void *arg) { + for(size_t i = 0; i < THREAD_CYCLES; i++) { + void *p = iso_alloc((i % 4096) + 1); + + if(p == NULL) { + LOG_AND_ABORT("iso_alloc failed in thread"); + } + + iso_free(p); + } + + return NULL; +} + +int main(int argc, char *argv[]) { + /* Hold most of a SMALL_SIZE_MAX zone so the profiler + * records usage for the largest possible chunk size. + * That index lands one past the end of the old map */ + void *held[HELD]; + + for(size_t i = 0; i < HELD; i++) { + held[i] = iso_alloc(SMALL_SIZE_MAX); + + if(held[i] == NULL) { + LOG_AND_ABORT("Failed to allocate %d byte chunk", SMALL_SIZE_MAX); + } + } + + for(size_t i = 0; i < CYCLES; i++) { + const size_t size = (i % 2048) + 1; + free_site_a(alloc_site_a(size)); + free_site_b(alloc_site_b(size)); + free_site_c(alloc_site_c(size)); + free_site_d(alloc_site_d(size)); + } + + /* The traces API must copy every recorded entry. Fill + * the output with a pattern so untouched entries show */ + iso_alloc_traces_t at[BACKTRACE_DEPTH_SZ]; + memset(at, FILL, sizeof(at)); + size_t alloc_trace_count = iso_get_alloc_traces(at); + + /* Four sites each sampled many times over must yield + * more than one distinct trace */ + if(alloc_trace_count < 2 || alloc_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad alloc trace count %d", alloc_trace_count); + } + + iso_alloc_traces_t pattern; + memset(&pattern, FILL, sizeof(pattern)); + + for(size_t i = 0; i < alloc_trace_count; i++) { + if(memcmp(&at[i], &pattern, sizeof(pattern)) == 0) { + LOG_AND_ABORT("Alloc trace %d of %d was not copied", i, alloc_trace_count); + } + + if(at[i].call_count == 0) { + LOG_AND_ABORT("Alloc trace %d has no calls", i); + } + } + + iso_free_traces_t ft[BACKTRACE_DEPTH_SZ]; + memset(ft, FILL, sizeof(ft)); + size_t free_trace_count = iso_get_free_traces(ft); + + if(free_trace_count < 2 || free_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad free trace count %d", free_trace_count); + } + + iso_free_traces_t fpattern; + memset(&fpattern, FILL, sizeof(fpattern)); + + for(size_t i = 0; i < free_trace_count; i++) { + if(memcmp(&ft[i], &fpattern, sizeof(fpattern)) == 0) { + LOG_AND_ABORT("Free trace %d of %d was not copied", i, free_trace_count); + } + + if(ft[i].call_count == 0) { + LOG_AND_ABORT("Free trace %d has no calls", i); + } + } + + iso_alloc_reset_traces(); + + if(iso_get_alloc_traces(at) != 0 || iso_get_free_traces(ft) != 0) { + LOG_AND_ABORT("Traces were not reset"); + } + +#if THREAD_SUPPORT + /* Concurrent frees exercise the lock around the free + * profiler. A race here corrupts the backtrace arrays */ + pthread_t threads[THREADS]; + + for(size_t i = 0; i < THREADS; i++) { + if(pthread_create(&threads[i], NULL, churn, NULL) != 0) { + LOG_AND_ABORT("pthread_create failed"); + } + } + + for(size_t i = 0; i < THREADS; i++) { + pthread_join(threads[i], NULL); + } + + free_trace_count = iso_get_free_traces(ft); + + if(free_trace_count == 0 || free_trace_count > BACKTRACE_DEPTH_SZ) { + LOG_AND_ABORT("Bad free trace count %d after threads", free_trace_count); + } +#endif + + for(size_t i = 0; i < HELD; i++) { + iso_free(held[i]); + } + + iso_flush_caches(); + iso_verify_zones(); + + return 0; +} diff --git a/tests/tag_refresh_test.c b/tests/tag_refresh_test.c new file mode 100644 index 0000000..2cff353 --- /dev/null +++ b/tests/tag_refresh_test.c @@ -0,0 +1,125 @@ +/* iso_alloc tag_refresh_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for _refresh_zone_mem_tags. Once a zone + * has no live allocations and has seen more than a quarter + * of its retirement threshold in allocations every free + * regenerates every tag in the zone. The condition for this + * was previously miscomputed so it never fired, and when it + * did it wrote through the masked user pages pointer. This + * test fills a small zone, drives it past the threshold, and + * then checks that a single free changed the tag of every + * other chunk in the zone */ +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if !MEMORY_TAGGING +#error "This test intended to be run with -DMEMORY_TAGGING=1" +#endif + +/* Large chunks keep the zone small so the refresh + * threshold is reached in a few hundred cycles */ +#define SIZE SMALL_SIZE_MAX +#define CHUNKS (ZONE_USER_SIZE / SIZE) + +int main(int argc, char *argv[]) { + iso_alloc_zone_handle *_zone_handle = iso_alloc_new_zone(SIZE); + + if(_zone_handle == NULL) { + abort(); + } + + /* Allocate every chunk in the zone so we know their + * addresses, then free them all */ + void *chunks[CHUNKS]; + size_t count = 0; + + for(size_t i = 0; i < CHUNKS; i++) { + void *p = iso_alloc_from_zone(_zone_handle); + + if(p == NULL) { + break; + } + + chunks[count++] = p; + } + + if(count < (CHUNKS / 2)) { + LOG_AND_ABORT("Only allocated %d of %d chunks", count, CHUNKS); + } + + for(size_t i = 0; i < count; i++) { + iso_free_from_zone(chunks[i], _zone_handle); + } + + /* The refresh fires on a free once the zone has no live + * chunks and alloc_count exceeds a quarter of the + * retirement threshold. Overshoot it to be safe */ + const size_t threshold = (CHUNKS * ZONE_ALLOC_RETIRE) / 4; + + for(size_t i = 0; i < threshold + 64; i++) { + void *p = iso_alloc_from_zone_tagged(_zone_handle); + + if(p == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk in cycle %d", i); + } + + iso_alloc_verify_ptr_tag(p, _zone_handle); + iso_free_from_zone(p, _zone_handle); + } + + /* Allocate one chunk and snapshot the tags of every other + * chunk. The free below must refresh all of them */ + void *q = iso_alloc_from_zone_tagged(_zone_handle); + + if(q == NULL) { + LOG_AND_ABORT("Failed to allocate the final tagged chunk"); + } + + void *uq = iso_alloc_untag_ptr(q, _zone_handle); + uint8_t before[CHUNKS]; + + for(size_t i = 0; i < count; i++) { + before[i] = iso_alloc_get_mem_tag(chunks[i], _zone_handle); + } + + iso_free_from_zone(uq, _zone_handle); + + size_t unchanged = 0; + + for(size_t i = 0; i < count; i++) { + if(chunks[i] == uq) { + continue; + } + + if(iso_alloc_get_mem_tag(chunks[i], _zone_handle) == before[i]) { + unchanged++; + } + } + + if(unchanged != 0) { + LOG_AND_ABORT("Zone tag refresh left %d of %d chunk tags unchanged", unchanged, count); + } + + /* The stale pointer must no longer untag to its chunk + * and a fresh pointer must still verify after a refresh */ + if(iso_alloc_untag_ptr(q, _zone_handle) == uq) { + LOG_AND_ABORT("Stale pointer %p untagged to its original address after refresh", q); + } + + void *fresh = iso_alloc_from_zone_tagged(_zone_handle); + + if(fresh == NULL) { + LOG_AND_ABORT("Failed to allocate after refresh"); + } + + iso_alloc_verify_ptr_tag(fresh, _zone_handle); + iso_free_from_zone(fresh, _zone_handle); + + iso_flush_caches(); + iso_alloc_destroy_zone(_zone_handle); + + return 0; +} diff --git a/tests/tests.cpp b/tests/tests.cpp index c166fe6..dffd547 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #if THREAD_SUPPORT #include #endif @@ -20,6 +23,124 @@ static const uint32_t array_sizes[] = {16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}; int32_t alloc_count; +int32_t new_handler_calls; + +struct alignas(256) Aligned256 { + uint8_t data[300]; +}; + +struct alignas(4096) AlignedPage { + uint8_t data[100]; +}; + +static void check_alignment(void *p, size_t alignment) { + if(p == NULL) { + LOG_AND_ABORT("Aligned new returned NULL for alignment %d", alignment); + } + + if((reinterpret_cast(p) & (alignment - 1)) != 0) { + LOG_AND_ABORT("Pointer %p is not %d byte aligned", p, alignment); + } +} + +/* The aligned new operators previously ignored the requested + * alignment, and the throwing operators could return NULL + * instead of calling the new handler and throwing bad_alloc */ +static void aligned_new_tests(void) { + auto *a = new Aligned256(); + check_alignment(a, alignof(Aligned256)); + memset(a->data, 0x41, sizeof(a->data)); + delete a; + + auto *aa = new Aligned256[8]; + check_alignment(aa, alignof(Aligned256)); + + for(size_t i = 0; i < 8; i++) { + check_alignment(&aa[i], alignof(Aligned256)); + memset(aa[i].data, 0x41, sizeof(aa[i].data)); + } + + delete[] aa; + + auto *pg = new AlignedPage(); + check_alignment(pg, alignof(AlignedPage)); + delete pg; + + /* Alignments larger than a page go through the big zone path */ + const size_t big_alignment = (1 << 20); + void *b = ::operator new(1000, std::align_val_t(big_alignment)); + check_alignment(b, big_alignment); + memset(b, 0x41, 1000); + ::operator delete(b, std::align_val_t(big_alignment)); + + void *n = ::operator new(100, std::align_val_t(512), std::nothrow); + check_alignment(n, 512); + ::operator delete(n, std::align_val_t(512), std::nothrow); + + /* A throwing new must throw bad_alloc rather than return NULL. + * These impossible sizes make the allocator log a failed big + * zone allocation for each attempt, which is expected */ + LOG("The following big zone allocation failures are expected"); + static volatile size_t huge = (SIZE_MAX >> 1); + bool caught = false; + + try { + auto *h = new uint8_t[huge]; + delete[] h; + } catch(const std::bad_alloc &) { + caught = true; + } + + if(caught == false) { + LOG_AND_ABORT("new did not throw bad_alloc for an impossible size"); + } + + caught = false; + + try { + void *h = ::operator new(huge, std::align_val_t(4096)); + ::operator delete(h, std::align_val_t(4096)); + } catch(const std::bad_alloc &) { + caught = true; + } + + if(caught == false) { + LOG_AND_ABORT("aligned new did not throw bad_alloc for an impossible size"); + } + + /* The nothrow variants return NULL instead */ + if(new(std::nothrow) uint8_t[huge] != NULL) { + LOG_AND_ABORT("nothrow new returned memory for an impossible size"); + } + + if(::operator new(huge, std::align_val_t(4096), std::nothrow) != NULL) { + LOG_AND_ABORT("nothrow aligned new returned memory for an impossible size"); + } + + /* An installed new handler must be called once before + * bad_alloc is thrown. The handler uninstalls itself so + * the retry fails and the exception follows */ + new_handler_calls = 0; + std::set_new_handler([]() { + new_handler_calls++; + std::set_new_handler(nullptr); + }); + + caught = false; + + try { + auto *h = new uint8_t[huge]; + delete[] h; + } catch(const std::bad_alloc &) { + caught = true; + } + + std::set_new_handler(nullptr); + + if(caught == false || new_handler_calls != 1) { + LOG_AND_ABORT("new handler was called %d times, bad_alloc caught = %d", new_handler_calls, caught); + } +} class Base { public: @@ -132,6 +253,8 @@ int main(int argc, char *argv[]) { } #endif + aligned_new_tests(); + iso_verify_zones(); return 0; diff --git a/tests/valid_tag_ptr_test.c b/tests/valid_tag_ptr_test.c new file mode 100644 index 0000000..3a4b77e --- /dev/null +++ b/tests/valid_tag_ptr_test.c @@ -0,0 +1,85 @@ +/* iso_alloc valid_tag_ptr_test.c + * Copyright 2026 - chris.rohlf@gmail.com */ + +/* Regression test for iso_alloc_verify_ptr_tag. The tag + * comparison previously compared the 8 bit tag against the + * unshifted top byte of the pointer so every valid tagged + * pointer failed verification. The other tagging tests only + * check that verification aborts on a stale pointer, this + * one checks that it succeeds on a valid pointer. It also + * checks that freeing a chunk always changes its tag so a + * stale tagged pointer can never untag to the same address */ +#include +#include +#include "iso_alloc.h" +#include "iso_alloc_internal.h" + +#if !MEMORY_TAGGING +#error "This test intended to be run with -DMEMORY_TAGGING=1" +#endif + +#define SIZE 256 +#define HELD 64 +#define CYCLES 4096 + +int main(int argc, char *argv[]) { + iso_alloc_zone_handle *_zone_handle = iso_alloc_new_zone(SIZE); + + if(_zone_handle == NULL) { + abort(); + } + + void *held[HELD]; + + /* Every live tagged pointer must verify */ + for(size_t i = 0; i < HELD; i++) { + held[i] = iso_alloc_from_zone_tagged(_zone_handle); + + if(held[i] == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk %d", i); + } + + iso_alloc_verify_ptr_tag(held[i], _zone_handle); + + void *up = iso_alloc_untag_ptr(held[i], _zone_handle); + memset(up, 0x41, SIZE); + } + + for(size_t i = 0; i < HELD; i++) { + iso_alloc_verify_ptr_tag(held[i], _zone_handle); + iso_free_from_zone(held[i], _zone_handle); + } + + /* A tag must change on every free. If it did not then + * untagging the stale pointer yields the original chunk + * address and a use after free would go undetected */ + for(size_t i = 0; i < CYCLES; i++) { + void *p = iso_alloc_from_zone_tagged(_zone_handle); + + if(p == NULL) { + LOG_AND_ABORT("Failed to allocate tagged chunk in cycle %d", i); + } + + iso_alloc_verify_ptr_tag(p, _zone_handle); + + void *up = iso_alloc_untag_ptr(p, _zone_handle); + const uint8_t old_tag = iso_alloc_get_mem_tag(up, _zone_handle); + + iso_free_from_zone(up, _zone_handle); + + const uint8_t new_tag = iso_alloc_get_mem_tag(up, _zone_handle); + + if(old_tag == new_tag) { + LOG_AND_ABORT("Tag %x for chunk %p did not change on free in cycle %d", old_tag, up, i); + } + + if(iso_alloc_untag_ptr(p, _zone_handle) == up) { + LOG_AND_ABORT("Stale pointer %p untagged to its original address in cycle %d", p, i); + } + } + + iso_flush_caches(); + iso_alloc_destroy_zone(_zone_handle); + + return 0; +} diff --git a/utils/run_profiler_tests.sh b/utils/run_profiler_tests.sh new file mode 100755 index 0000000..10c8780 --- /dev/null +++ b/utils/run_profiler_tests.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Runs the heap profiler test and checks the data file +# the profiler writes when the process exits +$(echo '' > profiler_test_output.txt) + +$(ulimit -c 0) + +export LD_LIBRARY_PATH=build/ +export ISO_ALLOC_PROFILER_FILE_PATH=build/iso_alloc_profiler.data +rm -f $ISO_ALLOC_PROFILER_FILE_PATH + +failure=0 + +echo -n "Running profiler_test test" +echo "Running profiler_test test" >> profiler_test_output.txt 2>&1 +$(build/profiler_test >> profiler_test_output.txt 2>&1) +ret=$? + +if [ $ret -ne 0 ]; then + echo "... Failed" + failure=$((failure+1)) +else + echo "... Succeeded" +fi + +# The profile must record sampled allocations and frees +# and a row for the largest chunk size. That row lands +# one entry past the end of an undersized map +echo -n "Checking profiler data file" + +if [ ! -s $ISO_ALLOC_PROFILER_FILE_PATH ]; then + echo "... Failed (no data file)" + failure=$((failure+1)) +elif ! grep -qE '^alloc_sampled=[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^free_sampled=[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^alloc_backtrace=' $ISO_ALLOC_PROFILER_FILE_PATH || + ! grep -qE '^65536,[0-9]+,[1-9]' $ISO_ALLOC_PROFILER_FILE_PATH; then + echo "... Failed (unexpected contents)" + cat $ISO_ALLOC_PROFILER_FILE_PATH + failure=$((failure+1)) +else + echo "... Succeeded" +fi + +unset LD_LIBRARY_PATH +unset LD_PRELOAD +unset ISO_ALLOC_PROFILER_FILE_PATH + +if [ $failure -ne 0 ]; then + cat profiler_test_output.txt + exit -1 +else + exit 0 +fi diff --git a/utils/run_tagging_tests.sh b/utils/run_tagging_tests.sh index 1327583..b78a4ba 100755 --- a/utils/run_tagging_tests.sh +++ b/utils/run_tagging_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash $(echo '' > tagging_test_output.txt) -tests=("tagged_ptr_test" "tagged_ptr_test_cpp") +tests=("tagged_ptr_test" "tagged_ptr_test_cpp" "valid_tag_ptr_test" "tag_refresh_test") failure=0 succeeded=0 diff --git a/utils/run_tests.sh b/utils/run_tests.sh index 4f2c529..107232e 100755 --- a/utils/run_tests.sh +++ b/utils/run_tests.sh @@ -4,7 +4,7 @@ $(echo '' > test_output.txt) tests=("tests" "big_tests" "big_zone_chunksz" "zone_retirement" "strdup_test" "interfaces_test" "thread_tests" "pool_test" - "rand_freelist") + "rand_freelist" "aligned_alloc_test") failure=0 succeeded=0 start=$SECONDS