summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--libguile/whippet/api/gc-api.h4
-rw-r--r--libguile/whippet/src/bdw.c41
-rw-r--r--libguile/whippet/src/copy-space.h21
-rw-r--r--libguile/whippet/src/gc-platform-gnu-linux.c8
-rw-r--r--libguile/whippet/src/mmc.c85
-rw-r--r--libguile/whippet/src/nofl-space.h5
-rw-r--r--libguile/whippet/src/pcc.c78
-rw-r--r--libguile/whippet/src/semi.c61
8 files changed, 204 insertions, 99 deletions
diff --git a/libguile/whippet/api/gc-api.h b/libguile/whippet/api/gc-api.h
index 603b38030..c95c33a65 100644
--- a/libguile/whippet/api/gc-api.h
+++ b/libguile/whippet/api/gc-api.h
@@ -42,6 +42,10 @@ struct gc_heap_roots;
GC_API_ void gc_heap_set_roots(struct gc_heap *heap,
struct gc_heap_roots *roots);
+GC_API_ void gc_heap_set_allocation_failure_handler(struct gc_heap *heap,
+ void* (*)(struct gc_heap*,
+ size_t));
+
struct gc_extern_space;
GC_API_ void gc_heap_set_extern_space(struct gc_heap *heap,
struct gc_extern_space *space);
diff --git a/libguile/whippet/src/bdw.c b/libguile/whippet/src/bdw.c
index 83a157d61..3e7bcf17c 100644
--- a/libguile/whippet/src/bdw.c
+++ b/libguile/whippet/src/bdw.c
@@ -60,6 +60,7 @@ struct gc_heap {
struct gc_finalizer_state *finalizer_state;
gc_finalizer_callback have_finalizers;
void *event_listener_data;
+ void* (*allocation_failure)(struct gc_heap *, size_t);
};
struct gc_mutator {
@@ -116,11 +117,8 @@ allocate_small(void **freelist, size_t idx, enum gc_inline_kind kind) {
size_t bytes = gc_inline_freelist_object_size(idx);
GC_generic_malloc_many(bytes, kind, freelist);
head = *freelist;
- if (GC_UNLIKELY (!head)) {
- fprintf(stderr, "ran out of space, heap size %zu\n",
- GC_get_heap_size());
- GC_CRASH();
- }
+ if (GC_UNLIKELY (!head))
+ return __the_bdw_gc_heap->allocation_failure(__the_bdw_gc_heap, bytes);
}
*freelist = *(void **)(head);
@@ -152,13 +150,20 @@ void* gc_allocate_slow(struct gc_mutator *mut, size_t size,
} else {
switch (kind) {
case GC_ALLOCATION_TAGGED:
- case GC_ALLOCATION_UNTAGGED_CONSERVATIVE:
- return GC_malloc(size);
+ case GC_ALLOCATION_UNTAGGED_CONSERVATIVE: {
+ void *ret = GC_malloc(size);
+ if (GC_LIKELY (ret != NULL))
+ return ret;
+ return __the_bdw_gc_heap->allocation_failure(__the_bdw_gc_heap, size);
+ }
case GC_ALLOCATION_TAGGED_POINTERLESS:
case GC_ALLOCATION_UNTAGGED_POINTERLESS: {
void *ret = GC_malloc_atomic(size);
- memset(ret, 0, size);
- return ret;
+ if (GC_LIKELY (ret != NULL)) {
+ memset(ret, 0, size);
+ return ret;
+ }
+ return __the_bdw_gc_heap->allocation_failure(__the_bdw_gc_heap, size);
}
default:
GC_CRASH();
@@ -521,6 +526,22 @@ uint64_t gc_allocation_counter(struct gc_heap *heap) {
return GC_get_total_bytes();
}
+static void* allocation_failure(struct gc_heap *heap, size_t size) {
+ fprintf(stderr, "ran out of space, heap size %zu\n", GC_get_heap_size());
+ GC_CRASH();
+ return NULL;
+}
+
+static void* oom_fn(size_t nbytes) {
+ return NULL;
+}
+
+void gc_heap_set_allocation_failure_handler(struct gc_heap *heap,
+ void* (*handler)(struct gc_heap*,
+ size_t)) {
+ heap->allocation_failure = handler;
+}
+
int gc_init(const struct gc_options *options, struct gc_stack_addr stack_base,
struct gc_heap **heap, struct gc_mutator **mutator,
struct gc_event_listener event_listener,
@@ -607,6 +628,8 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr stack_base,
HEAP_EVENT(init, GC_get_heap_size());
GC_set_on_collection_event(on_collection_event);
GC_set_on_heap_resize(on_heap_resize);
+ GC_set_oom_fn (oom_fn);
+ (*heap)->allocation_failure = allocation_failure;
*mutator = add_mutator(*heap);
diff --git a/libguile/whippet/src/copy-space.h b/libguile/whippet/src/copy-space.h
index 7f262c221..c6e3ccd61 100644
--- a/libguile/whippet/src/copy-space.h
+++ b/libguile/whippet/src/copy-space.h
@@ -287,7 +287,7 @@ copy_space_request_release_memory(struct copy_space *space, size_t bytes) {
return atomic_fetch_add(&space->bytes_to_page_out, bytes) + bytes;
}
-static int
+static ssize_t
copy_space_page_out_blocks_until_memory_released(struct copy_space *space) {
ssize_t pending = atomic_load(&space->bytes_to_page_out);
struct gc_lock lock = copy_space_lock(space);
@@ -299,7 +299,7 @@ copy_space_page_out_blocks_until_memory_released(struct copy_space *space) {
- COPY_SPACE_BLOCK_SIZE);
}
gc_lock_release(&lock);
- return pending <= 0;
+ return pending;
}
static ssize_t
@@ -409,11 +409,23 @@ copy_space_allocator_acquire_block(struct copy_space_allocator *alloc,
return 0;
}
+static struct copy_space_block*
+copy_space_obtain_empty_block_during_gc(struct copy_space *space,
+ const struct gc_lock *lock) {
+ GC_ASSERT(!copy_space_pop_empty_block(space, lock));
+ struct copy_space_block *block = copy_space_page_in_block(space, lock);
+ if (block)
+ atomic_fetch_add(&space->bytes_to_page_out, COPY_SPACE_BLOCK_SIZE);
+ return block;
+}
+
static int
copy_space_allocator_acquire_empty_block(struct copy_space_allocator *alloc,
struct copy_space *space) {
struct gc_lock lock = copy_space_lock(space);
struct copy_space_block *block = copy_space_pop_empty_block(space, &lock);
+ if (!block && space->in_gc)
+ block = copy_space_obtain_empty_block_during_gc(space, &lock);
gc_lock_release(&lock);
if (copy_space_allocator_acquire_block(alloc, block, space->active_region)) {
block->in_core = 1;
@@ -925,7 +937,10 @@ static int
copy_space_init(struct copy_space *space, size_t size, uint32_t flags,
struct gc_background_thread *thread) {
size = align_up(size, COPY_SPACE_BLOCK_SIZE);
- size_t reserved = align_up(size, COPY_SPACE_SLAB_SIZE);
+ // Reserve a few extra blocks to handle the fragmentation problem
+ // (https://wingolog.org/archives/2024/07/10/copying-collectors-with-block-structured-heaps-are-unreliable).
+ size_t reserved = size + COPY_SPACE_BLOCK_SIZE * 16;
+ reserved = align_up(reserved, COPY_SPACE_SLAB_SIZE);
if (flags & COPY_SPACE_ALIGNED)
reserved = copy_space_round_up_power_of_two(reserved);
size_t nslabs = reserved / COPY_SPACE_SLAB_SIZE;
diff --git a/libguile/whippet/src/gc-platform-gnu-linux.c b/libguile/whippet/src/gc-platform-gnu-linux.c
index 3ace1890d..e5fcb9804 100644
--- a/libguile/whippet/src/gc-platform-gnu-linux.c
+++ b/libguile/whippet/src/gc-platform-gnu-linux.c
@@ -162,12 +162,12 @@ gc_platform_acquire_memory_from_reservation(struct gc_reservation reservation,
GC_ASSERT(size <= reservation.size);
GC_ASSERT(offset <= reservation.size - size);
- void *mem = mmap((void*)(reservation.base + offset), size,
- PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
- if (mem == MAP_FAILED) {
- perror("mmap failed");
+ void *mem = (void*)(reservation.base + offset);
+ if (mprotect(mem, size, PROT_READ|PROT_WRITE)) {
+ perror("mprotect failed");
return NULL;
}
+ // FIXME: Should we gc_platform_populate_memory() here?
return mem;
}
diff --git a/libguile/whippet/src/mmc.c b/libguile/whippet/src/mmc.c
index d2bc4df9d..6a5f13bc2 100644
--- a/libguile/whippet/src/mmc.c
+++ b/libguile/whippet/src/mmc.c
@@ -66,6 +66,7 @@ struct gc_heap {
struct gc_heap_sizer sizer;
struct gc_event_listener event_listener;
void *event_listener_data;
+ void* (*allocation_failure)(struct gc_heap*, size_t);
};
#define HEAP_EVENT(heap, event, ...) do { \
@@ -511,20 +512,18 @@ heap_estimate_live_data_after_gc(struct gc_heap *heap,
return bytes;
}
-static void
-detect_out_of_memory(struct gc_heap *heap, uintptr_t allocation_since_last_gc) {
- if (heap->sizer.policy != GC_HEAP_SIZE_FIXED)
- return;
-
- if (allocation_since_last_gc > nofl_space_fragmentation(heap_nofl_space(heap)))
- return;
-
- if (heap->gc_kind == GC_COLLECTION_MINOR)
- return;
+static int
+compute_progress(struct gc_heap *heap, uintptr_t allocation_since_last_gc) {
+ struct nofl_space *nofl = heap_nofl_space(heap);
+ return allocation_since_last_gc > nofl_space_fragmentation(nofl);
+}
- // No allocation since last gc: out of memory.
- fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
- GC_CRASH();
+static int
+compute_success(struct gc_heap *heap, enum gc_collection_kind gc_kind,
+ int progress) {
+ return progress
+ || gc_kind == GC_COLLECTION_MINOR
+ || heap->sizer.policy != GC_HEAP_SIZE_FIXED;
}
static double
@@ -750,12 +749,10 @@ sweep_ephemerons(struct gc_heap *heap) {
return gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1);
}
-static void collect(struct gc_mutator *mut,
- enum gc_collection_kind requested_kind,
- int requested_by_user) GC_NEVER_INLINE;
-static void
-collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind,
- int requested_by_user) {
+static int collect(struct gc_mutator *mut,
+ enum gc_collection_kind requested_kind) GC_NEVER_INLINE;
+static int
+collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
struct gc_heap *heap = mutator_heap(mut);
struct nofl_space *nofl_space = heap_nofl_space(heap);
struct large_object_space *lospace = heap_large_object_space(heap);
@@ -773,8 +770,7 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind,
nofl_space_add_to_allocation_counter(nofl_space, &allocation_counter);
large_object_space_add_to_allocation_counter(lospace, &allocation_counter);
heap->total_allocated_bytes_at_last_gc += allocation_counter;
- if (!requested_by_user)
- detect_out_of_memory(heap, allocation_counter);
+ int progress = compute_progress(heap, allocation_counter);
enum gc_collection_kind gc_kind =
determine_collection_kind(heap, requested_kind);
int is_minor = gc_kind == GC_COLLECTION_MINOR;
@@ -821,12 +817,12 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind,
heap->size_at_last_gc = heap->size;
HEAP_EVENT(heap, restarting_mutators);
allow_mutators_to_continue(heap);
+ return compute_success(heap, gc_kind, progress);
}
-static void
+static int
trigger_collection(struct gc_mutator *mut,
- enum gc_collection_kind requested_kind,
- int requested_by_user) {
+ enum gc_collection_kind requested_kind) {
struct gc_heap *heap = mutator_heap(mut);
int prev_kind = -1;
gc_stack_capture_hot(&mut->stack);
@@ -836,14 +832,16 @@ trigger_collection(struct gc_mutator *mut,
heap_lock(heap);
while (mutators_are_stopping(heap))
prev_kind = pause_mutator_for_collection(heap, mut);
+ int success = 1;
if (prev_kind < (int)requested_kind)
- collect(mut, requested_kind, requested_by_user);
+ success = collect(mut, requested_kind);
heap_unlock(heap);
+ return success;
}
void
gc_collect(struct gc_mutator *mut, enum gc_collection_kind kind) {
- trigger_collection(mut, kind, 1);
+ trigger_collection(mut, kind);
}
int*
@@ -903,8 +901,10 @@ allocate_large(struct gc_mutator *mut, size_t size,
nofl_space_request_release_memory(nofl_space,
npages << lospace->page_size_log2);
- while (!nofl_space_shrink(nofl_space, 0))
- trigger_collection(mut, GC_COLLECTION_COMPACTING, 0);
+ while (!nofl_space_shrink(nofl_space, 0)) {
+ if (!trigger_collection(mut, GC_COLLECTION_COMPACTING))
+ return heap->allocation_failure(heap, size);
+ }
atomic_fetch_add(&heap->large_object_pages, npages);
void *ret = large_object_space_alloc(lospace, npages, kind);
@@ -919,7 +919,7 @@ allocate_large(struct gc_mutator *mut, size_t size,
static void
collect_for_small_allocation(void *mut) {
- trigger_collection(mut, GC_COLLECTION_ANY, 0);
+ trigger_collection(mut, GC_COLLECTION_ANY);
}
void*
@@ -930,10 +930,16 @@ gc_allocate_slow(struct gc_mutator *mut, size_t size,
if (size > gc_allocator_large_threshold())
return allocate_large(mut, size, compute_trace_kind(kind));
- return gc_ref_heap_object(nofl_allocate(&mut->allocator,
- heap_nofl_space(mutator_heap(mut)),
- size, collect_for_small_allocation,
- mut, kind));
+ struct gc_heap *heap = mutator_heap(mut);
+ while (1) {
+ struct gc_ref ret = nofl_allocate(&mut->allocator, heap_nofl_space(heap),
+ size, kind);
+ if (!gc_ref_is_null(ret))
+ return gc_ref_heap_object(ret);
+ if (trigger_collection(mut, GC_COLLECTION_ANY))
+ continue;
+ return heap->allocation_failure(heap, size);
+ }
}
void
@@ -1109,6 +1115,18 @@ static void set_heap_size_from_thread(struct gc_heap *heap, size_t size) {
pthread_mutex_unlock(&heap->lock);
}
+static void* allocation_failure(struct gc_heap *heap, size_t size) {
+ fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
+ GC_CRASH();
+ return NULL;
+}
+
+void gc_heap_set_allocation_failure_handler(struct gc_heap *heap,
+ void* (*handler)(struct gc_heap*,
+ size_t)) {
+ heap->allocation_failure = handler;
+}
+
static int
heap_init(struct gc_heap *heap, const struct gc_options *options) {
// *heap is already initialized to 0.
@@ -1143,6 +1161,7 @@ heap_init(struct gc_heap *heap, const struct gc_options *options) {
allocation_counter_from_thread,
set_heap_size_from_thread,
heap->background_thread);
+ heap->allocation_failure = allocation_failure;
return 1;
}
diff --git a/libguile/whippet/src/nofl-space.h b/libguile/whippet/src/nofl-space.h
index 2668232c6..56e226bd4 100644
--- a/libguile/whippet/src/nofl-space.h
+++ b/libguile/whippet/src/nofl-space.h
@@ -880,8 +880,7 @@ nofl_allocator_next_hole(struct nofl_allocator *alloc,
static struct gc_ref
nofl_allocate(struct nofl_allocator *alloc, struct nofl_space *space,
- size_t size, void (*gc)(void*), void *gc_data,
- enum gc_allocation_kind kind) {
+ size_t size, enum gc_allocation_kind kind) {
GC_ASSERT(size > 0);
GC_ASSERT(size <= gc_allocator_large_threshold());
size = align_up(size, NOFL_GRANULE_SIZE);
@@ -894,7 +893,7 @@ nofl_allocate(struct nofl_allocator *alloc, struct nofl_space *space,
break;
}
if (!hole)
- gc(gc_data);
+ return gc_ref_null();
}
}
diff --git a/libguile/whippet/src/pcc.c b/libguile/whippet/src/pcc.c
index 0b137f56b..19d0a1ef3 100644
--- a/libguile/whippet/src/pcc.c
+++ b/libguile/whippet/src/pcc.c
@@ -73,6 +73,7 @@ struct gc_heap {
struct gc_heap_sizer sizer;
struct gc_event_listener event_listener;
void *event_listener_data;
+ void* (*allocation_failure)(struct gc_heap *, size_t);
};
#define HEAP_EVENT(heap, event, ...) do { \
@@ -862,10 +863,26 @@ copy_spaces_allocated_bytes(struct gc_heap *heap)
: heap_mono_space(heap)->allocated_bytes_at_last_gc;
}
-static enum gc_collection_kind
+static int
+resolve_pending_large_allocation_and_compute_success(struct gc_heap *heap,
+ int is_minor_gc) {
+ struct copy_space *space = heap_resizable_space(heap);
+ ssize_t deficit = copy_space_page_out_blocks_until_memory_released(space);
+ if (is_minor_gc)
+ return 1;
+ if (deficit <= 0)
+ return copy_space_can_allocate(space, gc_allocator_large_threshold());
+ deficit = align_up(deficit, COPY_SPACE_BLOCK_SIZE);
+ if (heap->sizer.policy == GC_HEAP_SIZE_FIXED)
+ return 0;
+ resize_heap(heap, heap->size + deficit);
+ return 1;
+}
+
+static int
collect(struct gc_mutator *mut,
enum gc_collection_kind requested_kind) GC_NEVER_INLINE;
-static enum gc_collection_kind
+static int
collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
struct gc_heap *heap = mutator_heap(mut);
struct large_object_space *lospace = heap_large_object_space(heap);
@@ -920,32 +937,28 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
HEAP_EVENT(heap, live_data_size, live_size);
gc_heap_sizer_on_gc(heap->sizer, heap->size, live_size, pause_ns,
resize_heap);
- {
- struct copy_space *space = heap_resizable_space(heap);
- if (!copy_space_page_out_blocks_until_memory_released(space)
- && heap->sizer.policy == GC_HEAP_SIZE_FIXED) {
- fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
- GC_CRASH();
- }
- }
+ int success =
+ resolve_pending_large_allocation_and_compute_success(heap, is_minor_gc);
HEAP_EVENT(heap, restarting_mutators);
allow_mutators_to_continue(heap);
- return gc_kind;
+ return success;
}
-static void trigger_collection(struct gc_mutator *mut,
- enum gc_collection_kind requested_kind) {
+static int trigger_collection(struct gc_mutator *mut,
+ enum gc_collection_kind requested_kind) {
struct gc_heap *heap = mutator_heap(mut);
copy_space_allocator_finish(&mut->allocator, heap_allocation_space(heap));
if (GC_GENERATIONAL)
gc_field_set_writer_release_buffer(mutator_field_logger(mut));
heap_lock(heap);
int prev_kind = -1;
+ int success = 1;
while (mutators_are_stopping(heap))
prev_kind = pause_mutator_for_collection(heap, mut);
if (prev_kind < (int)requested_kind)
- collect(mut, requested_kind);
+ success = collect(mut, requested_kind);
heap_unlock(heap);
+ return success;
}
void gc_collect(struct gc_mutator *mut, enum gc_collection_kind kind) {
@@ -955,13 +968,18 @@ void gc_collect(struct gc_mutator *mut, enum gc_collection_kind kind) {
static void* allocate_large(struct gc_mutator *mut, size_t size) {
struct gc_heap *heap = mutator_heap(mut);
struct large_object_space *space = heap_large_object_space(heap);
+ struct copy_space *copy_space = heap_resizable_space(heap);
size_t npages = large_object_space_npages(space, size);
+ size_t page_bytes = npages << space->page_size_log2;
+
+ copy_space_request_release_memory(copy_space, page_bytes);
+ if (copy_space_page_out_blocks_until_memory_released(copy_space) > 0
+ && !trigger_collection(mut, GC_COLLECTION_COMPACTING)) {
+ copy_space_maybe_reacquire_memory(copy_space, page_bytes);
+ return heap->allocation_failure(heap, size);
+ }
- copy_space_request_release_memory(heap_resizable_space(heap),
- npages << space->page_size_log2);
- while (!copy_space_page_out_blocks_until_memory_released(heap_resizable_space(heap)))
- trigger_collection(mut, GC_COLLECTION_COMPACTING);
atomic_fetch_add(&heap->large_object_pages, npages);
void *ret = large_object_space_alloc(space, npages, GC_TRACE_PRECISELY);
@@ -974,10 +992,6 @@ static void* allocate_large(struct gc_mutator *mut, size_t size) {
return ret;
}
-static void get_more_empty_blocks_for_mutator(void *mut) {
- trigger_collection(mut, GC_COLLECTION_MINOR);
-}
-
void* gc_allocate_slow(struct gc_mutator *mut, size_t size,
enum gc_allocation_kind kind) {
if (GC_UNLIKELY(kind != GC_ALLOCATION_TAGGED
@@ -996,10 +1010,11 @@ void* gc_allocate_slow(struct gc_mutator *mut, size_t size,
ret = copy_space_allocate(&mut->allocator,
heap_allocation_space(mutator_heap(mut)),
size);
- if (gc_ref_is_null(ret))
- trigger_collection(mut, GC_COLLECTION_MINOR);
- else
+ if (!gc_ref_is_null(ret))
break;
+ if (trigger_collection(mut, GC_COLLECTION_MINOR))
+ continue;
+ return mutator_heap(mut)->allocation_failure(mutator_heap(mut), size);
}
return gc_ref_heap_object(ret);
@@ -1178,6 +1193,18 @@ static void set_heap_size_from_thread(struct gc_heap *heap, size_t size) {
pthread_mutex_unlock(&heap->lock);
}
+static void* allocation_failure(struct gc_heap *heap, size_t size) {
+ fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
+ GC_CRASH();
+ return NULL;
+}
+
+void gc_heap_set_allocation_failure_handler(struct gc_heap *heap,
+ void* (*handler)(struct gc_heap*,
+ size_t)) {
+ heap->allocation_failure = handler;
+}
+
static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
// *heap is already initialized to 0.
@@ -1214,6 +1241,7 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
allocation_counter_from_thread,
set_heap_size_from_thread,
heap->background_thread);
+ heap->allocation_failure = allocation_failure;
return 1;
}
diff --git a/libguile/whippet/src/semi.c b/libguile/whippet/src/semi.c
index a4ee85f39..27037df66 100644
--- a/libguile/whippet/src/semi.c
+++ b/libguile/whippet/src/semi.c
@@ -52,6 +52,7 @@ struct gc_heap {
struct gc_heap_sizer sizer;
struct gc_event_listener event_listener;
void *event_listener_data;
+ void* (*allocation_failure)(struct gc_heap *, size_t);
};
// One mutator per space, can just store the heap in the mutator.
struct gc_mutator {
@@ -103,9 +104,9 @@ static uintptr_t align_up(uintptr_t addr, size_t align) {
static size_t min_size(size_t a, size_t b) { return a < b ? a : b; }
static size_t max_size(size_t a, size_t b) { return a < b ? b : a; }
-static void collect(struct gc_mutator *mut, size_t for_alloc) GC_NEVER_INLINE;
-static void collect_for_alloc(struct gc_mutator *mut,
- size_t bytes) GC_NEVER_INLINE;
+static void collect(struct gc_mutator *mut) GC_NEVER_INLINE;
+static int collect_for_alloc(struct gc_mutator *mut,
+ size_t bytes) GC_NEVER_INLINE;
static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data);
@@ -385,7 +386,7 @@ static uintptr_t resolve_finalizers(struct gc_heap *heap, uintptr_t grey) {
return grey;
}
-static void collect(struct gc_mutator *mut, size_t for_alloc) {
+static void collect(struct gc_mutator *mut) {
struct gc_heap *heap = mutator_heap(mut);
int is_minor = 0;
int is_compacting = 1;
@@ -429,7 +430,6 @@ static void collect(struct gc_mutator *mut, size_t for_alloc) {
gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1);
size_t live_size = semi->live_bytes_at_last_gc;
live_size += large_object_space_size_at_last_collection(large);
- live_size += for_alloc;
uint64_t pause_ns = gc_platform_monotonic_nanoseconds() - start_ns;
HEAP_EVENT(heap, live_data_size, live_size);
DEBUG("gc %zu: live size %zu, heap size %zu\n", heap->count, live_size,
@@ -443,30 +443,29 @@ static void collect(struct gc_mutator *mut, size_t for_alloc) {
// fprintf(stderr, "%zd bytes copied\n", (space->size>>1)-(space->limit-space->hp));
}
-static void collect_for_alloc(struct gc_mutator *mut, size_t bytes) {
- collect(mut, bytes);
+static int collect_for_alloc(struct gc_mutator *mut, size_t bytes) {
+ collect(mut);
struct semi_space *space = mutator_semi_space(mut);
- if (bytes < space->limit - space->hp)
- return;
+ if (bytes <= space->limit - space->hp)
+ return 1;
struct gc_heap *heap = mutator_heap(mut);
if (heap->options->common.heap_size_policy != GC_HEAP_SIZE_FIXED) {
// Each collection can potentially resize only the inactive
// fromspace, so if we really run out of space we will need to
// collect again in order to resize the other half.
- collect(mut, bytes);
- if (bytes < space->limit - space->hp)
- return;
+ collect(mut);
+ if (bytes <= space->limit - space->hp)
+ return 1;
}
- fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
- GC_CRASH();
+ return 0;
}
void gc_collect(struct gc_mutator *mut,
enum gc_collection_kind requested_kind) {
// Ignore requested kind, because we always compact.
- collect(mut, 0);
+ collect(mut);
}
int gc_object_is_old_generation_slow(struct gc_mutator *mut,
@@ -482,8 +481,11 @@ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
int* gc_safepoint_flag_loc(struct gc_mutator *mut) { GC_CRASH(); }
void gc_safepoint_slow(struct gc_mutator *mut) { GC_CRASH(); }
-static void collect_for_large_alloc(struct gc_mutator *mut, size_t npages) {
- collect_for_alloc(mut, npages * mutator_semi_space(mut)->page_size);
+static int collect_for_large_alloc(struct gc_mutator *mut, size_t npages) {
+ size_t bytes = npages * mutator_semi_space(mut)->page_size;
+ // Large object pages don't need a copy reserve.
+ bytes /= 2;
+ return collect_for_alloc(mut, bytes);
}
static void* allocate_large(struct gc_mutator *mut, size_t size) {
@@ -492,8 +494,9 @@ static void* allocate_large(struct gc_mutator *mut, size_t size) {
struct semi_space *semi_space = heap_semi_space(heap);
size_t npages = large_object_space_npages(space, size);
- while (!semi_space_steal_pages(semi_space, npages))
- collect_for_large_alloc(mut, npages);
+ if (!semi_space_steal_pages(semi_space, npages)
+ && !collect_for_large_alloc(mut, npages))
+ return heap->allocation_failure(heap, size);
void *ret = large_object_space_alloc(space, npages, GC_TRACE_PRECISELY);
@@ -522,9 +525,10 @@ void* gc_allocate_slow(struct gc_mutator *mut, size_t size,
uintptr_t addr = space->hp;
uintptr_t new_hp = align_up (addr + size, GC_ALIGNMENT);
if (space->limit < new_hp) {
- // The factor of 2 is for both regions.
- collect_for_alloc(mut, size * 2);
- continue;
+ if (collect_for_alloc(mut, size))
+ continue;
+ struct gc_heap *heap = mutator_heap(mut);
+ return heap->allocation_failure(heap, size);
}
space->hp = new_hp;
return (void *)addr;
@@ -626,6 +630,18 @@ static void ignore_async_heap_size_adjustment(struct gc_heap *heap,
size_t size) {
}
+static void* allocation_failure(struct gc_heap *heap, size_t size) {
+ fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
+ GC_CRASH();
+ return NULL;
+}
+
+void gc_heap_set_allocation_failure_handler(struct gc_heap *heap,
+ void* (*handler)(struct gc_heap*,
+ size_t)) {
+ heap->allocation_failure = handler;
+}
+
static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
heap->extern_space = NULL;
heap->pending_ephemerons_size_factor = 0.01;
@@ -642,6 +658,7 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
get_allocation_counter,
ignore_async_heap_size_adjustment,
NULL);
+ heap->allocation_failure = allocation_failure;
return heap_prepare_pending_ephemerons(heap);
}