From 105ed8d3fac5d7ad26a7591416a18a02316f57c2 Mon Sep 17 00:00:00 2001 From: Olle Fredriksson Date: Thu, 30 May 2024 23:56:53 +0200 Subject: [PATCH] New RTS --- package.yaml | 4 +- rts/garbage_collector.c | 292 ---------------------------------------- rts/garbage_collector.h | 34 ----- rts/memory.c | 47 +++++++ rts/memory.h | 12 ++ src/Compiler.hs | 12 +- src/LowToLLVM.hs | 6 +- 7 files changed, 71 insertions(+), 336 deletions(-) delete mode 100644 rts/garbage_collector.c delete mode 100644 rts/garbage_collector.h create mode 100644 rts/memory.c create mode 100644 rts/memory.h diff --git a/package.yaml b/package.yaml index ff817a12..c62beb42 100644 --- a/package.yaml +++ b/package.yaml @@ -14,9 +14,9 @@ data-files: - builtin/Builtin.vix - rts/Sixten.Builtin.c - rts/Sixten.Builtin.ll -- rts/garbage_collector.c -- rts/garbage_collector.h - rts/main.ll +- rts/memory.c +- rts/memory.h ghc-options: - -Wall diff --git a/rts/garbage_collector.c b/rts/garbage_collector.c deleted file mode 100644 index 1e06687c..00000000 --- a/rts/garbage_collector.c +++ /dev/null @@ -1,292 +0,0 @@ -#define _DEFAULT_SOURCE -#include "garbage_collector.h" - -#include -#include -#include -#include -#include -#include - -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#define debug_printf(...) // printf(__VA_ARGS__) - -const uintptr_t INLINE_SIZE_MASK = 0xFF << 3; - -// heap pointer: | 45 bits pointer data | 8 bits constructor tag | 8 bits word size | 2 bits object type | 1 | - -static -void print_heap_object(uintptr_t heap_object) { - char* pointer = heap_object_pointer(heap_object); - debug_printf("pointer: 0x%" PRIxPTR, (uintptr_t)pointer); - uintptr_t constructor_tag = heap_object_constructor_tag(heap_object); - debug_printf(", constructor_tag: %" PRIuPTR, constructor_tag); - uintptr_t inline_size = heap_object & INLINE_SIZE_MASK; - debug_printf(", inline_size: %" PRIuPTR, inline_size); - uintptr_t object_type = heap_object & ~(~0ul << 3); - debug_printf(", object_type: %" PRIuPTR, object_type); -} - -struct collector_info { - char* heap_start; - uintptr_t last_occupied_size; -}; - -static -uintptr_t round_up_to_multiple_of(uintptr_t multiple, uintptr_t x) { - return ((x + multiple - 1) / multiple) * multiple; -} - -uintptr_t page_size() { - static uintptr_t result = 0; - if (result == 0) { - result = sysconf(_SC_PAGESIZE); - } - return result; -} - -struct init_result init_garbage_collector() { - uintptr_t size = round_up_to_multiple_of(page_size(), 4096); - char* heap_start = mmap(0, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (heap_start == MAP_FAILED) { - debug_printf("init_garbage_collector mmap failed"); - exit(EXIT_FAILURE); - } - // Put collector info right after the heap limit. - char* heap_end = heap_start + size; - char* heap_limit = heap_end - sizeof(struct collector_info); - struct collector_info* collector_info = (struct collector_info*) heap_limit; - *collector_info = (struct collector_info) { - .heap_start = heap_start, - .last_occupied_size = (heap_limit - heap_start) / 2, - }; - return (struct init_result) { - .heap_pointer = heap_start, - .heap_limit = heap_limit, - }; -} - -static -uintptr_t get_forwarded_object_or_0(uintptr_t object, char* new_heap_start, char* new_heap_end) { - char* object_data = heap_object_pointer(object); - uintptr_t first_word = *(uintptr_t*)object_data; - if (is_heap_pointer(first_word)) { - char* pointer = heap_object_pointer(first_word); - if (new_heap_start <= pointer && pointer < new_heap_end) { - return first_word; - } - } - return 0; -} - -static -uintptr_t copy(uintptr_t heap_object, char** new_heap_pointer_pointer, char* new_heap_start, char* new_heap_end) { - debug_printf("copying heap object "); - print_heap_object(heap_object); - debug_printf("\n"); - uintptr_t size = heap_object_size(heap_object); - debug_printf("heap object size: %" PRIuPTR "\n", size); - // If the size is 0 we don't have space to leave a forwarding pointer, but we - // also have nothing to copy. :) - if (size == 0) { - return heap_object; - } - // If we have a forwarding pointer, we're already done. - uintptr_t forwarded_object = get_forwarded_object_or_0(heap_object, new_heap_start, new_heap_end); - if (forwarded_object) { - debug_printf("already copied; returning forwarding pointer\n"); - return forwarded_object; - } - debug_printf("copied heap object "); - // If the size is larger than the inline size cutoff we store the size just - // before the copied new heap object. - if (unlikely(size >= INLINE_SIZE_MASK)) { - *(uintptr_t*)(*new_heap_pointer_pointer) = size; - *new_heap_pointer_pointer += sizeof(char*); - } - char* copied_object_start = *new_heap_pointer_pointer; - *new_heap_pointer_pointer = copied_object_start + size; - char* object_data_pointer = heap_object_pointer(heap_object); - // Copy old to new heap data. - memcpy(copied_object_start, object_data_pointer, size); - // Construct new heap object from new pointer + old metadata. - uintptr_t new_heap_object = (heap_object & ~(~0ul << 19)) | ((uintptr_t)copied_object_start << 16); - print_heap_object(new_heap_object); - debug_printf("\n"); - // Install forwarding pointer in old heap object data. - *(uintptr_t*)object_data_pointer = new_heap_object; - return new_heap_object; -} - -struct collection_result { - char* heap_pointer; - char* heap_limit; -}; - -static -struct collection_result collect(struct shadow_stack_frame* shadow_stack, char* heap_pointer, char* heap_limit, uintptr_t minimum_free_space) { - debug_printf("Starting collection\n"); - struct collector_info* collector_info = (struct collector_info*) heap_limit; - debug_printf("last occupied size: %" PRIuPTR "\n", collector_info->last_occupied_size); - char* heap_start = collector_info->heap_start; - uintptr_t old_size = heap_pointer - heap_start; - debug_printf("old size: %" PRIuPTR "\n", old_size + sizeof(struct collector_info)); - // We're aiming at allocating 2x the occupied space, but we don't know yet - // how much space will actually be occupied after the collection, so at this - // point we just ensure that we have at least minimum_free_space. - uintptr_t new_size = - round_up_to_multiple_of( - page_size(), - old_size + minimum_free_space + sizeof(struct collector_info)); - debug_printf("new size: %" PRIuPTR "\n", new_size); - - char* new_heap_start = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (new_heap_start == MAP_FAILED) { - exit(EXIT_FAILURE); - } - char* new_heap_pointer = new_heap_start; - char* new_heap_end = new_heap_start + new_size; - // Copy stack roots. - while (shadow_stack) { - uintptr_t entry_count = shadow_stack->entry_count; - debug_printf("scanning shadow stack frame of size %" PRIuPTR "\n", entry_count); - for (uintptr_t entry_index = 0; entry_index < entry_count; ++entry_index) { - struct shadow_stack_frame_entry* entry = &shadow_stack->entries[entry_index]; - uintptr_t* entry_end = (uintptr_t*)(entry->data + entry->size); - for (uintptr_t* entry_word_pointer = (uintptr_t*)entry->data; entry_word_pointer < entry_end; ++entry_word_pointer) { - uintptr_t entry_word = *entry_word_pointer; - if (is_heap_pointer(entry_word)) { - *entry_word_pointer = copy(entry_word, &new_heap_pointer, new_heap_start, new_heap_end); - } - } - } - shadow_stack = shadow_stack->previous; - } - // Scan copied heap objects. - char* scan_pointer = new_heap_start; - debug_printf("scanning copied heap objects\n"); - while (scan_pointer < new_heap_pointer) { - uintptr_t scan_word = *(uintptr_t*)scan_pointer; - if (is_heap_pointer(scan_word)) { - *(uintptr_t*)scan_pointer = copy(scan_word, &new_heap_pointer, new_heap_start, new_heap_end); - } - scan_pointer += sizeof(char*); - } - debug_printf("done scanning\n"); - uintptr_t occupied_size = new_heap_pointer - new_heap_start; - debug_printf("occupied size: %" PRIuPTR "\n", occupied_size); - - uintptr_t old_mmap_size = (char*)(collector_info + 1) - heap_start; - debug_printf("unmapping the from-space of size %" PRIuPTR " bytes \n", old_mmap_size); - if (munmap(heap_start, old_mmap_size) != 0) { - debug_printf("unmapping failed\n"); - exit(EXIT_FAILURE); - } - // Now we know the exact occupied size, so we see if we should allocate or - // deallocate some space to reach 2x that. We should still ensure that we - // have minimum_free_space. - uintptr_t desired_size = round_up_to_multiple_of(page_size(), occupied_size * 2 + minimum_free_space + sizeof(struct collector_info)); - if (desired_size > new_size) { - debug_printf("growing the to-space to %" PRIuPTR " bytes\n", desired_size); - uintptr_t extra_size = desired_size - new_size; - char* result = mmap(new_heap_end, extra_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (result != MAP_FAILED) { - new_heap_end += extra_size; - } - else { - debug_printf("growing failed\n"); - } - } - else if (desired_size < new_size) { - debug_printf("shrinking the to-space to %" PRIuPTR " bytes\n", desired_size); - uintptr_t extra_size = new_size - desired_size; - if (munmap(new_heap_end - extra_size, extra_size) == 0) { - new_heap_end -= extra_size; - } - else { - debug_printf("shrinking failed\n"); - } - } - // Store the collector info just past the new heap limit. - char* new_heap_limit = new_heap_end - sizeof(struct collector_info); - *(struct collector_info*)new_heap_limit = (struct collector_info) { - .heap_start = new_heap_start, - .last_occupied_size = occupied_size, - }; - return (struct collection_result) { - .heap_pointer = new_heap_pointer, - .heap_limit = new_heap_limit, - }; -} - -struct heap_alloc_result __attribute((regcall)) heap_alloc(struct shadow_stack_frame* shadow_stack, char* heap_pointer, char* heap_limit, char constructor_tag, uintptr_t size) { - debug_printf("heap allocating %" PRIuPTR " bytes \n", size); - debug_printf("free space %" PRIuPTR " bytes \n", heap_limit - heap_pointer); - uintptr_t inline_size = size; - uintptr_t object_size = size; - char* object_pointer = heap_pointer; - // If size is too large to be stored in free bits in the pointer, make room - // for it to be stored just before the heap object's data. - if (unlikely(size >= INLINE_SIZE_MASK)) { - inline_size = INLINE_SIZE_MASK; - object_size = size + sizeof(char*); - object_pointer = heap_pointer + sizeof(char*); - } - char* new_heap_pointer = heap_pointer + object_size; - // We're out of space, so trigger a collection. - if (unlikely(new_heap_pointer > heap_limit)) { - struct collection_result collection_result = collect(shadow_stack, heap_pointer, heap_limit, object_size); - heap_pointer = collection_result.heap_pointer; - heap_limit = collection_result.heap_limit; - object_pointer = heap_pointer; - if (unlikely(size >= INLINE_SIZE_MASK)) { - object_pointer = heap_pointer + sizeof(char*); - } - new_heap_pointer = heap_pointer + object_size; - } - // Actually store the size before the heap object if the size was too large - // to be inline. - if (unlikely(size >= INLINE_SIZE_MASK)) { - *(uintptr_t*)heap_pointer = size; - } - uintptr_t result - = ((uintptr_t)object_pointer << 16) - | ((uintptr_t)constructor_tag << 11) - | (uintptr_t)inline_size - | 1; - debug_printf("heap allocated object "); - print_heap_object(result); - debug_printf("\n"); - return (struct heap_alloc_result) { - .result = result, - .heap_pointer = new_heap_pointer, - .heap_limit = heap_limit, - }; -} - -int is_heap_pointer(uintptr_t word) { - return word & 0x1; -} - -uintptr_t heap_object_size(uintptr_t word) { - uintptr_t inline_size = word & INLINE_SIZE_MASK; - if (unlikely(inline_size == INLINE_SIZE_MASK)) { - return *(uintptr_t*)(heap_object_pointer(word) - sizeof(char*)); - } - return inline_size; -} - -uintptr_t heap_object_constructor_tag(uintptr_t word) { - return (word >> 11) & 0xFF; -} - -char* heap_object_pointer(uintptr_t word) { - return (char*)((uintptr_t)((intptr_t)word >> 19) << 3); -} - -// If we know that the constructor tag is <= 5 bits, we can get the pointer -// with one instead of two shifts. -char* heap_object_pointer_5bit_tag(uintptr_t word) { - return (char*)((intptr_t)word >> 16); -} diff --git a/rts/garbage_collector.h b/rts/garbage_collector.h deleted file mode 100644 index e9cce850..00000000 --- a/rts/garbage_collector.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include - -struct init_result { - char* heap_pointer; - char* heap_limit; -}; - -struct init_result init_garbage_collector(); - -struct heap_alloc_result { - uintptr_t result; - char* heap_pointer; - char* heap_limit; -}; - -struct shadow_stack_frame_entry { - uintptr_t size; - char* data; -}; - -struct shadow_stack_frame { - struct shadow_stack_frame* previous; - uintptr_t entry_count; - struct shadow_stack_frame_entry entries[]; -}; - -struct heap_alloc_result __attribute__((regcall)) heap_alloc(struct shadow_stack_frame* shadow_stack, char* heap_pointer, char* heap_limit, char constructor_tag, uintptr_t size); -int is_heap_pointer(uintptr_t word); -uintptr_t heap_object_size(uintptr_t word); -char* heap_object_pointer(uintptr_t word); -char* heap_object_pointer_5bit_tag(uintptr_t word); -uintptr_t heap_object_constructor_tag(uintptr_t word); diff --git a/rts/memory.c b/rts/memory.c new file mode 100644 index 00000000..461ff88f --- /dev/null +++ b/rts/memory.c @@ -0,0 +1,47 @@ +#include "memory.h" + +#include + +static const uintptr_t TAG_BITS = 16; +static const uintptr_t TAG_MASK = ((uintptr_t)1 << TAG_BITS) - 1; + +struct header { + uint32_t pointers; +}; + +uintptr_t sixten_heap_allocate(uint64_t tag, uint32_t pointers, uint32_t non_pointer_bytes) { + uintptr_t bytes = (uintptr_t)pointers * sizeof(void*) + (uintptr_t)non_pointer_bytes; + uint8_t* pointer = 0; + + if (bytes > 0) { + pointer = malloc(sizeof(struct header) + bytes); + } + + struct header* header = (struct header*)pointer; + header->pointers = pointers; + + pointer += sizeof(struct header); + + return (uintptr_t)pointer << TAG_BITS | (uintptr_t)(tag & TAG_MASK); +} + +struct sixten_reference sixten_heap_payload(uintptr_t heap_object) { + uint8_t* pointer = (uint8_t*)((intptr_t)heap_object >> TAG_BITS); + if (pointer == 0) { + return (struct sixten_reference) { + .pointers = 0, + .non_pointers = 0, + }; + } + + struct header* header = (struct header*)(pointer - sizeof(struct header)); + + return (struct sixten_reference) { + .pointers = pointer, + .non_pointers = pointer + header->pointers * sizeof(void*), + }; +} + +uint64_t sixten_heap_tag(uintptr_t heap_object) { + return (uint64_t)(heap_object & TAG_MASK); +} diff --git a/rts/memory.h b/rts/memory.h new file mode 100644 index 00000000..639ef471 --- /dev/null +++ b/rts/memory.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +struct sixten_reference { + uint8_t* pointers; + uint8_t* non_pointers; +}; + +uintptr_t sixten_heap_allocate(uint64_t tag, uint32_t pointers, uint32_t non_pointer_bytes); +struct sixten_reference sixten_heap_payload(uintptr_t heap_object); +uint64_t sixten_heap_tag(uintptr_t heap_object); diff --git a/src/Compiler.hs b/src/Compiler.hs index 9cae9d37..7f521da6 100644 --- a/src/Compiler.hs +++ b/src/Compiler.hs @@ -53,7 +53,7 @@ compile assemblyDir saveAssembly outputExecutableFile maybeOptimisationLevel pri builtinLLVMFile <- liftIO $ Paths.getDataFileName "rts/Sixten.Builtin.ll" builtinCFile <- liftIO $ Paths.getDataFileName "rts/Sixten.Builtin.c" mainLLVMFile <- liftIO $ Paths.getDataFileName "rts/main.ll" - garbageCollectorCFile <- liftIO $ Paths.getDataFileName "rts/garbage_collector.c" + memoryCFile <- liftIO $ Paths.getDataFileName "rts/memory.c" let llvmFiles = mainLLVMFile : builtinLLVMFile : moduleInitLLVMFile : moduleLLVMFiles let optimisationArgs = @@ -68,15 +68,15 @@ compile assemblyDir saveAssembly outputExecutableFile maybeOptimisationLevel pri assemblyDir "program-opt" <.> "ll" builtinCLLFile = assemblyDir "Sixten.Builtin" <.> "c" <.> "ll" - garbageCollectorLLFile = - assemblyDir "garbage_collector" <.> "ll" + memoryLLFile = + assemblyDir "memory" <.> "ll" llvmBin <- liftIO llvmBinPath callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-S", "-emit-llvm", "-o", builtinCLLFile, builtinCFile] - callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-S", "-emit-llvm", "-o", garbageCollectorLLFile, garbageCollectorCFile] - callProcess (llvmBin "llvm-link") $ ["-S", "-o", linkedProgramName, builtinCLLFile, garbageCollectorLLFile] <> llvmFiles + callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-S", "-emit-llvm", "-o", memoryLLFile, memoryCFile] + callProcess (llvmBin "llvm-link") $ ["-S", "-o", linkedProgramName, builtinCLLFile, memoryLLFile] <> llvmFiles callProcess (llvmBin "opt") $ optimisationArgs <> ["-S", "-o", optimisedProgramName, linkedProgramName] callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-o", outputExecutableFile, linkedProgramName] - else callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-o", outputExecutableFile, builtinCFile, garbageCollectorCFile] <> llvmFiles + else callProcess clang $ optimisationArgs <> ["-fPIC", "-Wno-override-module", "-o", outputExecutableFile, builtinCFile, memoryCFile] <> llvmFiles supportedLlvmVersions :: [Int] supportedLlvmVersions = [17, 16, 15] diff --git a/src/LowToLLVM.hs b/src/LowToLLVM.hs index 50601799..d66a2734 100644 --- a/src/LowToLLVM.hs +++ b/src/LowToLLVM.hs @@ -441,16 +441,18 @@ assembleTerm env nameSuggestion passBy = \case result <- constructTuple (fromMaybe "alloca_result" nameSuggestion) "ptr" allocaBytes "ptr" nonPointerPointer pure (Local result, Just stack) Syntax.HeapAllocate constr size -> do - declareLLVMGlobal "sixten_heap_allocate" "declare i64 @sixten_heap_allocate(i64, i64)" + declareLLVMGlobal "sixten_heap_allocate" "declare i64 @sixten_heap_allocate(i64, i32, i32)" var <- freshVar $ fromMaybe "heap_allocation" nameSuggestion (_, maybeTag) <- fetch $ Query.ConstructorRepresentation constr size' <- assembleOperand env size + (pointers, nonPointerBytes) <- extractSizeParts size' emitInstruction $ varName var <> " = call i64 @sixten_heap_allocate" <> parens [ "i64 " <> Builder.intDec (fromMaybe 0 maybeTag) - , typedOperand size' + , "i32 " <> varName pointers + , "i32 " <> varName nonPointerBytes ] pure (Local var, Nothing) Syntax.HeapPayload pointer -> do