From a5f5982c13572f6b4e157bb118b5b58206398ce1 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Fri, 27 Mar 2026 09:35:12 +0100 Subject: [PATCH 1/7] remove deprecated python2 imports --- test/integration/kernel/kprint/test.py | 3 +-- test/integration/net/http/test.py | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/test/integration/kernel/kprint/test.py b/test/integration/kernel/kprint/test.py index 1a2ec699c..079fd36e7 100755 --- a/test/integration/kernel/kprint/test.py +++ b/test/integration/kernel/kprint/test.py @@ -3,7 +3,6 @@ from __future__ import division from __future__ import print_function from builtins import str -from past.utils import old_div import sys import os @@ -27,7 +26,7 @@ def set_format_string_size(line): def check_truncation(line): assert(format_string_size) - print("Received truncated string: ", line, "of size", len(line), "(format size * ", old_div(len(line),format_string_size),")") + print("Received truncated string: ", line, "of size", len(line), "(format size * ", len(line)//format_string_size,")") assert(len(line) <= format_string_size * 2) # truncated outputs are unacceptable :) assert(line.strip().split(" ")[-1] == "END") diff --git a/test/integration/net/http/test.py b/test/integration/net/http/test.py index 5e0d61429..8e31b5ec0 100755 --- a/test/integration/net/http/test.py +++ b/test/integration/net/http/test.py @@ -1,19 +1,17 @@ #!/usr/bin/env python3 -from future import standard_library -standard_library.install_aliases() -from builtins import str -import sys -import os import _thread +import http.server +import sys +import urllib.error +import urllib.parse +import urllib.request from vmrunner import vmrunner HOST = '' PORT = 9011 -import http.server - DO_SERVE = True class RequestHandler(http.server.BaseHTTPRequestHandler): def do_GET(s): @@ -36,7 +34,6 @@ def Client_test(): _thread.start_new_thread(Client_test, ()) -import urllib.request, urllib.error, urllib.parse def Server_test(triggerline): res = urllib.request.urlopen("http://10.0.0.46:8080").read() assert(res.decode('utf-8') == "Hello") From ff12ecbaf022cc996fcb093ef62c2c09bd3ed62a Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Tue, 24 Mar 2026 14:31:59 +0100 Subject: [PATCH 2/7] support memory above 4G The multiboot specification explicitly states that `mem_upper` is not guaranteed to be the correct value for mem_upper. Additionally, placing the heap in-between `mem_lower` and `mem_upper` only really makes sense on 32-bit systems. By using memory mapped pages we can pick the largest consecutive region of memory available. Selecting 1_000_000 as the minimum address is a temporary hack since it's not immediately obvious how to get the correct address for virtual mappings that are still not initialized (such as LiveUpdate and SystemLog). The correct solution here would be to make these regions dynamic in size, initialize them earlier so the regions are already claimed, or re-initialize the heap. This is only necessary for systems that have less than 4G of memory. See-Also: https://www.gnu.org/software/grub/manual/multiboot/multiboot.html See-Also: https://wiki.osdev.org/Multiboot --- api/boot/multiboot.h | 11 ++++- src/arch/x86_64/paging.cpp | 5 +- src/kernel/multiboot.cpp | 97 ++++++++++++++++++++++++++++---------- src/platform/x86_pc/os.cpp | 37 ++++++++++----- 4 files changed, 111 insertions(+), 39 deletions(-) diff --git a/api/boot/multiboot.h b/api/boot/multiboot.h index ab4318778..240d6b9b6 100644 --- a/api/boot/multiboot.h +++ b/api/boot/multiboot.h @@ -195,12 +195,19 @@ typedef struct multiboot_info multiboot_info_t; struct multiboot_mmap_entry { - multiboot_uint32_t size; + multiboot_uint32_t size; // size of struct multiboot_uint64_t addr; - multiboot_uint64_t len; + multiboot_uint64_t len; // bytes available #define MULTIBOOT_MEMORY_AVAILABLE 1 #define MULTIBOOT_MEMORY_RESERVED 2 + #define MULTIBOOT_MEMORY_ACPI_RECLAIMABLE 3 + #define MULTIBOOT_MEMORY_NVS 4 + #define MULTIBOOT_MEMORY_BADRAM 5 multiboot_uint32_t type; + + [[nodiscard]] constexpr bool is_available() const noexcept { + return type == MULTIBOOT_MEMORY_AVAILABLE; + } } __attribute__((packed)); typedef struct multiboot_mmap_entry multiboot_memory_map_t; diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index cd3c78918..24ddbcba9 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -376,6 +376,9 @@ uintptr_t mem::active_page_size(uintptr_t addr){ void allow_executable() { + // this sets the region where the unikernel's executable code is + // loaded into by the linker (src/arch/x86_64/linker.ld) as executable + INFO2("* Allowing execute on %p -> %p", (void*) __exec_begin, (void*)__exec_end); @@ -391,7 +394,7 @@ void allow_executable() m.page_sizes = os::mem::Map::any_size; m.flags = os::mem::Access::execute | os::mem::Access::read; - os::mem::map(m, "ELF .text"); + os::mem::map(m, "ELF Executable (Unikernel service)"); } /* TODO: Compiler warning unused diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 295982016..491447430 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -22,6 +22,7 @@ #include #include #include +#include template static inline void _kfmt(fmt::string_view prefix, fmt::format_string fmtstr, Args&&... args) { @@ -122,9 +123,41 @@ uintptr_t _multiboot_free_begin(uintptr_t boot_addr) return multi_end; } +constexpr static inline const char* multiboot_memory_type_str(uint32_t type) noexcept { + // TODO: convert multiboot types to enum class + switch (type) { + case MULTIBOOT_MEMORY_AVAILABLE: + return "Available"; + case MULTIBOOT_MEMORY_ACPI_RECLAIMABLE: + return "ACPI Reclaimable"; + case MULTIBOOT_MEMORY_NVS: + return "ACPI Non-volatile Storage"; + case MULTIBOOT_MEMORY_BADRAM: + return "Bad RAM"; + case MULTIBOOT_MEMORY_RESERVED: + return "Reserved"; + default: + return "UNKNOWN"; + } +} + + +std::span _multiboot_memory_maps() { + auto* info = kernel::bootinfo(); + + auto* hardware_map = reinterpret_cast(info->mmap_addr); + const size_t entry_count = static_cast(info->mmap_length / sizeof(multiboot_memory_map_t)); + + return std::span { hardware_map, entry_count }; +} + void kernel::multiboot(uint32_t boot_addr) { - MYINFO("Booted with multiboot"); +#if defined(__x86_64) + MYINFO("Booted with multiboot x86_64"); +#else + MYINFO("Booted with multiboot x86"); +#endif auto* info = ::bootinfo(boot_addr); INFO2("* Boot flags: {:#x}", info->flags); @@ -152,36 +185,50 @@ void kernel::multiboot(uint32_t boot_addr) } if (info->flags & MULTIBOOT_INFO_MEM_MAP) { - INFO2("* Multiboot provided memory map ({} entries @ {})", - info->mmap_length / sizeof(multiboot_memory_map_t), - (const void*)(uintptr_t)info->mmap_addr); - std::span mmap { - reinterpret_cast(info->mmap_addr), - static_cast(info->mmap_length / sizeof(multiboot_memory_map_t)) - }; - - for (auto map : mmap) + auto* hardware_map = reinterpret_cast(info->mmap_addr); + const size_t entry_count = static_cast(info->mmap_length / sizeof(multiboot_memory_map_t)); + + INFO2("* Multiboot provided memory map ({} entries @ {})\n", entry_count, reinterpret_cast(hardware_map)); + + for (auto map : std::span{ hardware_map, entry_count }) { - const char* str_type = map.type & MULTIBOOT_MEMORY_AVAILABLE ? "FREE" : "RESERVED"; - const uintptr_t addr = map.addr; + const uintptr_t start = map.addr; const uintptr_t size = map.len; - INFO2(" {:#x} - {:#x} {} ({} KiB)", addr, addr + size - 1, str_type, size / 1024); - - if (not (map.type & MULTIBOOT_MEMORY_AVAILABLE)) { + const uintptr_t end = start + size - 1; - if (util::bits::is_aligned<4_KiB>(map.addr)) { - os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, - "Reserved (Multiboot)"); - continue; - } + INFO2(" {:#16x} - {:#16x} ({} KiB): {}", start, end, size / 1024, multiboot_memory_type_str(map.type)); - // For non-aligned addresses, assign - os::mem::vmmap().assign_range({addr, addr + size - 1, "Reserved (Multiboot)"}); + // os::mem::map() does not accept non-aligned page addresses + if (not util::bits::is_aligned<4_KiB>(map.addr)) { + os::mem::vmmap().assign_range({start, start + size - 1, "UNALIGNED"}); + continue; } - else + + os::mem::Map rw_map = { /*.linear=*/start, /*.physical=*/start, /*.fl=*/os::mem::Access::read | os::mem::Access::write, /*.sz=*/size }; + switch (map.type) { - // Map as free memory - //os::mem::map_avail({map.addr, map.addr, {os::mem::Access::read | os::mem::Access::write}, map.len}, "Reserved (Multiboot)"); + case MULTIBOOT_MEMORY_ACPI_RECLAIMABLE: + os::mem::map(rw_map, "Multiboot (ACPI Reclaimable)"); + break; + case MULTIBOOT_MEMORY_NVS: + os::mem::map(rw_map, "Multiboot (ACPI Non-volatile Storage)"); + break; + case MULTIBOOT_MEMORY_BADRAM: + os::mem::map(rw_map, "Multiboot (Bad RAM)"); + break; + case MULTIBOOT_MEMORY_RESERVED: + os::mem::map(rw_map, "Multiboot (Reserved)"); + break; + + case MULTIBOOT_MEMORY_AVAILABLE: { + // these are mapped in src/platform/${platform}/os.cpp + break; + } + default: { + char buf[32]; // libc is not entirely initialized at this point + std::snprintf(buf, sizeof(buf), "Unknown memory map type: %d", map.type); + os::panic(buf); + } } } INFO2(""); diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 9b2097889..e70e67ac5 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -48,6 +48,8 @@ extern uintptr_t _ELF_END_; // in kernel/os.cpp extern bool os_default_stdout; +extern std::span _multiboot_memory_maps(); + struct alignas(SMP_ALIGN) OS_CPU { uint64_t cycles_hlt = 0; }; @@ -125,10 +127,6 @@ void kernel::start(uint32_t boot_magic, uint32_t boot_addr) MYINFO("Total memory detected as %s ", util::Byte_r(kernel::memory_end()).to_string().c_str()); - // Give the rest of physical memory to heap - kernel::state().heap_max = kernel::memory_end() - 1; - assert(kernel::heap_begin() != 0x0 and kernel::heap_max() != 0x0); - PROFILE("Memory map"); // Assign memory ranges used by the kernel auto& memmap = os::mem::vmmap(); @@ -145,13 +143,30 @@ void kernel::start(uint32_t boot_magic, uint32_t boot_addr) memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #endif - // heap (physical) area - uintptr_t span_max = std::numeric_limits::max(); - uintptr_t heap_range_max_ = std::min(span_max, kernel::heap_max()); + multiboot_memory_map_t heap_map = {0,0,0,0}; + for (auto entry : _multiboot_memory_maps()) + { + if (not entry.is_available()) continue; + + if (entry.len > heap_map.len) { + heap_map = entry; + } + } + uintptr_t end = heap_map.addr + heap_map.len - 1; + + // NOTE: this hard-coded address stems from LiveUpdate and SystemLog using part of this space + // and should ideally be resolved by refactoring those subsystems to not use hardcoded addresses + if (heap_map.addr < 0x1'000'000) { + kernel::state().heap_begin = std::max((uintptr_t)0x1'000'000, (uintptr_t)heap_map.addr); + kernel::state().heap_max = std::min(kernel::heap_max(), end); + } else { + kernel::state().heap_begin = heap_map.addr; + kernel::state().heap_max = end; + } - INFO2("* Assigning heap 0x%zx -> 0x%zx", kernel::heap_begin(), heap_range_max_); - memmap.assign_range({kernel::heap_begin(), heap_range_max_, - "Dynamic memory", kernel::heap_usage }); + + INFO2("* Assigning heap 0x%lx -> 0x%lx", kernel::heap_begin(), kernel::heap_max()); + memmap.assign_range({kernel::heap_begin(), kernel::heap_max(), "Heap", kernel::heap_usage }); MYINFO("Virtual memory map"); for (const auto& entry : memmap) @@ -182,7 +197,7 @@ void os::event_loop() __arch_poweroff(); } - +/* legacy boot is used when MULTIBOOT_BOOTLOADER_MAGIC is unset, see x86_pc/kernel_start.cpp */ void kernel::legacy_boot() { // Fetch CMOS memory info (unfortunately this is maximally 10^16 kb) From 30201be735687a1cf2eb4d5ee6e66a5fe81be7a2 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Fri, 27 Mar 2026 09:00:09 +0100 Subject: [PATCH 3/7] add missing namespace --- api/util/bitops.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/util/bitops.hpp b/api/util/bitops.hpp index 4c9c0e617..76e9088ac 100644 --- a/api/util/bitops.hpp +++ b/api/util/bitops.hpp @@ -247,7 +247,7 @@ inline bool is_aligned(uintptr_t A, uintptr_t ptr) noexcept return (ptr & (A - 1)) == 0; } -inline size_t upercent(size_t a, size_t b) noexcept +inline std::size_t upercent(std::size_t a, std::size_t b) noexcept { return (100 * a + b / 2) / b; } From 6a3eb188f5d7ea8672ad510b35f15e300144475b Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Tue, 24 Mar 2026 14:43:19 +0100 Subject: [PATCH 4/7] update example unikernel to show mounted mappings --- example/src/main.cpp | 15 ++++++++++++--- example/src/vm.json | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 example/src/vm.json diff --git a/example/src/main.cpp b/example/src/main.cpp index dabb79362..c99727f64 100644 --- a/example/src/main.cpp +++ b/example/src/main.cpp @@ -1,9 +1,18 @@ +#include "kernel/memory.hpp" #include #include void Service::start(const std::string& args){ - printf("Args = %s\n", args.c_str()); - printf("Try giving the service less memory, eg. 10MB in vm.json\n"); - printf("Service done. Shutting down...\n"); + std::println("Hello from the example unikernel!"); + std::println(); + + std::println("Current virtual mappings:"); + for (const auto& entry : os::mem::vmmap()) + std::println(" {}", entry.second.to_string()); + std::println(); + + std::println("Tip: Try changing how much memory you give to the service in vm.json"); + std::println("Service done. Shutting down..."); + os::shutdown(); } diff --git a/example/src/vm.json b/example/src/vm.json new file mode 100644 index 000000000..a45e2c02c --- /dev/null +++ b/example/src/vm.json @@ -0,0 +1,3 @@ +{ + "mem": 128 +} From c64903631194826fe2b50adced4c8fdeb178c176 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Tue, 24 Mar 2026 14:44:07 +0100 Subject: [PATCH 5/7] add support for unassigning a mapping This will permit repurposing an already assigned mapping, allowing multiboot to preemptively assigning "free memory" regions, resizing ranges by deleting and recreating them, or making temporary regions. Could also be useful for NUMA. --- api/kernel/memmap.hpp | 6 ++++++ src/kernel/memmap.cpp | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/api/kernel/memmap.hpp b/api/kernel/memmap.hpp index 1009c2388..ac6cef6c0 100644 --- a/api/kernel/memmap.hpp +++ b/api/kernel/memmap.hpp @@ -339,6 +339,12 @@ class Memory_map { */ Fixed_memory_range& assign_range(const Fixed_memory_range::size_type size); + /** + * Removes a memory range previously defined + * Useful to redefine the purpose of a range + */ + void unassign_range(const Fixed_memory_range& range); + /** * Check if an address is within a range in the map * diff --git a/src/kernel/memmap.cpp b/src/kernel/memmap.cpp index 38ca1dd00..1c52cabfb 100644 --- a/src/kernel/memmap.cpp +++ b/src/kernel/memmap.cpp @@ -173,6 +173,21 @@ Fixed_memory_range& Memory_map::assign_range(Fixed_memory_range&& rng) { return new_entry.first->second; } +/////////////////////////////////////////////////////////////////////////////// +void Memory_map::unassign_range(const Fixed_memory_range& range) { + auto it = map_.find(range.addr_start()); + + if (it != map_.end()) { + if (it->second.addr_end() == range.addr_end()) { + map_.erase(it); + } else { + throw Memory_range_exception{"Range mismatch at address " + std::to_string(range.addr_start())}; + } + } else { + throw Memory_range_exception{"No range found to erase at " + std::to_string(range.addr_start())}; + } +} + /////////////////////////////////////////////////////////////////////////////// Fixed_memory_range& Memory_map::at(const Key key) { return const_cast(static_cast(this)->at(key)); From f0cacb3e93796a7b01b000e8edb03c438245e666 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Wed, 8 Apr 2026 19:08:19 +0200 Subject: [PATCH 6/7] fixup! fix printf-type specifiers --- src/kernel/multiboot.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 491447430..6e8bf3b21 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -169,7 +169,7 @@ void kernel::multiboot(uint32_t boot_addr) uint32_t mem_high_end = mem_high_start + (info->mem_upper * 1024) - 1; uint32_t mem_high_kb = info->mem_upper; - INFO2("* Valid memory (%i KiB):", mem_low_kb + mem_high_kb); + INFO2("* Valid memory ({} KiB):", mem_low_kb + mem_high_kb); INFO2(" 0x{:08x} - 0x{:08x} ({} KiB)", mem_low_start, mem_low_end, mem_low_kb); INFO2(" 0x{:08x} - 0x{:08x} ({} KiB)", mem_high_start, mem_high_end, mem_high_kb); INFO2(""); From b5734031ed7b7b29ee8daa4bf3f9b69d27e166b2 Mon Sep 17 00:00:00 2001 From: Mazunki Hoksaas Date: Wed, 8 Apr 2026 23:54:44 +0200 Subject: [PATCH 7/7] include integration tests for booting and allocation on differently sized vms --- .../memory/available/CMakeLists.txt | 19 +++ test/integration/memory/available/service.cpp | 76 +++++++++++ test/integration/memory/available/test.py | 119 ++++++++++++++++++ .../integration/memory/available/vm-128m.json | 3 + test/integration/memory/available/vm-16g.json | 3 + test/integration/memory/available/vm-2g.json | 3 + test/integration/memory/available/vm-4g.json | 3 + test/integration/memory/available/vm-6g.json | 3 + .../memory/available/vm-default.json | 2 + 9 files changed, 231 insertions(+) create mode 100644 test/integration/memory/available/CMakeLists.txt create mode 100644 test/integration/memory/available/service.cpp create mode 100644 test/integration/memory/available/test.py create mode 100644 test/integration/memory/available/vm-128m.json create mode 100644 test/integration/memory/available/vm-16g.json create mode 100644 test/integration/memory/available/vm-2g.json create mode 100644 test/integration/memory/available/vm-4g.json create mode 100644 test/integration/memory/available/vm-6g.json create mode 100644 test/integration/memory/available/vm-default.json diff --git a/test/integration/memory/available/CMakeLists.txt b/test/integration/memory/available/CMakeLists.txt new file mode 100644 index 000000000..15e3eacf5 --- /dev/null +++ b/test/integration/memory/available/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.31.6) +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) +project(service) +include(os) +set(SOURCES + service.cpp +) +os_add_executable(memory_available "Memory available test" ${SOURCES}) +os_add_stdout(memory_available default_stdout) + +configure_file(test.py ${CMAKE_CURRENT_BINARY_DIR}) + +configure_file(vm-default.json ${CMAKE_CURRENT_BINARY_DIR}) +configure_file(vm-128m.json ${CMAKE_CURRENT_BINARY_DIR}) +configure_file(vm-2g.json ${CMAKE_CURRENT_BINARY_DIR}) +configure_file(vm-4g.json ${CMAKE_CURRENT_BINARY_DIR}) +configure_file(vm-6g.json ${CMAKE_CURRENT_BINARY_DIR}) +configure_file(vm-16g.json ${CMAKE_CURRENT_BINARY_DIR}) + diff --git a/test/integration/memory/available/service.cpp b/test/integration/memory/available/service.cpp new file mode 100644 index 000000000..2131cabef --- /dev/null +++ b/test/integration/memory/available/service.cpp @@ -0,0 +1,76 @@ +#include +#include +#include +#include +#include +#include + +static void* try_mmap(std::size_t size) { + try { + return os::mem::raw_allocator().allocate(size); + } catch (...) { + return MAP_FAILED; + } +} + +static void try_munmap(void *p, size_t size) { + if (p != MAP_FAILED) os::mem::raw_allocator().deallocate(p, size); +} + +void Service::start() +{ + std::uintptr_t heap_start = 0; + std::uintptr_t heap_end = 0; + + for (const auto& [addr, entry] : os::mem::vmmap()) { + if (entry.name() == std::string_view{"Heap"}) { + heap_start = entry.addr_start(); + heap_end = entry.addr_end(); + break; + } + } + + const std::size_t heap_size = heap_end - heap_start + 1; + const std::size_t heap_mib = heap_size / (1024UL * 1024UL); + + std::println("HEAP_START: {:#x}", heap_start); + std::println("HEAP_END: {:#x}", heap_end); + std::println("HEAP_MiB: {}", heap_mib); + + // small allocation, always expected to succeed + { + constexpr std::size_t small = 1UL * 1024UL * 1024UL; // 1 MiB + std::println("bytes free: {}", os::mem::raw_allocator().bytes_free()); + + void* p = try_mmap(small); + std::println("HEAP_ALLOC_SMALL (1 MiB): {}", (p != MAP_FAILED) ? "OK" : "FAIL"); + try_munmap(p, small); + } + + std::println("HEAP_FREE: {}", heap_mib); + // large allocation + { + const size_t PERCENTAGE = (heap_mib >= 256) ? 30 : 10; + const std::size_t large = heap_size * PERCENTAGE / 100; + const std::size_t large_mib = large / (1024UL * 1024UL); + void* p = try_mmap(large); + + std::println("bytes free: {}", os::mem::raw_allocator().bytes_free()); + std::println("HEAP_ALLOC_LARGE ({} MiB = {}% of heap): {}", large_mib, PERCENTAGE, (p != MAP_FAILED) ? "OK" : "FAIL"); + try_munmap(p, large); + } + + // oversized allocation, must fail + { + const std::size_t oversize = os::mem::raw_allocator().bytes_free() + 16*1024UL*1024UL; // + 16MiB + const std::size_t oversize_mib = oversize / (1024UL * 1024UL); + + std::println("bytes free: {}", os::mem::raw_allocator().bytes_free()); + void* p = try_mmap(oversize); // this should fail, i.e. MAP_FAILED => ok + + std::println("HEAP_ALLOC_OVERSIZED ({} MiB): {}", oversize_mib, (p == MAP_FAILED) ? "OK" : "FAIL"); + } + + std::println("Test done."); + os::shutdown(); +} diff --git a/test/integration/memory/available/test.py b/test/integration/memory/available/test.py new file mode 100644 index 000000000..12dfc207a --- /dev/null +++ b/test/integration/memory/available/test.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +from __future__ import print_function +import sys +import re +from vmrunner import vmrunner + +IMAGE = "memory_available.elf.bin" + +CONFIGS = [ + { + "name": "default", + "config": "vm-default.json", + "heap_start_min": 0x1_000_000, # leaves room for LiveUpdate and SystemLog + "heap_start_max": 0xFFFF_FFFF, # stays below the 32-bit boundary + "heap_mib_min": 64, # observed to be 79 MiB + "heap_mib_max": 128, # heap can't be bigger than the provided memory + }, + { + "name": "128 MiB", # should match default + "config": "vm-128m.json", + "heap_start_min": 0x1_000_000, + "heap_start_max": 0xFFFF_FFFF, + "heap_mib_min": 64, + "heap_mib_max": 128, + }, + { + "name": "2 GiB", + "config": "vm-2g.json", + "heap_start_min": 0x1_000_000, + "heap_start_max": 0xFFFF_FFFF, + "heap_mib_min": 1400, # observed to be 1519 MiB + "heap_mib_max": 2048, + }, + { + "name": "4 GiB", + "config": "vm-4g.json", + "heap_start_min": 0x1_000_000, + "heap_start_max": 0xFFFF_FFFF, + "heap_mib_min": 2048, # observed to be 2287 MiB + "heap_mib_max": 4096, + }, + { + "name": "6 GiB", + "config": "vm-6g.json", + "heap_start_min": 0x1_0000_0000, # above 4G: heap will be located after the 32-bit area + "heap_start_max": None, + "heap_mib_min": 2800, # observed to be 3072 MiB + "heap_mib_max": 6144, + }, + # 16GiB fails because buddy doesn't support large allocations + # { + # "name": "16 GiB", + # "config": "vm-16g.json", + # "heap_start_min": 0x1_0000_0000, + # "heap_start_max": None, + # "heap_mib_min": 12288, # observed to be 13312 MiB + # "heap_mib_max": 16384, + # }, +] + + +def boot_chain(configs): + if not configs: return + cfg, *rest = configs + + vm = vmrunner.vm(config=cfg["config"]) + + def on_heap_start(line): + m = re.search(r'HEAP_START:\s+(0x[0-9a-fA-F]+)', line) + if not m: return + addr = int(m.group(1), 16) + + assert addr >= cfg["heap_start_min"], \ + f"[{cfg['name']}] Heap start {addr:#x} below minimum {cfg['heap_start_min']:#x}" + + if cfg["heap_start_max"] is not None: + assert addr <= cfg["heap_start_max"], \ + f"[{cfg['name']}] Heap start {addr:#x} unexpectedly above 4G" + + def on_heap_mib(line): + m = re.search(r'HEAP_MiB:\s+(\d+)', line) + if not m: return + mib = int(m.group(1)) + + assert mib >= cfg["heap_mib_min"], \ + f"[{cfg['name']}] Heap size {mib} MiB below minimum {cfg['heap_mib_min']} MiB" + if cfg["heap_mib_max"] is not None: + assert mib <= cfg["heap_mib_max"], \ + f"[{cfg['name']}] Heap size {mib} MiB above maximum {cfg['heap_mib_max']} MiB; " \ + f"default config should give same heap as explicit 128m" + + def on_alloc_small(line): + assert "HEAP_ALLOC_SMALL (1 MiB): OK" in line, \ + f"[{cfg['name']}] Small allocation failed: {line}" + + def on_alloc_large(line): + assert re.search(r'HEAP_ALLOC_LARGE \(\d+ MiB = \d+% of heap\): OK', line), \ + f"[{cfg['name']}] Large allocation failed: {line}" + + def on_alloc_oversize(line): + m = re.search(r'HEAP_ALLOC_OVERSIZED \(\d+ MiB\): (OK|FAIL)', line) + # TODO: hard-fail here after buddy is replaced + if m and m.group(1) == "FAIL": + print("WARNING: oversized allocation succeeded --- known limitation of current buddy allocator") + + vm.on_output("HEAP_START:", on_heap_start) + vm.on_output("HEAP_MiB:", on_heap_mib) + vm.on_output("HEAP_ALLOC_SMALL", on_alloc_small) + vm.on_output("HEAP_ALLOC_LARGE", on_alloc_large) + vm.on_output("HEAP_ALLOC_OVERSIZED", on_alloc_oversize) + vm.on_exit_success(lambda: boot_chain(rest)) + + print(f"Booting VM {len(CONFIGS) - len(configs) + 1}/{len(CONFIGS)}: {cfg['name']}") + vm.boot(image_name=IMAGE) + return vm + + + +boot_chain(CONFIGS) diff --git a/test/integration/memory/available/vm-128m.json b/test/integration/memory/available/vm-128m.json new file mode 100644 index 000000000..a45e2c02c --- /dev/null +++ b/test/integration/memory/available/vm-128m.json @@ -0,0 +1,3 @@ +{ + "mem": 128 +} diff --git a/test/integration/memory/available/vm-16g.json b/test/integration/memory/available/vm-16g.json new file mode 100644 index 000000000..7b976178d --- /dev/null +++ b/test/integration/memory/available/vm-16g.json @@ -0,0 +1,3 @@ +{ + "mem": 16384 +} diff --git a/test/integration/memory/available/vm-2g.json b/test/integration/memory/available/vm-2g.json new file mode 100644 index 000000000..e708c43e4 --- /dev/null +++ b/test/integration/memory/available/vm-2g.json @@ -0,0 +1,3 @@ +{ + "mem": 2048 +} diff --git a/test/integration/memory/available/vm-4g.json b/test/integration/memory/available/vm-4g.json new file mode 100644 index 000000000..56f9714a1 --- /dev/null +++ b/test/integration/memory/available/vm-4g.json @@ -0,0 +1,3 @@ +{ + "mem": 4096 +} diff --git a/test/integration/memory/available/vm-6g.json b/test/integration/memory/available/vm-6g.json new file mode 100644 index 000000000..9c4dd2988 --- /dev/null +++ b/test/integration/memory/available/vm-6g.json @@ -0,0 +1,3 @@ +{ + "mem": 6144 +} diff --git a/test/integration/memory/available/vm-default.json b/test/integration/memory/available/vm-default.json new file mode 100644 index 000000000..2c63c0851 --- /dev/null +++ b/test/integration/memory/available/vm-default.json @@ -0,0 +1,2 @@ +{ +}