From 3625bbd4162a8c70aa855a6877b008c62ecf4881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 30 Mar 2026 17:46:48 +0200 Subject: [PATCH 1/9] Add unpresent and fault handler callback --- lib/tinykvm/amd64/paging.cpp | 22 +++++++++ lib/tinykvm/common.hpp | 5 +++ lib/tinykvm/machine.hpp | 2 + lib/tinykvm/machine_utils.cpp | 21 +++++---- lib/tinykvm/memory.cpp | 12 +++-- lib/tinykvm/memory.hpp | 4 ++ lib/tinykvm/vcpu.cpp | 20 +++++++++ lib/tinykvm/vcpu_run.cpp | 84 ++++++++++++++++++++--------------- 8 files changed, 122 insertions(+), 48 deletions(-) diff --git a/lib/tinykvm/amd64/paging.cpp b/lib/tinykvm/amd64/paging.cpp index d5d99442..dd419799 100644 --- a/lib/tinykvm/amd64/paging.cpp +++ b/lib/tinykvm/amd64/paging.cpp @@ -17,6 +17,7 @@ #define CLPRINT(...) /* ... */ #endif #define PDE64_CLONEABLE (1ul << 11) +#define PDE64_PRESENTABLE (1ul << 10) namespace tinykvm { @@ -734,6 +735,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl assert(!is_copy_on_write(pml4[i]) && (pml4[i] & PDE64_PRESENT)); } const uint64_t j = index_from_pdpt_entry(addr); + if ((pdpt[j] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) { + pdpt[j] |= PDE64_PRESENT; + pdpt[j] &= ~PDE64_PRESENTABLE; + const uint64_t paddr = pdpt_base | (j << 30); + if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr); + throw RetryException(); + } if (pdpt[j] & PDE64_PRESENT) { const auto [pd_base, pd_mem, pd_size] = pd_from_index(j, pdpt_base, pdpt); auto* pd = memory.page_at(pd_mem); @@ -748,6 +756,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl } } const uint64_t k = index_from_pd_entry(addr); + if ((pd[k] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) { + pd[k] |= PDE64_PRESENT; + pd[k] &= ~PDE64_PRESENTABLE; + const uint64_t paddr = pd_base | (k << 21); + if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr); + throw RetryException(); + } if (pd[k] & (PDE64_PRESENT | PDE64_CLONEABLE)) { const auto [pt_base, pt_mem, pt_size] = pt_from_index(k, pd_base, pd); uint64_t* pt; @@ -855,6 +870,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl } const uint64_t e = index_from_pt_entry(addr); + if ((pt[e] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) { + pt[e] |= PDE64_PRESENT; + pt[e] &= ~PDE64_PRESENTABLE; + const uint64_t paddr = pt_base | (e << 12); + if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr); + throw RetryException(); + } if (pt[e] & (PDE64_PRESENT | PDE64_CLONEABLE)) { // 4KB page const auto [pte_base, pte_mem, pte_size] = pte_from_index(e, pt_base, pt); uint64_t* data; diff --git a/lib/tinykvm/common.hpp b/lib/tinykvm/common.hpp index 59c2e977..6b23cd80 100644 --- a/lib/tinykvm/common.hpp +++ b/lib/tinykvm/common.hpp @@ -166,6 +166,11 @@ namespace tinykvm bool m_is_oom = false; /* True if the exception was caused by OOM */ }; + class RetryException: public MachineException { + public: + RetryException() : MachineException("Retry", 0) {} + }; + template constexpr std::false_type always_false {}; template diff --git a/lib/tinykvm/machine.hpp b/lib/tinykvm/machine.hpp index b8d6b9d9..c64e33ee 100644 --- a/lib/tinykvm/machine.hpp +++ b/lib/tinykvm/machine.hpp @@ -266,6 +266,8 @@ struct Machine be used after preparation. */ void prepare_copy_on_write(size_t max_work_mem = 0, uint64_t shared_memory_boundary = UINT64_MAX, bool split_accessed_hugepages = false); + void make_unpresented_with_callback(vMemory::page_presentable_callback_t on_presentable); + void restore_unpresented_pages(); void set_main_memory_writable(bool v) { memory.main_memory_writes = v; } bool is_forked() const noexcept { return m_forked; } bool uses_cow_memory() const noexcept { return m_forked || m_prepped; } diff --git a/lib/tinykvm/machine_utils.cpp b/lib/tinykvm/machine_utils.cpp index 7f0d5aa1..1132e6e3 100644 --- a/lib/tinykvm/machine_utils.cpp +++ b/lib/tinykvm/machine_utils.cpp @@ -56,16 +56,21 @@ void Machine::copy_to_guest(address_t addr, const void* vsrc, size_t len, bool z WritablePageOptions opts; opts.allow_dirty = full_page; opts.zeroes = zeroes; - // Get a writable page, possibly allocating a new one - WritablePage page = writable_page_at(memory, addr & ~PageMask(), memory.expectedUsermodeFlags(), opts); - // Page is always dirty - page.set_dirty(); - // Copy data to the page - char* page_data = page.page; - std::memcpy(&page_data[offset], src, size); + try { + // Get a writable page, possibly allocating a new one + WritablePage page = writable_page_at(memory, addr & ~PageMask(), memory.expectedUsermodeFlags(), opts); + // Page is always dirty + page.set_dirty(); + // Copy data to the page + char* page_data = page.page; + std::memcpy(&page_data[offset], src, size); #if defined(TINYKVM_ARCH_ARM64) - __builtin___clear_cache(&page_data[offset], &page_data[offset + size]); + __builtin___clear_cache(&page_data[offset], &page_data[offset + size]); #endif + } catch (const RetryException& e) { + // Retry the operation + continue; + } addr += size; src += size; diff --git a/lib/tinykvm/memory.cpp b/lib/tinykvm/memory.cpp index 9a111ca3..612514e6 100644 --- a/lib/tinykvm/memory.cpp +++ b/lib/tinykvm/memory.cpp @@ -532,11 +532,15 @@ char* vMemory::get_writable_page(uint64_t addr, uint64_t flags, bool zeroes, boo WritablePageOptions zero_opts; zero_opts.zeroes = zeroes; - auto writable_page = writable_page_at(*this, addr, flags, zero_opts); - if (dirty) { - writable_page.set_dirty(); + try { + auto writable_page = writable_page_at(*this, addr, flags, zero_opts); + if (dirty) { + writable_page.set_dirty(); + } + return writable_page.page; + } catch (const RetryException& e) { + return get_writable_page(addr, flags, zeroes, dirty); } - return writable_page.page; } char* vMemory::get_kernelpage_at(uint64_t addr) const diff --git a/lib/tinykvm/memory.hpp b/lib/tinykvm/memory.hpp index c4ce3e9b..d8f39bf3 100644 --- a/lib/tinykvm/memory.hpp +++ b/lib/tinykvm/memory.hpp @@ -3,6 +3,7 @@ #include "memory_bank.hpp" #include "virtual_mem.hpp" #include +#include #include #include @@ -45,6 +46,9 @@ struct vMemory { bool split_hugepages = true; /* Executable heap */ bool executable_heap = false; + /* Callback for PRESENTABLE pages during VM snapshot profiling */ + using page_presentable_callback_t = std::function; + page_presentable_callback_t on_page_presentable; /* Enable file-backed memory mappings for large files */ bool mmap_backed_files = true; /* Dynamic page memory */ diff --git a/lib/tinykvm/vcpu.cpp b/lib/tinykvm/vcpu.cpp index 52dca9ef..efd43580 100644 --- a/lib/tinykvm/vcpu.cpp +++ b/lib/tinykvm/vcpu.cpp @@ -533,6 +533,26 @@ void Machine::setup_cow_mode(const Machine* other) }); } } +void Machine::make_unpresented_with_callback(vMemory::page_presentable_callback_t on_presentable) +{ + #define PDE64_PRESENTABLE address_t(1ul << 10) + this->memory.on_page_presentable = std::move(on_presentable); + foreach_page(this->memory, [this] (auto addr, auto& entry, auto) { + if (addr >= this->m_kernel_end && (entry & PDE64_PRESENT) != 0) { + entry &= ~PDE64_PRESENT; + entry |= PDE64_PRESENTABLE; + } + }, false); +} +void Machine::restore_unpresented_pages() +{ + foreach_page(this->memory, [] (auto addr, auto& entry, auto) { + if (entry & PDE64_PRESENTABLE) { + entry |= PDE64_PRESENT; + entry &= ~PDE64_PRESENTABLE; + } + }, false); +} void Machine::print_pagetables() const { tinykvm::print_pagetables(this->memory); diff --git a/lib/tinykvm/vcpu_run.cpp b/lib/tinykvm/vcpu_run.cpp index 5c7db5ae..3e22fd22 100644 --- a/lib/tinykvm/vcpu_run.cpp +++ b/lib/tinykvm/vcpu_run.cpp @@ -322,7 +322,14 @@ long vCPU::run_once() // Since it's foreign memory, we try to handle it in the remote VM WritablePageOptions zero_opts; zero_opts.zeroes = false; - (void)writable_page_at(machine().remote().memory, addr, PDE64_USER | PDE64_RW, zero_opts); + try { + (void)writable_page_at(machine().remote().memory, addr, PDE64_USER | PDE64_RW, zero_opts); + } catch (const RetryException&) { + // The page was presentable, but not present. We need to retry the instruction now that the page is present. + regs.rax = 0; /* Indicate that it was local */ + this->set_registers(regs); + return KVM_EXIT_IO; + } // Remember that this address caused a fault, so that we don't loop infinitely if (this->last_fault_address == addr) { // This address already caused a fault @@ -362,49 +369,54 @@ long vCPU::run_once() this->set_registers(regs); return KVM_EXIT_IO; } else { - regs.rax = 0; /* Indicate that it was local */ + regs.rax = 0; /* Indicate that it was local */ } this->set_registers(regs); WritablePageOptions zero_opts; zero_opts.zeroes = false; - auto result = writable_page_at(memory, addr, PDE64_USER | PDE64_RW, zero_opts); - if (machine().has_remote() && machine().remote().is_foreign_address(addr) && machine().remote().is_remote_connected()) { - // If a new gigapage was created, we need to update the - // PML4[0] 512GB page table entry in the caller VM too - machine().remote().remote_update_gigapage_mappings(machine()); - } - if (this->last_fault_address == addr) { - // This address already caused a fault - this->handle_exception(intr); - Machine::machine_exception("Page fault repeat on same address", intr); - } - this->last_fault_address = addr; - if constexpr (false) { - char buffer[256]; - PRINTER(machine().m_printer, buffer, - "Page fault on 0x%lX handled, mapped to host page %p\n", addr, result.page); - PRINTER(machine().m_printer, buffer, - " Entry value 0x%lX\n", result.entry); - PRINTER(machine().m_printer, buffer, - " Our bank arena: begin=0x%lX\n", - memory.banks.arena_begin()); - static uint64_t last_reported = 0; - static int count = 0; - if (last_reported == addr) { - count++; + try { + WritablePage result = writable_page_at(memory, addr, PDE64_USER | PDE64_RW, zero_opts); + if (machine().has_remote() && machine().remote().is_foreign_address(addr) && machine().remote().is_remote_connected()) { + // If a new gigapage was created, we need to update the + // PML4[0] 512GB page table entry in the caller VM too + machine().remote().remote_update_gigapage_mappings(machine()); + } + if (this->last_fault_address == addr) { + // This address already caused a fault + this->handle_exception(intr); + Machine::machine_exception("Page fault repeat on same address", intr); + } + this->last_fault_address = addr; + if constexpr (false) { + char buffer[256]; PRINTER(machine().m_printer, buffer, - " Page fault repeats %d times, address=0x%lX\n", - count, last_reported); - if (count > 2) { - print_pagetables(memory); - this->handle_exception(intr); - Machine::machine_exception("Too many page faults", intr); + "Page fault on 0x%lX handled, mapped to host page %p\n", addr, result.page); + PRINTER(machine().m_printer, buffer, + " Entry value 0x%lX\n", result.entry); + PRINTER(machine().m_printer, buffer, + " Our bank arena: begin=0x%lX\n", + memory.banks.arena_begin()); + static uint64_t last_reported = 0; + static int count = 0; + if (last_reported == addr) { + count++; + PRINTER(machine().m_printer, buffer, + " Page fault repeats %d times, address=0x%lX\n", + count, last_reported); + if (count > 2) { + print_pagetables(memory); + this->handle_exception(intr); + Machine::machine_exception("Too many page faults", intr); + } + } else { + last_reported = addr; + count = 0; } - } else { - last_reported = addr; - count = 0; } + } catch (const RetryException&) { + // The page was presentable, but not present. + return KVM_EXIT_IO; } return KVM_EXIT_IO; } From 9435518489a386fe0f8db9e53bf7cca29f27c714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 30 Mar 2026 18:31:10 +0200 Subject: [PATCH 2/9] Iterate presentable entries when restoring unpresented pages --- lib/tinykvm/amd64/paging.cpp | 15 ++++++++------- lib/tinykvm/arm64/paging.cpp | 10 +++++++--- lib/tinykvm/paging.hpp | 4 ++-- lib/tinykvm/vcpu.cpp | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/tinykvm/amd64/paging.cpp b/lib/tinykvm/amd64/paging.cpp index dd419799..06db0ec7 100644 --- a/lib/tinykvm/amd64/paging.cpp +++ b/lib/tinykvm/amd64/paging.cpp @@ -505,12 +505,13 @@ void print_pagetables(const vMemory& memory) } } -void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses) +void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent) { + const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : (PDE64_PRESENT); auto* pml4 = memory.page_at(memory.page_tables); for (size_t i = 0; i < 512; i++) { - if (pml4[i] & PDE64_PRESENT) { + if (pml4[i] & present_mask) { const auto [pdpt_base, pdpt_mem, pdpt_size] = pdpt_from_index(i, pml4); callback(pdpt_base, pml4[i], pdpt_size); @@ -521,7 +522,7 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres auto* pdpt = memory.page_at(pdpt_mem); for (uint64_t j = 0; j < 512; j++) { - if (pdpt[j] & PDE64_PRESENT) { + if (pdpt[j] & present_mask) { const auto [pd_base, pd_mem, pd_size] = pd_from_index(j, pdpt_base, pdpt); callback(pd_base, pdpt[j], pd_size); @@ -537,7 +538,7 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres auto* pd = memory.page_at(pd_mem); for (uint64_t k = 0; k < 512; k++) { - if (pd[k] & PDE64_PRESENT) { + if (pd[k] & present_mask) { const auto [pt_base, pt_mem, pt_size] = pt_from_index(k, pd_base, pd); const bool is_2mb_page = (pd[k] & PDE64_PS) != 0; callback(pt_base, pd[k], pt_size); @@ -545,7 +546,7 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres auto* pt = memory.page_at(pt_mem); for (uint64_t e = 0; e < 512; e++) { const auto [pte_base, pte_mem, pte_size] = pte_from_index(e, pt_base, pt); - if (pt[e] & PDE64_PRESENT) { // 4KB page + if (pt[e] & present_mask) { // 4KB page callback(pte_base, pt[e], pte_size); } } // e @@ -557,9 +558,9 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres } } // i } // foreach_page -void foreach_page(const vMemory& mem, foreach_page_t callback, bool skip_oob_addresses) +void foreach_page(const vMemory& mem, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent) { - foreach_page(const_cast(mem), std::move(callback), skip_oob_addresses); + foreach_page(const_cast(mem), std::move(callback), skip_oob_addresses, include_unpresent); } void foreach_page_makecow(vMemory& mem, uint64_t kernel_end, diff --git a/lib/tinykvm/arm64/paging.cpp b/lib/tinykvm/arm64/paging.cpp index b4428bed..8af9d111 100644 --- a/lib/tinykvm/arm64/paging.cpp +++ b/lib/tinykvm/arm64/paging.cpp @@ -347,7 +347,11 @@ void print_pagetables(const vMemory& memory) }); } -void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses) +/* include_unpresent is an amd64-only concept: it also visits entries kept + around with the present bit cleared (PDE64_PRESENTABLE) by the unpresent + fault handler. ARM64 has no such lazily-unpresented entries, so an invalid + descriptor here is genuinely absent and the flag has nothing to add. */ +void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool) { auto* l1 = memory.page_at(memory.page_tables); for (uint64_t i = 0; i < 512; i++) { @@ -381,9 +385,9 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres } } -void foreach_page(const vMemory& memory, foreach_page_t callback, bool skip_oob_addresses) +void foreach_page(const vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent) { - foreach_page(const_cast(memory), std::move(callback), skip_oob_addresses); + foreach_page(const_cast(memory), std::move(callback), skip_oob_addresses, include_unpresent); } void foreach_page_makecow(vMemory& memory, uint64_t kernel_end, diff --git a/lib/tinykvm/paging.hpp b/lib/tinykvm/paging.hpp index b01d07a2..9b9fead2 100644 --- a/lib/tinykvm/paging.hpp +++ b/lib/tinykvm/paging.hpp @@ -12,8 +12,8 @@ extern uint64_t setup_amd64_paging(vMemory&, extern void print_pagetables(const vMemory&); using foreach_page_t = std::function; -extern void foreach_page(vMemory&, foreach_page_t callback, bool skip_oob_addresses = true); -extern void foreach_page(const vMemory&, foreach_page_t callback, bool skip_oob_addresses = true); +extern void foreach_page(vMemory&, foreach_page_t callback, bool skip_oob_addresses = true, bool include_unpresent = false); +extern void foreach_page(const vMemory&, foreach_page_t callback, bool skip_oob_addresses = true, bool include_unpresent = false); extern void foreach_page_makecow(vMemory&, uint64_t kernel_end, uint64_t shared_memory_boundary, bool split_accessed_hugepages = false); extern std::vector> get_accessed_pages(const vMemory& memory); diff --git a/lib/tinykvm/vcpu.cpp b/lib/tinykvm/vcpu.cpp index efd43580..6212f076 100644 --- a/lib/tinykvm/vcpu.cpp +++ b/lib/tinykvm/vcpu.cpp @@ -551,7 +551,7 @@ void Machine::restore_unpresented_pages() entry |= PDE64_PRESENT; entry &= ~PDE64_PRESENTABLE; } - }, false); + }, false, true); } void Machine::print_pagetables() const { From 031adc5e58716ee5cf999e74f0b5b3f7e27e7a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Sun, 12 Apr 2026 10:14:42 +0200 Subject: [PATCH 3/9] Add snapshot support for reordering memory to be sequential during loading --- lib/tinykvm/amd64/paging.cpp | 133 ++++++++++++++++++++++++++++ lib/tinykvm/machine.hpp | 3 + lib/tinykvm/machine_state.cpp | 160 ++++++++++++++++++++++++++++++++++ lib/tinykvm/paging.hpp | 13 +++ 4 files changed, 309 insertions(+) diff --git a/lib/tinykvm/amd64/paging.cpp b/lib/tinykvm/amd64/paging.cpp index 06db0ec7..99dd9e10 100644 --- a/lib/tinykvm/amd64/paging.cpp +++ b/lib/tinykvm/amd64/paging.cpp @@ -1098,4 +1098,137 @@ size_t paging_merge_leaf_pages_into_hugepages(vMemory& memory, bool merge_if_dir return merged_pages; } // paging_merge_leaf_pages_into_hugepages() +std::vector collect_all_pages(const vMemory& memory, bool include_unpresent) +{ + std::vector pages; + const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : PDE64_PRESENT; + const uint64_t arena_base = MemoryBanks::ARENA_BASE_ADDRESS; + const uint64_t mem_end = memory.physbase + memory.size; + + // A page is "relevant" if it's in main memory OR in a memory bank. + // Bank pages (branch nodes from CoW, kernel data like IST) need to be + // flattened into main memory during snapshot reordering. + auto is_relevant = [&](uint64_t paddr) { + if (paddr >= arena_base) + return true; // bank page + return paddr >= memory.physbase && paddr < mem_end; + }; + + // Include the PML4 page itself (may be in a bank after setup_cow_mode) + pages.push_back({memory.page_tables, PAGE_SIZE, true}); + + auto* pml4 = memory.page_at(memory.page_tables); + for (size_t i = 0; i < 512; i++) { + if (!(pml4[i] & present_mask)) + continue; + const uint64_t pdpt_paddr = pml4[i] & PDE64_ADDR_MASK; + if (is_relevant(pdpt_paddr)) + pages.push_back({pdpt_paddr, PAGE_SIZE, true}); + + auto* pdpt = memory.page_at(pdpt_paddr); + for (size_t j = 0; j < 512; j++) { + if (!(pdpt[j] & present_mask)) + continue; + // 1GB leaf page + if (pdpt[j] & PDE64_PS) { + const uint64_t paddr = pdpt[j] & PDE64_ADDR_MASK; + if (is_relevant(paddr)) + pages.push_back({paddr, 1ULL << 30, false}); + continue; + } + const uint64_t pd_paddr = pdpt[j] & PDE64_ADDR_MASK; + if (is_relevant(pd_paddr)) + pages.push_back({pd_paddr, PAGE_SIZE, true}); + + auto* pd = memory.page_at(pd_paddr); + for (size_t k = 0; k < 512; k++) { + if (!(pd[k] & present_mask)) + continue; + // 2MB leaf page + if (pd[k] & PDE64_PS) { + const uint64_t paddr = pd[k] & PDE64_ADDR_MASK; + if (is_relevant(paddr)) + pages.push_back({paddr, 1ULL << 21, false}); + continue; + } + const uint64_t pt_paddr = pd[k] & PDE64_ADDR_MASK; + if (is_relevant(pt_paddr)) + pages.push_back({pt_paddr, PAGE_SIZE, true}); + + auto* pt = memory.page_at(pt_paddr); + for (size_t e = 0; e < 512; e++) { + if (!(pt[e] & present_mask)) + continue; + const uint64_t paddr = pt[e] & PDE64_ADDR_MASK; + if (is_relevant(paddr)) + pages.push_back({paddr, PAGE_SIZE, false}); + } + } + } + } + return pages; +} + +void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root, + const std::unordered_map& translation, bool include_unpresent) +{ + const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : PDE64_PRESENT; + + auto translate = [&](uint64_t& entry, uint64_t addr_mask) { + const uint64_t old_paddr = entry & addr_mask; + auto it = translation.find(old_paddr); + if (it != translation.end()) { + entry = (entry & ~addr_mask) | it->second; + } + }; + auto page_at = [&](uint64_t paddr) -> uint64_t* { + return (uint64_t*)(base_ptr + (paddr - physbase)); + }; + + auto* pml4 = page_at(new_root); + for (size_t i = 0; i < 512; i++) { + if (!(pml4[i] & present_mask)) + continue; + // Translate PML4 entry (points to PDPT page) + translate(pml4[i], ~(uint64_t)0xFFF); + const uint64_t pdpt_paddr = pml4[i] & ~(uint64_t)0xFFF; + + auto* pdpt = page_at(pdpt_paddr); + for (size_t j = 0; j < 512; j++) { + if (!(pdpt[j] & present_mask)) + continue; + if (pdpt[j] & PDE64_PS) { + // 1GB leaf — translate data address + translate(pdpt[j], PDE64_ADDR_MASK); + continue; + } + // Translate PDPT entry (points to PD page) + translate(pdpt[j], PDE64_ADDR_MASK); + const uint64_t pd_paddr = pdpt[j] & PDE64_ADDR_MASK; + + auto* pd = page_at(pd_paddr); + for (size_t k = 0; k < 512; k++) { + if (!(pd[k] & present_mask)) + continue; + if (pd[k] & PDE64_PS) { + // 2MB leaf — translate data address + translate(pd[k], PDE64_ADDR_MASK); + continue; + } + // Translate PD entry (points to PT page) + translate(pd[k], PDE64_ADDR_MASK); + const uint64_t pt_paddr = pd[k] & PDE64_ADDR_MASK; + + auto* pt = page_at(pt_paddr); + for (size_t e = 0; e < 512; e++) { + if (!(pt[e] & present_mask)) + continue; + // 4KB leaf — translate data address + translate(pt[e], PDE64_ADDR_MASK); + } + } + } + } +} + } // tinykvm diff --git a/lib/tinykvm/machine.hpp b/lib/tinykvm/machine.hpp index c64e33ee..ebd67033 100644 --- a/lib/tinykvm/machine.hpp +++ b/lib/tinykvm/machine.hpp @@ -302,6 +302,9 @@ struct Machine write-fault VM exits. CoW state is rebuilt by every fork/reset_to, so re-apply after each. Returns the number of pages made writable. */ size_t prefetch_pages(const std::vector>& pages); + /* Reorder snapshot memory so pages are sequential in fault order. + Rewires page tables to reflect the new physical layout. */ + void reorder_snapshot_memory(const std::vector& fault_order); /* Remote VM through address space merging */ void remote_connect(Machine& other, bool connect_now = false); diff --git a/lib/tinykvm/machine_state.cpp b/lib/tinykvm/machine_state.cpp index 92bfd9b0..6e1b95ec 100644 --- a/lib/tinykvm/machine_state.cpp +++ b/lib/tinykvm/machine_state.cpp @@ -1,5 +1,6 @@ #include "machine.hpp" +#include #include #include #include @@ -7,9 +8,12 @@ #include #include #include +#include +#include #include #ifdef TINYKVM_ARCH_AMD64 #include "amd64/amd64.hpp" +#include "amd64/memory_layout.hpp" #include "amd64/paging.hpp" #endif #include "linux/fds.hpp" @@ -211,6 +215,162 @@ bool Machine::load_snapshot_state() } return true; } +void Machine::reorder_snapshot_memory(const std::vector& fault_order) +{ + const uint64_t physbase = this->memory.physbase; + const uint64_t mem_size = this->memory.size; + const uint64_t kernel_end = this->kernel_end_address(); + const uint64_t FIXED_REGION_END = kernel_end; + const uint64_t arena_base = MemoryBanks::ARENA_BASE_ADDRESS; + char* const ptr = this->memory.ptr; + + // Step 1: Collect all pages from the page tables + auto all_pages = collect_all_pages(this->memory, true); + + // Deduplicate pages (a physical address may appear as both branch and leaf + // in edge cases, or the same page table page may be referenced multiple times) + std::unordered_set seen; + std::vector unique_pages; + unique_pages.reserve(all_pages.size()); + for (auto& pi : all_pages) { + if (seen.insert(pi.paddr).second) { + unique_pages.push_back(pi); + } + } + + // Step 2: Sort pages into categories + // Build fault order lookup: paddr -> order index + std::unordered_map fault_index; + fault_index.reserve(fault_order.size()); + for (size_t i = 0; i < fault_order.size(); i++) { + fault_index.insert_or_assign(fault_order[i], i); + } + + // Separate into categories. + // Pages below FIXED_REGION_END are fixed kernel structures (GDT, TSS, IDT, etc.) + // and must remain at their exact physical addresses. + // Bank pages (>= ARENA_BASE_ADDRESS) are CoW copies from setup_cow_mode that + // need to be flattened into main memory — they go into branch_pages. + std::vector fixed_pages, kernel_pages, faulted_pages, unfaulted_pages, branch_pages; + for (auto& pi : unique_pages) { + if (pi.paddr >= arena_base) { + // Bank page — must be flattened into main memory + branch_pages.push_back(pi); + } else if (pi.paddr < FIXED_REGION_END) { + fixed_pages.push_back(pi); + } else if (pi.is_branch) { + branch_pages.push_back(pi); + } else if (pi.paddr < kernel_end) { + kernel_pages.push_back(pi); + } else { + auto it = fault_index.find(pi.paddr); + if (it != fault_index.end()) { + faulted_pages.push_back(pi); + } else { + unfaulted_pages.push_back(pi); + } + } + } + + // Sort kernel pages by address + std::sort(kernel_pages.begin(), kernel_pages.end(), + [](const PageInfo& a, const PageInfo& b) { return a.paddr < b.paddr; }); + // Sort faulted pages by fault order + std::sort(faulted_pages.begin(), faulted_pages.end(), + [&](const PageInfo& a, const PageInfo& b) { + return fault_index[a.paddr] < fault_index[b.paddr]; + }); + // Sort unfaulted pages by address + std::sort(unfaulted_pages.begin(), unfaulted_pages.end(), + [](const PageInfo& a, const PageInfo& b) { return a.paddr < b.paddr; }); + // Sort branch pages by address + std::sort(branch_pages.begin(), branch_pages.end(), + [](const PageInfo& a, const PageInfo& b) { return a.paddr < b.paddr; }); + + // Concatenate in order: kernel, faulted user, unfaulted user, branch (page tables) + // Fixed pages are excluded — they keep their original addresses. + std::vector ordered; + ordered.reserve(unique_pages.size()); + ordered.insert(ordered.end(), kernel_pages.begin(), kernel_pages.end()); + ordered.insert(ordered.end(), faulted_pages.begin(), faulted_pages.end()); + ordered.insert(ordered.end(), unfaulted_pages.begin(), unfaulted_pages.end()); + ordered.insert(ordered.end(), branch_pages.begin(), branch_pages.end()); + + // Step 3: Assign new sequential addresses + // Fixed pages keep their identity mapping + std::unordered_map translation; + translation.reserve(ordered.size() + fixed_pages.size()); + for (auto& pi : fixed_pages) { + translation[pi.paddr] = pi.paddr; // identity — no move + } + // Movable pages start after the fixed region. + // Assign addresses sequentially, but stop accepting pages if we'd + // exceed the allocation. Pages that don't fit are dropped — unfaulted + // pages at the tail are the ones most likely trimmed, since they + // weren't accessed during the probing request anyway. + const uint64_t mem_limit = physbase + mem_size; + uint64_t cursor = FIXED_REGION_END; + size_t pages_placed = 0; + for (auto& pi : ordered) { + uint64_t aligned = (cursor + (pi.size - 1)) & ~(pi.size - 1); + if (aligned + pi.size > mem_limit) + break; + translation[pi.paddr] = aligned; + cursor = aligned + pi.size; + pages_placed++; + } + // Trim ordered to only the pages that fit + const size_t pages_dropped = ordered.size() - pages_placed; + ordered.resize(pages_placed); + if (pages_dropped > 0) { + printf("reorder_snapshot_memory: dropped %zu pages that didn't fit after alignment\n", + pages_dropped); + } + + // Step 4: Copy pages to temporary buffer in new order + char* tmp = (char*)mmap(NULL, mem_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (tmp == MAP_FAILED) { + fprintf(stderr, "reorder_snapshot_memory: failed to allocate temp buffer, skipping\n"); + return; + } + + // Copy fixed region as-is (pages below FIXED_REGION_END stay in place) + std::memcpy(tmp, ptr, FIXED_REGION_END - physbase); + // Copy movable pages to their new locations. + // Bank pages (>= ARENA_BASE_ADDRESS) must be read via memory.page_at() + // since they're not in the main memory mmap. + for (auto& pi : ordered) { + uint64_t new_paddr = translation[pi.paddr]; + const char* src; + if (pi.paddr >= arena_base) { + src = (const char*)this->memory.page_at(pi.paddr); + } else { + src = ptr + (pi.paddr - physbase); + } + std::memcpy(tmp + (new_paddr - physbase), src, pi.size); + } + + // Step 5: Rewire page tables in the temp buffer + uint64_t new_root = translation.at(this->memory.page_tables); + rewire_page_tables(tmp, physbase, new_root, translation, true); + + // Step 6: Copy back and update metadata + std::memcpy(ptr, tmp, mem_size); + munmap(tmp, mem_size); + this->memory.page_tables = new_root; + + // Update KVM CR3 to match the new page table root. + // setup_cow_mode set CR3 to a bank address which won't exist on load. + auto sregs = this->get_special_registers(); + sregs.cr3 = new_root; + this->set_special_registers(sregs); + + printf("Reordered snapshot memory: %zu pages (%zu fixed, %zu kernel, %zu faulted, %zu unfaulted, %zu branch)\n", + ordered.size() + fixed_pages.size(), fixed_pages.size(), kernel_pages.size(), + faulted_pages.size(), unfaulted_pages.size(), branch_pages.size()); +} + void Machine::save_snapshot_state_now(const std::vector>& populate_pages) const { if (this->is_forked()) { diff --git a/lib/tinykvm/paging.hpp b/lib/tinykvm/paging.hpp index 9b9fead2..6ae4e652 100644 --- a/lib/tinykvm/paging.hpp +++ b/lib/tinykvm/paging.hpp @@ -1,6 +1,7 @@ #pragma once #include "memory.hpp" #include +#include namespace tinykvm { @@ -41,6 +42,18 @@ extern uint64_t paging_address_mask(); code (e.g. Machine::memzero). */ extern uint64_t paging_dirty_bit(); +struct PageInfo { + uint64_t paddr; + uint64_t size; + bool is_branch; // true = page table node, false = leaf data page +}; +// Collect all pages (leaf + branch) from the page tables. +extern std::vector collect_all_pages(const vMemory& memory, bool include_unpresent = true); +// Rewire all physical addresses in page tables using a translation map. +// Operates on raw memory at base_ptr, using new_root as the PML4 physical address. +extern void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root, + const std::unordered_map& translation, bool include_unpresent = true); + static inline bool page_is_zeroed(const uint64_t* page) { for (size_t i = 0; i < 512; i += 8) { if ((page[i+0] | page[i+1] | page[i+2] | page[i+3]) != 0 || From 3bbd66176a9f9f729a4430cd9ac3e6e9bafd7f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Sun, 12 Apr 2026 11:16:20 +0200 Subject: [PATCH 4/9] Get populate pages from the pgtbl reordering --- lib/tinykvm/machine.hpp | 5 +++-- lib/tinykvm/machine_state.cpp | 37 ++++++++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/lib/tinykvm/machine.hpp b/lib/tinykvm/machine.hpp index ebd67033..f03adca8 100644 --- a/lib/tinykvm/machine.hpp +++ b/lib/tinykvm/machine.hpp @@ -303,8 +303,9 @@ struct Machine so re-apply after each. Returns the number of pages made writable. */ size_t prefetch_pages(const std::vector>& pages); /* Reorder snapshot memory so pages are sequential in fault order. - Rewires page tables to reflect the new physical layout. */ - void reorder_snapshot_memory(const std::vector& fault_order); + Rewires page tables to reflect the new physical layout. + Returns post-reorder populate pages (new paddr, size) for madvise on load. */ + std::vector> reorder_snapshot_memory(const std::vector& fault_order); /* Remote VM through address space merging */ void remote_connect(Machine& other, bool connect_now = false); diff --git a/lib/tinykvm/machine_state.cpp b/lib/tinykvm/machine_state.cpp index 6e1b95ec..114baebb 100644 --- a/lib/tinykvm/machine_state.cpp +++ b/lib/tinykvm/machine_state.cpp @@ -147,22 +147,36 @@ bool Machine::load_snapshot_state() void* current = state.current; // Load populate pages - madvise(this->memory.ptr, kernel_end_address(), MADV_WILLNEED | MADV_RANDOM); - static const uint64_t step = 1024*1024; + madvise(this->memory.ptr, kernel_end_address(), MADV_WILLNEED | MADV_SEQUENTIAL); + static constexpr uint64_t step = 2*1024*1024; + static constexpr uint64_t madvise_max_total = 40 * 1024 * 1024; + uint64_t madvised_total = 0; + int madvised_total_calls = 0; for (unsigned i = 0; i < state.num_access_ranges; i++) { ColdStartAccessedRange* range = state.next(current); if (range->start >= MemoryBanks::ARENA_BASE_ADDRESS || range->start < kernel_end_address()) continue; + if (madvised_total >= madvise_max_total) + continue; try { //printf("Populating pages from 0x%lX -> 0x%lX\n", range->start, range->end); for (uint64_t start = range->start; start < range->end; start += step) { - madvise(this->memory.ptr + start, std::min(range->end - start, step), MADV_WILLNEED | MADV_RANDOM); + madvise(this->memory.ptr + start, std::min(range->end - start, step), MADV_WILLNEED | MADV_SEQUENTIAL); + madvised_total += std::min(range->end - start, step); + madvised_total_calls++; + //printf("Madvised pages from 0x%lX -> 0x%lX (total madvised: %zu MiB)\n", + // start, std::min(start + step, range->end), madvised_total / (1024 * 1024)); + if (madvised_total >= madvise_max_total) { + break; + } } } catch (const std::exception& e) { fprintf(stderr, "Failed to access page at 0x%lX: %s\n", range->start, e.what()); continue; } } + //printf("Madvised a total of %zu MiB of pages in %d calls\n", + // madvised_total / (1024 * 1024), madvised_total_calls); // Load the thread states ColdStartThreads* threads = state.next(current); @@ -215,7 +229,7 @@ bool Machine::load_snapshot_state() } return true; } -void Machine::reorder_snapshot_memory(const std::vector& fault_order) +std::vector> Machine::reorder_snapshot_memory(const std::vector& fault_order) { const uint64_t physbase = this->memory.physbase; const uint64_t mem_size = this->memory.size; @@ -332,7 +346,7 @@ void Machine::reorder_snapshot_memory(const std::vector& fault_order) MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); if (tmp == MAP_FAILED) { fprintf(stderr, "reorder_snapshot_memory: failed to allocate temp buffer, skipping\n"); - return; + return {}; } // Copy fixed region as-is (pages below FIXED_REGION_END stay in place) @@ -369,6 +383,19 @@ void Machine::reorder_snapshot_memory(const std::vector& fault_order) printf("Reordered snapshot memory: %zu pages (%zu fixed, %zu kernel, %zu faulted, %zu unfaulted, %zu branch)\n", ordered.size() + fixed_pages.size(), fixed_pages.size(), kernel_pages.size(), faulted_pages.size(), unfaulted_pages.size(), branch_pages.size()); + + // Build post-reorder populate pages from the placed pages. + // Since kernel + faulted pages are packed sequentially, these should + // merge into very few contiguous ranges. + std::vector> populate_pages; + populate_pages.reserve(ordered.size()); + for (auto& pi : ordered) { + auto it = translation.find(pi.paddr); + if (it != translation.end()) { + populate_pages.push_back({it->second, pi.size}); + } + } + return populate_pages; } void Machine::save_snapshot_state_now(const std::vector>& populate_pages) const From 4010cb2e19a94423527f39ce8a243cce329ee54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Sun, 12 Apr 2026 21:27:32 +0200 Subject: [PATCH 5/9] Populate only faulted/accessed pages --- lib/tinykvm/machine_state.cpp | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/tinykvm/machine_state.cpp b/lib/tinykvm/machine_state.cpp index 114baebb..06370bcf 100644 --- a/lib/tinykvm/machine_state.cpp +++ b/lib/tinykvm/machine_state.cpp @@ -148,8 +148,8 @@ bool Machine::load_snapshot_state() void* current = state.current; // Load populate pages madvise(this->memory.ptr, kernel_end_address(), MADV_WILLNEED | MADV_SEQUENTIAL); - static constexpr uint64_t step = 2*1024*1024; - static constexpr uint64_t madvise_max_total = 40 * 1024 * 1024; + static constexpr uint64_t step = 1*1024*1024; + static constexpr uint64_t madvise_max_total = 32 * 1024 * 1024; uint64_t madvised_total = 0; int madvised_total_calls = 0; for (unsigned i = 0; i < state.num_access_ranges; i++) { @@ -384,17 +384,24 @@ std::vector> Machine::reorder_snapshot_memory(cons ordered.size() + fixed_pages.size(), fixed_pages.size(), kernel_pages.size(), faulted_pages.size(), unfaulted_pages.size(), branch_pages.size()); - // Build post-reorder populate pages from the placed pages. - // Since kernel + faulted pages are packed sequentially, these should - // merge into very few contiguous ranges. + // Build post-reorder populate pages from only the pages that were + // actually accessed during the probing request (+ kernel and branch + // pages needed for page table infrastructure). Unfaulted pages are + // still present in the snapshot but should not be prefetched — they + // can be demand-paged if a future request happens to need them. std::vector> populate_pages; - populate_pages.reserve(ordered.size()); - for (auto& pi : ordered) { - auto it = translation.find(pi.paddr); - if (it != translation.end()) { - populate_pages.push_back({it->second, pi.size}); + populate_pages.reserve(kernel_pages.size() + faulted_pages.size() + branch_pages.size()); + auto add_translated = [&](const std::vector& pages) { + for (auto& pi : pages) { + auto it = translation.find(pi.paddr); + if (it != translation.end()) { + populate_pages.push_back({it->second, pi.size}); + } } - } + }; + add_translated(kernel_pages); + add_translated(faulted_pages); + add_translated(branch_pages); return populate_pages; } From 0dfc50a2d726eafe4c123f13367762b69922c93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 13 Apr 2026 10:56:30 +0200 Subject: [PATCH 6/9] Use pagewalk for reordered VM snapshots --- lib/tinykvm/amd64/paging.cpp | 14 +++++++------- lib/tinykvm/machine_state.cpp | 13 ++++++++++--- lib/tinykvm/memory.cpp | 2 -- lib/tinykvm/memory.hpp | 5 +++++ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/tinykvm/amd64/paging.cpp b/lib/tinykvm/amd64/paging.cpp index 99dd9e10..7c0c7878 100644 --- a/lib/tinykvm/amd64/paging.cpp +++ b/lib/tinykvm/amd64/paging.cpp @@ -1181,19 +1181,19 @@ void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root, entry = (entry & ~addr_mask) | it->second; } }; - auto page_at = [&](uint64_t paddr) -> uint64_t* { + auto get_page_at = [&](uint64_t paddr) -> uint64_t* { return (uint64_t*)(base_ptr + (paddr - physbase)); }; - auto* pml4 = page_at(new_root); + auto* pml4 = get_page_at(new_root); for (size_t i = 0; i < 512; i++) { if (!(pml4[i] & present_mask)) continue; // Translate PML4 entry (points to PDPT page) - translate(pml4[i], ~(uint64_t)0xFFF); - const uint64_t pdpt_paddr = pml4[i] & ~(uint64_t)0xFFF; + translate(pml4[i], PDE64_ADDR_MASK); + const uint64_t pdpt_paddr = pml4[i] & PDE64_ADDR_MASK; - auto* pdpt = page_at(pdpt_paddr); + auto* pdpt = get_page_at(pdpt_paddr); for (size_t j = 0; j < 512; j++) { if (!(pdpt[j] & present_mask)) continue; @@ -1206,7 +1206,7 @@ void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root, translate(pdpt[j], PDE64_ADDR_MASK); const uint64_t pd_paddr = pdpt[j] & PDE64_ADDR_MASK; - auto* pd = page_at(pd_paddr); + auto* pd = get_page_at(pd_paddr); for (size_t k = 0; k < 512; k++) { if (!(pd[k] & present_mask)) continue; @@ -1219,7 +1219,7 @@ void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root, translate(pd[k], PDE64_ADDR_MASK); const uint64_t pt_paddr = pd[k] & PDE64_ADDR_MASK; - auto* pt = page_at(pt_paddr); + auto* pt = get_page_at(pt_paddr); for (size_t e = 0; e < 512; e++) { if (!(pt[e] & present_mask)) continue; diff --git a/lib/tinykvm/machine_state.cpp b/lib/tinykvm/machine_state.cpp index 06370bcf..cc967b53 100644 --- a/lib/tinykvm/machine_state.cpp +++ b/lib/tinykvm/machine_state.cpp @@ -87,6 +87,7 @@ struct SnapshotState { Machine::address_t m_page_tables; bool main_memory_writes; + bool memory_reordered; uint32_t num_access_ranges; char current[0]; @@ -143,6 +144,7 @@ bool Machine::load_snapshot_state() this->m_kernel_end = state.m_kernel_end; this->m_mmap_cache.current() = state.mmap_current; this->memory.main_memory_writes = state.main_memory_writes; + this->memory.memory_reordered = state.memory_reordered; this->memory.page_tables = state.m_page_tables; void* current = state.current; @@ -159,7 +161,8 @@ bool Machine::load_snapshot_state() if (madvised_total >= madvise_max_total) continue; try { - //printf("Populating pages from 0x%lX -> 0x%lX\n", range->start, range->end); + //const size_t num_pages = ((range->end - range->start + 0xFFF) & ~0xFFFLL) / 0x1000; + //printf("Populating pages from 0x%lX -> 0x%lX (%zu pages)\n", range->start, range->end, num_pages); for (uint64_t start = range->start; start < range->end; start += step) { madvise(this->memory.ptr + start, std::min(range->end - start, step), MADV_WILLNEED | MADV_SEQUENTIAL); madvised_total += std::min(range->end - start, step); @@ -175,8 +178,10 @@ bool Machine::load_snapshot_state() continue; } } - //printf("Madvised a total of %zu MiB of pages in %d calls\n", - // madvised_total / (1024 * 1024), madvised_total_calls); + if constexpr (false) { + printf("Madvised a total of %zu MiB of pages in %d calls\n", + madvised_total / (1024 * 1024), madvised_total_calls); + } // Load the thread states ColdStartThreads* threads = state.next(current); @@ -373,6 +378,7 @@ std::vector> Machine::reorder_snapshot_memory(cons std::memcpy(ptr, tmp, mem_size); munmap(tmp, mem_size); this->memory.page_tables = new_root; + this->memory.memory_reordered = true; // Update KVM CR3 to match the new page table root. // setup_cow_mode set CR3 to a bank address which won't exist on load. @@ -431,6 +437,7 @@ void Machine::save_snapshot_state_now(const std::vectorm_kernel_end; state.mmap_current = this->m_mmap_cache.current(); state.main_memory_writes = this->memory.main_memory_writes; + state.memory_reordered = this->memory.memory_reordered; state.m_page_tables = this->memory.page_tables; void* current = state.current; diff --git a/lib/tinykvm/memory.cpp b/lib/tinykvm/memory.cpp index 612514e6..02501398 100644 --- a/lib/tinykvm/memory.cpp +++ b/lib/tinykvm/memory.cpp @@ -202,8 +202,6 @@ bool vMemory::fork_reset(const Machine& main_vm, const MachineOptions& options) //fprintf(stderr, "Copying virtual page %016lx from physical %016lx with size %lu\n", // addr, bank_addr, page_size); - // This is a writable page, we will copy it using the "real" - // address from the master VM. auto* our_page = this->safely_at(bank_addr, page_size); // Find the page in the main VM *through its page tables*. // Only main memory is identity-mapped; the mmap physical diff --git a/lib/tinykvm/memory.hpp b/lib/tinykvm/memory.hpp index d8f39bf3..f67edb79 100644 --- a/lib/tinykvm/memory.hpp +++ b/lib/tinykvm/memory.hpp @@ -42,6 +42,11 @@ struct vMemory { /* Use memory banks only for page tables, write directly to main memory. Used with is_forkable_master(). */ bool main_memory_writes = false; + /* Set when reorder_snapshot_memory has moved pages, breaking + the identity mapping (vaddr == paddr), so any physical address + must be resolved through the page tables rather than assumed + equal to the virtual one. Carried across snapshot save/restore. */ + bool memory_reordered = false; /* Split into small pages (4K) when reaching a leaf hugepage. */ bool split_hugepages = true; /* Executable heap */ From 917174023ebdf84ade1937622706c7afa16236ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 22 Jun 2026 15:28:17 +0200 Subject: [PATCH 7/9] Add page-fault measuring snapshot benchmark --- CMakeLists.txt | 5 + guest/glibc/glibc.cpp | 56 +++++++- src/snapshot_bench.cpp | 301 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 356 insertions(+), 6 deletions(-) create mode 100644 src/snapshot_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b51220a..0ecae687 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,11 @@ if (TINYKVM_ARCH STREQUAL "AMD64") ) target_link_libraries(storagekvm tinykvm) + add_executable(snapshot_bench + src/snapshot_bench.cpp + ) + target_link_libraries(snapshot_bench tinykvm) + add_executable(pipekvm src/pipe.cpp ) diff --git a/guest/glibc/glibc.cpp b/guest/glibc/glibc.cpp index aa643b99..981c9c6f 100644 --- a/guest/glibc/glibc.cpp +++ b/guest/glibc/glibc.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -7,23 +8,66 @@ static void test_threads(); extern "C" int gettid(); static int threads_test_suite_ok = 0; + +/* ------------------------------------------------------------------------ + * Snapshot-ordering workload. + * + * g_buffer is a large buffer that lives in BSS, so it is part of the booted + * snapshot image. main() touches every page (making it resident in the + * snapshot file), and test() then streams over it in a FIXED, SCATTERED page + * order (g_order). Because the access order is decorrelated from the buffer's + * virtual/physical layout, the on-disk page ordering decides whether the cold + * page faults during test() become one sequential file read (fault-order + * snapshot) or thousands of random reads (no-order snapshot). + * ------------------------------------------------------------------------ */ +static constexpr size_t WS_PAGE = 4096; +static constexpr size_t WS_BYTES = 128UL * 1024 * 1024; /* 128 MiB working set */ +static constexpr size_t WS_PAGES = WS_BYTES / WS_PAGE; + +static uint8_t g_buffer[WS_BYTES]; +static uint32_t g_order[WS_PAGES]; + +/* Deterministic Fisher-Yates shuffle (fixed seed) so that the fault-order + capture run and every replay run touch the pages in the identical order. */ +static void build_workset() +{ + for (size_t i = 0; i < WS_PAGES; i++) + g_order[i] = (uint32_t)i; + + uint64_t s = 0x9E3779B97F4A7C15ULL; + for (size_t i = WS_PAGES - 1; i > 0; i--) { + s = s * 6364136223846793005ULL + 1442695040888963407ULL; + const size_t j = (size_t)((s >> 33) % (i + 1)); + const uint32_t t = g_order[i]; g_order[i] = g_order[j]; g_order[j] = t; + } + /* Make every page resident in the snapshot with a non-zero, page-unique + value so the file is not sparse for this region. */ + for (size_t i = 0; i < WS_PAGES; i++) + g_buffer[i * WS_PAGE] = (uint8_t)(1 + (i & 0xFF)); +} + int main() { - char* test = (char *)malloc(14); - strcpy(test, "Hello World!\n"); - printf("%.*s", 13, test); + char* hello = (char *)malloc(14); + strcpy(hello, "Hello World!\n"); + printf("%.*s", 13, hello); + build_workset(); test_threads(); // Prevent global destructors std::quick_exit(0); } +/* The replayed request: stream the working set in scattered page order. The + returned checksum keeps the reads from being optimised away. */ extern "C" __attribute__((used)) -void test() +uint64_t test() { - /* Verify that the threads test-suite passed */ - assert(threads_test_suite_ok == 1); + uint64_t sum = 0; + for (size_t i = 0; i < WS_PAGES; i++) + sum += g_buffer[(size_t)g_order[i] * WS_PAGE]; + return sum; } #include diff --git a/src/snapshot_bench.cpp b/src/snapshot_bench.cpp new file mode 100644 index 00000000..a2882904 --- /dev/null +++ b/src/snapshot_bench.cpp @@ -0,0 +1,301 @@ +/** + * Snapshot-load ordering benchmark. + * + * Measures the cost of cold-loading a VM snapshot under three different + * physical page-ordering strategies: + * + * 1. NO ORDER - snapshot saved with no prefetch hints. Pages are + * demand-faulted from the file in whatever order the + * workload touches them (scattered random file IO). + * 2. ACCESS ORDER - prefetch the accessed working set discovered via the + * page-table accessed bits (get_accessed_pages()). The + * pages stay at their original physical offsets, so the + * prefetched ranges are scattered across the file. + * 3. FAULT ORDER - reorder_snapshot_memory() physically relocates the + * faulted pages so they are contiguous and in first-touch + * order. Prefetch then reads one sequential file region. + * + * Each strategy produces its own snapshot file. The load phase drops the OS + * page cache before every trial (needs root) so each measurement reflects a + * genuine cold start from disk, then loads the snapshot and brings the booted + * working set resident, timing the whole thing in wall-clock. + */ +#include +#include +#include +#include +#include +#include +#include +#include "load_file.hpp" + +#define GUEST_MEMORY 1024UL * 1024 * 1024 /* 1024MB main memory */ +#define GUEST_WORK_MEM 256UL * 1024 * 1024 /* 256MB working memory */ +static const std::string ld_linux_so = "/lib64/ld-linux-x86-64.so.2"; +static const char* DEFAULT_GUEST = "../guest/glibc/glibc.static"; +static constexpr int TRIALS = 8; + +/* Wall-clock, not CPU time: cold page faults block on IO and that time must + be counted. timing.hpp uses CLOCK_THREAD_CPUTIME_ID which would hide it. */ +static inline timespec time_now() +{ + timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t; +} +static inline double seconds_between(timespec a, timespec b) +{ + return (b.tv_sec - a.tv_sec) + (b.tv_nsec - a.tv_nsec) / 1e9; +} + +struct Stats { + double avg = 0, median = 0, p90 = 0, min = 0, max = 0; +}; +static Stats summarize(std::vector v) +{ + Stats s; + if (v.empty()) return s; + std::sort(v.begin(), v.end()); + double total = 0; + for (double x : v) total += x; + s.avg = total / v.size(); + s.median = v[v.size() / 2]; + s.p90 = v[static_cast(v.size() * 0.9)]; + s.min = v.front(); + s.max = v.back(); + return s; +} + +/* Drop the OS page cache so the next snapshot load is a genuine cold start. + Requires root (or CAP_SYS_ADMIN). Returns false if it could not be done. */ +static bool drop_caches() +{ + sync(); + int fd = open("/proc/sys/vm/drop_caches", O_WRONLY); + if (fd < 0) + return false; + const char* three = "3\n"; + ssize_t w = write(fd, three, 2); + close(fd); + return w == 2; +} + +enum Strategy { NO_ORDER = 0, ACCESS_ORDER = 1, FAULT_ORDER = 2 }; +static const char* strategy_name(Strategy s) +{ + switch (s) { + case NO_ORDER: return "no-order "; + case ACCESS_ORDER: return "access-order "; + case FAULT_ORDER: return "fault-order "; + } + return "?"; +} + +static std::vector g_binary; // ld-linux.so or the static guest +static std::vector g_args; // guest argv +static bool g_is_dynamic = false; +static std::string g_entry = "test"; // workload entrypoint to replay +static uint64_t g_entry_addr = 0x0; // resolved VA (snapshots skip symbol load) + +/* Working set captured (by virtual address) from the freshly-booted master. + The same set is touched on every loaded snapshot so all three strategies + are measured bringing in the identical logical pages, differing only by + physical layout and prefetch hint. */ +static std::vector> g_working_set; + +static tinykvm::MachineOptions base_options(const std::string& snapshot_file, + tinykvm::MachineOptions::SnapshotMode mode) +{ + tinykvm::MachineOptions options { + .max_mem = GUEST_MEMORY, + .max_cow_mem = GUEST_WORK_MEM, + .dylink_address_hint = 0x400000, + .verbose_loader = false, + // Force 4KB pages so the workload's per-page access order actually + // determines the physical/file layout (2MB identity pages would make + // a per-4KB shuffle meaningless — 512 sub-pages share one mapping). + .split_all_hugepages_during_loading = true, + .executable_heap = g_is_dynamic, + .mmap_backed_files = false, // incompatible with snapshot files + }; + options.snapshot_file = snapshot_file; + options.snapshot_mode = mode; + return options; +} + +/* Boot a master VM, apply the ordering strategy, and persist a snapshot file. + Captures g_working_set on the first (NO_ORDER) build. */ +static void build_snapshot(Strategy strat, const std::string& path) +{ + auto options = base_options(path, tinykvm::MachineOptions::SnapshotMode::Create); + tinykvm::Machine master {g_binary, options}; + master.fds().set_open_readable_callback([] (std::string&) -> bool { return true; }); + master.setup_linux(g_args, {"LC_TYPE=C", "LC_ALL=C", "USER=root"}); + + /* Boot to main() */ + master.run(8.0f); + + const uint64_t entry = master.address_of(g_entry); + g_entry_addr = entry; // stable VA, reused on restored snapshots (no symbols) + + std::vector> populate; + size_t fault_pages = 0; + + if (strat == ACCESS_ORDER) { + populate = master.get_accessed_pages(); + } + else if (strat == FAULT_ORDER) { + /* Record the order in which the workload first touches each physical + page, by clearing the present bit on every user page and letting the + replayed request fault them back in one by one. */ + std::vector fault_order; + master.make_unpresented_with_callback( + [&fault_order] (uint64_t paddr, uint64_t /*vaddr*/) { + fault_order.push_back(paddr); + }); + if (entry != 0x0) { + try { + master.vmcall(entry); + } catch (const std::exception& e) { + fprintf(stderr, " (fault-order replay of '%s' threw: %s)\n", + g_entry.c_str(), e.what()); + } + } + master.restore_unpresented_pages(); + fault_pages = fault_order.size(); + populate = master.reorder_snapshot_memory(fault_order); + } + + /* Capture the working set once, from a clean (non-reordered) boot. */ + if (strat == NO_ORDER) { + g_working_set = master.get_accessed_pages(); + } + + master.save_snapshot_state_now(populate); + + printf(" built %s: %zu accessed pages, %zu fault-order pages, %zu prefetch ranges\n", + strategy_name(strat), + (strat == NO_ORDER) ? g_working_set.size() : master.get_accessed_pages().size(), + fault_pages, populate.size()); + // master destroyed here -> MAP_SHARED memory flushed to the snapshot file +} + +/* Replay the request entrypoint, which streams the working set in its fixed + scattered page order. This is the measured consumer: it brings pages + resident in the SAME order the fault-order snapshot was laid out for, so a + sequential physical layout turns into sequential file IO (and a scattered + layout into random file IO). Returns the pages faulted in by the request. */ +static size_t replay_request(tinykvm::Machine& vm) +{ + const uint64_t entry = g_entry_addr; + if (entry == 0x0) + return 0; + try { + vm.vmcall(entry); + } catch (const std::exception& e) { + fprintf(stderr, " (replay of '%s' threw: %s)\n", g_entry.c_str(), e.what()); + } + // Pages actually brought resident by the request, via accessed bits. + size_t pages = 0; + for (const auto& [vaddr, size] : vm.get_accessed_pages()) + { (void)vaddr; pages += size / 0x1000; } + return pages; +} + +/* Cold-load a snapshot and run the request against it, timed end to end. */ +static double timed_cold_load(const std::string& path, bool can_drop, size_t* out_touched) +{ + if (can_drop) + drop_caches(); + + auto options = base_options(path, tinykvm::MachineOptions::SnapshotMode::Open); + + asm("" ::: "memory"); + auto t0 = time_now(); + asm("" ::: "memory"); + + tinykvm::Machine restored {g_binary, options}; + if (!restored.has_snapshot_state()) + fprintf(stderr, " WARNING: VM did not load from snapshot state!\n"); + size_t touched = replay_request(restored); + + asm("" ::: "memory"); + auto t1 = time_now(); + asm("" ::: "memory"); + + if (out_touched) *out_touched = touched; + return seconds_between(t0, t1); +} + +int main(int argc, char** argv) +{ + setvbuf(stdout, nullptr, _IONBF, 0); + const std::string guest_path = (argc > 1) ? argv[1] : DEFAULT_GUEST; + if (const char* e = getenv("ENTRY")) g_entry = e; + + auto original = load_file(guest_path); + const tinykvm::DynamicElf dyn = tinykvm::is_dynamic_elf( + std::string_view{(const char*)original.data(), original.size()}); + g_is_dynamic = dyn.is_dynamic; + if (g_is_dynamic) { + g_binary = load_file(ld_linux_so); + g_args.push_back(ld_linux_so); + } else { + g_binary = std::move(original); + } + g_args.push_back(guest_path); + + printf(">>> Guest: %s (%s), replay entry '%s', %d trials\n", + guest_path.c_str(), g_is_dynamic ? "dynamic" : "static", + g_entry.c_str(), TRIALS); + + tinykvm::Machine::init(); + tinykvm::Machine::setup_linux_system_calls(); + tinykvm::Machine::install_unhandled_syscall_handler( + [] (tinykvm::vCPU& cpu, unsigned scall) { + if (scall == 0x10000) { cpu.stop(); return; } + auto regs = cpu.registers(); + regs.rax = -ENOSYS; + cpu.set_registers(regs); + }); + + /* Verify we can actually measure cold loads. */ + const bool can_drop = drop_caches(); + if (!can_drop) { + fprintf(stderr, + "WARNING: cannot drop the page cache (need root). Results will be\n" + " WARM and will not reflect cold-start IO differences.\n" + " Re-run with sudo for meaningful numbers.\n"); + } + + /* Build all three snapshot files. */ + const char* paths[3] = { + "/tmp/tinykvm-snap-noorder", + "/tmp/tinykvm-snap-access", + "/tmp/tinykvm-snap-fault", + }; + printf("\n=== Building snapshots ===\n"); + for (int s = 0; s < 3; s++) { + unlink(paths[s]); + build_snapshot(Strategy(s), paths[s]); + } + printf("Working set to touch on load: %zu ranges\n", g_working_set.size()); + + /* Load phase. */ + printf("\n=== Cold load (%d trials each) ===\n", TRIALS); + for (int s = 0; s < 3; s++) { + std::vector times; + size_t touched = 0; + for (int t = 0; t < TRIALS; t++) + times.push_back(timed_cold_load(paths[s], can_drop, &touched)); + Stats st = summarize(times); + printf("%s med %7.2f ms avg %7.2f ms p90 %7.2f ms min %7.2f ms max %7.2f ms (%zu pages)\n", + strategy_name(Strategy(s)), + st.median * 1e3, st.avg * 1e3, st.p90 * 1e3, + st.min * 1e3, st.max * 1e3, touched); + } + + for (int s = 0; s < 3; s++) + unlink(paths[s]); + return 0; +} From 78e482be69efe4420ca6068816117b611b3acad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Wed, 29 Jul 2026 09:15:27 +0200 Subject: [PATCH 8/9] tests: regression test for fork_reset's mmap-backed source pages PR #88 fixed reset_keep_all_work_memory resolving the master's copy-back source through main_memory().safely_at(vaddr), which is only correct for identity-mapped memory. File-backed mmap regions live at MMAP_PHYS_BASE (32GB), so any recorded page backed there was restored from an unrelated identity-mapped page -- in practice zeros -- and the fork came back from reset with silently wrong contents. It shipped without a test. The test maps a 4MB pattern file into a master, forks, writes to the mapping, resets with reset_keep_all_work_memory and requires the pattern back. It fails on the parent of #88 (reads 0x00, expected 0x5A) and passes on #88. mmap_backed_area only installs MMAP_PHYS_BASE-backed memory for the part of the range that is 2MB-aligned and 2MB-sized; a smaller mapping falls back to preadv into ordinary identity-mapped memory, where the bug does not reproduce and the test would pass for the wrong reason. Hence the 4MB file, the 2MB-aligned base, and the explicit page-table check that the leaf really does resolve above MMAP_PHYS_BASE. Co-Authored-By: Claude Opus 5 (1M context) --- tests/unit/mmap.cpp | 78 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/unit/mmap.cpp b/tests/unit/mmap.cpp index 05ac9a6e..27121f7e 100644 --- a/tests/unit/mmap.cpp +++ b/tests/unit/mmap.cpp @@ -1,9 +1,15 @@ #include +#include +#include +#include #include +#include +#include extern std::vector build_and_load(const std::string &code); /* mmap tests touch mapped pages from guest code; keep enough physical headroom. */ static const uint64_t MAX_MEMORY = 64ul << 20; /* 64MB */ +static const uint64_t MAX_COWMEM = 16ul << 20; /* 16MB */ static const std::vector env{ "LC_TYPE=C", "LC_ALL=C", "USER=root"}; @@ -239,3 +245,75 @@ int do_munmap(void* addr, size_t size) { } } } + +TEST_CASE("Fork reset restores file-backed mmap pages from the master", "[MMAP]") +{ + /* A file-backed mmap region lives at vMemory::MMAP_PHYS_BASE, which is the + one part of the guest address space that is *not* identity-mapped. A fork + that writes such a page records it in cow_written_pages, and the + reset_keep_all_work_memory copy-back has to resolve the master's source + page through the page tables. Reading it as main_memory().safely_at(vaddr) + lands on an unrelated (typically zero) identity-mapped page, so the fork + silently comes back from reset with the wrong contents. */ + const auto binary = build_and_load(R"M( +int main() { return 666; } +long peek(char* p) { return (unsigned char)p[0]; } +long poke(char* p, int v) { p[0] = (unsigned char)v; return (unsigned char)p[0]; } +)M"); + + char path[] = "/tmp/tinykvm-mmap-forkreset-XXXXXX"; + const int fd = mkstemp(path); + REQUIRE(fd >= 0); + /* mmap_backed_area only installs MMAP_PHYS_BASE-backed memory for the 2MB- + aligned-and-down part of the range, so the file has to be at least 2MB. */ + const std::vector pattern(4ul << 20, 0x5A); + REQUIRE(write(fd, pattern.data(), pattern.size()) == (ssize_t)pattern.size()); + + tinykvm::Machine machine{binary, {.max_mem = MAX_MEMORY, .split_hugepages = true}}; + machine.setup_linux({"program"}, env); + machine.run(2.0f); + REQUIRE(machine.return_value() == 666); + + uint64_t area = machine.mmap_allocate(8ul << 20); + area = (area + 0x1FFFFF) & ~0x1FFFFFULL; + REQUIRE(machine.mmap_backed_area(fd, 0, PROT_READ | PROT_WRITE, area, 4ul << 20)); + + /* Guard the premise: if this ever resolves to an identity-mapped page the + test still passes for the wrong reason, which is exactly the trap here. */ + auto& mem = machine.main_memory(); + uint64_t entry = 0; + tinykvm::page_at(mem, area, + [&](uint64_t, uint64_t& e, size_t) { entry = e; }, true); + REQUIRE((entry & tinykvm::paging_address_mask()) >= tinykvm::vMemory::MMAP_PHYS_BASE); + + machine.prepare_copy_on_write(65536); + const auto peek = machine.address_of("peek"); + const auto poke = machine.address_of("poke"); + REQUIRE(peek != 0x0); + REQUIRE(poke != 0x0); + + auto fork = tinykvm::Machine{machine, + {.max_mem = MAX_MEMORY, .max_cow_mem = MAX_COWMEM, .split_hugepages = true}}; + + fork.timed_vmcall(peek, 4.0f, area); + REQUIRE(fork.return_value() == 0x5A); + + for (int i = 0; i < 3; i++) + { + fork.timed_vmcall(poke, 4.0f, area, 0x7E); + REQUIRE(fork.return_value() == 0x7E); + + fork.reset_to(machine, { + .max_mem = MAX_MEMORY, + .max_cow_mem = MAX_COWMEM, + .reset_keep_all_work_memory = true, + }); + + /* The master still holds the file contents, so the fork must too. */ + fork.timed_vmcall(peek, 4.0f, area); + REQUIRE(fork.return_value() == 0x5A); + } + + close(fd); + unlink(path); +} From 59d6d1bfb2fde35878803d52b8f08faacef07264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Wed, 29 Jul 2026 09:15:43 +0200 Subject: [PATCH 9/9] memory: walk the fork baseline page tables in fork_reset, not the master's live root PR #88 correctly changed fork_reset to resolve the master's copy-back source through the page tables, but it walks from main_memory().page_tables. For a master with working memory that is the banked PML4 the master itself executes on, whose entries have been redirected to bank pages holding writes the master made *after* prepare_copy_on_write(). setup_cow_mode() deliberately roots forks at physbase + PT_ADDR to exclude exactly those pages ("we use the fixed PT_ADDR directly in order to avoid duplicating the memory banked page tables"). So a fork never sees the master's post-prepare writes -- but after #88 a reset_keep_all_work_memory reset did, restoring a state no fresh fork and no full reset produces. Observable as "Fork w/working memory sanity checks" in tests/unit/fork.cpp: after the master runs get_value() once, a fresh fork and a full reset both see value == 0, while a keep-work-memory reset saw 1. Pass the baseline root explicitly. readable_page_at() takes an optional root (0 = memory.page_tables, unchanged for every other caller). #88's actual fix is untouched: mmap regions are installed in the PT_ADDR-rooted tables too, since that is the only view forks ever get. Co-Authored-By: Claude Opus 5 (1M context) --- lib/tinykvm/amd64/paging.cpp | 4 ++-- lib/tinykvm/arm64/paging.cpp | 4 ++-- lib/tinykvm/memory.cpp | 10 +++++++++- lib/tinykvm/paging.hpp | 8 +++++++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/tinykvm/amd64/paging.cpp b/lib/tinykvm/amd64/paging.cpp index 7c0c7878..a795296d 100644 --- a/lib/tinykvm/amd64/paging.cpp +++ b/lib/tinykvm/amd64/paging.cpp @@ -929,10 +929,10 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl memory_exception("page_at: pml4 entry not present", addr, PDE64_PDPT_SIZE); } -char * readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags) +char * readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags, uint64_t root) { CLPRINT("Resolving a readable page for 0x%lX\n", addr); - auto* pml4 = memory.page_at(memory.page_tables); + auto* pml4 = memory.page_at(root ? root : memory.page_tables); const uint64_t i = (addr >> 39) & 511; if (is_flagged_page(flags, pml4[i])) { const auto [pdpt_base, pdpt_mem, pdpt_size] = pdpt_from_index(i, pml4); diff --git a/lib/tinykvm/arm64/paging.cpp b/lib/tinykvm/arm64/paging.cpp index 8af9d111..cd71d002 100644 --- a/lib/tinykvm/arm64/paging.cpp +++ b/lib/tinykvm/arm64/paging.cpp @@ -518,9 +518,9 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl return WritablePage{.page = (char*)data, .entry = e3, .size = L3_PAGE_SIZE}; } -char* readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags) +char* readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags, uint64_t root) { - auto* l1 = memory.page_at(memory.page_tables); + auto* l1 = memory.page_at(root ? root : memory.page_tables); const uint64_t e1 = l1[l1_index(addr)]; if (!is_valid(e1)) memory_exception("readable_page_at: l1 entry not present", addr, L1_BLOCK_SIZE); diff --git a/lib/tinykvm/memory.cpp b/lib/tinykvm/memory.cpp index 02501398..fbbdff88 100644 --- a/lib/tinykvm/memory.cpp +++ b/lib/tinykvm/memory.cpp @@ -217,13 +217,21 @@ bool vMemory::fork_reset(const Machine& main_vm, const MachineOptions& options) #else constexpr uint64_t flags = 1ULL; // DESC_VALID #endif + // Walk from the *fork baseline* root (physbase + PT_ADDR), not + // main_memory().page_tables: a master with working memory runs + // on a banked PML4 whose entries point at bank pages holding + // writes it made after prepare_copy_on_write(). setup_cow_mode() + // deliberately roots forks at PT_ADDR to exclude those, so a + // reset must use the same root or it restores a state no fresh + // fork would ever see. constexpr uint64_t granule = vMemory::PageSize(); const uint64_t vbase = addr & ~(page_size - 1); + const uint64_t master_root = main_vm.main_memory().physbase + PT_ADDR; for (size_t e = 0; e < page_size / granule; e++) { auto* dest = (uint64_t*)our_page + e * (granule / sizeof(uint64_t)); try { auto* master_page = tinykvm::readable_page_at( - main_vm.main_memory(), vbase + e * granule, flags); + main_vm.main_memory(), vbase + e * granule, flags, master_root); page_duplicate(dest, (const uint64_t*)master_page); } catch (const MemoryException&) { // The master (frozen since fork) has no present page diff --git a/lib/tinykvm/paging.hpp b/lib/tinykvm/paging.hpp index 6ae4e652..59bd9ec3 100644 --- a/lib/tinykvm/paging.hpp +++ b/lib/tinykvm/paging.hpp @@ -32,7 +32,13 @@ struct WritablePageOptions { bool allow_dirty = false; }; extern WritablePage writable_page_at(vMemory&, uint64_t addr, uint64_t flags, WritablePageOptions = {}); -extern char * readable_page_at(const vMemory&, uint64_t addr, uint64_t flags); +/* Resolve a readable page through the page tables. When root is non-zero the + walk starts there instead of memory.page_tables; callers that need the + *fork baseline* view of a master must pass physbase + PT_ADDR, because a + master with working memory executes on a banked PML4 whose entries have + been redirected to bank pages holding writes made after + prepare_copy_on_write() -- writes no fork ever sees (see setup_cow_mode). */ +extern char * readable_page_at(const vMemory&, uint64_t addr, uint64_t flags, uint64_t root = 0); extern size_t paging_merge_leaf_pages_into_hugepages(vMemory&, bool merge_if_dirty = false); extern uint64_t paging_default_usermode_flags(bool executable_heap); extern uint64_t paging_address_mask();