Fault handler rebase

CMakeLists.txt

@@ -37,6 +37,11 @@ if (TINYKVM_ARCH STREQUAL "AMD64")
 	)
 	target_link_libraries(storagekvm tinykvm)
 
+	add_executable(snapshot_bench
+		src/snapshot_bench.cpp
+	)
+	target_link_libraries(snapshot_bench tinykvm)
+
 	add_executable(pipekvm
 		src/pipe.cpp
 	)

guest/glibc/glibc.cpp

@@ -1,29 +1,73 @@
 #include <cassert>
 #include <malloc.h>
+#include <cstdint>
 #include <cstdio>
 #include <cstring>
 #include <cstdlib>
 static void test_threads();
 extern "C" int gettid();
 
 static int threads_test_suite_ok = 0;
+
+/* ------------------------------------------------------------------------
+ * Snapshot-ordering workload.
+ *
+ * g_buffer is a large buffer that lives in BSS, so it is part of the booted
+ * snapshot image. main() touches every page (making it resident in the
+ * snapshot file), and test() then streams over it in a FIXED, SCATTERED page
+ * order (g_order). Because the access order is decorrelated from the buffer's
+ * virtual/physical layout, the on-disk page ordering decides whether the cold
+ * page faults during test() become one sequential file read (fault-order
+ * snapshot) or thousands of random reads (no-order snapshot).
+ * ------------------------------------------------------------------------ */
+static constexpr size_t WS_PAGE  = 4096;
+static constexpr size_t WS_BYTES = 128UL * 1024 * 1024; /* 128 MiB working set */
+static constexpr size_t WS_PAGES = WS_BYTES / WS_PAGE;
+
+static uint8_t  g_buffer[WS_BYTES];
+static uint32_t g_order[WS_PAGES];
+
+/* Deterministic Fisher-Yates shuffle (fixed seed) so that the fault-order
+   capture run and every replay run touch the pages in the identical order. */
+static void build_workset()
+{
+	for (size_t i = 0; i < WS_PAGES; i++)
+		g_order[i] = (uint32_t)i;
+
+	uint64_t s = 0x9E3779B97F4A7C15ULL;
+	for (size_t i = WS_PAGES - 1; i > 0; i--) {
+		s = s * 6364136223846793005ULL + 1442695040888963407ULL;
+		const size_t j = (size_t)((s >> 33) % (i + 1));
+		const uint32_t t = g_order[i]; g_order[i] = g_order[j]; g_order[j] = t;
+	}
+	/* Make every page resident in the snapshot with a non-zero, page-unique
+	   value so the file is not sparse for this region. */
+	for (size_t i = 0; i < WS_PAGES; i++)
+		g_buffer[i * WS_PAGE] = (uint8_t)(1 + (i & 0xFF));
+}
+
 int main()
 {
-	char* test = (char *)malloc(14);
-	strcpy(test, "Hello World!\n");
-	printf("%.*s", 13, test);
+	char* hello = (char *)malloc(14);
+	strcpy(hello, "Hello World!\n");
+	printf("%.*s", 13, hello);
 
+	build_workset();
 	test_threads();
 
 	// Prevent global destructors
 	std::quick_exit(0);
 }
 
+/* The replayed request: stream the working set in scattered page order. The
+   returned checksum keeps the reads from being optimised away. */
 extern "C" __attribute__((used))
-void test()
+uint64_t test()
 {
-	/* Verify that the threads test-suite passed */
-	assert(threads_test_suite_ok == 1);
+	uint64_t sum = 0;
+	for (size_t i = 0; i < WS_PAGES; i++)
+		sum += g_buffer[(size_t)g_order[i] * WS_PAGE];
+	return sum;
 }
 
 #include <pthread.h>

lib/tinykvm/amd64/paging.cpp

@@ -17,6 +17,7 @@
 #define CLPRINT(...) /* ... */
 #endif
 #define PDE64_CLONEABLE  (1ul << 11)
+#define PDE64_PRESENTABLE (1ul << 10)
 
 namespace tinykvm {
 
@@ -504,12 +505,13 @@ void print_pagetables(const vMemory& memory)
 	}
 }
 
-void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses)
+void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent)
 {
+	const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : (PDE64_PRESENT);
 	auto* pml4 = memory.page_at(memory.page_tables);
 	for (size_t i = 0; i < 512; i++)
 	{
-		if (pml4[i] & PDE64_PRESENT) {
+		if (pml4[i] & present_mask) {
 			const auto [pdpt_base, pdpt_mem, pdpt_size] = pdpt_from_index(i, pml4);
 			callback(pdpt_base, pml4[i], pdpt_size);
 
@@ -520,7 +522,7 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres
 			auto* pdpt = memory.page_at(pdpt_mem);
 			for (uint64_t j = 0; j < 512; j++)
 			{
-				if (pdpt[j] & PDE64_PRESENT) {
+				if (pdpt[j] & present_mask) {
 					const auto [pd_base, pd_mem, pd_size] = pd_from_index(j, pdpt_base, pdpt);
 					callback(pd_base, pdpt[j], pd_size);
 
@@ -536,15 +538,15 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres
 					auto* pd = memory.page_at(pd_mem);
 					for (uint64_t k = 0; k < 512; k++)
 					{
-						if (pd[k] & PDE64_PRESENT) {
+						if (pd[k] & present_mask) {
 							const auto [pt_base, pt_mem, pt_size] = pt_from_index(k, pd_base, pd);
 							const bool is_2mb_page = (pd[k] & PDE64_PS) != 0;
 							callback(pt_base, pd[k], pt_size);
 							if (!is_2mb_page) {
 								auto* pt = memory.page_at(pt_mem);
 								for (uint64_t e = 0; e < 512; e++) {
 									const auto [pte_base, pte_mem, pte_size] = pte_from_index(e, pt_base, pt);
-									if (pt[e] & PDE64_PRESENT) { // 4KB page
+									if (pt[e] & present_mask) { // 4KB page
 										callback(pte_base, pt[e], pte_size);
 									}
 								} // e
@@ -556,9 +558,9 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres
 		}
 	} // i
 } // foreach_page
-void foreach_page(const vMemory& mem, foreach_page_t callback, bool skip_oob_addresses)
+void foreach_page(const vMemory& mem, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent)
 {
-	foreach_page(const_cast<vMemory&>(mem), std::move(callback), skip_oob_addresses);
+	foreach_page(const_cast<vMemory&>(mem), std::move(callback), skip_oob_addresses, include_unpresent);
 }
 
 void foreach_page_makecow(vMemory& mem, uint64_t kernel_end,
@@ -734,6 +736,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl
 			assert(!is_copy_on_write(pml4[i]) && (pml4[i] & PDE64_PRESENT));
 		}
 		const uint64_t j = index_from_pdpt_entry(addr);
+		if ((pdpt[j] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) {
+			pdpt[j] |= PDE64_PRESENT;
+			pdpt[j] &= ~PDE64_PRESENTABLE;
+			const uint64_t paddr = pdpt_base | (j << 30);
+			if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr);
+			throw RetryException();
+		}
 		if (pdpt[j] & PDE64_PRESENT) {
 			const auto [pd_base, pd_mem, pd_size] = pd_from_index(j, pdpt_base, pdpt);
 			auto* pd = memory.page_at(pd_mem);
@@ -748,6 +757,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl
 				}
 			}
 			const uint64_t k = index_from_pd_entry(addr);
+			if ((pd[k] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) {
+				pd[k] |= PDE64_PRESENT;
+				pd[k] &= ~PDE64_PRESENTABLE;
+				const uint64_t paddr = pd_base | (k << 21);
+				if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr);
+				throw RetryException();
+			}
 			if (pd[k] & (PDE64_PRESENT | PDE64_CLONEABLE)) {
 				const auto [pt_base, pt_mem, pt_size] = pt_from_index(k, pd_base, pd);
 				uint64_t* pt;
@@ -855,6 +871,13 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl
 				}
 
 				const uint64_t e = index_from_pt_entry(addr);
+				if ((pt[e] & (PDE64_PRESENT | PDE64_PRESENTABLE)) == PDE64_PRESENTABLE) {
+					pt[e] |= PDE64_PRESENT;
+					pt[e] &= ~PDE64_PRESENTABLE;
+					const uint64_t paddr = pt_base | (e << 12);
+					if (memory.on_page_presentable) memory.on_page_presentable(paddr, addr);
+					throw RetryException();
+				}
 				if (pt[e] & (PDE64_PRESENT | PDE64_CLONEABLE)) { // 4KB page
 					const auto [pte_base, pte_mem, pte_size] = pte_from_index(e, pt_base, pt);
 					uint64_t* data;
@@ -906,10 +929,10 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl
 	memory_exception("page_at: pml4 entry not present", addr, PDE64_PDPT_SIZE);
 }
 
-char * readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags)
+char * readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags, uint64_t root)
 {
 	CLPRINT("Resolving a readable page for 0x%lX\n", addr);
-	auto* pml4 = memory.page_at(memory.page_tables);
+	auto* pml4 = memory.page_at(root ? root : memory.page_tables);
 	const uint64_t i = (addr >> 39) & 511;
 	if (is_flagged_page(flags, pml4[i])) {
 		const auto [pdpt_base, pdpt_mem, pdpt_size] = pdpt_from_index(i, pml4);
@@ -1075,4 +1098,137 @@ size_t paging_merge_leaf_pages_into_hugepages(vMemory& memory, bool merge_if_dir
 	return merged_pages;
 } // paging_merge_leaf_pages_into_hugepages()
 
+std::vector<PageInfo> collect_all_pages(const vMemory& memory, bool include_unpresent)
+{
+	std::vector<PageInfo> pages;
+	const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : PDE64_PRESENT;
+	const uint64_t arena_base = MemoryBanks::ARENA_BASE_ADDRESS;
+	const uint64_t mem_end = memory.physbase + memory.size;
+
+	// A page is "relevant" if it's in main memory OR in a memory bank.
+	// Bank pages (branch nodes from CoW, kernel data like IST) need to be
+	// flattened into main memory during snapshot reordering.
+	auto is_relevant = [&](uint64_t paddr) {
+		if (paddr >= arena_base)
+			return true; // bank page
+		return paddr >= memory.physbase && paddr < mem_end;
+	};
+
+	// Include the PML4 page itself (may be in a bank after setup_cow_mode)
+	pages.push_back({memory.page_tables, PAGE_SIZE, true});
+
+	auto* pml4 = memory.page_at(memory.page_tables);
+	for (size_t i = 0; i < 512; i++) {
+		if (!(pml4[i] & present_mask))
+			continue;
+		const uint64_t pdpt_paddr = pml4[i] & PDE64_ADDR_MASK;
+		if (is_relevant(pdpt_paddr))
+			pages.push_back({pdpt_paddr, PAGE_SIZE, true});
+
+		auto* pdpt = memory.page_at(pdpt_paddr);
+		for (size_t j = 0; j < 512; j++) {
+			if (!(pdpt[j] & present_mask))
+				continue;
+			// 1GB leaf page
+			if (pdpt[j] & PDE64_PS) {
+				const uint64_t paddr = pdpt[j] & PDE64_ADDR_MASK;
+				if (is_relevant(paddr))
+					pages.push_back({paddr, 1ULL << 30, false});
+				continue;
+			}
+			const uint64_t pd_paddr = pdpt[j] & PDE64_ADDR_MASK;
+			if (is_relevant(pd_paddr))
+				pages.push_back({pd_paddr, PAGE_SIZE, true});
+
+			auto* pd = memory.page_at(pd_paddr);
+			for (size_t k = 0; k < 512; k++) {
+				if (!(pd[k] & present_mask))
+					continue;
+				// 2MB leaf page
+				if (pd[k] & PDE64_PS) {
+					const uint64_t paddr = pd[k] & PDE64_ADDR_MASK;
+					if (is_relevant(paddr))
+						pages.push_back({paddr, 1ULL << 21, false});
+					continue;
+				}
+				const uint64_t pt_paddr = pd[k] & PDE64_ADDR_MASK;
+				if (is_relevant(pt_paddr))
+					pages.push_back({pt_paddr, PAGE_SIZE, true});
+
+				auto* pt = memory.page_at(pt_paddr);
+				for (size_t e = 0; e < 512; e++) {
+					if (!(pt[e] & present_mask))
+						continue;
+					const uint64_t paddr = pt[e] & PDE64_ADDR_MASK;
+					if (is_relevant(paddr))
+						pages.push_back({paddr, PAGE_SIZE, false});
+				}
+			}
+		}
+	}
+	return pages;
+}
+
+void rewire_page_tables(char* base_ptr, uint64_t physbase, uint64_t new_root,
+	const std::unordered_map<uint64_t, uint64_t>& translation, bool include_unpresent)
+{
+	const uint64_t present_mask = include_unpresent ? (PDE64_PRESENT | PDE64_PRESENTABLE) : PDE64_PRESENT;
+
+	auto translate = [&](uint64_t& entry, uint64_t addr_mask) {
+		const uint64_t old_paddr = entry & addr_mask;
+		auto it = translation.find(old_paddr);
+		if (it != translation.end()) {
+			entry = (entry & ~addr_mask) | it->second;
+		}
+	};
+	auto get_page_at = [&](uint64_t paddr) -> uint64_t* {
+		return (uint64_t*)(base_ptr + (paddr - physbase));
+	};
+
+	auto* pml4 = get_page_at(new_root);
+	for (size_t i = 0; i < 512; i++) {
+		if (!(pml4[i] & present_mask))
+			continue;
+		// Translate PML4 entry (points to PDPT page)
+		translate(pml4[i], PDE64_ADDR_MASK);
+		const uint64_t pdpt_paddr = pml4[i] & PDE64_ADDR_MASK;
+
+		auto* pdpt = get_page_at(pdpt_paddr);
+		for (size_t j = 0; j < 512; j++) {
+			if (!(pdpt[j] & present_mask))
+				continue;
+			if (pdpt[j] & PDE64_PS) {
+				// 1GB leaf — translate data address
+				translate(pdpt[j], PDE64_ADDR_MASK);
+				continue;
+			}
+			// Translate PDPT entry (points to PD page)
+			translate(pdpt[j], PDE64_ADDR_MASK);
+			const uint64_t pd_paddr = pdpt[j] & PDE64_ADDR_MASK;
+
+			auto* pd = get_page_at(pd_paddr);
+			for (size_t k = 0; k < 512; k++) {
+				if (!(pd[k] & present_mask))
+					continue;
+				if (pd[k] & PDE64_PS) {
+					// 2MB leaf — translate data address
+					translate(pd[k], PDE64_ADDR_MASK);
+					continue;
+				}
+				// Translate PD entry (points to PT page)
+				translate(pd[k], PDE64_ADDR_MASK);
+				const uint64_t pt_paddr = pd[k] & PDE64_ADDR_MASK;
+
+				auto* pt = get_page_at(pt_paddr);
+				for (size_t e = 0; e < 512; e++) {
+					if (!(pt[e] & present_mask))
+						continue;
+					// 4KB leaf — translate data address
+					translate(pt[e], PDE64_ADDR_MASK);
+				}
+			}
+		}
+	}
+}
+
 } // tinykvm

lib/tinykvm/arm64/paging.cpp

@@ -347,7 +347,11 @@ void print_pagetables(const vMemory& memory)
 	});
 }
 
-void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses)
+/* include_unpresent is an amd64-only concept: it also visits entries kept
+   around with the present bit cleared (PDE64_PRESENTABLE) by the unpresent
+   fault handler. ARM64 has no such lazily-unpresented entries, so an invalid
+   descriptor here is genuinely absent and the flag has nothing to add. */
+void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool)
 {
 	auto* l1 = memory.page_at(memory.page_tables);
 	for (uint64_t i = 0; i < 512; i++) {
@@ -381,9 +385,9 @@ void foreach_page(vMemory& memory, foreach_page_t callback, bool skip_oob_addres
 	}
 }
 
-void foreach_page(const vMemory& memory, foreach_page_t callback, bool skip_oob_addresses)
+void foreach_page(const vMemory& memory, foreach_page_t callback, bool skip_oob_addresses, bool include_unpresent)
 {
-	foreach_page(const_cast<vMemory&>(memory), std::move(callback), skip_oob_addresses);
+	foreach_page(const_cast<vMemory&>(memory), std::move(callback), skip_oob_addresses, include_unpresent);
 }
 
 void foreach_page_makecow(vMemory& memory, uint64_t kernel_end,
@@ -514,9 +518,9 @@ WritablePage writable_page_at(vMemory& memory, uint64_t addr, uint64_t verify_fl
 	return WritablePage{.page = (char*)data, .entry = e3, .size = L3_PAGE_SIZE};
 }
 
-char* readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags)
+char* readable_page_at(const vMemory& memory, uint64_t addr, uint64_t flags, uint64_t root)
 {
-	auto* l1 = memory.page_at(memory.page_tables);
+	auto* l1 = memory.page_at(root ? root : memory.page_tables);
 	const uint64_t e1 = l1[l1_index(addr)];
 	if (!is_valid(e1))
 		memory_exception("readable_page_at: l1 entry not present", addr, L1_BLOCK_SIZE);

lib/tinykvm/common.hpp

@@ -166,6 +166,11 @@ namespace tinykvm
 		bool m_is_oom = false; /* True if the exception was caused by OOM */
 	};
 
+	class RetryException: public MachineException {
+	public:
+		RetryException() : MachineException("Retry", 0) {}
+	};
+
 	template <class...> constexpr std::false_type always_false {};
 
 	template<typename T>