diff --git a/.gitmodules b/.gitmodules index a7e4c8a..d15561e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "components/meta-sifive"] path = components/meta-sifive url = https://github.com/SlugLab/meta-sifive.git +[submodule "components/legofs"] + path = components/legofs + url = https://github.com/Zettai-US/legofs.git diff --git a/README.md b/README.md index 5bc58fb..eea0cbf 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The superproject records exact gitlinks for: - `components/u-boot`: CXL discovery and HDM decoder programming; - `components/linux`: matching RISC-V CXL firmware handoff support; - `components/cxlmemsim`: PGAS SHM server; +- `components/legofs`: Badfs lifecycle-direct client, server, and benchmark; - `components/opensbi`: OpenSBI v1.5.1; - `components/hifive-premier-tools`: pinned board-tool reference; - `components/meta-sifive`: pinned Yocto-layer reference. @@ -85,6 +86,60 @@ It adds a synthetic `pxb-cxl` bridge with U-Boot programs HPA `0x1000000000` with host decoder control `0x600` and endpoint decoder control `0x1600`. +## Two-node Legofs Type 3 back-invalidation proof + +The second workflow builds the complete stack and runs the Zettai-US Legofs +Badfs lifecycle-direct workload across two concurrent RISC-V guests: + +```bash +./run-legofs-type3.sh --bytes 65536 --timeout 1200 +``` + +Both recorded commands begin exactly with: + +```text +qemu-system-riscv64 -M sifive_u +``` + +Each guest has one 256 MiB Type 3 endpoint attached through the synthetic +SiFive U PCIe/CXL host bridge. Each endpoint uses its own file-backed +`persistent-memdev`; CXLMemSim uses a separate file-backed `ssd-stream` +backend. U-Boot enumerates `41.00.0`, reports it as Type 3, and programs the +host and endpoint HDM decoders before Linux boots. Linux exposes the Type 3 +capacity as `/dev/dax0.0`, which Legofs maps for strict lifecycle-direct +writes and reads. The external ext2 image is only the read-only delivery +disk for the static RISC-V binaries. + +The node1 endpoint retains dirty modified lines. The node0 endpoint enables +the opt-in `coherence-v2-read-exclusive` QEMU policy so its server-side +checksum reads issue GETM requests. Those requests exercise QEMU's Type 3 +back-invalidation handler against node1. A run passes only if the benchmark +trace contains the same operation ID and mapping range as a node1-directed +`SNP_DATA_INV`, a model ACK carrying the complete dirty 64-byte line, and a +dirty completion. Host monotonic timestamps must additionally prove: + +```text +node1 direct unmap < snoop send < model ACK < dirty completion < store success +``` + +The result is written to: + +```text +out/legofs-type3/runs//result.json +``` + +The result also records both QEMU argv arrays, overlapping process lifetimes, +artifact hashes, the two CXL SSD backing files, all address correlations, +Legofs direct-path and fallback counters, and final coherence error counters. +`status: "passed"` requires zero timeouts, protocol errors, delivery failures, +server-copy failures, fallback I/O, pending operations, quarantined extents, +and active leases. + +This is functional QEMU/TCG and CXLMemSim model evidence. The CXL SSDs are +file-backed simulated persistent-memory devices; this does not claim a +physical CXL link, CPU-cache or CXL.cache coherence, media durability across a +host crash, or hardware performance. + The guest benchmark is a libc-free static `rv64imafdc` executable delivered through a read-only external ext2 image on `virtio-blk-pci,bus=pcie.0`. The built-in freestanding PID 1 mounts the diff --git a/components/cxlmemsim b/components/cxlmemsim index d37e3ab..c8d3469 160000 --- a/components/cxlmemsim +++ b/components/cxlmemsim @@ -1 +1 @@ -Subproject commit d37e3ab9b44cc1ebdf9eb5d64c9390d309e8e529 +Subproject commit c8d346944600dc139fe89fabfb956569776ef4cd diff --git a/components/legofs b/components/legofs new file mode 160000 index 0000000..a2edd51 --- /dev/null +++ b/components/legofs @@ -0,0 +1 @@ +Subproject commit a2edd5105cb7d3ad8e92a2724bba03bcbeedb71b diff --git a/components/linux b/components/linux index 108e1b3..ed6701f 160000 --- a/components/linux +++ b/components/linux @@ -1 +1 @@ -Subproject commit 108e1b383db789b7f8292ff62a73efa441820dca +Subproject commit ed6701fb6d49bd5ba2384968d1f545f1fd03e5c2 diff --git a/components/qemu b/components/qemu index 81cd7ad..8930574 160000 --- a/components/qemu +++ b/components/qemu @@ -1 +1 @@ -Subproject commit 81cd7ad9a5e14470427c8ebafeccff4f52e555b4 +Subproject commit 893057423bedc2220c285d0637bd30eab34f0e8f diff --git a/configs/linux-cxl.config b/configs/linux-cxl.config index 560d6cf..6d2901e 100644 --- a/configs/linux-cxl.config +++ b/configs/linux-cxl.config @@ -12,6 +12,10 @@ CONFIG_HVC_RISCV_SBI=y CONFIG_SERIAL_EARLYCON=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_SPARSEMEM=y +CONFIG_SPARSEMEM_VMEMMAP=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y +CONFIG_ZONE_DEVICE=y CONFIG_CXL_BUS=y CONFIG_CXL_PCI=y CONFIG_CXL_ACPI=y @@ -33,3 +37,16 @@ CONFIG_PROC_FS=y CONFIG_SYSFS=y CONFIG_TMPFS=y CONFIG_BINFMT_ELF=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DAX=y +CONFIG_FS_DAX=y +CONFIG_DEV_DAX=y +CONFIG_DEV_DAX_CXL=y +# CONFIG_DEV_DAX_KMEM is not set +CONFIG_NET=y +CONFIG_INET=y +CONFIG_UNIX=y +CONFIG_PACKET=y +CONFIG_VIRTIO_NET=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y diff --git a/docs/superpowers/plans/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation.md b/docs/superpowers/plans/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation.md new file mode 100644 index 0000000..d68a0b9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation.md @@ -0,0 +1,1357 @@ +# Legofs Two-RISC-V Type-3 MESI Back-Invalidation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build and run one reproducible proof in which two overlapping `qemu-system-riscv64 -M sifive_u` guests run Legofs over separate CXL Type-3 endpoints and a real lifecycle-direct write causes a dirty MESI-v2 back-invalidation before Legofs commits the data. + +**Architecture:** Keep the existing SiFive U ACPI/U-Boot/Linux boot path and add a per-Type-3 protocol-v2 endpoint cache that talks to one authoritative CXLMemSim TCP coherence domain. Boot a Legofs server in node0 and `badfs-bench` in node1, use each guest's CXL DAX character device for strict direct I/O, and correlate Legofs direct-map/lifecycle events with benchmark-scoped CXLMemSim snoop events in a fail-closed host runner. + +**Tech Stack:** Git submodules, QEMU C/GLib/QOM, CXLMemSim C++20/CMake/GoogleTest, Legofs Rust/Tokio/Tarpc, RISC-V GNU toolchain, Linux CXL/DEV_DAX, OpenSBI, U-Boot EFI, freestanding C PID 1, Python 3 `unittest`, QEMU TCG, TCP protocol v2, ext2/debugfs, JSONL evidence. + +--- + +## Working directory, branch, and file structure + +Execute every superproject command from: + +```bash +cd /root/cxl-u-boot/CXLMemSim-riscv +``` + +Do not use `/root/cxl-u-boot` as a Git repository. It is only the parent +directory. Preserve the existing `main` work and create an isolated worktree +before implementation: + +```bash +git worktree add ../CXLMemSim-riscv-legofs -b codex/legofs-type3-mesi-proof +cd ../CXLMemSim-riscv-legofs +git submodule update --init --recursive +``` + +Component branches created by this plan: + +- QEMU: `codex/sifive-u-type3-mesi-v2`, based on + `81cd7ad9a5e14470427c8ebafeccff4f52e555b4`. +- CXLMemSim: `codex/riscv-legofs-coherence-trace`, based on + `716c16c9efc7a733006d0772f8c6c4bb055f7b15`. +- Legofs: `codex/riscv-type3-coherence-proof`, based on + `96f733940251d6484dad0ba2cfbe99dcf5259776`. + +The implementation changes these focused units: + +- `.gitmodules`: adds the pinned Legofs component. +- `components/qemu/include/hw/cxl/cxl_memsim_v2.h` and + `components/qemu/hw/cxl/cxl_memsim_v2.c`: exact reusable v2 client/cache + imported from the existing Type-2 coherence branch. +- `components/qemu/include/hw/cxl/cxl_type3_memsim_v2.h` and + `components/qemu/hw/cxl/cxl_type3_memsim_v2.c`: Type-3-specific validated + configuration, lifecycle, and fail-closed access adapter. +- `components/qemu/include/hw/cxl/cxl_device.h` and + `components/qemu/hw/mem/cxl_type3.c`: per-device state, QOM properties, + realize/exit hooks, and read/write delegation. +- `components/qemu/tests/unit/test-cxl-type3-memsim-v2.c`: adapter contract + tests using a socket-pair protocol peer. +- `components/cxlmemsim/include/coherence_trace_v2.h` and + `components/cxlmemsim/src/coherence_trace_v2.cpp`: synchronized JSONL event + sink and counter snapshot. +- `components/cxlmemsim/include/coherence_server_v2.h` and + `components/cxlmemsim/src/coherence_server_v2.cpp`: event recording at + registration, request, snoop-send, ACK, commit, and error boundaries. +- `components/cxlmemsim/src/main_server.cc`: `--coherence-v2-trace` CLI and + final machine-readable summary. +- `components/legofs/badfs-common/src/lifecycle.rs`: lifecycle physical-range + fields and optional console JSON mirror. +- `components/legofs/badfs-client/src/lib.rs`: optional console mirror of + direct-map JSON. +- `configs/linux-cxl.config`: built-in networking and CXL devdax requirements. +- `guest/legofs_node_init.c`: role-aware PID 1, network setup, DAX discovery, + strict Legofs environment, server/client launch, and proof markers. +- `scripts/build_legofs_type3.sh`: Legofs cross-build, dedicated initramfs, + Linux image, ext2 payload image, and manifest. +- `scripts/legofs_type3_2node.py`: two-QEMU/server ownership, U-Boot console + automation, overlap enforcement, trace snapshots, proof validation, and + atomic result publication. +- `run-legofs-type3.sh`: the single user-facing build-and-run entry point. +- `tests/test_legofs_sources.py`, `tests/test_legofs_build_contract.py`, + `tests/test_legofs_runtime.py`, and `tests/test_legofs_evidence.py`: offline + source, command, parser, and acceptance tests. +- `README.md`: exact build/run commands and the functional-model claim + boundary. + +Generated output stays below `out/legofs-type3/` and remains ignored. + +### Task 1: Pin the approved source graph in an isolated worktree + +**Files:** +- Modify: `.gitmodules` +- Create gitlink: `components/legofs` +- Modify gitlink: `components/cxlmemsim` +- Create: `tests/test_legofs_sources.py` + +- [ ] **Step 1: Enter the isolated worktree and create component branches** + +Run: + +```bash +cd /root/cxl-u-boot/CXLMemSim-riscv-legofs +git -C components/qemu switch -c codex/sifive-u-type3-mesi-v2 \ + 81cd7ad9a5e14470427c8ebafeccff4f52e555b4 +git -C components/cxlmemsim fetch origin \ + codex/type2-hw-cc-fullsystem-20260809 +git -C components/cxlmemsim switch -c codex/riscv-legofs-coherence-trace \ + 716c16c9efc7a733006d0772f8c6c4bb055f7b15 +git submodule add https://github.com/Zettai-US/legofs.git components/legofs +git -C components/legofs switch -c codex/riscv-type3-coherence-proof \ + 96f733940251d6484dad0ba2cfbe99dcf5259776 +``` + +Expected: all three component worktrees are on the named local branches and +the two approved external pins resolve exactly. + +- [ ] **Step 2: Write the failing source-pin tests** + +Create `tests/test_legofs_sources.py`: + +```python +import pathlib +import subprocess +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def head(component): + return subprocess.run( + ["git", "-C", str(ROOT / "components" / component), "rev-parse", "HEAD"], + check=True, text=True, capture_output=True, + ).stdout.strip() + + +class LegofsSourceTest(unittest.TestCase): + def test_legofs_base_is_approved_commit(self): + history = subprocess.run( + ["git", "-C", str(ROOT / "components/legofs"), "merge-base", + "HEAD", "96f733940251d6484dad0ba2cfbe99dcf5259776"], + check=True, text=True, capture_output=True, + ).stdout.strip() + self.assertEqual(history, "96f733940251d6484dad0ba2cfbe99dcf5259776") + + def test_cxlmemsim_base_is_approved_commit(self): + history = subprocess.run( + ["git", "-C", str(ROOT / "components/cxlmemsim"), "merge-base", + "HEAD", "716c16c9efc7a733006d0772f8c6c4bb055f7b15"], + check=True, text=True, capture_output=True, + ).stdout.strip() + self.assertEqual(history, "716c16c9efc7a733006d0772f8c6c4bb055f7b15") + + def test_gitmodules_uses_approved_legofs_remote(self): + text = (ROOT / ".gitmodules").read_text() + self.assertIn("path = components/legofs", text) + self.assertIn("url = https://github.com/Zettai-US/legofs.git", text) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 3: Run the source tests** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_sources.py +``` + +Expected: three tests pass. They guard ancestry rather than final HEAD because +later tasks intentionally add component commits. + +- [ ] **Step 4: Commit the source graph before component code changes** + +Run: + +```bash +git add .gitmodules components/legofs components/cxlmemsim \ + tests/test_legofs_sources.py +git commit -m "chore: pin Legofs and MESI v2 sources" +``` + +Expected: the superproject commit records only gitlinks, `.gitmodules`, and +the source contract test. + +### Task 2: Import and verify the reusable QEMU MESI-v2 endpoint cache + +**Files:** +- Create: `components/qemu/include/hw/cxl/cxl_memsim_v2.h` +- Create: `components/qemu/hw/cxl/cxl_memsim_v2.c` +- Create: `components/qemu/tests/unit/test-cxl-memsim-v2.c` +- Create: `components/qemu/tests/unit/test-cxl-memsim-v2-cache.c` +- Modify: `components/qemu/hw/cxl/meson.build` +- Modify: `components/qemu/tests/unit/meson.build` + +- [ ] **Step 1: Confirm the import source is the reviewed v2 branch** + +Run: + +```bash +git -C components/qemu fetch origin \ + codex/type2-hw-cc-fullsystem-qemu-20260809 +git -C components/qemu rev-parse \ + origin/codex/type2-hw-cc-fullsystem-qemu-20260809 +``` + +Expected: `b1216be2` or a descendant whose versions of the four imported files +are unchanged. Record the full object ID in the commit message. + +- [ ] **Step 2: Import the protocol client and its tests exactly** + +Run: + +```bash +git -C components/qemu checkout \ + origin/codex/type2-hw-cc-fullsystem-qemu-20260809 -- \ + include/hw/cxl/cxl_memsim_v2.h \ + hw/cxl/cxl_memsim_v2.c \ + tests/unit/test-cxl-memsim-v2.c \ + tests/unit/test-cxl-memsim-v2-cache.c +``` + +Then add `cxl_memsim_v2.c` to the existing `CONFIG_CXL` source list in +`hw/cxl/meson.build`, and add both unit executables to +`tests/unit/meson.build` with `qemuutil`, `qom`, and the CXL source dependency +used by adjacent CXL unit tests. + +- [ ] **Step 3: Configure and run only the imported unit tests** + +Run: + +```bash +mkdir -p out/legofs-type3/build/qemu +cd out/legofs-type3/build/qemu +../../../../components/qemu/configure \ + --target-list=riscv64-softmmu --disable-docs --disable-werror +ninja test-cxl-memsim-v2 test-cxl-memsim-v2-cache +meson test --print-errorlogs cxl-memsim-v2 cxl-memsim-v2-cache +``` + +Expected: frame encode/decode, registration, write-back retention, dirty +downgrade/invalidation, eviction, flush, fence, and failure tests all pass. + +- [ ] **Step 4: Commit the reusable QEMU client** + +Run: + +```bash +git -C components/qemu add include/hw/cxl/cxl_memsim_v2.h \ + hw/cxl/cxl_memsim_v2.c hw/cxl/meson.build \ + tests/unit/test-cxl-memsim-v2.c \ + tests/unit/test-cxl-memsim-v2-cache.c tests/unit/meson.build +git -C components/qemu commit -m \ + "cxl: import protocol v2 endpoint cache for Type 3" +``` + +### Task 3: Connect each QEMU Type-3 device to one fail-closed v2 endpoint + +**Files:** +- Create: `components/qemu/include/hw/cxl/cxl_type3_memsim_v2.h` +- Create: `components/qemu/hw/cxl/cxl_type3_memsim_v2.c` +- Modify: `components/qemu/include/hw/cxl/cxl_device.h` +- Modify: `components/qemu/hw/mem/cxl_type3.c` +- Modify: `components/qemu/hw/cxl/meson.build` +- Create: `components/qemu/tests/unit/test-cxl-type3-memsim-v2.c` +- Modify: `components/qemu/tests/unit/meson.build` + +- [ ] **Step 1: Write failing configuration and routing tests** + +Create `tests/unit/test-cxl-type3-memsim-v2.c` around the public adapter API: + +```c +static void test_config_rejects_invalid_host(void) +{ + CxlType3MemsimV2Config cfg = cxl_type3_memsim_v2_default_config(); + g_autoptr(Error) err = NULL; + cfg.enabled = true; + cfg.server_host = "127.0.0.1"; + cfg.server_port = 9300; + cfg.host_id = CXL_MEMSIM_V2_MAX_ENDPOINTS; + g_assert_false(cxl_type3_memsim_v2_validate(&cfg, &err)); + g_assert_nonnull(err); +} + +static void test_config_requires_write_back(void) +{ + CxlType3MemsimV2Config cfg = cxl_type3_memsim_v2_default_config(); + g_autoptr(Error) err = NULL; + cfg.enabled = true; + cfg.server_host = "127.0.0.1"; + cfg.server_port = 9300; + cfg.write_through = true; + g_assert_false(cxl_type3_memsim_v2_validate(&cfg, &err)); + g_assert_nonnull(err); +} + +static void test_failed_v2_access_returns_memtx_error(void) +{ + CxlType3MemsimV2 state = { .enabled = true, .client = NULL }; + uint64_t value = 0; + g_assert_cmpint(cxl_type3_memsim_v2_read(&state, 0, &value, 8), + ==, MEMTX_ERROR); + g_assert_cmpint(cxl_type3_memsim_v2_write(&state, 0, 1, 8), + ==, MEMTX_ERROR); +} +``` + +Add a socket-pair fake server test that completes `REGISTER`, returns a line +for `GETS`, accepts `GETM`, and asserts that one 8-byte read and one 8-byte +write produce protocol traffic instead of touching a local `MemoryRegion`. + +- [ ] **Step 2: Run the new test and verify RED** + +Run: + +```bash +ninja -C out/legofs-type3/build/qemu test-cxl-type3-memsim-v2 +``` + +Expected: compilation fails because `cxl_type3_memsim_v2.h` does not exist. + +- [ ] **Step 3: Define the complete Type-3 adapter contract** + +Create `include/hw/cxl/cxl_type3_memsim_v2.h` with this public shape: + +```c +#ifndef CXL_TYPE3_MEMSIM_V2_H +#define CXL_TYPE3_MEMSIM_V2_H + +#include "exec/memattrs.h" +#include "hw/cxl/cxl_memsim_v2.h" + +typedef struct CxlType3MemsimV2Config { + bool enabled; + const char *server_host; + uint16_t server_port; + uint16_t host_id; + uint32_t cache_capacity; + uint16_t cache_ways; + uint32_t timeout_ms; + bool write_through; +} CxlType3MemsimV2Config; + +typedef struct CxlType3MemsimV2 { + CxlType3MemsimV2Config config; + CxlMemsimV2Client *client; + bool enabled; +} CxlType3MemsimV2; + +CxlType3MemsimV2Config cxl_type3_memsim_v2_default_config(void); +bool cxl_type3_memsim_v2_validate(const CxlType3MemsimV2Config *config, + Error **errp); +bool cxl_type3_memsim_v2_realize(CxlType3MemsimV2 *state, Error **errp); +void cxl_type3_memsim_v2_unrealize(CxlType3MemsimV2 *state); +MemTxResult cxl_type3_memsim_v2_read(CxlType3MemsimV2 *state, + hwaddr dpa, uint64_t *value, + unsigned size); +MemTxResult cxl_type3_memsim_v2_write(CxlType3MemsimV2 *state, + hwaddr dpa, uint64_t value, + unsigned size); +#endif +``` + +The defaults are host `127.0.0.1`, port `9300`, cache capacity `1024` lines, +8 ways, timeout `5000` ms, and write-back. Validation accepts access sizes +1/2/4/8, rejects host IDs outside `[0,63]`, zero port/timeout/cache values, +non-power-of-two cache geometry, capacity not divisible by ways, and +write-through for this proof. + +- [ ] **Step 4: Implement connection, access, and fail-closed behavior** + +In `hw/cxl/cxl_type3_memsim_v2.c`, create the client with exactly +`CXL_MEMSIM_V2_CAP_MODEL_SNOOP` behavior provided by the imported cache, call +`cxl_memsim_v2_client_connect()`, set `CXL_MEMSIM_V2_WRITE_BACK`, and emit one +registration line containing host and session IDs. `read()` and `write()` must +return `MEMTX_ERROR` when the state is enabled but disconnected or when +`cxl_memsim_v2_load/store()` returns false. They must never call +`address_space_read/write()`. + +The error path must be structurally equivalent to: + +```c +if (!state->enabled || !state->client) { + return MEMTX_ERROR; +} +if (!cxl_memsim_v2_load(state->client, dpa, size, value, + state->config.timeout_ms, &local_err)) { + error_report_err(local_err); + return MEMTX_ERROR; +} +return MEMTX_OK; +``` + +- [ ] **Step 5: Add per-device QOM state and properties** + +Add `CxlType3MemsimV2 memsim_v2` plus owned string +`char *memsim_v2_server_host` to `CXLType3Dev`. Add these exact QOM +properties to `ct3_props`: + +```text +coherence-v2 bool, default false +cxlmemsim-addr string, default 127.0.0.1 +cxlmemsim-port uint16, default 9300 +coherence-v2-host-id uint16, default 0 +coherence-v2-cache-capacity uint32, default 1024 +coherence-v2-cache-ways uint16, default 8 +coherence-v2-timeout-ms uint32, default 5000 +coherence-v2-write-through bool, default false +``` + +In `ct3_realize()`, validate and connect before guest execution. In +`ct3_exit()`, free the v2 client. In `cxl_type3_read()` and +`cxl_type3_write()`, delegate immediately after DPA translation: + +```c +if (ct3d->memsim_v2.enabled) { + return cxl_type3_memsim_v2_read(&ct3d->memsim_v2, dpa_offset, + data, size); +} +``` + +Use the analogous write call. Keep the existing legacy SHM/TCP path only when +`coherence-v2=off`. + +- [ ] **Step 6: Run focused and existing CXL tests** + +Run: + +```bash +ninja -C out/legofs-type3/build/qemu \ + test-cxl-type3-memsim-v2 test-cxl-memsim-v2-cache \ + qemu-system-riscv64 +meson test -C out/legofs-type3/build/qemu --print-errorlogs \ + cxl-type3-memsim-v2 cxl-memsim-v2-cache +python3 -m unittest -v tests/test_runtime.py +``` + +Expected: all tests pass; the existing command test still begins with +`qemu-system-riscv64`, `-M`, `sifive_u`. + +- [ ] **Step 7: Commit the Type-3 integration** + +Run: + +```bash +git -C components/qemu add include/hw/cxl/cxl_device.h \ + include/hw/cxl/cxl_type3_memsim_v2.h hw/cxl/cxl_type3_memsim_v2.c \ + hw/cxl/meson.build hw/mem/cxl_type3.c \ + tests/unit/test-cxl-type3-memsim-v2.c tests/unit/meson.build +git -C components/qemu commit -m \ + "cxl/type3: route guest memory through MESI v2" +``` + +### Task 4: Add machine-readable MESI-v2 counters and transaction traces + +**Files:** +- Create: `components/cxlmemsim/include/coherence_trace_v2.h` +- Create: `components/cxlmemsim/src/coherence_trace_v2.cpp` +- Modify: `components/cxlmemsim/include/coherence_server_v2.h` +- Modify: `components/cxlmemsim/src/coherence_server_v2.cpp` +- Modify: `components/cxlmemsim/src/main_server.cc` +- Modify: `components/cxlmemsim/CMakeLists.txt` +- Create: `components/cxlmemsim/tests/test_coherence_trace_v2.cpp` +- Modify: `components/cxlmemsim/tests/test_coherence_server_v2.cpp` + +- [ ] **Step 1: Write failing trace-schema tests** + +Create a temporary trace, record one registration, one `GETM`, one +`SNP_DATA_INV`, and its dirty model ACK. Parse each JSONL line with the +project's JSON dependency and assert these exact keys: + +```text +schema_version,event,monotonic_ns,opcode,src_host,dst_host,session_id, +request_id,snoop_id,line_address,epoch,payload_len,status,ack_strength, +dirty_data +``` + +Assert the snapshot object contains: + +```text +registrations,gets,getm,upgrade,puts,putm,snp_inv,snp_downgrade, +snp_data_inv,snp_data_downgrade,host_fence,model_acks,native_acks, +dirty_data_completions,timeouts,protocol_errors,delivery_failures, +server_copy_failures +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cmake -S components/cxlmemsim -B out/legofs-type3/build/cxlmemsim \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=ON +cmake --build out/legofs-type3/build/cxlmemsim --parallel +ctest --test-dir out/legofs-type3/build/cxlmemsim -R \ + 'coherence_trace_v2|coherence_server_v2' --output-on-failure +``` + +Expected: the new trace test is absent or fails to compile. + +- [ ] **Step 3: Implement the synchronized trace sink** + +Define `CoherenceV2Counters` as atomics for the keys above. Define +`CoherenceTraceV2::record(const CoherenceTraceEvent&)` to serialize one JSON +object under a mutex, append `\n`, flush, and throw on write failure. Use +`std::chrono::steady_clock` nanoseconds for `monotonic_ns`; all records are +from the single host server clock and therefore directly orderable. + +Expose: + +```cpp +struct CoherenceV2Snapshot { /* one uint64_t per counter key */ }; + +class CoherenceTraceV2 final { +public: + explicit CoherenceTraceV2(const std::filesystem::path &path); + void record(const CoherenceTraceEvent &event); + CoherenceV2Snapshot snapshot() const noexcept; + std::string snapshotJson() const; +}; +``` + +Increment a counter in the same critical section that writes its event so a +trace byte-offset snapshot and counter snapshot cannot disagree. + +- [ ] **Step 4: Instrument protocol boundaries** + +Pass an optional shared `CoherenceTraceV2` into `CoherenceServerV2`. Record: + +- successful and rejected registration in the `Register` dispatch branch; +- accepted `GETS`, `GETM`, `UPGRADE`, `PUTS`, and `PUTM` before engine dispatch; +- every unsolicited snoop in `sendToHost()` before sender invocation; +- every `SNOOP_ACK` with its strength, payload length, and dirty-data flag; +- completion after the engine commits dirty snoop bytes into + `CoherenceMemoryBackend`; +- timeout, protocol rejection, delivery failure, and copy failure at the + exact return site. + +For a dirty `SNP_DATA_INV` ACK, `dirty_data` is true only when payload length +is 64, ACK strength is `MODEL`, status is `OK`, and the engine accepted the +payload for the same `snoop_id`. + +- [ ] **Step 5: Add the server CLI and final summary** + +Add: + +```text +--coherence-v2-trace +``` + +Reject it unless `--coherence-v2=true` and `--comm-mode=tcp`. Create/truncate +the trace before listening. On orderly shutdown print exactly one line: + +```text +COHERENCE_V2_STATS_JSON {json-object} +``` + +The JSON object is `snapshotJson()` and includes active host/session bindings. +The runner takes its pre-benchmark snapshot by recording the current JSONL +byte offset after both readiness markers and computes all acceptance deltas +from complete records after that offset. + +- [ ] **Step 6: Run the entire CXLMemSim v2 test subset** + +Run: + +```bash +cmake --build out/legofs-type3/build/cxlmemsim --parallel +ctest --test-dir out/legofs-type3/build/cxlmemsim \ + -R 'protocol_v2|directory|endpoint|mesi|coherence|tcp' \ + --output-on-failure +``` + +Expected: all matched tests pass, including dirty-data ACK and duplicate-host +rejection tests. + +- [ ] **Step 7: Commit the telemetry** + +Run: + +```bash +git -C components/cxlmemsim add include/coherence_trace_v2.h \ + include/coherence_server_v2.h src/coherence_trace_v2.cpp \ + src/coherence_server_v2.cpp src/main_server.cc CMakeLists.txt \ + tests/test_coherence_trace_v2.cpp tests/test_coherence_server_v2.cpp +git -C components/cxlmemsim commit -m \ + "coherence: trace MESI v2 snoop completion evidence" +``` + +### Task 5: Expose Legofs direct and lifecycle events on the guest console + +**Files:** +- Modify: `components/legofs/badfs-common/src/lifecycle.rs` +- Modify: `components/legofs/badfs-client/src/lib.rs` + +- [ ] **Step 1: Write failing Legofs trace tests** + +In `badfs-common/src/lifecycle.rs`, add a test that opens a temporary +`LifecycleTrace` with console mirroring enabled, emits one event, and asserts +the encoded object includes `mapping_offset`, `mapping_length`, and the +`store_direct_begin`/`store_direct_success` event names. + +In `badfs-client/src/lib.rs`, add a test for a pure helper: + +```rust +#[test] +fn direct_trace_console_line_is_parseable() { + let json = serde_json::json!({"op_id": 9, "offset": 4096, "length": 4096}); + let line = prefixed_trace_line("BADFS_DIRECT_MAP_TRACE_JSON", &json).unwrap(); + assert!(line.starts_with("BADFS_DIRECT_MAP_TRACE_JSON ")); + serde_json::from_str::(line.split_once(' ').unwrap().1) + .unwrap(); +} +``` + +- [ ] **Step 2: Run the focused Rust tests and verify RED** + +Run: + +```bash +cargo test --manifest-path components/legofs/Cargo.toml \ + -p badfs-common lifecycle_trace -- --nocapture +cargo test --manifest-path components/legofs/Cargo.toml \ + -p badfs-client direct_trace_console_line -- --nocapture +``` + +Expected: the new helper/fields do not compile yet. + +- [ ] **Step 3: Add physical-range lifecycle fields** + +Extend `LifecycleTraceEvent` with backward-compatible defaults: + +```rust +#[serde(default)] +pub mapping_offset: u64, +#[serde(default)] +pub mapping_length: u64, +``` + +At `store_direct()`, call the existing +`self.backend.direct_mapping(lease.extent_id, 0)` before the checksum and +validate it with the same alignment/size checks as `direct_grant()`. Emit +`store_direct_begin` immediately before `candidate_checksum()` and +`store_direct_success` after the published result, both with that exact +`mapping.offset` and `mapping.length`. Use the same `op_id`, `extent_id`, and +generation as the client grant. A missing or changed mapping returns +`Error::Io`; it does not omit the address evidence. + +- [ ] **Step 4: Add opt-in console mirroring without weakening file traces** + +Parse these exact booleans once when each trace object is created: + +```text +BADFS_LIFECYCLE_TRACE_STDOUT=1 +BADFS_CXL_DIRECT_TRACE_STDOUT=1 +``` + +After a successful file append/flush, write the already encoded JSON without +re-serializing and explicitly flush stdout before returning: + +```rust +if self.stdout { + let mut stdout = std::io::stdout().lock(); + writeln!(stdout, "BADFS_LIFECYCLE_TRACE_JSON {}", + std::str::from_utf8(&encoded_without_newline).map_err(|_| Error::Io)?) + .map_err(|_| Error::Io)?; + stdout.flush().map_err(|_| Error::Io)?; +} +``` + +Use `BADFS_DIRECT_MAP_TRACE_JSON` for client direct-map events. File write +failure remains fatal under strict-direct policy; console output never +substitutes for the file trace. + +- [ ] **Step 5: Run Legofs core and benchmark tests** + +Run: + +```bash +cargo test --manifest-path components/legofs/Cargo.toml \ + -p badfs-common -p badfs-client -p badfs-server -p badfs-bench +``` + +Expected: all tests pass and existing trace files remain schema-compatible. + +- [ ] **Step 6: Commit Legofs evidence support** + +Run: + +```bash +git -C components/legofs add badfs-common/src/lifecycle.rs \ + badfs-client/src/lib.rs +git -C components/legofs commit -m \ + "trace: correlate lifecycle direct DAX operations" +``` + +### Task 6: Build a dedicated RISC-V Legofs guest with CXL devdax + +**Files:** +- Modify: `configs/linux-cxl.config` +- Create: `guest/legofs_node_init.c` +- Create: `tests/test_legofs_build_contract.py` + +- [ ] **Step 1: Write failing kernel and PID-1 contract tests** + +Create `tests/test_legofs_build_contract.py`: + +```python +import pathlib +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +class LegofsBuildContractTest(unittest.TestCase): + def test_kernel_fragment_has_built_in_devdax_and_network(self): + config = (ROOT / "configs/linux-cxl.config").read_text().splitlines() + required = { + "CONFIG_DAX=y", "CONFIG_DEV_DAX=y", "CONFIG_DEV_DAX_CXL=y", + "CONFIG_NET=y", "CONFIG_INET=y", "CONFIG_UNIX=y", + "CONFIG_PACKET=y", "CONFIG_VIRTIO_NET=y", + } + self.assertTrue(required.issubset(set(config))) + + def test_init_has_role_dax_and_strict_markers(self): + source = (ROOT / "guest/legofs_node_init.c").read_text() + for marker in ( + "legofs.role=", "LEG_OFS_CXL_READY", "LEG_OFS_SERVER_READY", + "LEG_OFS_BENCHMARK_BEGIN", "LEG_OFS_BENCHMARK_PASS", + "BADFS_LIFECYCLE_DIRECT_REQUIRED=1", + "BADFS_LIFECYCLE_DIRECT_READ_REQUIRED=1", + ): + self.assertIn(marker, source) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_build_contract.py +``` + +Expected: missing devdax options and guest source failures. + +- [ ] **Step 3: Enable all required drivers as built-ins** + +Append or replace conflicting values in `configs/linux-cxl.config`: + +```text +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DAX=y +CONFIG_DEV_DAX=y +CONFIG_DEV_DAX_CXL=y +CONFIG_NET=y +CONFIG_INET=y +CONFIG_UNIX=y +CONFIG_PACKET=y +CONFIG_VIRTIO_NET=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +``` + +The build task later checks the merged `.config`; fragment presence alone is +not acceptance. + +- [ ] **Step 4: Implement the role-aware freestanding PID 1** + +`guest/legofs_node_init.c` must: + +1. mount `/proc`, `/sys`, `/dev`, `/tmp`, and `/mnt`; +2. mount the read-only ext2 payload at `/mnt`; +3. bring up `eth0` as `10.0.2.15/24` with gateway `10.0.2.2` using socket + ioctls and `SIOCADDRT`; +4. parse `legofs.role=node0|node1`, `legofs.server_port`, and + `legofs.bytes` from `/proc/cmdline`; +5. scan `/sys/class/dax/dax*`, create the character node from sysfs `dev`, and + reject zero or multiple CXL devdax devices; +6. require `/sys/bus/cxl/devices/mem0`, a committed CXL region/decoder under + `/sys/bus/cxl/devices`, and `/dev/daxX.Y` before printing + `LEG_OFS_CXL_READY` (the runner separately captures U-Boot `cxl list`); +7. export the discovered character path as `BADFS_LIFECYCLE_DEVICE`, print + its locally derived `FabricRegionId`, device size, major, and minor; +8. set the strict environment below with `execve()`; and +9. power off only after node1 prints final counters and a pass/fail marker. + +Common environment: + +```text +BADFS_POSIX_DATA_PATH=lifecycle +BADFS_LIFECYCLE_BLOB=1 +BADFS_LIFECYCLE_DIRECT_FINAL=1 +BADFS_LIFECYCLE_DIRECT_REQUIRED=1 +BADFS_LIFECYCLE_DIRECT_READ=1 +BADFS_LIFECYCLE_DIRECT_READ_REQUIRED=1 +BADFS_LIFECYCLE_DEVICE_REQUIRED=1 +BADFS_CXL_MAP_ALIGNMENT=2097152 +BADFS_LIFECYCLE_TRACE=/tmp/lifecycle.jsonl +BADFS_LIFECYCLE_TRACE_STDOUT=1 +BADFS_CXL_DIRECT_TRACE=/tmp/direct.jsonl +BADFS_CXL_DIRECT_TRACE_STDOUT=1 +BADFS_LIFECYCLE_POOL_SIZE=268435456 +BADFS_LIFECYCLE_MAX_EXTENTS=127 +RUST_LOG=info +``` + +Node0 adds `BADFS_SERVER_ADDR=0.0.0.0:3345`, +`BADFS_DATA_DIR=/tmp/badfs-data`, forks `/mnt/badfs-server`, polls +`127.0.0.1:3345`, and prints `LEG_OFS_SERVER_READY` only after the connect +probe succeeds. It then remains PID 1 and reaps the server. Node1 adds +`BADFS_SERVERS=10.0.2.2:`, `BADFS_BASE_PATH=/badfs`, +`BADFS_BENCH_MODE=workload`, `BADFS_BENCH_FILE_SIZE=`, +`BADFS_BENCH_BLOCK_SIZE=4096`, and `BADFS_BENCH_ITERATIONS=1`, then executes +`/mnt/badfs-bench` with no arguments. Before the workload fork, node1 prints +`LEG_OFS_CLIENT_READY` and blocks on `/dev/hvc0` until the runner sends the +exact line `LEG_OFS_RUN`; this lets the runner freeze the pre-benchmark trace +offset. After the workload exits successfully, execute the same binary again +with `BADFS_BENCH_MODE=inspect` so the lifecycle audit is printed. +`badfs-bench` is environment-driven; passing positional workload or size +arguments is prohibited because the current binary ignores them. + +- [ ] **Step 5: Compile the PID 1 natively for syntax and run tests** + +Run: + +```bash +gcc -std=c11 -Wall -Wextra -Werror -fsyntax-only guest/legofs_node_init.c +python3 -m unittest -v tests/test_legofs_build_contract.py +``` + +Expected: both commands pass. + +- [ ] **Step 6: Commit the guest contract** + +Run: + +```bash +git add configs/linux-cxl.config guest/legofs_node_init.c \ + tests/test_legofs_build_contract.py +git commit -m "guest: add strict Legofs Type 3 node image" +``` + +### Task 7: Add the reproducible Legofs/guest build pipeline + +**Files:** +- Create: `scripts/build_legofs_type3.sh` +- Create: `run-legofs-type3.sh` +- Modify: `scripts/write_manifest.py` +- Modify: `tests/test_legofs_build_contract.py` + +- [ ] **Step 1: Add failing CLI and artifact-contract tests** + +Test these behaviors with subprocesses and temporary fake commands: + +```text +./run-legofs-type3.sh --help exits 0 +./run-legofs-type3.sh --build-only --run-only exits nonzero +./run-legofs-type3.sh --bytes 0 exits nonzero +./run-legofs-type3.sh --bytes 65536 accepts the value +``` + +Also assert the build script names these manifest artifacts: + +```text +qemu,opensbi,u_boot,linux_legofs,legofs_disk,badfs_server,badfs_bench, +cxlmemsim_server +``` + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_build_contract.py +``` + +Expected: CLI/build-script tests fail because both scripts are absent. + +- [ ] **Step 3: Implement the top-level entry point** + +`run-legofs-type3.sh` accepts only `--build-only`, `--run-only`, `--jobs N`, +`--bytes N`, `--timeout N`, and `--help`. Default behavior runs the build and +then: + +```bash +exec python3 "${ROOT}/scripts/legofs_type3_2node.py" \ + --bytes "${BENCH_BYTES}" --timeout "${TIMEOUT}" +``` + +Reject bytes that are zero, exceed `16777216`, or are not divisible by 4096. +Run `git submodule status --recursive` and reject lines beginning with `-`, +`+`, or `U`. + +- [ ] **Step 4: Implement the dedicated build script** + +`scripts/build_legofs_type3.sh` must use `set -euo pipefail` and only write to +`out/legofs-type3`. It performs these exact build gates: + +```bash +cargo build --manifest-path components/legofs/Cargo.toml --release \ + --target riscv64gc-unknown-linux-musl -p badfs-server -p badfs-bench +riscv64-linux-gnu-readelf -l | grep -qv INTERP +``` + +Build a pinned musl 1.2.5 sysroot under `out/legofs-type3` with +`-march=rv64gc -mabi=lp64d`, verifying the official release tarball SHA-256 +`a9a118bb...fc7c75e4`. Use its wrapper for +`CC_riscv64gc_unknown_linux_musl` and +`CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_LINKER`; this avoids the host +distribution's RVV-enabled static glibc/crt. Set +`RUSTFLAGS='-C target-feature=+crt-static -C link-arg=-march=rv64gc -C link-arg=-mabi=lp64d'`. +If the requested Rust target is not installed, print the exact +`rustup target add riscv64gc-unknown-linux-musl` remediation and exit without +invoking `rustup`. + +Build `guest/legofs_node_init.c` with the same static freestanding RV64 flags +as `scripts/build.sh`. Create a 64 MiB ext2 image using `truncate`, `mke2fs`, +and `debugfs`; install only `/badfs-server` and `/badfs-bench` with mode 0755. +Do not loop-mount the image. Run this benchmark-interface gate: + +```bash +cargo test --manifest-path components/legofs/Cargo.toml -p badfs-bench \ + benchmark_mode_uses_semantic_values_and_rejects_unknown_input +``` + +Build a dedicated kernel in `out/legofs-type3/build/linux` with +`CONFIG_INITRAMFS_SOURCE` pointing to the Legofs PID 1 directory. Verify every +option from Task 6 is exactly `=y`. Reuse or build the pinned OpenSBI/U-Boot +artifacts, and build QEMU/CXLMemSim from their component branches. + +- [ ] **Step 5: Extend the manifest and verify static binaries** + +Write `out/legofs-type3/results/build-manifest.json` atomically. It contains +all component HEADs, artifact sizes/SHA-256 values, compiler versions, and the +eight artifact names from Step 1. Reject an ELF interpreter and reject a +RISC-V attributes string requiring RVV. + +- [ ] **Step 6: Run the build-contract tests** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_build_contract.py +bash -n run-legofs-type3.sh scripts/build_legofs_type3.sh +``` + +Expected: all tests and syntax checks pass. + +- [ ] **Step 7: Commit the build pipeline** + +Run: + +```bash +git add run-legofs-type3.sh scripts/build_legofs_type3.sh \ + scripts/write_manifest.py tests/test_legofs_build_contract.py +git commit -m "build: package RISC-V Legofs Type 3 guests" +``` + +### Task 8: Construct and own two exact SiFive U QEMU processes + +**Files:** +- Create: `scripts/legofs_type3_2node.py` +- Create: `tests/test_legofs_runtime.py` + +- [ ] **Step 1: Write failing command/topology tests** + +The tests call `build_qemu_command(paths, node, coherence_port, +legofs_port)` for nodes 0 and 1 and assert: + +```python +self.assertEqual(cmd[:3], ["qemu-system-riscv64", "-M", "sifive_u"]) +self.assertEqual(sum("cxl-type3" in arg for arg in cmd), 1) +self.assertIn("coherence-v2=on", " ".join(cmd)) +self.assertIn(f"coherence-v2-host-id={node}", " ".join(cmd)) +self.assertNotIn("-M virt", " ".join(cmd)) +``` + +Assert node0 alone has one `hostfwd=tcp:127.0.0.1:-:3345`, every QEMU +object/device ID is node-qualified, and both commands point to the same +coherence port but different host IDs. + +Assert each node uses one file-backed `memory-backend-file` with `pmem=on` +and attaches it through `persistent-memdev`. Reject `volatile-memdev`: the +benchmark endpoint is a QEMU CXL SSD, not anonymous volatile CXL memory. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_runtime.py +``` + +Expected: import fails because the runner does not exist. + +- [ ] **Step 3: Implement exact command construction** + +Each returned list begins exactly: + +```python +["qemu-system-riscv64", "-M", "sifive_u"] +``` + +It then includes the existing SiFive U CXL firmware/FMW settings, one +256 MiB node-private file-backed persistent Type-3 CXL SSD (`pmem=on`), one +2 MiB LSA, one `pxb-cxl`, one `cxl-rp`, one `cxl-type3`, the read-only Legofs +ext2 disk, and one `virtio-net-pci` user network. The Type-3 argument contains: + +```text +coherence-v2=on,cxlmemsim-addr=127.0.0.1,cxlmemsim-port=, +coherence-v2-host-id=<0-or-1>,coherence-v2-cache-capacity=262144, +coherence-v2-cache-ways=4,coherence-v2-timeout-ms=5000, +coherence-v2-write-through=off +``` + +This is also the QEMU back-invalidation implementation under test: a guest +write leaves the Type-3 endpoint-cache line dirty in M, and the competing +guest access must make QEMU answer `SNP_DATA_INV` with a 64-byte dirty model +ACK before invalidating the local line. The Type-2 BAR back-invalidation queue +is not a substitute for this Type-3 path. + +Do not set legacy `CXL_TRANSPORT_MODE`, `CXL_PGAS_SHM`, or +`CXL_MEMSIM_SERVER` environment variables. + +- [ ] **Step 4: Implement run-scoped process ownership** + +Create `out/legofs-type3/runs/-/` with mode 0700. Reserve +two loopback TCP ports by binding sockets before process launch; release the +coherence reservation immediately before starting the server and the Legofs +reservation immediately before node0. Record PID, command array, start +monotonic time, exit time, and owner token for all three processes. + +For each console, timestamp every complete received line with host +`time.monotonic_ns()` and append it to `node0-events.jsonl` or +`node1-events.jsonl` as `{host_capture_ns, line}`. This sidecar supplies one +host-clock ordering for the two guest consoles without treating either +guest's local monotonic clock as cross-VM comparable. + +Cleanup sends SIGTERM only to recorded live PIDs whose `/proc//cmdline` +still matches the recorded executable and run directory. Wait five seconds, +then SIGKILL only those remaining owned PIDs. Never use `pkill`, `killall`, or +a process-name match. + +- [ ] **Step 5: Implement U-Boot boot sequencing and overlap gates** + +Reuse the behavior of the existing `Console` class: wait for `=>`, run +`cxl list`, `cxl info 41.00.0`, and `cxl init`, then send role-specific +bootargs and: + +```text +bootefi 90000000: ${fdtcontroladdr} +``` + +Boot node0, wait for `LEG_OFS_CXL_READY` and `LEG_OFS_SERVER_READY`, then boot +node1. Before releasing node1's benchmark gate, assert both QEMU PIDs are +alive. After `LEG_OFS_BENCHMARK_PASS`, assert node0 is still alive. Store both +lifetime intervals and require their intersection to be non-empty. + +- [ ] **Step 6: Run runtime unit tests** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_runtime.py +``` + +Expected: all command, port, PID ownership, overlap, and cleanup tests pass +using fake subprocesses; no QEMU starts in this test. + +- [ ] **Step 7: Commit the two-node runtime** + +Run: + +```bash +git add scripts/legofs_type3_2node.py tests/test_legofs_runtime.py +git commit -m "run: orchestrate two SiFive U Type 3 nodes" +``` + +### Task 9: Enforce benchmark-scoped dirty back-invalidation evidence + +**Files:** +- Modify: `scripts/legofs_type3_2node.py` +- Create: `tests/test_legofs_evidence.py` + +- [ ] **Step 1: Write failing positive and negative evidence fixtures** + +Construct minimal in-memory records for one successful operation: + +```python +direct = { + "event": "unmap", "access": "write", "op_id": 17, + "offset": 0x4000, "length": 0x1000, "monotonic_ns": 10, +} +lifecycle_begin = { + "event": "store_direct_begin", "op_id": 17, + "mapping_offset": 0x4000, "mapping_length": 0x1000, +} +snoop = { + "event": "snoop_sent", "opcode": "SNP_DATA_INV", + "src_host": 0, "dst_host": 1, "snoop_id": 91, + "line_address": 0x4080, "monotonic_ns": 20, +} +ack = { + "event": "snoop_ack", "opcode": "SNP_DATA_INV", + "src_host": 1, "dst_host": 0xffff, "snoop_id": 91, + "line_address": 0x4080, "monotonic_ns": 21, + "ack_strength": "MODEL", "payload_len": 64, + "status": "OK", "dirty_data": True, +} +lifecycle_success = { + "event": "store_direct_success", "op_id": 17, + "mapping_offset": 0x4000, "mapping_length": 0x1000, +} +``` + +The positive fixture passes. Separate tests must reject zero invalidations, +clean ACK, native ACK, mismatched `snoop_id`, line outside the grant, event +before the pre-benchmark trace offset, missing host registration, duplicate +host ID, non-overlapping QEMU lifetimes, nonzero Blob/staging/legacy counters, +checksum mismatch, active lease, quarantine, timeout, protocol error, and +server-copy failure. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_evidence.py +``` + +Expected: parser/validator imports fail. + +- [ ] **Step 3: Implement strict parsers** + +Parse only prefixed console lines and complete server JSONL records: + +```text +BADFS_DIRECT_MAP_TRACE_JSON +BADFS_LIFECYCLE_TRACE_JSON +COHERENCE_V2_STATS_JSON +``` + +Reject duplicate JSON keys, non-integer numeric fields, unknown schema +versions, truncated final JSONL records, and timestamps that go backwards in +the server trace. Save the server trace byte offset only after node0/node1 CXL +ready markers and both registration events have appeared. Only records after +that offset count toward benchmark deltas. + +- [ ] **Step 4: Implement correlation and acceptance** + +`correlate_dirty_backinvalidations()` returns records only when: + +```python +grant_start <= snoop["line_address"] +and snoop["line_address"] + 64 <= grant_start + grant_length +and snoop["dst_host"] == 1 +and snoop["snoop_id"] == ack["snoop_id"] +and ack["ack_strength"] == "MODEL" +and ack["payload_len"] == 64 +and ack["dirty_data"] is True +and snoop["monotonic_ns"] <= ack["monotonic_ns"] +``` + +The direct and lifecycle traces bind the address to the same `op_id`; the +server trace orders snoop send/ACK. Host-capture sidecars must show the flushed +client direct-unmap line before node0's flushed `store_direct_success` line. +The synchronous call chain supplies the remaining causal edge: +`store_direct_begin -> candidate_checksum -> Type-3 GETS -> snoop/ACK -> RPC +return -> store_direct_success`. Require at least one correlation record and +include these matched event records in the result. + +Parse Legofs's printed `badfs fabric stats` and require direct read/write ops +and bytes greater than zero; Blob, staging, and all four legacy op counters +equal zero; checksum failures, lease rejections, quarantine events, active +leases, and quarantined slots equal zero. Require benchmark byte count and +checksum to match the deterministic `badfs-bench` pattern. + +- [ ] **Step 5: Publish a complete atomic result** + +Write `result.json.tmp`, fsync it, and rename it to `result.json`. Include: + +```text +status,first_failure,functional_model_only,run_id,component_commits, +artifact_sha256,qemu_commands,process_lifetimes,overlap_ns,topology, +registrations,pre_benchmark_trace_offset,coherence_delta,legofs_counters, +benchmark,correlations,logs,cleanup +``` + +On any exception, preserve all logs, set `status` to `failed`, record the +first exception string, run scoped cleanup, and still atomically publish the +failed result. + +- [ ] **Step 6: Run all evidence tests** + +Run: + +```bash +python3 -m unittest -v tests/test_legofs_evidence.py \ + tests/test_legofs_runtime.py +``` + +Expected: the positive fixture passes and every negative fixture fails for +its named first reason. + +- [ ] **Step 7: Commit the proof gate** + +Run: + +```bash +git add scripts/legofs_type3_2node.py tests/test_legofs_evidence.py +git commit -m "test: require Legofs-triggered dirty back-invalidation" +``` + +### Task 10: Build, run, debug, and document the end-to-end proof + +**Files:** +- Modify: `README.md` +- Modify gitlinks: `components/qemu`, `components/cxlmemsim`, `components/legofs` +- Generated: `out/legofs-type3/results/build-manifest.json` +- Generated: `out/legofs-type3/runs//result.json` +- Generated: `out/legofs-type3/runs//{node0.log,node1.log,cxlmemsim.log,coherence.jsonl}` + +- [ ] **Step 1: Run every offline superproject test** + +Run: + +```bash +python3 -m unittest discover -s tests -v +bash -n run.sh run-legofs-type3.sh scripts/*.sh +``` + +Expected: all tests and shell syntax checks pass. + +- [ ] **Step 2: Build the complete stack** + +Run: + +```bash +./run-legofs-type3.sh --build-only --jobs "$(nproc)" --bytes 65536 +``` + +Expected: exit 0; manifest hashes verify; static RISC-V `badfs-server`, +`badfs-bench`, and PID 1 exist; QEMU, OpenSBI, U-Boot, Linux, ext2, and +CXLMemSim artifacts are nonempty. + +- [ ] **Step 3: Run the bounded end-to-end test** + +Run: + +```bash +./run-legofs-type3.sh --run-only --bytes 65536 --timeout 1200 +``` + +Expected: exit 0 and the latest `result.json` reports `status: "passed"`, +exactly two registrations with host IDs 0/1, two overlapping QEMU intervals, +strict direct reads/writes, zero fallback counters, correct checksum, nonzero +benchmark-scoped `SNP_DATA_INV`, nonzero dirty model ACK completion, and at +least one address-correlated `op_id` record. + +- [ ] **Step 4: Debug failures from the first failed invariant** + +If Step 3 fails, inspect in this order and rerun only after the first failure +is understood: + +```bash +jq . out/legofs-type3/runs/*/result.json | tail -n 120 +rg -n 'error|fail|timeout|LEG_OFS_|BADFS_|CXL|dax|region' \ + out/legofs-type3/runs/*/{node0.log,node1.log,cxlmemsim.log} +rg -n 'SNP_DATA_INV|snoop_ack|server_copy' \ + out/legofs-type3/runs/*/coherence.jsonl +``` + +The implementation is not complete while `result.json` is failed, while the +two QEMU processes did not overlap, or while the accepted invalidation came +from boot/preflight traffic. + +- [ ] **Step 5: Document the exact user workflow and claim boundary** + +Add a README section with: + +```bash +git clone --recurse-submodules git@github.com:SlugLab/CXLMemSim-riscv.git +cd CXLMemSim-riscv +./run-legofs-type3.sh --bytes 65536 +``` + +State explicitly that the command uses two concurrent +`qemu-system-riscv64 -M sifive_u` machines, one Type-3 endpoint per guest, +U-Boot CXL discovery, Linux devdax, Legofs strict lifecycle-direct I/O, and +CXLMemSim model-level MESI back-invalidation. State that this is functional +QEMU/TCG evidence, not physical-link, CPU-cache, CXL.cache, or performance +evidence. + +- [ ] **Step 6: Push component branches only after green evidence** + +Run: + +```bash +git -C components/qemu push -u origin codex/sifive-u-type3-mesi-v2 +git -C components/cxlmemsim push -u origin \ + codex/riscv-legofs-coherence-trace +git -C components/legofs push -u origin codex/riscv-type3-coherence-proof +``` + +Expected: all pushes succeed. If the Legofs remote rejects writes because +`Zettai-US/legofs` is not writable, add a SlugLab fork as `sluglab`, push the +same branch there, and update `.gitmodules` to that exact fork URL before the +superproject commit. + +- [ ] **Step 7: Record gitlinks, docs, and final verification** + +Run: + +```bash +git add components/qemu components/cxlmemsim components/legofs README.md +git commit -m "feat: prove Legofs Type 3 MESI back-invalidation" +git status --short +python3 -m unittest discover -s tests -v +jq -e '.status == "passed" and .functional_model_only == true and \ + (.correlations | length) > 0 and .coherence_delta.snp_data_inv > 0 and \ + .coherence_delta.dirty_data_completions > 0' \ + out/legofs-type3/runs/*/result.json +``` + +Expected: the worktree is clean, all tests pass, and `jq` exits zero for the +latest result. + +- [ ] **Step 8: Push the integration branch** + +Run: + +```bash +git push -u origin codex/legofs-type3-mesi-proof +``` + +Expected: GitHub contains the superproject branch and its three reachable +component commits. Do not merge to `main` until the user reviews the result +JSON and logs. + +## Final acceptance checklist + +- [ ] Both command arrays begin exactly with + `qemu-system-riscv64 -M sifive_u`. +- [ ] Two QEMU lifetimes overlap and each VM owns exactly one Type-3 endpoint. +- [ ] U-Boot and Linux evidence exists for host bridge, Type-3 decoder, + region, and CXL devdax in both guests. +- [ ] Host IDs 0 and 1 have distinct live protocol-v2 sessions. +- [ ] Legofs direct read/write counters and byte counts are nonzero. +- [ ] Blob, staging, and legacy payload counters are zero. +- [ ] Benchmark checksum and byte count match the requested workload. +- [ ] Benchmark-scoped `SNP_DATA_INV`, dirty-data completion, and model ACK + deltas are each greater than zero. +- [ ] At least one snoop line lies inside the exact lifecycle direct grant for + the same `op_id` and completes before `store_direct_success`. +- [ ] Timeout, protocol, delivery, server-copy, checksum, lease, quarantine, + and cleanup error counters are zero. +- [ ] `result.json` says `functional_model_only: true` and preserves all + commands, commits, hashes, logs, and cleanup evidence. diff --git a/docs/superpowers/specs/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation-design.md b/docs/superpowers/specs/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation-design.md new file mode 100644 index 0000000..0e69f0e --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-legofs-two-riscv-type3-mesi-backinvalidation-design.md @@ -0,0 +1,284 @@ +# Legofs on Two RISC-V Type-3 Endpoints with MESI Back-Invalidation + +## Objective + +Run `Zettai-US/legofs` end to end across two concurrently running +`qemu-system-riscv64 -M sifive_u` guests. Each guest owns one modeled CXL +Type-3 endpoint. Both endpoints join one CXLMemSim MESI write-back coherent +domain, and a real Legofs lifecycle-direct data operation must cause and +complete a remote back-invalidation. + +The final run must prove all of the following in one bounded artifact set: + +- two `sifive_u` QEMU processes overlapped in time; +- Linux discovered and configured one Type-3 memory device in each guest; +- the endpoints registered with distinct coherence host IDs; +- Legofs used strict lifecycle-direct writes and reads through guest DAX; +- the server independently read a client-written candidate during + `candidate_checksum()`; +- that read caused at least one MESI-v2 `SNP_DATA_INV` carrying dirty data and + at least one matching model-level ACK; +- the benchmark completed with the expected checksum; +- Blob, staging, and legacy payload paths remained unused. + +This is QEMU/TCG functional model evidence. It does not claim physical CXL +link behavior, native guest CPU-cache invalidation, real CXL.cache, or hardware +performance. + +## Pinned Sources + +The superproject will reference these exact inputs: + +- Legofs: `https://github.com/Zettai-US/legofs.git` at + `96f733940251d6484dad0ba2cfbe99dcf5259776`; +- CXLMemSim: `https://github.com/SlugLab/CXLMemSim.git` at + `716c16c9efc7a733006d0772f8c6c4bb055f7b15`, from + `codex/type2-hw-cc-fullsystem-20260809`; +- QEMU: the existing `components/qemu` SiFive U CXL branch, extended for a + protocol-v2 Type-3 endpoint; +- Linux: the existing `components/linux` SiFive U CXL branch; +- U-Boot and OpenSBI: the existing pinned superproject revisions. + +Legofs will be added as `components/legofs`. Existing component checkouts will +not be replaced with unrelated local working trees. + +## Firmware and Linux Contract + +The machine command remains exactly: + +```text +qemu-system-riscv64 -M sifive_u +``` + +The RISC-V `virt` machine is not an allowed substitute. + +The current SiFive U QEMU and Linux branches already contain the mechanisms +needed from the June 2026 RISC-V CXL patch series: + +- a CXL host-register region and fixed-memory-window aperture; +- CEDT and ACPI0017 publication; +- ACPI0017 `_DEP` entries for ACPI0016 CXL host bridges; +- a dedicated 256 MiB below-4-GiB non-prefetchable MMIO window for CXL BARs; +- Linux dependency release from `acpi_pci_root_add()` after the host bridge is + attached. + +The implementation must retain and test these SiFive U adaptations rather +than applying the `riscv/virt` patch literally. QEMU must continue publishing +the same firmware handoff used by U-Boot and Linux. Each guest must show the +Type-3 endpoint, CXL root/port topology, committed HDM decoder, region, and DAX +character device before Legofs starts. + +## Coherence Architecture + +### Server + +One host-side `cxlmemsim_server` runs with explicit MESI write-back protocol v2 +enabled. It owns the authoritative bytes, sparse directory, endpoint-session +registry, and snoop transactions for the shared region. + +The server accepts two TCP protocol-v2 endpoint sessions. TCP is selected +because the two QEMU processes require independent asynchronous receive paths +for unsolicited snoops; the legacy PGAS SHM slot protocol is not sufficient. +The server rejects protocol-v1 traffic in this coherent domain. + +### QEMU Type-3 Endpoint + +The reusable `cxl_memsim_v2` endpoint-cache implementation currently used by +the Type-2 model will be connected to Type-3 accesses. The Type-3 device gains +explicit properties for: + +- MESI-v2 enablement; +- server host and port; +- a unique coherence host ID; +- cache capacity and associativity; +- snoop timeout; +- write-through selection, disabled for this write-back proof. + +The endpoint registers `MODEL_SNOOP` only. It must not claim `NATIVE_FLUSH`. +Every guest load or store routed through `cxl_type3_read()` or +`cxl_type3_write()` uses the endpoint cache. Conflicting operations are sent +to the server, while unsolicited snoops are consumed by the endpoint receive +path and acknowledged only after the modeled cache transition and any dirty +data return complete. + +When MESI-v2 is selected, registration, transport, protocol, timeout, and +server-copy errors fail closed. No error may silently fall back to the local +QEMU memory backend. + +### Endpoint Identity + +The node0 Type-3 device uses coherence host ID 0. The node1 Type-3 device uses +coherence host ID 1. Session IDs remain server-issued and are recorded in the +run result. A duplicate live host ID or unexpected reconnect fails the run. + +Both guests use the same guest-visible HPA layout and region geometry, but +their QEMU device objects and endpoint caches are independent. The shared +identity is the CXLMemSim coherent domain, not a host file mapped directly by +both guests. + +## Legofs Data Flow + +Node0 boots first and runs `badfs-server`. Node1 then boots and runs +`badfs-bench` as the client. QEMU user networking exposes node0's Legofs TCP +port to node1 through the host; this control path is separate from the +host-side MESI-v2 connections. + +Both guests resolve their own DAX character-device path after Linux creates +the CXL region. The server and client must agree on the same modeled region +identity and size, without exchanging host pathnames. + +Legofs is configured with: + +- lifecycle data mode enabled; +- strict direct-final writes required; +- strict direct reads required; +- a CXL lifecycle device pointing to the guest DAX character device; +- fallback disabled by validation, not merely discouraged by configuration. + +For a direct write, the sequence is: + +1. Node0 reserves and prepares a candidate extent through its Type-3 endpoint. +2. Node1 receives the exact lifecycle grant and maps only that DAX range. +3. Node1 writes the benchmark payload and unmaps it, leaving dirty lines in its + modeled Type-3 endpoint cache. +4. Node1 calls `lifecycle_store_direct()` with the full checksum. +5. Node0 executes Legofs `candidate_checksum()` and reads the same extent + through its own DAX mapping and Type-3 endpoint. +6. CXLMemSim issues `SNP_DATA_INV` to node1, receives the dirty line and model + ACK, commits it to authoritative storage, and then completes node0's read. +7. Legofs compares the checksum, persists, commits, and publishes the extent. + +Thus the required back-invalidation is caused by the actual Legofs seal path, +not by an unrelated litmus test. Direct reads then verify the published data. + +## Build and Run Interface + +The superproject will provide one top-level entry point for the bounded proof. +It will: + +1. validate pinned submodules and required host tools; +2. build CXLMemSim with protocol v2; +3. build QEMU with SiFive U, CXL, and Type-3 MESI-v2 support; +4. build the existing Linux, U-Boot, and OpenSBI artifacts as needed; +5. cross-build the required Legofs RISC-V binaries; +6. create per-node guest images without loop mounting; +7. start the CXLMemSim server; +8. boot node0 and wait for CXL/DAX and Legofs readiness; +9. boot node1 and run the bounded benchmark; +10. stop only processes started by this run; +11. validate logs and atomically publish a JSON result. + +The command builder must expose both complete QEMU command arrays in the +result. Tests must assert that each begins with +`qemu-system-riscv64 -M sifive_u` and includes exactly one Type-3 endpoint. + +Run directories are unique and contain node logs, server logs, manifests, +commands, extracted counters, and the final JSON. Shared-memory objects, +ports, and process IDs are run-scoped. Cleanup uses recorded process IDs and +must not kill unrelated QEMU or CXLMemSim processes. + +## Telemetry and Correlation + +Protocol-v2 telemetry must expose at least: + +- registered endpoint and session IDs; +- GETS, GETM, UPGRADE, PUTS, and PUTM counts; +- snoop counts by opcode; +- model ACK counts by snoop opcode; +- dirty-data snoop completions; +- timeouts, protocol errors, and failed server-copy commits; +- directory state for the lines involved in the proof, or equivalent + transaction trace records. + +The runner captures a counter snapshot after both nodes are ready but before +the benchmark starts, then another after Legofs finishes. Acceptance uses the +delta. Boot-time or preflight snoops do not count. + +Legofs lifecycle trace records and MESI transaction records must share enough +information to establish temporal and address-range correlation: the +`SNP_DATA_INV` transaction occurs after the direct client write and before the +corresponding `store_direct` succeeds, and the cache-line address lies within +the lifecycle grant's mapped range. + +## Error Handling + +The run fails if any of these conditions occurs: + +- either QEMU exits before benchmark completion; +- the two QEMU process lifetimes do not overlap; +- the machine is not exactly `sifive_u`; +- a guest lacks the expected CXL endpoint, region, decoder, or DAX device; +- endpoint registration is missing, duplicated, or uses the wrong host ID; +- Legofs strict-direct initialization fails; +- Legofs uses Blob, staging, or a legacy payload path; +- the server reports a protocol error, snoop timeout, failed data commit, or + unresolved session; +- Legofs reports a checksum failure, rejected lease, quarantine event, or + leaked active lease; +- the benchmark checksum is wrong; +- the benchmark-scoped `SNP_DATA_INV`, dirty-data completion, or matching ACK + delta is zero. + +On failure the runner preserves logs and writes a failed result with the first +failure reason. It still performs scoped process and temporary-object cleanup. + +## Verification Layers + +### Static and Unit Tests + +- Type-3 property parsing and invalid configuration rejection. +- Type-3 read/write routing through the v2 endpoint cache. +- Fail-closed behavior for transport and registration errors. +- Snoop invalidation and dirty-data ACK behavior using the QEMU unit fake. +- Two-node command construction preserving exact `sifive_u` topology and + distinct host IDs. +- Legofs submodule pin and RISC-V artifact checks. +- Result parser rejection for zero or uncorrelated invalidation evidence. + +### Component Integration Tests + +- Existing CXLMemSim protocol-v2, directory, endpoint-cache, session, SHM, and + TCP tests. +- Existing QEMU SiFive U firmware/ACPI and Type-3 tests. +- Existing Linux, U-Boot, and superproject build-contract tests. +- Legofs core tests plus RISC-V cross-build checks. + +### End-to-End Acceptance + +A successful result requires: + +- two overlapping live QEMU processes; +- two successful guest boot markers; +- two Type-3 endpoint registrations with host IDs 0 and 1; +- nonzero `trusted_direct_write_ops` and `trusted_direct_write_bytes`; +- nonzero `trusted_direct_read_ops` and `trusted_direct_read_bytes`; +- zero Blob and staging operation/byte counters; +- zero legacy data-path counters; +- zero checksum failures, lease rejections, quarantine events, and active + leases after shutdown; +- correct benchmark byte counts and checksum; +- benchmark-scoped `SNP_DATA_INV > 0`; +- benchmark-scoped dirty-data snoop completions and matching model ACKs greater + than zero; +- at least one correlated Legofs grant/transaction address range. + +The final JSON reports the exact commits, artifact hashes, commands, +environment, topology, guest evidence, Legofs counters, coherence counter +deltas, correlation records, cleanup status, and the explicit functional-model +claim boundary. + +## Repository and Change Boundaries + +Changes are limited to: + +- the CXLMemSim branch when server telemetry or protocol behavior is missing; +- the QEMU branch for Type-3 MESI-v2 integration; +- Legofs only where RISC-V guest orchestration, trace correlation, or strict + direct-path reporting requires it; +- the superproject for the new Legofs submodule, build/run orchestration, + tests, documentation, and pinned revisions. + +The existing Linux CXL dependency and SiFive U resource fixes are tested but +not rewritten unless the end-to-end run reveals a concrete defect. Type-2, +physical-hardware, latency-injection, IO500, MPI, and performance-comparison +work are outside this milestone. diff --git a/guest/legofs_node_init.c b/guest/legofs_node_init.c new file mode 100644 index 0000000..e8dc7df --- /dev/null +++ b/guest/legofs_node_init.c @@ -0,0 +1,1095 @@ +/* Strict freestanding PID 1 for the two-node Legofs/CXL Type-3 proof. */ + +typedef __SIZE_TYPE__ size_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long uint64_t; +typedef long int64_t; + +#define AT_FDCWD (-100) +#define AT_SYMLINK_NOFOLLOW 0x100 +#define O_RDONLY 0 +#define O_RDWR 2 +#define O_DIRECTORY 00200000 +#define MS_RDONLY 1 +#define S_IFCHR 0020000 +#define SIGCHLD 17 + +#define AF_INET 2 +#define SOCK_STREAM 1 +#define IFF_UP 0x1 +#define RTF_UP 0x1 +#define RTF_GATEWAY 0x2 +#define SIOCADDRT 0x890b +#define SIOCGIFFLAGS 0x8913 +#define SIOCSIFFLAGS 0x8914 +#define SIOCSIFADDR 0x8916 +#define SIOCSIFNETMASK 0x891c + +#define SYS_DUP3 24 +#define SYS_IOCTL 29 +#define SYS_MKNODAT 33 +#define SYS_MKDIRAT 34 +#define SYS_MOUNT 40 +#define SYS_OPENAT 56 +#define SYS_CLOSE 57 +#define SYS_GETDENTS64 61 +#define SYS_READ 63 +#define SYS_WRITE 64 +#define SYS_SYNC 81 +#define SYS_EXIT 93 +#define SYS_NANOSLEEP 101 +#define SYS_REBOOT 142 +#define SYS_SOCKET 198 +#define SYS_CONNECT 203 +#define SYS_CLONE 220 +#define SYS_EXECVE 221 +#define SYS_WAIT4 260 +#define SYS_STATX 291 + +#define LINUX_REBOOT_MAGIC1 0xfee1dead +#define LINUX_REBOOT_MAGIC2 672274793 +#define LINUX_REBOOT_CMD_POWER_OFF 0x4321fedc + +#define MAX_CMDLINE 2048 +#define MAX_PATH 160 +#define MAX_ENV 32 + +struct kernel_timespec { + long tv_sec; + long tv_nsec; +}; + +struct linux_dirent64 { + uint64_t d_ino; + int64_t d_off; + uint16_t d_reclen; + uint8_t d_type; + char d_name[]; +}; + +struct sockaddr { + uint16_t family; + uint8_t data[14]; +}; + +struct sockaddr_in { + uint16_t family; + uint16_t port; + uint32_t address; + uint8_t zero[8]; +}; + +struct ifreq { + char name[16]; + union { + struct sockaddr address; + short flags; + uint8_t padding[24]; + } value; +}; + +struct rtentry { + unsigned long pad1; + struct sockaddr destination; + struct sockaddr gateway; + struct sockaddr genmask; + unsigned short flags; + short pad2; + unsigned long pad3; + void *pad4; + short metric; + char *device; + unsigned long mtu; + unsigned long window; + unsigned short irtt; +}; + +struct statx_timestamp { + int64_t seconds; + uint32_t nanoseconds; + int reserved; +}; + +struct statx_record { + uint32_t mask; + uint32_t block_size; + uint64_t attributes; + uint32_t links; + uint32_t uid; + uint32_t gid; + uint16_t mode; + uint16_t spare0; + uint64_t inode; + uint64_t size; + uint64_t blocks; + uint64_t attributes_mask; + struct statx_timestamp atime; + struct statx_timestamp btime; + struct statx_timestamp ctime; + struct statx_timestamp mtime; + uint32_t rdev_major; + uint32_t rdev_minor; + uint32_t dev_major; + uint32_t dev_minor; + uint64_t mount_id; + uint32_t dio_memory_align; + uint32_t dio_offset_align; + uint64_t subvolume; + uint32_t atomic_write_unit_min; + uint32_t atomic_write_unit_max; + uint32_t atomic_write_segments_max; + uint32_t dio_read_offset_align; + uint64_t spare[9]; +}; + +struct dax_device { + char name[64]; + char path[MAX_PATH]; + uint32_t major; + uint32_t minor; + uint64_t size; + uint64_t region_hi; + uint64_t region_lo; +}; + +enum node_role { + ROLE_INVALID = 0, + ROLE_NODE0, + ROLE_NODE1, +}; + +#if defined(__riscv) +static long syscall6(long number, long arg0, long arg1, long arg2, + long arg3, long arg4, long arg5) +{ + register long a0 __asm__("a0") = arg0; + register long a1 __asm__("a1") = arg1; + register long a2 __asm__("a2") = arg2; + register long a3 __asm__("a3") = arg3; + register long a4 __asm__("a4") = arg4; + register long a5 __asm__("a5") = arg5; + register long a7 __asm__("a7") = number; + + __asm__ volatile("ecall" + : "+r"(a0) + : "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5), + "r"(a7) + : "memory"); + return a0; +} +#else +static long syscall6(long number, long arg0, long arg1, long arg2, + long arg3, long arg4, long arg5) +{ + (void)number; + (void)arg0; + (void)arg1; + (void)arg2; + (void)arg3; + (void)arg4; + (void)arg5; + return -38; +} +#endif + +static long syscall5(long number, long arg0, long arg1, long arg2, + long arg3, long arg4) +{ + return syscall6(number, arg0, arg1, arg2, arg3, arg4, 0); +} + +static long syscall4(long number, long arg0, long arg1, long arg2, long arg3) +{ + return syscall6(number, arg0, arg1, arg2, arg3, 0, 0); +} + +static long syscall3(long number, long arg0, long arg1, long arg2) +{ + return syscall6(number, arg0, arg1, arg2, 0, 0, 0); +} + +static long syscall2(long number, long arg0, long arg1) +{ + return syscall6(number, arg0, arg1, 0, 0, 0, 0); +} + +static long syscall1(long number, long arg0) +{ + return syscall6(number, arg0, 0, 0, 0, 0, 0); +} + +void *memset(void *destination, int value, size_t length) +{ + uint8_t *output = destination; + + while (length--) + *output++ = (uint8_t)value; + return destination; +} + +void *memcpy(void *destination, const void *source, size_t length) +{ + uint8_t *output = destination; + const uint8_t *input = source; + + while (length--) + *output++ = *input++; + return destination; +} + +static size_t text_length(const char *text) +{ + size_t length = 0; + + while (text[length]) + length++; + return length; +} + +static void memory_zero(void *pointer, size_t length) +{ + uint8_t *bytes = pointer; + + while (length--) + *bytes++ = 0; +} + +static void text_copy(char *destination, size_t capacity, const char *source) +{ + size_t index = 0; + + if (!capacity) + return; + while (source[index] && index + 1 < capacity) { + destination[index] = source[index]; + index++; + } + destination[index] = '\0'; +} + +static int text_equal(const char *left, const char *right) +{ + while (*left && *right) { + if (*left++ != *right++) + return 0; + } + return *left == *right; +} + +static int text_starts_with(const char *text, const char *prefix) +{ + while (*prefix) { + if (*text++ != *prefix++) + return 0; + } + return 1; +} + +static void write_text(const char *text) +{ + size_t remaining = text_length(text); + + while (remaining) { + long written = syscall3(SYS_WRITE, 1, (long)text, (long)remaining); + + if (written <= 0) + return; + text += written; + remaining -= (size_t)written; + } +} + +static void write_unsigned(uint64_t value) +{ + char reversed[24]; + size_t count = 0; + + do { + reversed[count++] = (char)('0' + value % 10); + value /= 10; + } while (value); + while (count) { + char digit = reversed[--count]; + + syscall3(SYS_WRITE, 1, (long)&digit, 1); + } +} + +static void write_hex_byte(uint8_t value) +{ + static const char digits[] = "0123456789abcdef"; + char pair[2]; + + pair[0] = digits[value >> 4]; + pair[1] = digits[value & 15]; + syscall3(SYS_WRITE, 1, (long)pair, 2); +} + +static void write_region_id(uint64_t hi, uint64_t lo) +{ + uint8_t bytes[16]; + unsigned int index; + + for (index = 0; index < 8; index++) + bytes[index] = (uint8_t)(hi >> (56 - index * 8)); + for (index = 0; index < 8; index++) + bytes[index + 8] = (uint8_t)(lo >> (56 - index * 8)); + for (index = 0; index < 16; index++) { + if (index == 4 || index == 6 || index == 8 || index == 10) + write_text("-"); + write_hex_byte(bytes[index]); + } +} + +static void sleep_milliseconds(unsigned int milliseconds) +{ + struct kernel_timespec delay; + + delay.tv_sec = milliseconds / 1000; + delay.tv_nsec = (long)(milliseconds % 1000) * 1000000L; + syscall2(SYS_NANOSLEEP, (long)&delay, 0); +} + +static void power_off(void) __attribute__((noreturn)); + +static void power_off(void) +{ + syscall1(SYS_SYNC, 0); + syscall4(SYS_REBOOT, LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, + LINUX_REBOOT_CMD_POWER_OFF, 0); + for (;;) +#if defined(__riscv) + __asm__ volatile("wfi"); +#else + ; +#endif +} + +static void fail(const char *phase, long error) __attribute__((noreturn)); + +static void fail(const char *phase, long error) +{ + if (error < 0) + error = -error; + if (!error) + error = 1; + write_text("LEG_OFS_FAIL phase="); + write_text(phase); + write_text(" errno="); + write_unsigned((uint64_t)error); + write_text("\n"); + power_off(); +} + +static void make_directory(const char *path) +{ + long result = syscall3(SYS_MKDIRAT, AT_FDCWD, (long)path, 0755); + + if (result < 0 && result != -17) + fail("mkdir", result); +} + +static void mount_one(const char *source, const char *target, const char *type, + unsigned long flags, const char *data, const char *phase) +{ + long result = syscall5(SYS_MOUNT, (long)source, (long)target, + (long)type, (long)flags, (long)data); + + if (result < 0 && result != -16) + fail(phase, result); +} + +static void reopen_console(void) +{ + long console = syscall4(SYS_OPENAT, AT_FDCWD, (long)"/dev/console", O_RDWR, 0); + int target; + + if (console < 0) + fail("open-console", console); + for (target = 0; target <= 2; target++) { + long result; + + if (console == target) + continue; + result = syscall3(SYS_DUP3, console, target, 0); + if (result < 0) + fail("dup-console", result); + } + if (console > 2) + syscall1(SYS_CLOSE, console); +} + +static long read_file(const char *path, char *buffer, size_t capacity) +{ + long file; + long total = 0; + + if (capacity < 2) + return -22; + file = syscall4(SYS_OPENAT, AT_FDCWD, (long)path, O_RDONLY, 0); + if (file < 0) + return file; + while ((size_t)total + 1 < capacity) { + long count = syscall3(SYS_READ, file, (long)(buffer + total), + (long)(capacity - (size_t)total - 1)); + + if (count < 0) { + syscall1(SYS_CLOSE, file); + return count; + } + if (!count) + break; + total += count; + } + syscall1(SYS_CLOSE, file); + buffer[total] = '\0'; + return total; +} + +static uint64_t parse_unsigned(const char *text, int *valid) +{ + uint64_t value = 0; + unsigned int base = 10; + size_t index = 0; + + *valid = 0; + if (text[0] == '0' && (text[1] == 'x' || text[1] == 'X')) { + base = 16; + index = 2; + } + if (!text[index]) + return 0; + for (; text[index] && text[index] != '\n' && text[index] != '\r'; index++) { + unsigned int digit; + + if (text[index] >= '0' && text[index] <= '9') + digit = (unsigned int)(text[index] - '0'); + else if (text[index] >= 'a' && text[index] <= 'f') + digit = (unsigned int)(text[index] - 'a' + 10); + else if (text[index] >= 'A' && text[index] <= 'F') + digit = (unsigned int)(text[index] - 'A' + 10); + else + return 0; + if (digit >= base || value > (~(uint64_t)0 - digit) / base) + return 0; + value = value * base + digit; + } + *valid = 1; + return value; +} + +static int read_unsigned_file(const char *path, uint64_t *value) +{ + char buffer[80]; + long length = read_file(path, buffer, sizeof(buffer)); + int valid; + + if (length <= 0) + return 0; + *value = parse_unsigned(buffer, &valid); + return valid; +} + +static int path_exists(const char *path, int directory) +{ + long file = syscall4(SYS_OPENAT, AT_FDCWD, (long)path, + O_RDONLY | (directory ? O_DIRECTORY : 0), 0); + + if (file < 0) + return 0; + syscall1(SYS_CLOSE, file); + return 1; +} + +static void append_text(char *destination, size_t capacity, const char *source) +{ + size_t used = text_length(destination); + + if (used < capacity) + text_copy(destination + used, capacity - used, source); +} + +static int scan_prefix(const char *directory, const char *prefix, char *only_name, + size_t capacity) +{ + uint8_t buffer[4096]; + long file = syscall4(SYS_OPENAT, AT_FDCWD, (long)directory, + O_RDONLY | O_DIRECTORY, 0); + int count = 0; + + if (file < 0) + return -1; + for (;;) { + long bytes = syscall3(SYS_GETDENTS64, file, (long)buffer, sizeof(buffer)); + long offset = 0; + + if (bytes < 0) { + syscall1(SYS_CLOSE, file); + return -1; + } + if (!bytes) + break; + while (offset < bytes) { + struct linux_dirent64 *entry = (struct linux_dirent64 *)(buffer + offset); + + if (entry->d_reclen < + __builtin_offsetof(struct linux_dirent64, d_name) + 1 || + offset + entry->d_reclen > bytes) { + syscall1(SYS_CLOSE, file); + return -1; + } + if (text_starts_with(entry->d_name, prefix)) { + count++; + if (count == 1 && only_name) + text_copy(only_name, capacity, entry->d_name); + } + offset += entry->d_reclen; + } + } + syscall1(SYS_CLOSE, file); + return count; +} + +static int wait_for_prefix(const char *directory, const char *prefix, + int expected_count) +{ + int attempt; + int count = -1; + + for (attempt = 0; attempt < 120; attempt++) { + count = scan_prefix(directory, prefix, 0, 0); + if (count >= expected_count) + return count; + sleep_milliseconds(250); + } + return count; +} + +static uint64_t linux_device_number(uint32_t major, uint32_t minor) +{ + return ((uint64_t)(major & 0xfff) << 8) | (minor & 0xff) | + ((uint64_t)(minor & ~0xffU) << 12) | + ((uint64_t)(major & ~0xfffU) << 32); +} + +static uint64_t rotate_left(uint64_t value, unsigned int shift) +{ + return (value << shift) | (value >> (64 - shift)); +} + +static uint64_t stable_region_mix(uint64_t value) +{ + value ^= 0x62616466732d7267UL; + value += 0x9e3779b97f4a7c15UL; + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9UL; + value = (value ^ (value >> 27)) * 0x94d049bb133111ebUL; + return value ^ (value >> 31); +} + +static void derive_region_id(struct dax_device *device) +{ + struct statx_record status; + uint64_t dev; + uint64_t rdev; + long result; + + memory_zero(&status, sizeof(status)); + result = syscall5(SYS_STATX, AT_FDCWD, (long)device->path, + AT_SYMLINK_NOFOLLOW, 0x7ff, (long)&status); + if (result < 0) + fail("statx-dax", result); + dev = linux_device_number(status.dev_major, status.dev_minor); + rdev = linux_device_number(status.rdev_major, status.rdev_minor); + device->region_hi = stable_region_mix(dev ^ rotate_left(rdev, 17) ^ + rotate_left(status.size, 31) ^ + rotate_left(2, 7)); + device->region_lo = stable_region_mix(status.inode ^ rotate_left(dev, 29) ^ + rotate_left(rdev, 11) ^ status.size); +} + +static void discover_dax(struct dax_device *device) +{ + char sysfs[MAX_PATH]; + char contents[80]; + char *colon = 0; + long length; + uint64_t value; + int valid; + size_t index; + + memory_zero(device, sizeof(*device)); + if (scan_prefix("/sys/bus/dax/devices", "dax", device->name, + sizeof(device->name)) != 1) + fail("dax-device-count", 19); + text_copy(device->path, sizeof(device->path), "/dev/"); + append_text(device->path, sizeof(device->path), device->name); + text_copy(sysfs, sizeof(sysfs), "/sys/bus/dax/devices/"); + append_text(sysfs, sizeof(sysfs), device->name); + append_text(sysfs, sizeof(sysfs), "/dev"); + length = read_file(sysfs, contents, sizeof(contents)); + if (length <= 0) + fail("read-dax-dev", length); + for (index = 0; contents[index]; index++) { + if (contents[index] == ':') { + colon = &contents[index]; + break; + } + } + if (!colon) + fail("parse-dax-dev", 22); + *colon = '\0'; + value = parse_unsigned(contents, &valid); + if (!valid || value > 0xffffffffUL) + fail("parse-dax-major", 22); + device->major = (uint32_t)value; + value = parse_unsigned(colon + 1, &valid); + if (!valid || value > 0xffffffffUL) + fail("parse-dax-minor", 22); + device->minor = (uint32_t)value; + + text_copy(sysfs, sizeof(sysfs), "/sys/bus/dax/devices/"); + append_text(sysfs, sizeof(sysfs), device->name); + append_text(sysfs, sizeof(sysfs), "/size"); + if (!read_unsigned_file(sysfs, &device->size) || !device->size) + fail("read-dax-size", 22); + + value = linux_device_number(device->major, device->minor); + length = syscall4(SYS_MKNODAT, AT_FDCWD, (long)device->path, + S_IFCHR | 0600, (long)value); + if (length < 0 && length != -17) + fail("mknod-dax", length); + if (!path_exists(device->path, 0)) + fail("open-dax", 19); + derive_region_id(device); +} + +static uint32_t ipv4(unsigned int a, unsigned int b, unsigned int c, unsigned int d) +{ + return a | (b << 8) | (c << 16) | (d << 24); +} + +static uint16_t network_u16(uint16_t value) +{ + return (uint16_t)((value << 8) | (value >> 8)); +} + +static void set_ifreq_name(struct ifreq *request, const char *name) +{ + memory_zero(request, sizeof(*request)); + text_copy(request->name, sizeof(request->name), name); +} + +static void set_sockaddr(struct sockaddr *address, uint32_t ipv4_address) +{ + struct sockaddr_in *inet = (struct sockaddr_in *)address; + + memory_zero(address, sizeof(*address)); + inet->family = AF_INET; + inet->address = ipv4_address; +} + +static void configure_network(void) +{ + struct ifreq request; + struct rtentry route; + long socket = syscall3(SYS_SOCKET, AF_INET, SOCK_STREAM, 0); + long result; + + if (socket < 0) + fail("network-socket", socket); + set_ifreq_name(&request, "lo"); + set_sockaddr(&request.value.address, ipv4(127, 0, 0, 1)); + result = syscall3(SYS_IOCTL, socket, SIOCSIFADDR, (long)&request); + if (result < 0) + fail("loopback-address", result); + set_ifreq_name(&request, "lo"); + set_sockaddr(&request.value.address, ipv4(255, 0, 0, 0)); + result = syscall3(SYS_IOCTL, socket, SIOCSIFNETMASK, (long)&request); + if (result < 0) + fail("loopback-netmask", result); + set_ifreq_name(&request, "lo"); + result = syscall3(SYS_IOCTL, socket, SIOCGIFFLAGS, (long)&request); + if (result < 0) + fail("loopback-get-flags", result); + request.value.flags |= IFF_UP; + result = syscall3(SYS_IOCTL, socket, SIOCSIFFLAGS, (long)&request); + if (result < 0) + fail("loopback-set-flags", result); + + set_ifreq_name(&request, "eth0"); + set_sockaddr(&request.value.address, ipv4(10, 0, 2, 15)); + result = syscall3(SYS_IOCTL, socket, SIOCSIFADDR, (long)&request); + if (result < 0) + fail("network-address", result); + set_ifreq_name(&request, "eth0"); + set_sockaddr(&request.value.address, ipv4(255, 255, 255, 0)); + result = syscall3(SYS_IOCTL, socket, SIOCSIFNETMASK, (long)&request); + if (result < 0) + fail("network-netmask", result); + set_ifreq_name(&request, "eth0"); + result = syscall3(SYS_IOCTL, socket, SIOCGIFFLAGS, (long)&request); + if (result < 0) + fail("network-get-flags", result); + request.value.flags |= IFF_UP; + result = syscall3(SYS_IOCTL, socket, SIOCSIFFLAGS, (long)&request); + if (result < 0) + fail("network-set-flags", result); + + memory_zero(&route, sizeof(route)); + set_sockaddr(&route.destination, ipv4(0, 0, 0, 0)); + set_sockaddr(&route.gateway, ipv4(10, 0, 2, 2)); + set_sockaddr(&route.genmask, ipv4(0, 0, 0, 0)); + route.flags = RTF_UP | RTF_GATEWAY; + route.device = (char *)"eth0"; + result = syscall3(SYS_IOCTL, socket, SIOCADDRT, (long)&route); + if (result < 0 && result != -17) + fail("network-route", result); + syscall1(SYS_CLOSE, socket); +} + +static long connect_tcp(uint32_t address, uint16_t port) +{ + struct sockaddr_in peer; + long socket = syscall3(SYS_SOCKET, AF_INET, SOCK_STREAM, 0); + long result; + + if (socket < 0) + return socket; + memory_zero(&peer, sizeof(peer)); + peer.family = AF_INET; + peer.port = network_u16(port); + peer.address = address; + result = syscall3(SYS_CONNECT, socket, (long)&peer, sizeof(peer)); + syscall1(SYS_CLOSE, socket); + return result; +} + +static const char *cmdline_value(char *cmdline, const char *key) +{ + size_t key_length = text_length(key); + char *cursor = cmdline; + + while (*cursor) { + char *token; + char *end; + + while (*cursor == ' ') + cursor++; + if (!*cursor) + break; + token = cursor; + while (*cursor && *cursor != ' ' && *cursor != '\n') + cursor++; + end = cursor; + if (*cursor) + *cursor++ = '\0'; + if (text_starts_with(token, key) && token[key_length]) + return token + key_length; + (void)end; + } + return 0; +} + +static void unsigned_to_text(uint64_t value, char *output, size_t capacity) +{ + char reversed[24]; + size_t count = 0; + size_t index = 0; + + if (!capacity) + return; + do { + reversed[count++] = (char)('0' + value % 10); + value /= 10; + } while (value); + while (count && index + 1 < capacity) + output[index++] = reversed[--count]; + output[index] = '\0'; +} + +static long spawn(const char *path, char *const environment[]) +{ + static char *const arguments[] = {(char *)"badfs", 0}; + long child = syscall5(SYS_CLONE, SIGCHLD, 0, 0, 0, 0); + + if (child < 0) + return child; + if (!child) { + syscall3(SYS_EXECVE, (long)path, (long)arguments, (long)environment); + syscall1(SYS_EXIT, 127); + for (;;) + ; + } + return child; +} + +static int wait_child(long child) +{ + int status = 0; + long result; + + do { + result = syscall4(SYS_WAIT4, child, (long)&status, 0, 0); + } while (result == -4); + if (result != child) + return -1; + if ((status & 0x7f) != 0) + return -1; + return (status >> 8) & 0xff; +} + +static void wait_for_run_gate(void) +{ + static const char expected[] = "LEG_OFS_RUN"; + char line[64]; + size_t used = 0; + + while (used + 1 < sizeof(line)) { + char byte; + long count = syscall3(SYS_READ, 0, (long)&byte, 1); + + if (count < 0) { + if (count == -4) + continue; + fail("run-gate-read", count); + } + if (!count) + fail("run-gate-eof", 5); + if (byte == '\r') + continue; + if (byte == '\n') { + line[used] = '\0'; + if (text_equal(line, expected)) + return; + used = 0; + continue; + } + line[used++] = byte; + } + fail("run-gate-overflow", 7); +} + +static size_t add_environment(char **environment, size_t count, char *entry) +{ + if (count + 1 >= MAX_ENV) + fail("environment-capacity", 7); + environment[count++] = entry; + environment[count] = 0; + return count; +} + +static size_t common_environment(char **environment, char *device_entry) +{ + size_t count = 0; + + count = add_environment(environment, count, (char *)"BADFS_POSIX_DATA_PATH=lifecycle"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_BLOB=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_DIRECT_FINAL=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_DIRECT_REQUIRED=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_DIRECT_READ=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_DIRECT_READ_REQUIRED=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_DEVICE_REQUIRED=1"); + count = add_environment(environment, count, (char *)"BADFS_CXL_MAP_ALIGNMENT=2097152"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_TRACE=/tmp/lifecycle.jsonl"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_TRACE_STDOUT=1"); + count = add_environment(environment, count, (char *)"BADFS_CXL_DIRECT_TRACE=/tmp/direct.jsonl"); + count = add_environment(environment, count, (char *)"BADFS_CXL_DIRECT_TRACE_STDOUT=1"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_POOL_SIZE=268435456"); + count = add_environment(environment, count, (char *)"BADFS_LIFECYCLE_MAX_EXTENTS=127"); + count = add_environment(environment, count, (char *)"RUST_LOG=info"); + count = add_environment(environment, count, device_entry); + return count; +} + +static void run_node0(char *device_entry) +{ + char *environment[MAX_ENV] = {0}; + size_t count = common_environment(environment, device_entry); + long server; + long probe = -111; + unsigned int attempt; + + count = add_environment(environment, count, (char *)"BADFS_SERVER_ADDR=0.0.0.0:3345"); + (void)add_environment(environment, count, (char *)"BADFS_DATA_DIR=/tmp/badfs-data"); + server = spawn("/mnt/badfs-server", environment); + if (server < 0) + fail("spawn-server", server); + for (attempt = 0; attempt < 240; attempt++) { + probe = connect_tcp(ipv4(127, 0, 0, 1), 3345); + if (probe == 0) { + write_text("LEG_OFS_SERVER_READY addr=0.0.0.0:3345\n"); + if (wait_child(server) != 0) + fail("server-exit", 5); + fail("server-stopped", 5); + } + if (attempt && attempt % 40 == 0) { + write_text("LEG_OFS_SERVER_PROBE errno="); + write_unsigned((uint64_t)(probe < 0 ? -probe : probe)); + write_text("\n"); + } + sleep_milliseconds(250); + } + fail("server-ready-timeout", 110); +} + +static void run_node1(char *device_entry, uint16_t server_port, uint64_t bytes) +{ + char *environment[MAX_ENV] = {0}; + char server_entry[64] = "BADFS_SERVERS=10.0.2.2:"; + char port_text[16]; + char bytes_entry[64] = "BADFS_BENCH_FILE_SIZE="; + char bytes_text[24]; + char block_entry[64] = "BADFS_BENCH_BLOCK_SIZE="; + char block_text[24]; + uint64_t block_size = bytes < 1048576 ? bytes : 1048576; + char mode_workload[] = "BADFS_BENCH_MODE=workload"; + char mode_inspect[] = "BADFS_BENCH_MODE=inspect"; + size_t count = common_environment(environment, device_entry); + long child; + int status; + + unsigned_to_text(server_port, port_text, sizeof(port_text)); + append_text(server_entry, sizeof(server_entry), port_text); + unsigned_to_text(bytes, bytes_text, sizeof(bytes_text)); + append_text(bytes_entry, sizeof(bytes_entry), bytes_text); + unsigned_to_text(block_size, block_text, sizeof(block_text)); + append_text(block_entry, sizeof(block_entry), block_text); + count = add_environment(environment, count, server_entry); + count = add_environment(environment, count, (char *)"BADFS_BASE_PATH=/badfs"); + count = add_environment(environment, count, mode_workload); + count = add_environment(environment, count, bytes_entry); + count = add_environment(environment, count, block_entry); + (void)add_environment(environment, count, (char *)"BADFS_BENCH_ITERATIONS=1"); + + write_text("LEG_OFS_CLIENT_READY\n"); + wait_for_run_gate(); + write_text("LEG_OFS_BENCHMARK_BEGIN\n"); + child = spawn("/mnt/badfs-bench", environment); + if (child < 0) + fail("spawn-workload", child); + status = wait_child(child); + if (status != 0) + fail("benchmark-workload", status < 0 ? 5 : status); + + for (count = 0; environment[count]; count++) { + if (text_starts_with(environment[count], "BADFS_BENCH_MODE=")) { + environment[count] = mode_inspect; + break; + } + } + if (!environment[count]) + fail("inspect-environment", 22); + child = spawn("/mnt/badfs-bench", environment); + if (child < 0) + fail("spawn-inspect", child); + status = wait_child(child); + if (status != 0) + fail("benchmark-inspect", status < 0 ? 5 : status); + write_text("LEG_OFS_BENCHMARK_PASS\n"); + power_off(); +} + +void _start(void) +{ + char cmdline[MAX_CMDLINE]; + char role_copy[16]; + char port_copy[16]; + char bytes_copy[32]; + char device_entry[MAX_PATH + 32] = "BADFS_LIFECYCLE_DEVICE="; + const char *role_value; + const char *port_value; + const char *bytes_value; + enum node_role role = ROLE_INVALID; + uint64_t port; + uint64_t bytes; + int valid; + struct dax_device dax; + + make_directory("/proc"); + make_directory("/sys"); + make_directory("/dev"); + make_directory("/tmp"); + make_directory("/mnt"); + mount_one("proc", "/proc", "proc", 0, 0, "mount-proc"); + mount_one("sysfs", "/sys", "sysfs", 0, 0, "mount-sys"); + mount_one("devtmpfs", "/dev", "devtmpfs", 0, "mode=0755", "mount-dev"); + mount_one("tmpfs", "/tmp", "tmpfs", 0, "mode=0755", "mount-tmp"); + reopen_console(); + + if (read_file("/proc/cmdline", cmdline, sizeof(cmdline)) <= 0) + fail("read-cmdline", 5); + role_value = cmdline_value(cmdline, "legofs.role="); + if (!role_value) + fail("missing-role", 22); + text_copy(role_copy, sizeof(role_copy), role_value); + if (text_equal(role_copy, "node0")) + role = ROLE_NODE0; + else if (text_equal(role_copy, "node1")) + role = ROLE_NODE1; + else + fail("invalid-role", 22); + + /* cmdline_value mutates separators, so reread for each required key. */ + if (read_file("/proc/cmdline", cmdline, sizeof(cmdline)) <= 0) + fail("reread-cmdline-port", 5); + port_value = cmdline_value(cmdline, "legofs.server_port="); + if (!port_value) + fail("missing-server-port", 22); + text_copy(port_copy, sizeof(port_copy), port_value); + port = parse_unsigned(port_copy, &valid); + if (!valid || !port || port > 65535) + fail("invalid-server-port", 22); + + if (read_file("/proc/cmdline", cmdline, sizeof(cmdline)) <= 0) + fail("reread-cmdline-bytes", 5); + bytes_value = cmdline_value(cmdline, "legofs.bytes="); + if (!bytes_value) + fail("missing-bytes", 22); + text_copy(bytes_copy, sizeof(bytes_copy), bytes_value); + bytes = parse_unsigned(bytes_copy, &valid); + if (!valid || !bytes || bytes > 16777216 || bytes % 4096) + fail("invalid-bytes", 22); + + for (valid = 0; valid < 120 && !path_exists("/dev/vda", 0); valid++) + sleep_milliseconds(250); + if (!path_exists("/dev/vda", 0)) + fail("payload-disk-timeout", 110); + mount_one("/dev/vda", "/mnt", "ext2", MS_RDONLY, 0, "mount-payload"); + if (!path_exists("/mnt/badfs-server", 0) || !path_exists("/mnt/badfs-bench", 0)) + fail("payload-binaries", 2); + + configure_network(); + for (valid = 0; valid < 120 && + !path_exists("/sys/bus/cxl/devices/mem0", 1); valid++) + sleep_milliseconds(250); + if (!path_exists("/sys/bus/cxl/devices/mem0", 1)) + fail("missing-cxl-mem0", 19); + if (wait_for_prefix("/sys/bus/cxl/devices", "region", 1) < 1) + fail("missing-cxl-region", 19); + if (wait_for_prefix("/sys/bus/cxl/devices", "decoder", 1) < 1) + fail("missing-cxl-decoder", 19); + if (wait_for_prefix("/sys/bus/dax/devices", "dax", 1) < 1) + fail("missing-dax", 19); + discover_dax(&dax); + append_text(device_entry, sizeof(device_entry), dax.path); + + write_text("LEG_OFS_CXL_READY role="); + write_text(role == ROLE_NODE0 ? "node0" : "node1"); + write_text(" dax="); + write_text(dax.path); + write_text(" size="); + write_unsigned(dax.size); + write_text(" major="); + write_unsigned(dax.major); + write_text(" minor="); + write_unsigned(dax.minor); + write_text(" region_id="); + write_region_id(dax.region_hi, dax.region_lo); + write_text("\n"); + + if (role == ROLE_NODE0) + run_node0(device_entry); + run_node1(device_entry, (uint16_t)port, bytes); +} diff --git a/run-legofs-type3.sh b/run-legofs-type3.sh new file mode 100755 index 0000000..e561939 --- /dev/null +++ b/run-legofs-type3.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BUILD_ONLY=0 +RUN_ONLY=0 +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1\n')" +BENCH_BYTES=65536 +TIMEOUT=300 +BUILD_SCRIPT="${LEGOFS_BUILD_SCRIPT:-${ROOT}/scripts/build_legofs_type3.sh}" +RUNNER="${LEGOFS_RUNNER:-${ROOT}/scripts/legofs_type3_2node.py}" + +usage() +{ + cat <<'EOF' +Usage: ./run-legofs-type3.sh [OPTIONS] + + --build-only build artifacts without starting QEMU + --run-only run existing artifacts without rebuilding + --jobs N parallel build jobs (default: online CPUs) + --bytes N benchmark bytes, 4096-aligned and <= 16777216 + --timeout N end-to-end timeout in seconds + --help show this help +EOF +} + +die() +{ + printf 'error: %s\n' "$*" >&2 + exit 2 +} + +while (($#)); do + case "$1" in + --build-only) + BUILD_ONLY=1 + shift + ;; + --run-only) + RUN_ONLY=1 + shift + ;; + --jobs) + (($# >= 2)) || die "--jobs requires a value" + JOBS="$2" + shift 2 + ;; + --bytes) + (($# >= 2)) || die "--bytes requires a value" + BENCH_BYTES="$2" + shift 2 + ;; + --timeout) + (($# >= 2)) || die "--timeout requires a value" + TIMEOUT="$2" + shift 2 + ;; + --help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +((BUILD_ONLY == 0 || RUN_ONLY == 0)) || + die "--build-only and --run-only are mutually exclusive" +[[ "${JOBS}" =~ ^[1-9][0-9]*$ ]] || die "jobs must be a positive integer" +[[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || die "timeout must be a positive integer" +[[ "${BENCH_BYTES}" =~ ^[1-9][0-9]*$ ]] || die "bytes must be a positive integer" +((BENCH_BYTES <= 16777216)) || die "bytes must not exceed 16777216" +((BENCH_BYTES % 4096 == 0)) || die "bytes must be divisible by 4096" + +submodule_status="$(git -C "${ROOT}" submodule status)" || + die "unable to inspect submodule state" +while IFS= read -r line; do + [[ -z "${line}" ]] && continue + case "${line:0:1}" in + -|+|U) + die "submodule is not at its recorded gitlink: ${line}" + ;; + esac +done <<<"${submodule_status}" + +if ((RUN_ONLY == 0)); then + [[ -x "${BUILD_SCRIPT}" ]] || die "build script is not executable: ${BUILD_SCRIPT}" + "${BUILD_SCRIPT}" --jobs "${JOBS}" +fi + +if ((BUILD_ONLY == 0)); then + [[ -f "${RUNNER}" ]] || die "two-node runner is missing: ${RUNNER}" + exec python3 "${RUNNER}" --bytes "${BENCH_BYTES}" --timeout "${TIMEOUT}" +fi diff --git a/scripts/build_legofs_type3.sh b/scripts/build_legofs_type3.sh new file mode 100755 index 0000000..a231a9a --- /dev/null +++ b/scripts/build_legofs_type3.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${ROOT}/out/legofs-type3" +BUILD="${OUT}/build" +IMAGES="${OUT}/images" +RESULTS="${OUT}/results" +LOGS="${OUT}/logs" +INITRAMFS="${IMAGES}/initramfs" +CARGO_TARGET="${BUILD}/cargo" +LEGOFS_BIN="${BUILD}/legofs-bin" +CROSS_COMPILE="${CROSS_COMPILE:-riscv64-linux-gnu-}" +RUST_TARGET="riscv64gc-unknown-linux-musl" +MUSL_VERSION=1.2.5 +MUSL_SHA256=a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4 +MUSL_SOURCE_ROOT="${OUT}/toolchain-src" +MUSL_SOURCE="${MUSL_SOURCE_ROOT}/musl-${MUSL_VERSION}" +MUSL_TARBALL="${MUSL_SOURCE_ROOT}/musl-${MUSL_VERSION}.tar.gz" +MUSL_BUILD="${BUILD}/musl-rv64gc" +MUSL_PREFIX="${OUT}/toolchain/musl-rv64gc" +MUSL_CC="${MUSL_PREFIX}/bin/musl-gcc" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1\n')" + +die() +{ + printf 'error: %s\n' "$*" >&2 + exit 2 +} + +while (($#)); do + case "$1" in + --jobs) + (($# >= 2)) || die "--jobs requires a value" + JOBS="$2" + shift 2 + ;; + *) + die "unknown argument: $1" + ;; + esac +done +[[ "${JOBS}" =~ ^[1-9][0-9]*$ ]] || die "jobs must be a positive integer" + +for command in cargo rustc rustup "${CROSS_COMPILE}gcc" \ + "${CROSS_COMPILE}readelf" "${CROSS_COMPILE}strip" cmake ninja make mke2fs \ + debugfs truncate python3 wget sha256sum tar install stat cmp; do + command -v "${command}" >/dev/null || die "required command is missing: ${command}" +done +if ! rustup target list --installed | grep -qx "${RUST_TARGET}"; then + printf '%s\n' "error: Rust target ${RUST_TARGET} is not installed" >&2 + printf '%s\n' "remediation: rustup target add ${RUST_TARGET}" >&2 + exit 2 +fi + +mkdir -p \ + "${BUILD}/qemu" "${BUILD}/opensbi" "${BUILD}/u-boot" \ + "${BUILD}/linux" "${BUILD}/cxlmemsim" "${CARGO_TARGET}" \ + "${LEGOFS_BIN}" \ + "${MUSL_SOURCE_ROOT}" "${MUSL_BUILD}" "${MUSL_PREFIX}" \ + "${INITRAMFS}" "${IMAGES}" "${RESULTS}" "${LOGS}" +exec > >(tee -a "${LOGS}/build.log") 2>&1 + +printf '%s\n' '[legofs-build] pinned RV64GC musl sysroot' +if [[ ! -f "${MUSL_TARBALL}" ]]; then + wget -O "${MUSL_TARBALL}.tmp" \ + "https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz" + mv "${MUSL_TARBALL}.tmp" "${MUSL_TARBALL}" +fi +printf '%s %s\n' "${MUSL_SHA256}" "${MUSL_TARBALL}" | sha256sum -c - +if [[ ! -x "${MUSL_SOURCE}/configure" ]]; then + tar -xzf "${MUSL_TARBALL}" -C "${MUSL_SOURCE_ROOT}" +fi +if [[ ! -x "${MUSL_CC}" ]]; then + ( + cd "${MUSL_BUILD}" + "${MUSL_SOURCE}/configure" --prefix="${MUSL_PREFIX}" \ + --target=riscv64-linux-musl CROSS_COMPILE="${CROSS_COMPILE}" \ + CFLAGS='-O2 -march=rv64gc -mabi=lp64d' + make -j "${JOBS}" + make install + ) +fi + +export CARGO_TARGET_DIR="${CARGO_TARGET}" +export CC_riscv64gc_unknown_linux_musl="${MUSL_CC}" +export CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_LINKER="${MUSL_CC}" +export RUSTFLAGS='-C target-feature=+crt-static -C link-arg=-march=rv64gc -C link-arg=-mabi=lp64d' + +printf '%s\n' '[legofs-build] static RISC-V server and benchmark' +cargo build --manifest-path "${ROOT}/components/legofs/Cargo.toml" --release \ + --target "${RUST_TARGET}" -p badfs-server -p badfs-bench +RUSTFLAGS= cargo test --manifest-path "${ROOT}/components/legofs/Cargo.toml" -p badfs-bench \ + benchmark_mode_uses_semantic_values_and_rejects_unknown_input + +badfs_server_unstripped="${CARGO_TARGET}/${RUST_TARGET}/release/badfs-server" +badfs_bench_unstripped="${CARGO_TARGET}/${RUST_TARGET}/release/badfs-bench" +badfs_server="${LEGOFS_BIN}/badfs-server" +badfs_bench="${LEGOFS_BIN}/badfs-bench" +install -m 0755 "${badfs_server_unstripped}" "${badfs_server}" +install -m 0755 "${badfs_bench_unstripped}" "${badfs_bench}" +"${CROSS_COMPILE}strip" --strip-debug "${badfs_server}" "${badfs_bench}" +for binary in "${badfs_server}" "${badfs_bench}"; do + [[ -s "${binary}" ]] || die "missing Legofs binary: ${binary}" + if "${CROSS_COMPILE}readelf" -l "${binary}" | grep -q INTERP; then + die "Legofs binary has an ELF interpreter: ${binary}" + fi + if "${CROSS_COMPILE}readelf" -A "${binary}" | grep -q '_v'; then + die "Legofs binary unexpectedly requires RVV: ${binary}" + fi +done + +guest_flags=( + -O2 -std=c11 -Wall -Wextra -Werror -static -nostdlib -fno-builtin + -fno-stack-protector -fno-pie -no-pie -march=rv64imafdc -mabi=lp64d +) +printf '%s\n' '[legofs-build] freestanding role-aware PID 1' +"${CROSS_COMPILE}gcc" "${guest_flags[@]}" \ + "${ROOT}/guest/legofs_node_init.c" -o "${INITRAMFS}/init" +chmod 0755 "${INITRAMFS}/init" +if "${CROSS_COMPILE}readelf" -l "${INITRAMFS}/init" | grep -q INTERP; then + die "guest PID 1 has an ELF interpreter" +fi +if "${CROSS_COMPILE}readelf" -A "${INITRAMFS}/init" | grep -q '_v'; then + die "guest PID 1 unexpectedly requires RVV" +fi + +printf '%s\n' '[legofs-build] read-only ext2 payload image' +legofs_disk="${IMAGES}/legofs-type3.ext2" +payload_bytes=$(($(stat -c %s "${badfs_server}") + $(stat -c %s "${badfs_bench}"))) +image_mib=$(((payload_bytes + 32 * 1024 * 1024 + 1024 * 1024 - 1) / (1024 * 1024))) +truncate -s "${image_mib}M" "${legofs_disk}" +mke2fs -q -t ext2 -F "${legofs_disk}" +debugfs -w -R "write ${badfs_server} /badfs-server" "${legofs_disk}" +debugfs -w -R "set_inode_field /badfs-server mode 0100755" "${legofs_disk}" +debugfs -w -R "write ${badfs_bench} /badfs-bench" "${legofs_disk}" +debugfs -w -R "set_inode_field /badfs-bench mode 0100755" "${legofs_disk}" +debugfs -R 'stat /badfs-server' "${legofs_disk}" | grep -q 'Mode:.*0755' +debugfs -R 'stat /badfs-bench' "${legofs_disk}" | grep -q 'Mode:.*0755' +verify_server="${BUILD}/verify-badfs-server" +verify_bench="${BUILD}/verify-badfs-bench" +debugfs -R "dump /badfs-server ${verify_server}" "${legofs_disk}" +debugfs -R "dump /badfs-bench ${verify_bench}" "${legofs_disk}" +cmp "${badfs_server}" "${verify_server}" || die 'badfs-server ext2 payload is incomplete' +cmp "${badfs_bench}" "${verify_bench}" || die 'badfs-bench ext2 payload is incomplete' + +printf '%s\n' '[legofs-build] QEMU riscv64-softmmu with Type-3 MESI v2 BI' +( + cd "${BUILD}/qemu" + "${ROOT}/components/qemu/configure" --target-list=riscv64-softmmu \ + --disable-docs --disable-werror --extra-cflags=-Wno-error \ + --prefix="${BUILD}/qemu-install" +) +ninja -C "${BUILD}/qemu" -j "${JOBS}" qemu-system-riscv64 + +printf '%s\n' '[legofs-build] CXLMemSim MESI-v2 server' +cmake -S "${ROOT}/components/cxlmemsim" -B "${BUILD}/cxlmemsim" \ + -DCMAKE_BUILD_TYPE=Release +cmake --build "${BUILD}/cxlmemsim" --target cxlmemsim_server \ + --parallel "${JOBS}" + +printf '%s\n' '[legofs-build] OpenSBI and CXL U-Boot' +make -C "${ROOT}/components/opensbi" O="${BUILD}/opensbi" \ + CROSS_COMPILE="${CROSS_COMPILE}" PLATFORM=generic \ + 'platform-cflags-y=-std=gnu11' -j "${JOBS}" +make -C "${ROOT}/components/u-boot" O="${BUILD}/u-boot" \ + CROSS_COMPILE="${CROSS_COMPILE}" sifive_unleashed_qemu_cxl_defconfig +python3 "${ROOT}/scripts/prepare_uboot_pylibfdt.py" \ + --source "${ROOT}/components/u-boot/scripts/dtc/pylibfdt/libfdt.i_shipped" \ + --output "${BUILD}/u-boot/scripts/dtc/pylibfdt/libfdt.i" +make -C "${ROOT}/components/u-boot" O="${BUILD}/u-boot" \ + CROSS_COMPILE="${CROSS_COMPILE}" \ + OPENSBI="${BUILD}/opensbi/platform/generic/firmware/fw_dynamic.bin" \ + -j "${JOBS}" + +printf '%s\n' '[legofs-build] Linux CXL devdax image' +make -C "${ROOT}/components/linux" O="${BUILD}/linux" \ + ARCH=riscv CROSS_COMPILE="${CROSS_COMPILE}" defconfig +ARCH=riscv CROSS_COMPILE="${CROSS_COMPILE}" \ + "${ROOT}/components/linux/scripts/kconfig/merge_config.sh" -m \ + -O "${BUILD}/linux" "${BUILD}/linux/.config" \ + "${ROOT}/configs/linux-cxl.config" +"${ROOT}/components/linux/scripts/config" --file "${BUILD}/linux/.config" \ + --set-str CONFIG_INITRAMFS_SOURCE "${INITRAMFS}" +make -C "${ROOT}/components/linux" O="${BUILD}/linux" \ + ARCH=riscv CROSS_COMPILE="${CROSS_COMPILE}" olddefconfig + +required_kernel_options=( + CONFIG_PCI CONFIG_PCIEPORTBUS CONFIG_EFI CONFIG_EFI_STUB CONFIG_RISCV_SBI + CONFIG_NONPORTABLE CONFIG_HVC_RISCV_SBI CONFIG_CXL_BUS CONFIG_CXL_PCI + CONFIG_CXL_ACPI CONFIG_CXL_MEM CONFIG_CXL_PORT CONFIG_CXL_REGION + CONFIG_MEMORY_HOTPLUG CONFIG_MEMORY_HOTREMOVE CONFIG_SPARSEMEM_VMEMMAP + CONFIG_ZONE_DEVICE CONFIG_DAX CONFIG_FS_DAX CONFIG_DEV_DAX CONFIG_DEV_DAX_CXL CONFIG_NET CONFIG_INET + CONFIG_UNIX CONFIG_PACKET CONFIG_VIRTIO CONFIG_VIRTIO_PCI CONFIG_VIRTIO_BLK + CONFIG_VIRTIO_NET CONFIG_IP_PNP CONFIG_IP_PNP_DHCP CONFIG_EXT4_FS + CONFIG_EXT4_USE_FOR_EXT2 CONFIG_DEVTMPFS CONFIG_DEVTMPFS_MOUNT + CONFIG_BLK_DEV_INITRD CONFIG_PROC_FS CONFIG_SYSFS CONFIG_TMPFS CONFIG_BINFMT_ELF +) +for option in "${required_kernel_options[@]}"; do + grep -qx "${option}=y" "${BUILD}/linux/.config" || + die "required kernel option is not built in: ${option}" +done +grep -Fqx '# CONFIG_DEV_DAX_KMEM is not set' "${BUILD}/linux/.config" || + die 'CONFIG_DEV_DAX_KMEM must be disabled so CXL DAX binds device_dax' +grep -Fqx "CONFIG_INITRAMFS_SOURCE=\"${INITRAMFS}\"" "${BUILD}/linux/.config" || + die 'CONFIG_INITRAMFS_SOURCE does not match the Legofs PID 1 directory' +make -C "${ROOT}/components/linux" O="${BUILD}/linux" \ + ARCH=riscv CROSS_COMPILE="${CROSS_COMPILE}" -j "${JOBS}" Image + +qemu="${BUILD}/qemu/qemu-system-riscv64" +opensbi="${BUILD}/opensbi/platform/generic/firmware/fw_dynamic.bin" +u_boot="${BUILD}/u-boot/u-boot.bin" +linux_legofs="${BUILD}/linux/arch/riscv/boot/Image" +cxlmemsim_server="${BUILD}/cxlmemsim/cxlmemsim_server" +for artifact in "${qemu}" "${opensbi}" "${u_boot}" "${linux_legofs}" \ + "${legofs_disk}" "${badfs_server}" "${badfs_bench}" "${cxlmemsim_server}"; do + [[ -s "${artifact}" ]] || die "missing build artifact: ${artifact}" +done + +python3 "${ROOT}/scripts/write_manifest.py" \ + --root "${ROOT}" --output "${RESULTS}/build-manifest.json" \ + --compiler "rustc=rustc --version" \ + --compiler "cargo=cargo --version" \ + --compiler "riscv_musl_gcc=${MUSL_CC} --version" \ + --compiler "qemu=${qemu} --version" \ + --artifact "qemu=${qemu}" \ + --artifact "opensbi=${opensbi}" \ + --artifact "u_boot=${u_boot}" \ + --artifact "linux_legofs=${linux_legofs}" \ + --artifact "legofs_disk=${legofs_disk}" \ + --artifact "badfs_server=${badfs_server}" \ + --artifact "badfs_bench=${badfs_bench}" \ + --artifact "cxlmemsim_server=${cxlmemsim_server}" + +printf '%s\n' "[legofs-build] manifest ${RESULTS}/build-manifest.json" diff --git a/scripts/legofs_type3_2node.py b/scripts/legofs_type3_2node.py new file mode 100755 index 0000000..d402910 --- /dev/null +++ b/scripts/legofs_type3_2node.py @@ -0,0 +1,1025 @@ +#!/usr/bin/env python3 +"""Own two exact SiFive U guests for the Legofs Type-3 coherence proof.""" + +import argparse +import datetime +import hashlib +import json +import os +import pathlib +import re +import signal +import socket +import subprocess +import sys +import threading +import time +import uuid + + +HOST_DECODER = ( + "CXL host decoder0: HPA 0000001000000000 " + "size 0000000010000000 target 0 ctrl 00000600" +) +TYPE3_DECODER = ( + "41.00.0 Type 3 decoder0: HPA 0000001000000000 " + "size 0000000010000000 target 0 ctrl 00001600" +) +LEGACY_TRANSPORT_ENV = ( + "CXL_TRANSPORT_MODE", + "CXL_PGAS_SHM", + "CXL_MEMSIM_SERVER", +) + + +class RuntimePaths: + def __init__(self, root, run_dir): + self.root = pathlib.Path(root).resolve() + self.output = self.root / "out" / "legofs-type3" + self.build = self.output / "build" + self.images = self.output / "images" + self.results = self.output / "results" + self.run_dir = pathlib.Path(run_dir).resolve() + self.qemu = self.build / "qemu" / "qemu-system-riscv64" + self.opensbi = self.build / "opensbi/platform/generic/firmware/fw_dynamic.bin" + self.u_boot = self.build / "u-boot/u-boot.bin" + self.linux = self.build / "linux/arch/riscv/boot/Image" + self.legofs_disk = self.images / "legofs-type3.ext2" + self.cxlmemsim_server = self.build / "cxlmemsim/cxlmemsim_server" + self.manifest = self.results / "build-manifest.json" + self.topology = self.root / "components/cxlmemsim/qemu_integration/topology_simple.txt" + self.coherence_trace = self.run_dir / "coherence.jsonl" + self.server_log = self.run_dir / "cxlmemsim.log" + self.server_ssd = self.run_dir / "cxlmemsim-cxl-ssd.raw" + self.result = self.run_dir / "result.json" + + @classmethod + def create(cls, root): + root = pathlib.Path(root).resolve() + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y%m%dT%H%M%S.%fZ" + ) + run_dir = root / "out/legofs-type3/runs" / f"{timestamp}-{os.getpid()}" + run_dir.mkdir(parents=True, mode=0o700) + os.chmod(run_dir, 0o700) + return cls(root, run_dir) + + @classmethod + def for_test(cls, root): + root = pathlib.Path(root).resolve() + run_dir = root / "out/legofs-type3/runs/test" + return cls(root, run_dir) + + def cxl_ssd_path(self, node): + return self.run_dir / f"node{node}-cxl-ssd.raw" + + def lsa_path(self, node): + return self.run_dir / f"node{node}-lsa.raw" + + def console_log(self, node): + return self.run_dir / f"node{node}.log" + + def event_log(self, node): + return self.run_dir / f"node{node}-events.jsonl" + + +def build_qemu_command(paths, node, coherence_port, legofs_port): + if node not in (0, 1): + raise ValueError("node must be 0 or 1") + prefix = f"node{node}" + command = [ + "qemu-system-riscv64", + "-M", + "sifive_u", + "-machine", + "cxl=on", + "-machine", + "cxl-fmw.0.targets.0=cxl-node%d,cxl-fmw.0.size=4G," + "cxl-fmw.0.restrictions=0xe" % node, + "-smp", + "5", + "-m", + "2G", + "-display", + "none", + "-serial", + "stdio", + "-monitor", + "none", + "-no-reboot", + "-bios", + str(paths.opensbi), + "-kernel", + str(paths.u_boot), + "-device", + f"loader,file={paths.linux},addr=0x90000000,force-raw=on", + "-object", + ( + f"memory-backend-file,id=t3ssd-{prefix}," + f"mem-path={paths.cxl_ssd_path(node)},size=256M,share=on,pmem=on" + ), + "-object", + ( + f"memory-backend-file,id=t3lsa-{prefix}," + f"mem-path={paths.lsa_path(node)},size=2M,share=on" + ), + "-device", + f"pxb-cxl,bus=pcie.0,bus_nr=64,id=cxl-{prefix},hdm_for_passthrough=on", + "-device", + f"cxl-rp,bus=cxl-{prefix},port=0,id=rp-t3-{prefix},chassis=0,slot=0", + "-device", + ( + f"cxl-type3,bus=rp-t3-{prefix},persistent-memdev=t3ssd-{prefix}," + f"lsa=t3lsa-{prefix},id=t3-{prefix},coherence-v2=on," + f"cxlmemsim-addr=127.0.0.1,cxlmemsim-port={coherence_port}," + f"coherence-v2-host-id={node},coherence-v2-cache-capacity=8388608," + "coherence-v2-cache-ways=4,coherence-v2-timeout-ms=5000," + "coherence-v2-write-through=off," + f"coherence-v2-read-exclusive={'on' if node == 0 else 'off'}" + ), + "-drive", + f"file={paths.legofs_disk},if=none,format=raw,readonly=on,id=payload-{prefix}", + "-device", + f"virtio-blk-pci,drive=payload-{prefix},bus=pcie.0,id=payload-dev-{prefix}", + "-netdev", + ] + netdev = f"user,id=net-{prefix}" + if node == 0: + netdev += f",hostfwd=tcp:127.0.0.1:{legofs_port}-:3345" + command.extend( + [ + netdev, + "-device", + f"virtio-net-pci,netdev=net-{prefix},bus=pcie.0,id=nic-{prefix}", + ] + ) + return command + + +def build_server_command(paths, coherence_port): + return [ + str(paths.cxlmemsim_server), + "--comm-mode=tcp", + f"--port={coherence_port}", + "--capacity=256", + "--default_latency=100", + f"--topology={paths.topology}", + "--coherence-v2=true", + "--coherence-v2-snoop-timeout-ms=5000", + f"--coherence-v2-trace={paths.coherence_trace}", + "--backing-mode=ssd-stream", + f"--ssd-backing-file={paths.server_ssd}", + "--ssd-page-size=4096", + "--ssd-io-chunk-size=65536", + "--ssd-cache-mb=16", + "--ssd-read-ahead-pages=16", + "--ssd-io-uring=false", + "--ssd-odirect=false", + ] + + +def qemu_environment(paths, base=None): + environment = dict(os.environ if base is None else base) + for name in LEGACY_TRANSPORT_ENV: + environment.pop(name, None) + old_path = environment.get("PATH", "") + environment["PATH"] = str(paths.qemu.parent) + if old_path: + environment["PATH"] += os.pathsep + old_path + return environment + + +def overlap_ns(first, second): + overlap = min(first[1], second[1]) - max(first[0], second[0]) + if overlap <= 0: + raise ValueError("the two QEMU lifetimes did not overlap") + return overlap + + +def _reject_duplicate_pairs(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError(f"duplicate JSON key: {key}") + value[key] = item + return value + + +def strict_json_loads(text): + try: + value = json.loads(text, object_pairs_hook=_reject_duplicate_pairs) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON evidence: {error}") from error + if not isinstance(value, dict): + raise ValueError("JSON evidence must be an object") + return value + + +def _required_integer(record, name): + value = record.get(name) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"evidence field {name} must be an integer") + return value + + +def parse_prefixed_records(output, prefix, schema): + marker = prefix + " " + records = [] + for line in output.splitlines(): + line = line.strip() + if not line.startswith(marker): + continue + record = strict_json_loads(line[len(marker):]) + if record.get("schema_version") != schema: + raise ValueError(f"unsupported {prefix} schema") + records.append(record) + return records + + +def read_coherence_trace(path, start_offset=0): + path = pathlib.Path(path) + with path.open("rb") as source: + source.seek(start_offset) + raw = source.read() + if raw and not raw.endswith(b"\n"): + raise ValueError("truncated final coherence JSONL record") + records = [] + previous = None + for number, raw_line in enumerate(raw.splitlines(), 1): + try: + line = raw_line.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError(f"invalid UTF-8 in coherence record {number}") from error + record = strict_json_loads(line) + if record.get("schema_version") != 1: + raise ValueError("unsupported coherence trace schema") + now = _required_integer(record, "monotonic_ns") + if previous is not None and now < previous: + raise ValueError("coherence timestamps went backwards") + previous = now + records.append(record) + return records + + +def validate_registrations(records): + registrations = [ + record for record in records + if record.get("event") == "registration" and record.get("status") == "OK" + ] + if len(registrations) != 2: + raise ValueError(f"expected exactly two successful host registrations, got {len(registrations)}") + by_host = {} + sessions = set() + for record in registrations: + host = _required_integer(record, "src_host") + session = _required_integer(record, "session_id") + if host in by_host: + raise ValueError(f"duplicate coherence host ID: {host}") + if host not in (0, 1) or session == 0 or session in sessions: + raise ValueError("coherence registrations have invalid host/session identity") + by_host[host] = record + sessions.add(session) + if set(by_host) != {0, 1}: + raise ValueError("coherence registrations must contain host IDs 0 and 1") + return [by_host[0], by_host[1]] + + +def correlate_dirty_backinvalidations(direct_records, lifecycle_records, coherence_records): + direct_by_op = {} + for record in direct_records: + if ( + record.get("event") in ("drop", "unmap") + and record.get("access") in ("write", "read_write") + and record.get("rc") == 0 + ): + direct_by_op[_required_integer(record, "op_id")] = record + begin_by_op = { + _required_integer(record, "op_id"): record + for record in lifecycle_records if record.get("event") == "store_direct_begin" + } + success_by_op = { + _required_integer(record, "op_id"): record + for record in lifecycle_records if record.get("event") == "store_direct_success" + } + acks = { + _required_integer(record, "snoop_id"): record + for record in coherence_records if record.get("event") == "snoop_ack" + } + completions = { + _required_integer(record, "snoop_id"): record + for record in coherence_records if record.get("event") == "dirty_completion" + } + matches = [] + for snoop in coherence_records: + if snoop.get("event") != "snoop_send" or snoop.get("opcode") != "SNP_DATA_INV": + continue + snoop_id = _required_integer(snoop, "snoop_id") + line = _required_integer(snoop, "line_address") + if _required_integer(snoop, "dst_host") != 1: + continue + ack = acks.get(snoop_id) + completion = completions.get(snoop_id) + if not ack or not completion: + continue + if ( + ack.get("ack_strength") != "MODEL" + or ack.get("status") != "OK" + or ack.get("dirty_data") is not True + or _required_integer(ack, "payload_len") != 64 + or ack.get("opcode") != "SNOOP_ACK" + or _required_integer(ack, "src_host") != 1 + or _required_integer(ack, "dst_host") != 0xFFFF + or completion.get("opcode") != "SNOOP_ACK" + or completion.get("ack_strength") != "MODEL" + or completion.get("status") != "OK" + or completion.get("dirty_data") is not True + or _required_integer(completion, "payload_len") != 64 + or _required_integer(completion, "src_host") != 1 + or _required_integer(completion, "dst_host") != 0xFFFF + or _required_integer(ack, "line_address") != line + or _required_integer(completion, "line_address") != line + or _required_integer(ack, "session_id") != _required_integer(snoop, "session_id") + or _required_integer(completion, "session_id") != _required_integer(snoop, "session_id") + or _required_integer(ack, "epoch") != _required_integer(snoop, "epoch") + or _required_integer(completion, "epoch") != _required_integer(snoop, "epoch") + or _required_integer(snoop, "monotonic_ns") > _required_integer(ack, "monotonic_ns") + or _required_integer(ack, "monotonic_ns") > _required_integer(completion, "monotonic_ns") + ): + continue + for op_id, direct in direct_by_op.items(): + begin = begin_by_op.get(op_id) + success = success_by_op.get(op_id) + if not begin or not success: + continue + start = _required_integer(begin, "mapping_offset") + length = _required_integer(begin, "mapping_length") + if ( + _required_integer(success, "mapping_offset") != start + or _required_integer(success, "mapping_length") != length + or _required_integer(direct, "offset") != start + or _required_integer(direct, "length") != length + or not (start <= line and line + 64 <= start + length) + ): + continue + matches.append({ + "op_id": op_id, + "mapping_offset": start, + "mapping_length": length, + "direct_unmap": direct, + "lifecycle_begin": begin, + "snoop": snoop, + "ack": ack, + "completion": completion, + "lifecycle_success": success, + }) + if not matches: + raise ValueError("no address-correlated dirty SNP_DATA_INV MODEL completion") + return matches + + +def expected_benchmark_checksum(file_size, block_size, iterations): + total = 0 + offset = 0 + while offset < file_size: + length = min(block_size, file_size - offset) + total += sum(index % 251 for index in range(length)) + offset += length + return total * iterations + + +def _parse_scalar_fields(line): + fields = {} + for name, value in re.findall(r"([a-zA-Z0-9_]+)=([^\s]+)", line): + if name in fields: + raise ValueError(f"duplicate scalar field: {name}") + fields[name] = value + return fields + + +def validate_legofs_output(output, benchmark_bytes): + bench_lines = [line for line in output.splitlines() if line.startswith("badfs_bench ")] + if len(bench_lines) != 1: + raise ValueError(f"expected exactly one badfs benchmark line, got {len(bench_lines)}") + raw = _parse_scalar_fields(bench_lines[0]) + integer_names = ( + "file_size", "block_size", "iterations", "written_bytes", "read_bytes", "checksum" + ) + try: + benchmark = {name: int(raw[name], 10) for name in integer_names} + except (KeyError, ValueError) as error: + raise ValueError("badfs benchmark integer fields are invalid") from error + expected_block_size = min(benchmark_bytes, 1024 * 1024) + if ( + benchmark["file_size"] != benchmark_bytes + or benchmark["block_size"] != expected_block_size + or benchmark["iterations"] != 1 + or benchmark["written_bytes"] != benchmark_bytes + or benchmark["read_bytes"] != benchmark_bytes + or benchmark["checksum"] != expected_benchmark_checksum( + benchmark_bytes, expected_block_size, 1 + ) + ): + raise ValueError("badfs benchmark bytes or checksum mismatch") + inspections = parse_prefixed_records( + output, "badfs_lifecycle_inspection", "badfs.lifecycle.inspection.v1" + ) + if not inspections: + raise ValueError("missing badfs lifecycle inspection") + zero_fabric = ( + "staged_read_ops", "staged_read_bytes", "staged_write_ops", "staged_write_bytes", + "blob_read_ops", "blob_read_bytes", "blob_write_ops", "blob_write_bytes", + "legacy_read_file_block_ops", "legacy_write_file_block_ops", + "legacy_read_fabric_block_ops", "legacy_write_fabric_block_ops", + "stale_ref_rejections", "epoch_rejections", "checksum_failures", "lease_rejections", + "quarantine_events", "active_leases", "quarantined_slots", + ) + zero_audit = ( + "pending_operations", "quarantined_extents", "active_read_leases", + ) + direct_totals = { + "trusted_direct_read_ops": 0, + "trusted_direct_read_bytes": 0, + "trusted_direct_write_ops": 0, + "trusted_direct_write_bytes": 0, + } + for inspection in inspections: + fabric = inspection.get("fabric") + audit = inspection.get("audit") + if not isinstance(fabric, dict) or not isinstance(audit, dict): + raise ValueError("badfs lifecycle inspection lacks fabric/audit objects") + for name in zero_fabric: + if _required_integer(fabric, name) != 0: + raise ValueError(f"badfs fallback counter is nonzero: {name}") + for name in zero_audit: + if _required_integer(audit, name) != 0: + raise ValueError(f"badfs lifecycle audit is not clean: {name}") + if _required_integer(audit, "direct_mapped_extents") != 1: + raise ValueError("badfs lifecycle audit must retain exactly one published direct extent") + for name in direct_totals: + direct_totals[name] += _required_integer(fabric, name) + if any(value <= 0 for value in direct_totals.values()): + raise ValueError("badfs strict direct read/write counters must be positive") + return { + "benchmark": benchmark, + "direct_totals": direct_totals, + "inspections": inspections, + } + + +def read_event_sidecar(path): + records = [] + previous = None + raw = pathlib.Path(path).read_bytes() + if raw and not raw.endswith(b"\n"): + raise ValueError("truncated console event sidecar") + for line in raw.splitlines(): + record = strict_json_loads(line.decode("utf-8")) + capture = _required_integer(record, "host_capture_ns") + if previous is not None and capture < previous: + raise ValueError("console capture timestamps went backwards") + if not isinstance(record.get("line"), str): + raise ValueError("console event line must be text") + previous = capture + records.append(record) + return records + + +def validate_host_capture_order(paths, correlations): + node0 = read_event_sidecar(paths.event_log(0)) + node1 = read_event_sidecar(paths.event_log(1)) + for correlation in correlations: + op_id = correlation["op_id"] + direct_times = [] + success_times = [] + for event in node1: + line = event["line"] + if line.startswith("BADFS_DIRECT_MAP_TRACE_JSON "): + record = strict_json_loads(line.split(" ", 1)[1]) + if record.get("op_id") == op_id and record.get("event") in ("drop", "unmap"): + direct_times.append(event["host_capture_ns"]) + for event in node0: + line = event["line"] + if line.startswith("BADFS_LIFECYCLE_TRACE_JSON "): + record = strict_json_loads(line.split(" ", 1)[1]) + if record.get("op_id") == op_id and record.get("event") == "store_direct_success": + success_times.append(event["host_capture_ns"]) + snoop_ns = _required_integer(correlation["snoop"], "monotonic_ns") + ack_ns = _required_integer(correlation["ack"], "monotonic_ns") + completion_ns = _required_integer(correlation["completion"], "monotonic_ns") + for direct_ns in direct_times: + for success_ns in success_times: + if direct_ns < snoop_ns <= ack_ns <= completion_ns < success_ns: + return { + "op_id": op_id, + "direct_unmap_capture_ns": direct_ns, + "snoop_send_ns": snoop_ns, + "snoop_ack_ns": ack_ns, + "dirty_completion_ns": completion_ns, + "store_success_capture_ns": success_ns, + } + raise ValueError( + "host capture does not order direct unmap < snoop < ACK < " + "dirty completion < store_direct_success" + ) + + +def parse_server_stats(output): + marker = "COHERENCE_V2_STATS_JSON " + records = [ + strict_json_loads(line.strip()[len(marker):]) + for line in output.splitlines() if line.strip().startswith(marker) + ] + if len(records) != 1: + raise ValueError(f"expected one final coherence stats object, got {len(records)}") + stats = records[0] + for name in ("timeouts", "protocol_errors", "delivery_failures", "server_copy_failures", "active_bindings"): + if _required_integer(stats, name) != 0: + raise ValueError(f"coherence final error counter is nonzero: {name}") + return stats + + +def validate_runtime_evidence(paths, node0_output, node1_output, pre_benchmark_offset, benchmark_bytes): + all_coherence = read_coherence_trace(paths.coherence_trace) + registrations = validate_registrations(all_coherence) + benchmark_coherence = read_coherence_trace(paths.coherence_trace, pre_benchmark_offset) + for event in benchmark_coherence: + if event.get("event") in ("timeout", "protocol_error", "delivery_failure", "server_copy_failure"): + raise ValueError(f"benchmark coherence error event: {event.get('event')}") + direct = parse_prefixed_records( + node1_output, "BADFS_DIRECT_MAP_TRACE_JSON", "badfs.direct-map-trace.v1" + ) + lifecycle = parse_prefixed_records( + node0_output, "BADFS_LIFECYCLE_TRACE_JSON", "badfs.lifecycle.v1" + ) + correlations = correlate_dirty_backinvalidations(direct, lifecycle, benchmark_coherence) + host_order = validate_host_capture_order(paths, correlations) + legofs = validate_legofs_output(node1_output, benchmark_bytes) + server_stats = parse_server_stats(paths.server_log.read_text(encoding="utf-8", errors="replace")) + coherence_delta = { + "snp_data_inv": sum( + event.get("event") == "snoop_send" and event.get("opcode") == "SNP_DATA_INV" + for event in benchmark_coherence + ), + "model_acks": sum( + event.get("event") == "snoop_ack" and event.get("ack_strength") == "MODEL" + for event in benchmark_coherence + ), + "dirty_data_completions": sum(event.get("event") == "dirty_completion" for event in benchmark_coherence), + } + return { + "registrations": registrations, + "coherence_delta": coherence_delta, + "legofs_counters": legofs["direct_totals"], + "benchmark": legofs["benchmark"], + "inspections": legofs["inspections"], + "correlations": correlations, + "host_capture_order": host_order, + "coherence_final_stats": server_stats, + } + + +class PortReservation: + def __init__(self): + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.bind(("127.0.0.1", 0)) + self.port = self.socket.getsockname()[1] + self.released = False + + def release(self): + if not self.released: + self.socket.close() + self.released = True + + +class OwnedProcess: + def __init__(self, process, command, run_dir, owner_token, start_ns): + self.process = process + self.command = list(command) + self.run_dir = str(pathlib.Path(run_dir).resolve()) + self.owner_token = owner_token + self.start_ns = start_ns + self.end_ns = None + + def record_exit(self): + if self.end_ns is None and self.process.poll() is not None: + self.end_ns = time.monotonic_ns() + + def matches_live_pid(self): + if self.process.poll() is not None: + self.record_exit() + return False + try: + raw = pathlib.Path(f"/proc/{self.process.pid}/cmdline").read_bytes() + except OSError: + return False + fields = [field.decode(errors="replace") for field in raw.split(b"\0") if field] + if not fields: + return False + expected = pathlib.Path(self.command[0]).name + return pathlib.Path(fields[0]).name == expected and any( + self.run_dir in field for field in fields + ) + + def terminate_owned(self): + if not self.matches_live_pid(): + return + os.kill(self.process.pid, signal.SIGTERM) + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + if self.matches_live_pid(): + os.kill(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=5) + self.record_exit() + + def as_json(self): + self.record_exit() + return { + "pid": self.process.pid, + "command": self.command, + "start_monotonic_ns": self.start_ns, + "end_monotonic_ns": self.end_ns, + "owner_token": self.owner_token, + "returncode": self.process.poll(), + } + + +class Console: + def __init__(self, command, environment, log_path, event_path, run_dir, owner_token): + self.command = list(command) + self.output = "" + self.condition = threading.Condition() + self.closed = False + self.log = pathlib.Path(log_path).open("w", encoding="utf-8") + self.events = pathlib.Path(event_path).open("w", encoding="utf-8") + start_ns = time.monotonic_ns() + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + env=environment, + ) + self.owned = OwnedProcess(process, command, run_dir, owner_token, start_ns) + self.process = process + self.reader = threading.Thread(target=self._read_output, daemon=True) + self.reader.start() + + def _read_output(self): + pending = b"" + while True: + chunk = os.read(self.process.stdout.fileno(), 4096) + if not chunk: + break + decoded = chunk.decode(errors="replace") + with self.condition: + self.output += decoded + self.log.write(decoded) + self.log.flush() + self.condition.notify_all() + sys.stdout.write(decoded) + sys.stdout.flush() + pending += chunk + while b"\n" in pending: + raw_line, pending = pending.split(b"\n", 1) + self._capture_event(raw_line) + self.owned.record_exit() + with self.condition: + self.condition.notify_all() + + def _capture_event(self, raw_line): + decoded = raw_line.decode(errors="replace").rstrip("\r") + capture_ns = time.monotonic_ns() + with self.condition: + json.dump( + {"host_capture_ns": capture_ns, "line": decoded}, + self.events, + sort_keys=True, + ) + self.events.write("\n") + self.events.flush() + + def wait(self, text, timeout, start=0): + deadline = time.monotonic() + timeout + with self.condition: + while text not in self.output[start:]: + if self.process.poll() is not None: + raise RuntimeError( + f"QEMU exited with {self.process.returncode} while waiting for {text!r}" + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"timed out waiting for {text!r}") + self.condition.wait(min(remaining, 0.5)) + + def send(self, line): + if self.process.stdin is None or self.process.poll() is not None: + raise RuntimeError("QEMU console is not writable") + self.process.stdin.write((line + "\n").encode()) + self.process.stdin.flush() + + def command_until_prompt(self, command, timeout): + start = len(self.output) + self.send(command) + self.wait("=> ", timeout, start) + return self.output[start:] + + def close_files(self): + if self.closed: + return + self.closed = True + if self.process.poll() is not None: + self.reader.join(timeout=2) + if self.process.stdin is not None: + self.process.stdin.close() + if self.process.stdout is not None: + self.process.stdout.close() + self.log.close() + self.events.close() + + +def create_sparse_file(path, size): + path = pathlib.Path(path) + with path.open("xb") as output: + output.truncate(size) + + +def sha256_file(path): + digest = hashlib.sha256() + with pathlib.Path(path).open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verified_manifest(paths): + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + if manifest.get("schema_version") != 2: + raise ValueError("unsupported build manifest schema") + expected = { + "qemu": paths.qemu, + "opensbi": paths.opensbi, + "u_boot": paths.u_boot, + "linux_legofs": paths.linux, + "legofs_disk": paths.legofs_disk, + "cxlmemsim_server": paths.cxlmemsim_server, + } + for name, path in expected.items(): + entry = manifest.get("artifacts", {}).get(name) + if not path.is_file() or not isinstance(entry, dict): + raise FileNotFoundError(f"missing verified runtime artifact: {name}") + if entry.get("size") != path.stat().st_size or entry.get("sha256") != sha256_file(path): + raise ValueError(f"runtime artifact changed after build: {name}") + current = subprocess.run( + ["git", "-C", str(paths.root), "rev-parse", "HEAD"], + check=True, + text=True, + capture_output=True, + ).stdout.strip() + if manifest.get("superproject_commit") != current: + raise ValueError("superproject changed after the build manifest") + return manifest + + +def wait_for_log(path, marker, process, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"CXLMemSim exited before readiness: {process.returncode}") + try: + if marker in pathlib.Path(path).read_text(encoding="utf-8", errors="replace"): + return + except FileNotFoundError: + pass + time.sleep(0.05) + raise TimeoutError(f"timed out waiting for CXLMemSim marker {marker!r}") + + +def run_uboot(console, paths, node, legofs_port, benchmark_bytes, timeout): + console.wait("Hit any key to stop autoboot", timeout) + console.send("") + console.wait("=> ", timeout) + if HOST_DECODER not in console.output or TYPE3_DECODER not in console.output: + raise ValueError(f"node{node} preboot CXL decoder proof is missing") + listing = console.command_until_prompt("cxl list", timeout) + if "41.00.0" not in listing or "Type 3" not in listing: + raise ValueError(f"node{node} cxl list did not report Type 3") + information = console.command_until_prompt("cxl info 41.00.0", timeout) + if "41.00.0" not in information or "0000000010000000" not in information: + raise ValueError(f"node{node} cxl info is incomplete") + initialization = console.command_until_prompt("cxl init", timeout) + if HOST_DECODER not in initialization or TYPE3_DECODER not in initialization: + raise ValueError(f"node{node} cxl init did not reproduce decoder state") + console.command_until_prompt( + "setenv bootargs 'earlycon=sbi console=hvc0 loglevel=6 " + "cxl_core.pmem_as_dax=1 " + f"legofs.role=node{node} legofs.server_port={legofs_port} " + f"legofs.bytes={benchmark_bytes}'", + timeout, + ) + console.send(f"bootefi 90000000:{paths.linux.stat().st_size:x} ${{fdtcontroladdr}}") + + +def atomic_write_json(path, value): + path = pathlib.Path(path) + temporary = path.with_name(path.name + ".tmp") + with temporary.open("w", encoding="utf-8") as output: + json.dump(value, output, indent=2, sort_keys=True) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + + +def execute(paths, benchmark_bytes, timeout): + manifest = verified_manifest(paths) + coherence_reservation = PortReservation() + legofs_reservation = PortReservation() + owner_token = str(uuid.uuid4()) + owned = [] + owned_names = [] + consoles = [] + result = { + "schema_version": 1, + "status": "failed", + "first_failure": None, + "functional_model_only": True, + "run_id": paths.run_dir.name, + "owner_token": owner_token, + "component_commits": manifest["submodules"], + "artifact_sha256": { + name: entry["sha256"] for name, entry in manifest["artifacts"].items() + }, + "qemu_commands": [], + "process_lifetimes": {}, + "logs": { + "node0": str(paths.console_log(0)), + "node1": str(paths.console_log(1)), + "cxlmemsim": str(paths.server_log), + "coherence": str(paths.coherence_trace), + }, + } + try: + create_sparse_file(paths.server_ssd, 256 * 1024 * 1024) + for node in (0, 1): + create_sparse_file(paths.cxl_ssd_path(node), 256 * 1024 * 1024) + create_sparse_file(paths.lsa_path(node), 2 * 1024 * 1024) + + server_command = build_server_command(paths, coherence_reservation.port) + server_log = paths.server_log.open("w", encoding="utf-8") + coherence_reservation.release() + start_ns = time.monotonic_ns() + server_process = subprocess.Popen( + server_command, + stdout=server_log, + stderr=subprocess.STDOUT, + text=True, + ) + server = OwnedProcess( + server_process, server_command, paths.run_dir, owner_token, start_ns + ) + server.log_file = server_log + owned.append(server) + owned_names.append("cxlmemsim") + wait_for_log(paths.server_log, "Server listening on TCP port", server_process, timeout) + + environment = qemu_environment(paths) + commands = [ + build_qemu_command( + paths, node, coherence_reservation.port, legofs_reservation.port + ) + for node in (0, 1) + ] + result["qemu_commands"] = commands + + legofs_reservation.release() + node0 = Console( + commands[0], environment, paths.console_log(0), paths.event_log(0), + paths.run_dir, owner_token, + ) + consoles.append(node0) + owned.append(node0.owned) + owned_names.append("node0") + run_uboot(node0, paths, 0, legofs_reservation.port, benchmark_bytes, timeout) + node0.wait("LEG_OFS_CXL_READY role=node0", timeout) + node0.wait("LEG_OFS_SERVER_READY", timeout) + + node1 = Console( + commands[1], environment, paths.console_log(1), paths.event_log(1), + paths.run_dir, owner_token, + ) + consoles.append(node1) + owned.append(node1.owned) + owned_names.append("node1") + run_uboot(node1, paths, 1, legofs_reservation.port, benchmark_bytes, timeout) + node1.wait("LEG_OFS_CXL_READY role=node1", timeout) + node1.wait("LEG_OFS_CLIENT_READY", timeout) + if node0.process.poll() is not None or node1.process.poll() is not None: + raise RuntimeError("both QEMU processes must be live before benchmark release") + registrations = validate_registrations(read_coherence_trace(paths.coherence_trace)) + pre_benchmark_offset = paths.coherence_trace.stat().st_size + result["registrations"] = registrations + result["pre_benchmark_trace_offset"] = pre_benchmark_offset + node1.send("LEG_OFS_RUN") + node1.wait("LEG_OFS_BENCHMARK_PASS", timeout) + if node0.process.poll() is not None: + raise RuntimeError("node0 exited before node1 benchmark completion") + try: + node1.process.wait(timeout=10) + except subprocess.TimeoutExpired: + node1.owned.terminate_owned() + node1.owned.record_exit() + + node0.owned.terminate_owned() + server_process.send_signal(signal.SIGINT) + try: + server_process.wait(timeout=10) + except subprocess.TimeoutExpired: + server.terminate_owned() + server.record_exit() + server_log.close() + + for item in owned: + item.record_exit() + node0_interval = (node0.owned.start_ns, node0.owned.end_ns) + node1_interval = (node1.owned.start_ns, node1.owned.end_ns) + overlap = overlap_ns(node0_interval, node1_interval) + result["overlap_ns"] = overlap + result["process_lifetimes"] = dict( + zip(owned_names, (item.as_json() for item in owned)) + ) + evidence = validate_runtime_evidence( + paths, + node0.output, + node1.output, + pre_benchmark_offset, + benchmark_bytes, + ) + result.update(evidence) + result["topology"] = { + "machine": "sifive_u", + "nodes": 2, + "type3_endpoints": 2, + "type3_per_node": 1, + "cxl_ssd_bytes_per_node": 256 * 1024 * 1024, + "persistent_memdev": True, + "server_backing": "ssd-stream", + "coherence_transport": "tcp-mesi-v2", + "qemu_type3_backinvalidation": True, + } + result["status"] = "passed" + return result + except BaseException as error: + result["first_failure"] = str(error) + raise + finally: + coherence_reservation.release() + legofs_reservation.release() + for item in reversed(owned): + try: + item.terminate_owned() + except (OSError, subprocess.SubprocessError): + pass + for console in consoles: + console.close_files() + for item in owned: + log_file = getattr(item, "log_file", None) + if log_file is not None and not log_file.closed: + log_file.close() + result["process_lifetimes"] = dict( + zip(owned_names, (item.as_json() for item in owned)) + ) + result["cleanup"] = {"owned_processes_remaining": 0} + atomic_write_json(paths.result, result) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--bytes", type=int, default=65536) + parser.add_argument("--timeout", type=int, default=300) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + if args.bytes <= 0 or args.bytes > 16777216 or args.bytes % 4096: + raise ValueError("bytes must be positive, 4096-aligned, and no larger than 16777216") + if args.timeout <= 0: + raise ValueError("timeout must be positive") + root = pathlib.Path(__file__).resolve().parents[1] + paths = RuntimePaths.create(root) + try: + result = execute(paths, args.bytes, args.timeout) + except BaseException as error: + print(f"error: {error}", file=sys.stderr) + print(f"result: {paths.result}", file=sys.stderr) + return 1 + print(f"LEG_OFS_TWO_NODE_RUNTIME_COMPLETE {paths.result}") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/write_manifest.py b/scripts/write_manifest.py index f345f88..bb62565 100755 --- a/scripts/write_manifest.py +++ b/scripts/write_manifest.py @@ -4,6 +4,7 @@ import json import os import pathlib +import shlex import subprocess import sys @@ -18,6 +19,12 @@ def parse_args(argv=None): default=[], metavar="NAME=PATH", ) + parser.add_argument( + "--compiler", + action="append", + default=[], + metavar="NAME=COMMAND", + ) return parser.parse_args(argv) @@ -85,6 +92,32 @@ def parse_artifacts(root, specifications): return dict(sorted(artifacts.items())) +def parse_compilers(specifications): + compilers = {} + for specification in specifications: + name, separator, raw_command = specification.partition("=") + if not separator or not name or not raw_command: + raise ValueError( + f"compiler must use non-empty NAME=COMMAND: {specification}" + ) + if name in compilers: + raise ValueError(f"duplicate compiler name: {name}") + command = shlex.split(raw_command) + if not command: + raise ValueError(f"compiler command is empty: {specification}") + run = subprocess.run( + command, + check=True, + text=True, + capture_output=True, + ) + version = (run.stdout or run.stderr).strip() + if not version: + raise ValueError(f"compiler produced no version output: {name}") + compilers[name] = {"command": command, "version": version} + return dict(sorted(compilers.items())) + + def atomic_write_json(output, value): output.parent.mkdir(parents=True, exist_ok=True) temporary = output.with_name(output.name + ".tmp") @@ -110,9 +143,10 @@ def main(argv=None): if not (root / ".git").exists(): raise ValueError(f"root is not a Git checkout: {root}") manifest = { - "schema_version": 1, + "schema_version": 2, "superproject_commit": git_output(root, "rev-parse", "HEAD").strip(), "submodules": read_submodules(root), + "compilers": parse_compilers(args.compiler), "artifacts": parse_artifacts(root, args.artifact), } atomic_write_json(output, manifest) diff --git a/tests/test_legofs_build_contract.py b/tests/test_legofs_build_contract.py new file mode 100644 index 0000000..a151ae4 --- /dev/null +++ b/tests/test_legofs_build_contract.py @@ -0,0 +1,115 @@ +import pathlib +import os +import subprocess +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +RUN = ROOT / "run-legofs-type3.sh" +BUILD = ROOT / "scripts" / "build_legofs_type3.sh" + + +class LegofsBuildContractTest(unittest.TestCase): + def test_kernel_fragment_has_built_in_devdax_and_network(self): + config = (ROOT / "configs/linux-cxl.config").read_text().splitlines() + required = { + "CONFIG_DAX=y", + "CONFIG_FS_DAX=y", + "CONFIG_DEV_DAX=y", + "CONFIG_DEV_DAX_CXL=y", + "CONFIG_NET=y", + "CONFIG_INET=y", + "CONFIG_UNIX=y", + "CONFIG_PACKET=y", + "CONFIG_VIRTIO_NET=y", + } + self.assertTrue(required.issubset(set(config))) + + def test_init_has_role_dax_and_strict_markers(self): + source = (ROOT / "guest/legofs_node_init.c").read_text() + for marker in ( + "legofs.role=", + "LEG_OFS_CXL_READY", + "LEG_OFS_SERVER_READY", + "LEG_OFS_BENCHMARK_BEGIN", + "LEG_OFS_BENCHMARK_PASS", + "BADFS_LIFECYCLE_DIRECT_REQUIRED=1", + "BADFS_LIFECYCLE_DIRECT_READ_REQUIRED=1", + "BADFS_CXL_MAP_ALIGNMENT=2097152", + 'char block_entry[64] = "BADFS_BENCH_BLOCK_SIZE="', + ): + self.assertIn(marker, source) + self.assertNotIn("BADFS_CXL_MAP_ALIGNMENT=4096", source) + self.assertNotIn("BADFS_BENCH_BLOCK_SIZE=4096", source) + self.assertIn('set_ifreq_name(&request, "lo")', source) + self.assertIn("set_sockaddr(&request.value.address, ipv4(127, 0, 0, 1))", source) + self.assertIn("connect_tcp(ipv4(127, 0, 0, 1), 3345)", source) + self.assertIn("LEG_OFS_SERVER_PROBE errno=", source) + + def test_cxl_devdax_exposes_real_persistence_flush(self): + device = (ROOT / "components/linux/drivers/dax/device.c").read_text() + cxl = (ROOT / "components/linux/drivers/dax/cxl.c").read_text() + self.assertIn("static int dax_fsync", device) + self.assertIn("dax_flush(dev_dax->dax_dev", device) + self.assertIn(".fsync = dax_fsync", device) + self.assertIn(".persistent = true", cxl) + + def run_cli(self, *arguments, environment=None): + return subprocess.run( + [str(RUN), *arguments], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + def test_run_cli_validation_and_accepted_size(self): + self.assertTrue(RUN.is_file(), "run-legofs-type3.sh is missing") + help_run = self.run_cli("--help") + self.assertEqual(help_run.returncode, 0, help_run.stderr) + conflict = self.run_cli("--build-only", "--run-only") + self.assertNotEqual(conflict.returncode, 0) + invalid = self.run_cli("--bytes", "0") + self.assertNotEqual(invalid.returncode, 0) + + with tempfile.TemporaryDirectory() as temporary: + fake = pathlib.Path(temporary) / "build" + fake.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + fake.chmod(0o755) + environment = os.environ.copy() + environment["LEGOFS_BUILD_SCRIPT"] = str(fake) + accepted = self.run_cli( + "--build-only", "--bytes", "65536", environment=environment + ) + self.assertEqual(accepted.returncode, 0, accepted.stderr) + + def test_build_script_names_complete_manifest(self): + self.assertTrue(BUILD.is_file(), "scripts/build_legofs_type3.sh is missing") + source = BUILD.read_text(encoding="utf-8") + self.assertIn("riscv64gc-unknown-linux-musl", source) + self.assertIn( + "a9a118bbe84d8764da0ea0d28b3ab3fae8477fc7e4085d90102b8596fc7c75e4", + source, + ) + self.assertIn('"${CROSS_COMPILE}strip" --strip-debug', source) + self.assertIn('cmp "${badfs_server}" "${verify_server}"', source) + self.assertNotIn("./config.status", source) + self.assertNotIn("qemu_configure=", source) + self.assertIn('--compiler "qemu=${qemu} --version"', source) + for artifact in ( + "qemu", + "opensbi", + "u_boot", + "linux_legofs", + "legofs_disk", + "badfs_server", + "badfs_bench", + "cxlmemsim_server", + ): + with self.subTest(artifact=artifact): + self.assertIn(f'--artifact "{artifact}=', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_legofs_evidence.py b/tests/test_legofs_evidence.py new file mode 100644 index 0000000..8523826 --- /dev/null +++ b/tests/test_legofs_evidence.py @@ -0,0 +1,199 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +RUNNER = ROOT / "scripts" / "legofs_type3_2node.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("legofs_type3_2node_evidence", RUNNER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class LegofsEvidenceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.runner = load_runner() + + def records(self): + direct = [{ + "schema_version": "badfs.direct-map-trace.v1", + "event": "drop", "access": "read_write", "op_id": 17, + "offset": 0x4000, "length": 0x1000, "rc": 0, + }] + lifecycle = [ + { + "schema_version": "badfs.lifecycle.v1", + "event": "store_direct_begin", "op_id": 17, + "mapping_offset": 0x4000, "mapping_length": 0x1000, + }, + { + "schema_version": "badfs.lifecycle.v1", + "event": "store_direct_success", "op_id": 17, + "mapping_offset": 0x4000, "mapping_length": 0x1000, + }, + ] + coherence = [ + { + "schema_version": 1, "event": "snoop_send", + "opcode": "SNP_DATA_INV", "src_host": 0xFFFF, "dst_host": 1, + "session_id": 2, "epoch": 7, "snoop_id": 91, + "line_address": 0x4080, "monotonic_ns": 20, + "payload_len": 0, "status": "OK", "ack_strength": "NONE", + "dirty_data": False, + }, + { + "schema_version": 1, "event": "snoop_ack", + "opcode": "SNOOP_ACK", "src_host": 1, "dst_host": 0xFFFF, + "session_id": 2, "epoch": 7, "snoop_id": 91, + "line_address": 0x4080, "monotonic_ns": 21, + "payload_len": 64, "status": "OK", "ack_strength": "MODEL", + "dirty_data": True, + }, + { + "schema_version": 1, "event": "dirty_completion", + "opcode": "SNOOP_ACK", "src_host": 1, "dst_host": 0xFFFF, + "session_id": 2, "epoch": 7, "snoop_id": 91, + "line_address": 0x4080, "monotonic_ns": 22, + "payload_len": 64, "status": "OK", "ack_strength": "MODEL", + "dirty_data": True, + }, + ] + return direct, lifecycle, coherence + + def test_positive_dirty_backinvalidation_correlation(self): + direct, lifecycle, coherence = self.records() + matches = self.runner.correlate_dirty_backinvalidations( + direct, lifecycle, coherence + ) + self.assertEqual(len(matches), 1) + self.assertEqual(matches[0]["op_id"], 17) + self.assertEqual(matches[0]["snoop"]["snoop_id"], 91) + + def test_dirty_correlation_rejects_each_required_invariant(self): + mutations = { + "clean ACK": lambda d, l, c: c[1].update(dirty_data=False), + "native ACK": lambda d, l, c: c[1].update(ack_strength="NATIVE"), + "short ACK": lambda d, l, c: c[1].update(payload_len=0), + "wrong snoop": lambda d, l, c: c[1].update(snoop_id=92), + "wrong completion opcode": lambda d, l, c: c[2].update(opcode="SNP_DATA_INV"), + "wrong completion host": lambda d, l, c: c[2].update(src_host=0), + "wrong completion session": lambda d, l, c: c[2].update(session_id=3), + "outside grant": lambda d, l, c: c[0].update(line_address=0x5000), + "wrong host": lambda d, l, c: c[0].update(dst_host=0), + "not invalidation": lambda d, l, c: c[0].update(opcode="SNP_DATA_DOWNGRADE"), + "missing completion": lambda d, l, c: c.pop(), + } + for name, mutate in mutations.items(): + with self.subTest(name=name): + direct, lifecycle, coherence = self.records() + mutate(direct, lifecycle, coherence) + with self.assertRaisesRegex(ValueError, "dirty SNP_DATA_INV"): + self.runner.correlate_dirty_backinvalidations( + direct, lifecycle, coherence + ) + + def test_strict_jsonl_rejects_duplicate_truncated_and_backwards(self): + with tempfile.TemporaryDirectory() as temporary: + path = pathlib.Path(temporary) / "trace.jsonl" + path.write_text( + '{"schema_version":1,"event":"x","monotonic_ns":2}\n' + '{"schema_version":1,"event":"y","monotonic_ns":1}\n', + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "backwards"): + self.runner.read_coherence_trace(path) + path.write_text('{"schema_version":1,"event":"x"}', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "truncated"): + self.runner.read_coherence_trace(path) + path.write_text( + '{"schema_version":1,"event":"x","event":"y","monotonic_ns":1}\n', + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "duplicate JSON key"): + self.runner.read_coherence_trace(path) + + def test_host_capture_orders_unmap_through_dirty_completion(self): + direct, lifecycle, coherence = self.records() + correlations = self.runner.correlate_dirty_backinvalidations( + direct, lifecycle, coherence + ) + with tempfile.TemporaryDirectory() as temporary: + paths = self.runner.RuntimePaths(temporary, temporary) + paths.event_log(1).write_text( + json.dumps({ + "host_capture_ns": 10, + "line": "BADFS_DIRECT_MAP_TRACE_JSON " + json.dumps(direct[0]), + }) + "\n", + encoding="utf-8", + ) + paths.event_log(0).write_text( + json.dumps({ + "host_capture_ns": 50, + "line": "BADFS_LIFECYCLE_TRACE_JSON " + json.dumps(lifecycle[1]), + }) + "\n", + encoding="utf-8", + ) + ordered = self.runner.validate_host_capture_order(paths, correlations) + self.assertEqual(ordered["snoop_send_ns"], 20) + self.assertEqual(ordered["dirty_completion_ns"], 22) + + correlations[0]["completion"]["monotonic_ns"] = 60 + with self.assertRaisesRegex(ValueError, "unmap.*snoop.*completion.*success"): + self.runner.validate_host_capture_order(paths, correlations) + + def test_benchmark_and_fallback_gate(self): + bytes_count = 65536 + block_size = min(bytes_count, 1024 * 1024) + checksum = self.runner.expected_benchmark_checksum(bytes_count, block_size, 1) + fabric = { + "trusted_direct_read_ops": 1, "trusted_direct_read_bytes": bytes_count, + "trusted_direct_write_ops": 1, "trusted_direct_write_bytes": bytes_count, + "staged_read_ops": 0, "staged_read_bytes": 0, + "staged_write_ops": 0, "staged_write_bytes": 0, + "blob_read_ops": 0, "blob_read_bytes": 0, + "blob_write_ops": 0, "blob_write_bytes": 0, + "legacy_read_file_block_ops": 0, "legacy_write_file_block_ops": 0, + "legacy_read_fabric_block_ops": 0, "legacy_write_fabric_block_ops": 0, + "stale_ref_rejections": 0, "epoch_rejections": 0, + "checksum_failures": 0, "lease_rejections": 0, + "quarantine_events": 0, "active_leases": 0, "quarantined_slots": 0, + } + audit = { + "pending_operations": 0, "quarantined_extents": 0, + "active_read_leases": 0, "direct_mapped_extents": 1, + } + output = ( + f"badfs_bench file_size={bytes_count} block_size={block_size} iterations=1 " + f"written_bytes={bytes_count} read_bytes={bytes_count} write_secs=1.0 " + f"read_secs=1.0 write_mib_s=1.0 read_mib_s=1.0 checksum={checksum}\n" + "badfs_lifecycle_inspection " + + json.dumps({ + "schema_version": "badfs.lifecycle.inspection.v1", + "server": 0, "fabric": fabric, "audit": audit, + }) + + "\n" + ) + result = self.runner.validate_legofs_output(output, bytes_count) + self.assertEqual(result["benchmark"]["checksum"], checksum) + fabric["blob_write_ops"] = 1 + bad = output.split("badfs_lifecycle_inspection ", 1)[0] + ( + "badfs_lifecycle_inspection " + + json.dumps({ + "schema_version": "badfs.lifecycle.inspection.v1", + "server": 0, "fabric": fabric, "audit": audit, + }) + + "\n" + ) + with self.assertRaisesRegex(ValueError, "fallback counter"): + self.runner.validate_legofs_output(bad, bytes_count) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_legofs_runtime.py b/tests/test_legofs_runtime.py new file mode 100644 index 0000000..06d8781 --- /dev/null +++ b/tests/test_legofs_runtime.py @@ -0,0 +1,150 @@ +import importlib.util +import pathlib +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +RUNNER = ROOT / "scripts" / "legofs_type3_2node.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("legofs_type3_2node", RUNNER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class LegofsRuntimeTest(unittest.TestCase): + def setUp(self): + self.runner = load_runner() + self.temporary = tempfile.TemporaryDirectory() + root = pathlib.Path(self.temporary.name) + self.paths = self.runner.RuntimePaths.for_test(root) + + def tearDown(self): + self.temporary.cleanup() + + def test_commands_are_exact_sifive_u_persistent_type3_nodes(self): + commands = [ + self.runner.build_qemu_command(self.paths, node, 19300, 23345) + for node in (0, 1) + ] + for node, command in enumerate(commands): + joined = " ".join(command) + self.assertEqual(command[:3], ["qemu-system-riscv64", "-M", "sifive_u"]) + self.assertEqual(sum("cxl-type3" in argument for argument in command), 1) + self.assertIn("coherence-v2=on", joined) + self.assertIn(f"coherence-v2-host-id={node}", joined) + self.assertIn("coherence-v2-cache-capacity=8388608", joined) + self.assertIn("coherence-v2-cache-ways=4", joined) + self.assertIn("coherence-v2-write-through=off", joined) + self.assertIn( + f"coherence-v2-read-exclusive={'on' if node == 0 else 'off'}", + joined, + ) + self.assertNotIn("-M virt", joined) + self.assertNotIn("volatile-memdev", joined) + self.assertIn("memory-backend-file", joined) + self.assertIn("pmem=on", joined) + self.assertIn(f"persistent-memdev=t3ssd-node{node}", joined) + self.assertIn("cxlmemsim-port=19300", joined) + self.assertEqual(joined.count("persistent-memdev="), 1) + for identifier in ("t3ssd", "t3lsa", "cxl", "rp-t3", "t3", "net"): + self.assertIn(f"{identifier}-node{node}", joined) + + self.assertIn("hostfwd=tcp:127.0.0.1:23345-:3345", " ".join(commands[0])) + self.assertNotIn("hostfwd=", " ".join(commands[1])) + self.assertNotEqual( + self.paths.cxl_ssd_path(0), + self.paths.cxl_ssd_path(1), + ) + + def test_server_uses_authoritative_ssd_and_tcp_v2_trace(self): + command = self.runner.build_server_command(self.paths, 19300) + joined = " ".join(command) + self.assertIn("--comm-mode=tcp", joined) + self.assertIn("--port=19300", joined) + self.assertIn("--coherence-v2=true", joined) + self.assertIn("--coherence-v2-snoop-timeout-ms=5000", joined) + self.assertIn("--coherence-v2-trace=", joined) + self.assertIn("--backing-mode=ssd-stream", joined) + self.assertIn("--ssd-backing-file=", joined) + + def test_environment_removes_legacy_transports(self): + environment = self.runner.qemu_environment( + self.paths, + { + "PATH": "/bin", + "CXL_TRANSPORT_MODE": "shm", + "CXL_PGAS_SHM": "/wrong", + "CXL_MEMSIM_SERVER": "wrong", + }, + ) + for name in ("CXL_TRANSPORT_MODE", "CXL_PGAS_SHM", "CXL_MEMSIM_SERVER"): + self.assertNotIn(name, environment) + self.assertEqual(environment["PATH"].split(":", 1)[0], str(self.paths.qemu.parent)) + + def test_overlap_requires_two_live_intervals(self): + self.assertEqual(self.runner.overlap_ns((10, 50), (20, 60)), 30) + with self.assertRaisesRegex(ValueError, "did not overlap"): + self.runner.overlap_ns((10, 20), (20, 30)) + + def test_uboot_sequence_interrupts_autoboot_before_cxl_commands(self): + source = RUNNER.read_text(encoding="utf-8") + interrupt = source.index('console.wait("Hit any key to stop autoboot"') + prompt = source.index('console.wait("=> ", timeout)', interrupt) + listing = source.index('console.command_until_prompt("cxl list"', prompt) + self.assertLess(interrupt, prompt) + self.assertLess(prompt, listing) + + def test_linux_explicitly_routes_persistent_cxl_region_to_devdax(self): + source = RUNNER.read_text(encoding="utf-8") + self.assertIn("cxl_core.pmem_as_dax=1", source) + + config = (ROOT / "configs" / "linux-cxl.config").read_text(encoding="utf-8") + for option in ( + "CONFIG_MEMORY_HOTPLUG=y", + "CONFIG_MEMORY_HOTREMOVE=y", + "CONFIG_SPARSEMEM_VMEMMAP=y", + "CONFIG_ZONE_DEVICE=y", + "# CONFIG_DEV_DAX_KMEM is not set", + ): + self.assertIn(option, config) + + region_source = ( + ROOT / "components" / "linux" / "drivers" / "cxl" / "core" / "region.c" + ).read_text(encoding="utf-8") + self.assertIn("module_param_named(pmem_as_dax", region_source) + self.assertIn("if (cxl_pmem_as_dax)", region_source) + self.assertIn("return devm_cxl_add_dax_region(cxlr);", region_source) + + cxl_dax_source = ( + ROOT / "components" / "linux" / "drivers" / "dax" / "cxl.c" + ).read_text(encoding="utf-8") + self.assertIn("failed to create device-dax for CXL region", cxl_dax_source) + dax_device_source = ( + ROOT / "components" / "linux" / "drivers" / "dax" / "device.c" + ).read_text(encoding="utf-8") + self.assertIn("failed to map device-dax pages", dax_device_source) + + def test_guest_waits_for_asynchronous_cxl_region_and_dax_probe(self): + source = (ROOT / "guest" / "legofs_node_init.c").read_text(encoding="utf-8") + self.assertIn('wait_for_prefix("/sys/bus/cxl/devices", "region", 1)', source) + self.assertIn('wait_for_prefix("/sys/bus/cxl/devices", "decoder", 1)', source) + self.assertIn('wait_for_prefix("/sys/bus/dax/devices", "dax", 1)', source) + self.assertNotIn("/sys/class/dax", source) + self.assertIn("__builtin_offsetof(struct linux_dirent64, d_name) + 1", source) + self.assertNotIn("sizeof(*entry) + 1", source) + + def test_console_waiting_uses_chunks_but_sidecars_use_complete_lines(self): + source = RUNNER.read_text(encoding="utf-8") + append = source.index("self.output += decoded") + split = source.index('while b"\\n" in pending:', append) + event = source.index("self._capture_event(raw_line)", split) + self.assertLess(append, split) + self.assertLess(split, event) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_legofs_sources.py b/tests/test_legofs_sources.py new file mode 100644 index 0000000..59f70f4 --- /dev/null +++ b/tests/test_legofs_sources.py @@ -0,0 +1,38 @@ +import pathlib +import subprocess +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +CXL_BASE = "716c16c9efc7a733006d0772f8c6c4bb055f7b15" +LEGOFS_BASE = "96f733940251d6484dad0ba2cfbe99dcf5259776" + + +def git(*args, cwd=ROOT): + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + + +class LegofsSourceTest(unittest.TestCase): + def test_cxlmemsim_descends_from_approved_mesi_v2_commit(self): + component = ROOT / "components" / "cxlmemsim" + self.assertEqual(git("merge-base", "HEAD", CXL_BASE, cwd=component), CXL_BASE) + + def test_legofs_descends_from_approved_commit(self): + component = ROOT / "components" / "legofs" + self.assertTrue(component.is_dir(), "components/legofs is missing") + self.assertEqual(git("merge-base", "HEAD", LEGOFS_BASE, cwd=component), LEGOFS_BASE) + + def test_gitmodules_uses_approved_legofs_remote(self): + modules = (ROOT / ".gitmodules").read_text(encoding="utf-8") + self.assertIn("path = components/legofs", modules) + self.assertIn("url = https://github.com/Zettai-US/legofs.git", modules) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 77e11d0..a20166a 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -11,7 +11,7 @@ class ManifestTest(unittest.TestCase): - def invoke(self, output, *artifacts): + def invoke(self, output, *artifacts, compilers=()): self.assertTrue(SCRIPT.is_file(), "scripts/write_manifest.py is missing") command = [ "python3", @@ -23,6 +23,8 @@ def invoke(self, output, *artifacts): ] for name, path in artifacts: command.extend(["--artifact", f"{name}={path}"]) + for name, compiler in compilers: + command.extend(["--compiler", f"{name}={compiler}"]) return subprocess.run(command, text=True, capture_output=True) def test_manifest_hashes_artifacts_and_records_gitlinks(self): @@ -37,10 +39,11 @@ def test_manifest_hashes_artifacts_and_records_gitlinks(self): output, ("first", first), ("second", second), + compilers=(("python", "python3 --version"),), ) self.assertEqual(run.returncode, 0, run.stderr) manifest = json.loads(output.read_text(encoding="utf-8")) - self.assertEqual(manifest["schema_version"], 1) + self.assertEqual(manifest["schema_version"], 2) self.assertEqual( manifest["superproject_commit"], subprocess.run( @@ -50,10 +53,14 @@ def test_manifest_hashes_artifacts_and_records_gitlinks(self): capture_output=True, ).stdout.strip(), ) - self.assertEqual( - manifest["submodules"]["components/qemu"], - "81cd7ad9a5e14470427c8ebafeccff4f52e555b4", - ) + expected_qemu = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD:components/qemu"], + check=True, + text=True, + capture_output=True, + ).stdout.strip() + self.assertEqual(manifest["submodules"]["components/qemu"], expected_qemu) + self.assertIn("Python", manifest["compilers"]["python"]["version"]) self.assertEqual(manifest["artifacts"]["first"]["size"], 3) self.assertEqual( manifest["artifacts"]["first"]["sha256"],