Add whole-process four-engine benchmark memory diagnostics

Author: KSemenenko

README.md

@@ -143,9 +143,41 @@ Separately, native `llama-bench` reported 115.36 ± 5.94 tok/s for five
 excludes tokenization and sampling, so it is a kernel diagnostic, not a
 completion-process result.
 
+### Whole-process memory diagnostic (new run)
+
+A separate C# matrix runner rotated all four CPU subjects through 3 warm-up
+and 5 measured fresh-process rounds on this Mac, with 8 threads, the same
+Qwen GGUF and matching eight-token continuation. Medians below are from
+[the raw 32 samples](benchmarks/results/2026-09-28-m2-pro-qwen2.5-0.5b-q8_0-four-subject-memory-clr-smoke.json).
+Power/thermal and competing-load state were not captured for this run, so it
+has **no statistical winner verdict** and must not be merged with the earlier
+12-thread table.
+
+| Subject · CPU | Peak RSS MiB | Peak physical footprint MiB | CLR live heap MiB | Matrix process wall ms |
+|---|---:|---:|---:|---:|
+| Synapse | 560.8 | 39.9 | 20.4 | 895.1 |
+| dotLLM | 1,187.3 | 663.1 | not instrumented | 1,350.9 |
+| LLamaSharp | 1,274.9 | 606.9 | 1.1 | 962.6 |
+| direct llama.cpp | 1,258.6 | 594.6 | not applicable | 681.6 |
+
+Peak RSS includes managed, native, and file-backed resident pages. macOS
+physical footprint is a different accounting view; for example, Synapse's
+mapped model raises RSS far above its footprint. CLR heap is a diagnostic
+subset, **not** total .NET memory; subtracting it from RSS does not yield
+native allocations. The raw JSON also preserves virtual-size availability,
+sample counts, executable/model hashes, and each subject's original result.
+Profilers are run separately so their overhead does not contaminate this
+table.
+
+The [32-token quality diagnostic](benchmarks/results/2026-09-28-m2-pro-qwen2.5-0.5b-q8_0-32tok-quality-divergence-final.json)
+is `ineligible_quality_mismatch`: dotLLM's continuation diverged from
+LLamaSharp/direct llama.cpp after a shared prefix. Synapse currently emits
+token IDs without a repo-owned decoder, so its 32-token text parity cannot
+yet be asserted. These 32-token timings are **not** a four-engine speed result.
+
 | Catalog model / architecture | Synapse | dotLLM | LLamaSharp | native llama.cpp | MLX | ONNX Runtime |
 |---|---|---|---|---|---|---|
-| Qwen2.5 0.5B Q8_0 · Qwen2 | D8 | D8 | D8 | D8 + D128 | NR | NR |
+| Qwen2.5 0.5B Q8_0 · Qwen2 | D8 + D32 IDs | D8 + D32 mismatch | D8 + D32 | D8 + D32 + D128 | NR | NR |
 | SmolLM2 135M BF16 · Llama | NR | NR | NR | NR | NR | NR |
 | Qwen3 0.6B Q8_0 · Qwen3 | NR | NR | NR | NR | NR | NR |
 | Mamba 130M F32 · SSM | NR | NR | NR | NR | NR | NR |
@@ -237,6 +269,9 @@ docs/                        architecture, ADRs, features, commands, task regist
 | [llama.cpp](https://github.com/ggml-org/llama.cpp) | GGUF/quantization reference and direct native baseline | MIT; external CPU process pinned at `b29c606e2` |
 | [MLX Swift LM](https://github.com/ml-explore/mlx-swift-lm) | Candidate Python-free Apple Silicon/Metal baseline | External subject planned; no measurement yet |
 | [ONNX Runtime GenAI](https://onnxruntime.ai/docs/genai/api/csharp.html) | Candidate C# ONNX-format baseline | Preview API; verified ONNX package and measurement pending |
+| [dotnet/diagnostics](https://github.com/dotnet/diagnostics) | CLR heap, GC counters, and traces for managed allocation diagnosis | Separate profiling runs, never the clean timing baseline |
+| [samply](https://github.com/mstange/samply) | Mac/Linux CPU stack sampling across native hotspots | Separate profiling runs; not a memory allocation collector |
+| [KDE heaptrack](https://github.com/KDE/heaptrack) | Native heap allocation trace on Linux | Collector is Linux-only; not used for Mac claims |
 | [ZoneTree](https://github.com/ZoneTree/ZoneTree) | Durable cache metadata, prefix indexes, journals, evidence indexes | MIT; runtime package 1.9.8 |
 | [Microsoft Orleans](https://github.com/dotnet/orleans) | Request/control plane, leases, epochs, placement, recovery | Planned D3 dependency; never tensor/KV transport |
 | [Aspire](https://github.com/dotnet/aspire) | Multi-process topology, health, telemetry, test orchestration | Added only with the first real distributed topology |

benchmarks/results/2026-09-28-m2-pro-qwen2.5-0.5b-q8_0-32tok-quality-divergence-final.json

@@ -0,0 +1 @@
+{"status":"ineligible_quality_mismatch","model_sha256":"ca59ca7f13d0e15a8cfa77bd17e65d24f6844b554a7b6c12e07a5f89ff76844e","runner_sha256":"7572d665df9a98de422b224d3eb0ea8d7f3ccb49a32f47d9679de8068307171b","synapse_binary_sha256":"8ca7187e701f6ea9d832148d843e8e8562eb51674644060d4b8141215256ccad","dot_llm_binary_sha256":"231751c8772ab8503fd0697e8a541d6aa20d4041d40e1672e48044256c011cdd","llama_cpp_binary_sha256":"d268690c4ec3f7db8f99f970aefc600afe57e0fb9835d182c2b1232b4610a8c3","llama_sharp_assembly_sha256":"559fd41e6a9757ccd7aaec61255ecde33bc0294894510e10b94d095e02ad7c10","operating_system":"macOS 27.0.0","architecture":"Arm64","power_state":"not_recorded","prompt":"The capital of France is","prompt_token_ids":[785,6722,315,9625,374],"expected_token_ids":[12095,13,1084,374,279,7772,3283,304,4505,323,279,2086,7772,304,279,1879,13,1084,374,7407,304,279,4126,315,279,8585,92900,11,304,279,9806,315],"expected_text":" Paris. It is the largest city in Europe and the second largest in the world. It is located in the center of the French Alps, in the south of","threads":8,"max_tokens":32,"warmups":0,"measurements":1,"samples":[{"subject":"synapse","round":0,"warmup":false,"quality_matched":true,"subject_result":{"subject":"synapse-managed-qwen2-q8_0","model_path":"/Users/ksemenenko/Developer/Synapse/artifacts/models/qwen2.5-0.5b-instruct-q8_0/qwen2.5-0.5b-instruct-q8_0.gguf","prompt_tokens":[785,6722,315,9625,374],"generated_tokens":[12095,13,1084,374,279,7772,3283,304,4505,323,279,2086,7772,304,279,1879,13,1084,374,7407,304,279,4126,315,279,8585,92900,11,304,279,9806,315],"threads":8,"load_milliseconds":76.7954,"time_to_first_token_milliseconds":541.4696,"total_generation_milliseconds":1767.9345,"total_output_tokens_per_second":18.100218079346266,"decode_tokens_per_second":25.275896603319023,"subject_wall_milliseconds":1846.2697,"process_cpu_milliseconds":13974.154,"average_cpu_cores":7.568858439262693,"working_set_after_load_bytes":58458112,"maximum_observed_working_set_bytes":588070912,"managed_live_heap_after_load_bytes":14839768,"managed_live_heap_after_generation_bytes":17667896,"managed_allocated_during_subject_bytes":58384968},"peak_resident_bytes":590462976,"peak_physical_footprint_bytes":43812112,"memory_sample_count":379,"memory_metric_origin":"outer_direct_subject","matrix_process_wall_milliseconds":1937.3599,"outer_process_peak_resident_bytes":590462976},{"subject":"dotllm","round":0,"warmup":false,"quality_matched":false,"subject_result":{"subject":"dotllm","subject_version":"d88040451d7db56e5dfef9d5754ad0955b0f7fe5","backend":"cpu","model_path":"/Users/ksemenenko/Developer/Synapse/artifacts/models/qwen2.5-0.5b-instruct-q8_0/qwen2.5-0.5b-instruct-q8_0.gguf","prompt":"The capital of France is","max_tokens":32,"threads":8,"generated_tokens":32,"text":" Paris. It is the largest city in Europe and the second largest in the world. It is located in the south of France, on the Mediterranean Sea. It","load_milliseconds":373.6,"time_to_first_token_milliseconds":354,"total_generation_milliseconds":1300.4,"decode_tokens_per_second":33.38,"subject_wall_milliseconds":1815.8822,"process_cpu_milliseconds":9917.3333,"average_cpu_cores":5.4614408908243055,"working_set_after_load_bytes":null,"maximum_observed_working_set_bytes":1246609408,"measurement_scope":"dotllm_reported_plus_process_observed","prompt_token_ids":null,"native_prompt_eval_milliseconds":null,"native_eval_milliseconds":null,"native_eval_tokens_per_second":null,"maximum_observed_private_virtual_bytes":null,"maximum_observed_virtual_bytes":503086366720,"peak_physical_footprint_bytes":696960944,"memory_sample_count":357,"managed_live_heap_after_load_bytes":null,"managed_live_heap_after_generation_bytes":null,"managed_allocated_during_subject_bytes":null},"peak_resident_bytes":1246609408,"peak_physical_footprint_bytes":696960944,"memory_sample_count":357,"memory_metric_origin":"inner_direct_subject","matrix_process_wall_milliseconds":1893.1082,"outer_process_peak_resident_bytes":61341696},{"subject":"llamasharp","round":0,"warmup":false,"quality_matched":true,"subject_result":{"subject":"llamasharp","subject_version":"0.0.0.0","backend":"cpu","model_path":"/Users/ksemenenko/Developer/Synapse/artifacts/models/qwen2.5-0.5b-instruct-q8_0/qwen2.5-0.5b-instruct-q8_0.gguf","prompt":"The capital of France is","max_tokens":32,"threads":8,"generated_tokens":32,"text":" Paris. It is the largest city in Europe and the second largest in the world. It is located in the center of the French Alps, in the south of","load_milliseconds":716.7905,"time_to_first_token_milliseconds":16.553,"total_generation_milliseconds":281.1737,"decode_tokens_per_second":117.14880959803975,"subject_wall_milliseconds":1019.8107,"process_cpu_milliseconds":2856.837,"average_cpu_cores":2.801340484072191,"working_set_after_load_bytes":1191329792,"maximum_observed_working_set_bytes":1337245696,"measurement_scope":"managed_wrapper_observed","prompt_token_ids":null,"native_prompt_eval_milliseconds":null,"native_eval_milliseconds":null,"native_eval_tokens_per_second":null,"maximum_observed_private_virtual_bytes":null,"maximum_observed_virtual_bytes":503150051328,"peak_physical_footprint_bytes":636700424,"memory_sample_count":194,"managed_live_heap_after_load_bytes":915104,"managed_live_heap_after_generation_bytes":1588520,"managed_allocated_during_subject_bytes":1495552},"peak_resident_bytes":1337245696,"peak_physical_footprint_bytes":636700424,"memory_sample_count":194,"memory_metric_origin":"inner_direct_subject","matrix_process_wall_milliseconds":1111.8127,"outer_process_peak_resident_bytes":1337245696},{"subject":"llamacpp","round":0,"warmup":false,"quality_matched":true,"subject_result":{"subject":"llamacpp","subject_version":"b29c606e28a01b1bc8c1351026a0fa6e616bf6c4","backend":"cpu","model_path":"/Users/ksemenenko/Developer/Synapse/artifacts/models/qwen2.5-0.5b-instruct-q8_0/qwen2.5-0.5b-instruct-q8_0.gguf","prompt":"The capital of France is","max_tokens":32,"threads":8,"generated_tokens":32,"text":" Paris. It is the largest city in Europe and the second largest in the world. It is located in the center of the French Alps, in the south of","load_milliseconds":null,"time_to_first_token_milliseconds":null,"total_generation_milliseconds":null,"decode_tokens_per_second":null,"subject_wall_milliseconds":713.0621,"process_cpu_milliseconds":2105.5041,"average_cpu_cores":2.952764001901097,"working_set_after_load_bytes":null,"maximum_observed_working_set_bytes":1314439168,"measurement_scope":"native_internal_plus_process_observed","prompt_token_ids":[785,6722,315,9625,374],"native_prompt_eval_milliseconds":9.85,"native_eval_milliseconds":176.34,"native_eval_tokens_per_second":175.79675626630373,"maximum_observed_private_virtual_bytes":null,"maximum_observed_virtual_bytes":502262513664,"peak_physical_footprint_bytes":623199936,"memory_sample_count":135,"managed_live_heap_after_load_bytes":null,"managed_live_heap_after_generation_bytes":null,"managed_allocated_during_subject_bytes":null},"peak_resident_bytes":1314439168,"peak_physical_footprint_bytes":623199936,"memory_sample_count":135,"memory_metric_origin":"inner_direct_subject","matrix_process_wall_milliseconds":788.9413,"outer_process_peak_resident_bytes":61456384}]}

benchmarks/results/2026-09-28-m2-pro-qwen2.5-0.5b-q8_0-four-subject-memory-clr-smoke.json

@@ -23,6 +23,18 @@ generation, or process-wall measurements. A result is comparable only when
 revision, model digest, prompt token IDs, backend, hardware, and workload are
 recorded together. Short smoke runs carry no winner verdict.
 
+The memory headline is the operating-system observation of the **whole subject
+process**, including CLR and native allocations: peak resident working set on
+both platforms and physical footprint on macOS when `proc_pid_rusage` permits
+it. Sampling begins before model load and continues through generation; the
+report records the sample count and separately labels private/virtual address
+space, which is not resident memory. Synapse CLI and LLamaSharp additionally
+report CLR live heap and cumulative managed allocation as diagnostics. These are subsets or
+allocation-flow counters, not a partition of resident bytes. In particular,
+`resident - GC heap` is **not** reported as native allocation: mmap pages,
+shared libraries, runtime heaps, and driver memory make that subtraction
+invalid. A missing platform metric remains null with its scope recorded.
+
 ## Consequences
 
 The benchmark result schema adds nullable timing fields and a provenance label

docs/ADR/ADR-005-benchmark-timing-provenance.md

@@ -25,6 +25,37 @@ subject wall time, process CPU time, average CPU cores, and observed working
 set. Energy/power is a separate nullable status. `not_run_missing_privilege`
 or `not_run_missing_hardware` never acquires a numeric zero.
 
+The process memory envelope is collected before model load through completion
+for every reference subject. Peak resident working set includes managed and
+native pages; macOS physical footprint is recorded separately via
+`proc_pid_rusage`. Virtual/private address-space values are **not** interpreted
+as resident memory (private virtual size is unavailable from the current
+macOS process API and remains null). Synapse CLI and LLamaSharp also emit live
+CLR heap and cumulative managed allocation diagnostics. These cannot be subtracted from
+RSS to obtain native bytes. The sampler records its count; unavailable metrics
+remain null. The existing dated smoke samples predate this schema and are not
+retroactively relabeled.
+
+Profiling is a separate diagnostic run, not part of paired timing samples:
+open-source `dotnet/diagnostics` supplies CLR counters/traces/heap inspection,
+`samply` can inspect native CPU stacks on macOS and Linux, and installed Xcode
+Instruments supplies Mac Allocations/Leaks/Metal traces. Linux-only heaptrack
+is not a Mac collector. Profiling overhead and native allocator attribution
+must be reported with the trace, never silently merged into baseline timing.
+
+The C# `matrix` command in `experiments/Synapse.ReferenceBenchmarks` now runs
+Synapse, dotLLM, LLamaSharp, and direct llama.cpp in rotated fresh-process
+rounds. It requires explicit executable paths, source/model token IDs, a
+locked expected continuation, thread/output limits, and a new output path;
+defaults are three warm-ups and five measured rounds. The output is immutable
+raw JSON with binary/model SHA-256 digests, every subject's original JSON,
+quality status, OS memory peaks, and sample counts. A mismatch is retained as
+`ineligible_quality_mismatch`, not erased or awarded a throughput verdict.
+Run `dotnet experiments/Synapse.ReferenceBenchmarks/bin/Release/net10.0/Synapse.ReferenceBenchmarks.dll matrix`
+without arguments for the exact option list. This diagnostic runner is not yet
+the 30-pair randomized release benchmark, and 8-token decode timing remains
+too short for an optimization win claim.
+
 Two 10-turn modes use `benchmarks/scenarios/travel-planner-10-turns.json`:
 
 - **locked transcript** feeds the same pre-recorded assistant response into

docs/Features/Benchmarking.md

@@ -0,0 +1,97 @@
+using System.Globalization;
+
+internal sealed record DiagnosticMatrixArguments(
+    string ModelPath,
+    string Prompt,
+    int[] PromptTokenIds,
+    int[] ExpectedTokenIds,
+    string ExpectedText,
+    string SynapseExecutable,
+    string DotLlmExecutable,
+    string DotLlmVersion,
+    string LlamaCppExecutable,
+    string LlamaCppVersion,
+    int MaxTokens,
+    int Threads,
+    int Warmups,
+    int Measurements,
+    string OutputPath)
+{
+    public static DiagnosticMatrixArguments? Parse(string[] arguments)
+    {
+        if (arguments.Length == 0 || arguments.Length % 2 != 0)
+        {
+            return null;
+        }
+
+        var values = new Dictionary<string, string>(StringComparer.Ordinal);
+        for (var index = 0; index < arguments.Length; index += 2)
+        {
+            if (!arguments[index].StartsWith("--", StringComparison.Ordinal) ||
+                !values.TryAdd(arguments[index], arguments[index + 1]))
+            {
+                return null;
+            }
+        }
+
+        var model = Required(values, "--model");
+        var prompt = Required(values, "--prompt");
+        var expectedText = Required(values, "--expected-text");
+        var synapse = Required(values, "--synapse-executable");
+        var dotLlm = Required(values, "--dotllm-executable");
+        var dotLlmVersion = Required(values, "--dotllm-version");
+        var llamaCpp = Required(values, "--llamacpp-executable");
+        var llamaCppVersion = Required(values, "--llamacpp-version");
+        var output = Required(values, "--output");
+        var promptIds = ParseIds(Required(values, "--prompt-token-ids"));
+        var expectedIds = ParseIds(Required(values, "--expected-token-ids"));
+        var maxTokens = ParseCount(values, "--max-tokens", 8);
+        var threads = ParseCount(values, "--threads", Environment.ProcessorCount);
+        var warmups = ParseCount(values, "--warmups", 3);
+        var measurements = ParseCount(values, "--measurements", 5);
+        if (new[] { model, prompt, expectedText, synapse, dotLlm, dotLlmVersion,
+            llamaCpp, llamaCppVersion, output }.Any(string.IsNullOrWhiteSpace) ||
+            promptIds is not { Length: > 0 } || expectedIds is not { Length: > 0 } ||
+            maxTokens is <= 0 or > 512 || threads is <= 0 or > 256 ||
+            warmups is < 0 or > 10 || measurements is <= 0 or > 200 ||
+            expectedIds.Length != maxTokens ||
+            !new[] { model, synapse, dotLlm, llamaCpp }.All(File.Exists))
+        {
+            return null;
+        }
+
+        return new DiagnosticMatrixArguments(
+            model!, prompt!, promptIds, expectedIds, expectedText!, synapse!,
+            dotLlm!, dotLlmVersion!, llamaCpp!, llamaCppVersion!, maxTokens,
+            threads, warmups, measurements, output!);
+    }
+
+    private static string? Required(Dictionary<string, string> values, string name) =>
+        values.TryGetValue(name, out var value) && !string.IsNullOrWhiteSpace(value)
+            ? value
+            : null;
+
+    private static int[]? ParseIds(string? value)
+    {
+        if (value is null)
+        {
+            return null;
+        }
+
+        var parts = value.Split(',', StringSplitOptions.TrimEntries);
+        var ids = new int[parts.Length];
+        for (var index = 0; index < parts.Length; index++)
+        {
+            if (!int.TryParse(parts[index], NumberStyles.None, CultureInfo.InvariantCulture, out ids[index]))
+            {
+                return null;
+            }
+        }
+
+        return ids;
+    }
+
+    private static int ParseCount(Dictionary<string, string> values, string name, int fallback) =>
+        !values.TryGetValue(name, out var value) ? fallback :
+        int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var count) ? count : -1;
+}