Codex re-pin: get the rollout walk off the event loop

Dockerfile

@@ -138,7 +138,21 @@ COPY --chown=node:node session.bashrc /app/
 ENV PORT=7860 \
     DATA_DIR=/data \
     PUBLIC_DIR=/app/public \
-    DISABLE_AUTOUPDATER=1
+    DISABLE_AUTOUPDATER=1 \
+    UV_THREADPOOL_SIZE=16
+
+# Why 16 and not libuv's default 4: /data is a FUSE bucket, and moving the
+# re-pin walks off the event loop moved them onto that pool instead — one
+# outstanding op per watching pane, and panes launched together stay in phase,
+# so their beats land together. Measured on the bucket, one stat on an unrelated
+# file while N panes walk (p95 / worst):
+#
+#   pool  4:  N=4  0.2ms / 8ms      N=8  99.6ms / 177ms
+#   pool 16:  N=4  0.2ms / 5ms      N=8   0.3ms /  89ms
+#
+# At 4 the pool is oversubscribed by the walks and everything else fs-shaped in
+# the process queues behind them. The isolated worst-case outliers are the mount
+# hiccuping and survive any pool size — it's the p95 that this fixes.
 
 # Snapshot the env var NAMES present at build time. HF injects Space secrets and
 # variables only at runtime, so anything in the runtime env that's absent here

server/package.json

@@ -14,7 +14,7 @@
     "start": "node src/index.js",
     "dev": "node --watch src/index.js",
     "test:ui": "node terminal-ui.test.mjs",
-    "test": "node test/hidden.test.mjs && node test/spawn-group.test.mjs && node test/revive.test.mjs && node test/repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node test/trace-tail.test.mjs && node test/trace-window.test.mjs && node migration.test.mjs && node resize.test.mjs"
+    "test": "node test/hidden.test.mjs && node test/slowfs.test.mjs && node test/spawn-group.test.mjs && node test/revive.test.mjs && node test/repin.test.mjs && node test/codex-repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node test/trace-tail.test.mjs && node test/trace-window.test.mjs && node migration.test.mjs && node resize.test.mjs"
   },
   "engines": {
     "node": ">=20.19"

server/src/index.js

@@ -36,6 +36,11 @@ import { shareSession, shareNamespace, findTrace, shareAccess, grantAccess, revo
          importBundle, listBundles, SHAREABLE_CLIS } from './share.js';
 import * as backup from './backup.js';
 import * as runstate from './runstate.js';
+import { installSlowFsProbe } from './slowfs.js';
+
+// Before anything else touches the mount: a sync fs call to /data is ~85ms of
+// frozen event loop here, and nothing else in the stack can see it. See slowfs.js.
+installSlowFsProbe();
 
 ensureDirs();
 refreshVersions();

server/src/slowfs.js

@@ -0,0 +1,75 @@
+// Warn when a synchronous fs call blocks the event loop, and name the caller.
+//
+// On the Space, DATA_DIR / HOME / CLAUDE_CONFIG_DIR are on the FUSE bucket,
+// where a statSync costs ~85ms against 0.01ms on local disk. Node runs JS on one
+// thread, so one such call freezes every pane the server is carrying for that
+// long. This has bitten us three times now — claudeTranscriptsSince, then
+// codexRolloutsSince — and it is hard to spot because **the kernel's pressure
+// accounting cannot see it**: FUSE parks the caller in wait_event_interruptible
+// (state S), not io_schedule, so the wait is never counted as iowait and
+// /proc/pressure/io reads 0.00 while the loop is stuck. Hence a tripwire here.
+//
+// Silent on a normal filesystem. AM_SLOWFS_MS=0 disables it.
+import fs from 'node:fs';
+
+const THRESHOLD_MS = Number(process.env.AM_SLOWFS_MS ?? 50);
+// One line per call site per window: a bucket stall must not become a log storm.
+const REPEAT_MS = 10_000;
+
+const lastLogged = new Map(); // call site -> { at, suppressed }
+
+// The first frame outside this file that names real source. Internal callback
+// frames ("at Array.forEach (<anonymous>)") point at nothing actionable.
+function callSite(skip) {
+  const holder = {};
+  try { Error.captureStackTrace(holder, skip); } catch { return '?'; }
+  for (const raw of String(holder.stack || '').split('\n').slice(1)) {
+    const line = raw.trim();
+    if (!line.startsWith('at ') || line.includes('slowfs.js') || line.includes('node:')) continue;
+    if (!line.includes('file:') && !line.includes('/')) continue;
+    return line.slice(3);
+  }
+  return '?';
+}
+
+export function installSlowFsProbe({ thresholdMs = THRESHOLD_MS } = {}) {
+  if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) return false;
+
+  // Every *Sync method, rather than a hand-kept list that drifts as callers move.
+  for (const name of Object.keys(fs).filter((k) => k.endsWith('Sync'))) {
+    const original = fs[name];
+    if (typeof original !== 'function') continue;
+
+    function slowFsWrapper(...args) {
+      const t0 = performance.now();
+      try {
+        return original.apply(this, args);
+      } finally {
+        // This sits in front of every sync fs call in the process, including
+        // ones inside catch blocks: it must never throw and never swallow.
+        try {
+          const ms = performance.now() - t0;
+          if (ms >= thresholdMs) warn(name, args[0], ms, slowFsWrapper);
+        } catch { /* diagnostics must not break the app */ }
+      }
+    }
+    // realpathSync.native and friends hang off the function itself.
+    Object.assign(slowFsWrapper, original);
+    Object.defineProperty(slowFsWrapper, 'name', { value: name });
+    fs[name] = slowFsWrapper;
+  }
+  console.warn(`[slowfs] watching synchronous fs; logging calls over ${thresholdMs}ms`);
+  return true;
+}
+
+function warn(method, target, ms, skip) {
+  const site = callSite(skip);
+  const key = `${method} ${site}`;
+  const prev = lastLogged.get(key);
+  const now = Date.now();
+  if (prev && now - prev.at < REPEAT_MS) { prev.suppressed++; return; }
+  const extra = prev?.suppressed ? ` (+${prev.suppressed} more since last line)` : '';
+  lastLogged.set(key, { at: now, suppressed: 0 });
+  // target may be a Buffer, URL or fd; String() covers all three well enough.
+  console.warn(`[slowfs] ${ms.toFixed(0)}ms ${method} ${String(target)} — at ${site}${extra}`);
+}

server/test/codex-repin.test.mjs

@@ -0,0 +1,80 @@
+// The codex rollout walk, after being made async.
+//
+// It walks $CODEX_HOME/sessions, whose `sessions` child is a symlink onto the
+// FUSE bucket on the Space. Synchronously that froze every pane for 400-750ms on
+// every REPIN_MS beat, once per codex session. Run with:
+//   node test/codex-repin.test.mjs
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-repin-'));
+process.env.CODEX_HOME = path.join(TMP, 'codex-home');
+process.env.DATA_DIR = path.join(TMP, 'data');
+fs.mkdirSync(process.env.DATA_DIR, { recursive: true });
+
+const runner = await import('../src/runner.js');
+
+let pass = 0, fail = 0;
+const check = (name, got, want) => {
+  const ok = got === want;
+  ok ? pass++ : fail++;
+  console.log(`  ${ok ? 'PASS' : 'FAIL'}  ${name}${ok ? '' : `\n          got ${got}  want ${want}`}`);
+};
+
+const NOW = 1770000000000;
+const SESSIONS = path.join(process.env.CODEX_HOME, 'sessions');
+
+// Codex lays rollouts out under sessions/YYYY/MM/DD/, so the walk has to descend
+// three levels before it ever sees a file.
+function rollout(uuid, { day = '28', month = '07', mtimeMs }) {
+  const dir = path.join(SESSIONS, '2026', month, day);
+  fs.mkdirSync(dir, { recursive: true });
+  const p = path.join(dir, `rollout-2026-${month}-${day}T00-00-00-${uuid}.jsonl`);
+  fs.writeFileSync(p, `${JSON.stringify({ payload: { cwd: '/w', timestamp: '2026-07-28T00:00:00Z' } })}\n`);
+  fs.utimesSync(p, new Date(mtimeMs), new Date(mtimeMs));
+  return p;
+}
+
+const U = (n) => `0000000${n}-0000-4000-8000-000000000000`;
+
+console.log('an absent root is not an error');
+check('returns empty, does not throw', (await runner.codexRolloutsSince(0)).length, 0);
+
+console.log('\nfinds rollouts under the YYYY/MM/DD tree');
+const old = rollout(U(1), { mtimeMs: NOW - 60_000 });
+const mid = rollout(U(2), { mtimeMs: NOW - 30_000 });
+const New = rollout(U(3), { month: '08', day: '01', mtimeMs: NOW });
+
+let got = await runner.codexRolloutsSince(0);
+check('finds all three across months', got.length, 3);
+check('newest first', got[0].p, New);
+check('oldest last', got[2].p, old);
+
+console.log('\nfilters on mtime');
+got = await runner.codexRolloutsSince(NOW - 45_000);
+check('drops the one older than sinceMs', got.length, 2);
+check('keeps the newest', got[0].p, New);
+check('keeps the middle', got[1].p, mid);
+
+console.log('\nignores anything that is not a rollout');
+fs.writeFileSync(path.join(SESSIONS, '2026', '07', '28', 'notes.jsonl'), '{}\n');
+fs.writeFileSync(path.join(SESSIONS, '2026', '07', '28', 'rollout-x.txt'), 'x\n');
+check('still only the rollouts', (await runner.codexRolloutsSince(0)).length, 3);
+
+console.log('\nis actually async — the whole point of the change');
+const returned = runner.codexRolloutsSince(0);
+check('returns a promise', typeof returned.then, 'function');
+await returned;
+
+// A rollout nested past the depth cap must not be walked forever.
+console.log('\nrespects the depth cap');
+let deep = SESSIONS;
+for (let i = 0; i < 8; i++) deep = path.join(deep, `d${i}`);
+fs.mkdirSync(deep, { recursive: true });
+fs.writeFileSync(path.join(deep, `rollout-deep-${U(9)}.jsonl`), '{}\n');
+check('too-deep rollout ignored', (await runner.codexRolloutsSince(0)).length, 3);
+
+console.log(`\n${pass} passed, ${fail} failed`);
+fs.rmSync(TMP, { recursive: true, force: true });
+process.exit(fail ? 1 : 0);

server/test/repin.test.mjs

@@ -169,7 +169,8 @@ check('Codex exact id accepted', runner.breadcrumbVerdict(codexCrumb, 's1', {
   cli: 'codex', runId: RUN, workdir: WORKDIR, pinned: null, claimed: new Set(), pidTrusted: true,
 }).repin, CODEX_ID);
 check('Codex rollout path retained', runner.codexRolloutForBreadcrumb(codexCrumb), rollout);
-check('Codex null transcript resolves by exact id', runner.codexRolloutForId(CODEX_ID), rollout);
+// Awaited: the id lookup walks the rollout tree, which is async now.
+check('Codex null transcript resolves by exact id', await runner.codexRolloutForId(CODEX_ID), rollout);
 const CODEX_ALIAS = path.join(TMP, 'codex-alias');
 fs.mkdirSync(CODEX_ALIAS);
 fs.symlinkSync(path.join(CODEX, 'sessions'), path.join(CODEX_ALIAS, 'sessions'));

server/test/slowfs.test.mjs

@@ -0,0 +1,69 @@
+// The sync-fs tripwire: it must name the caller of a slow call without ever
+// changing what that call does. Run with: node test/slowfs.test.mjs
+//
+// Timing is deliberately not asserted on — a test needing a real 50ms stat would
+// want a FUSE mount or a sleep, and both are flaky. The threshold drops to ~0
+// instead, so every call is "slow" and the same path runs: wrap, time, attribute.
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'slowfs-'));
+const FILE = path.join(TMP, 'hello.txt');
+fs.writeFileSync(FILE, 'contents\n');
+
+let pass = 0, fail = 0;
+const check = (name, got, want) => {
+  const ok = got === want;
+  ok ? pass++ : fail++;
+  console.log(`  ${ok ? 'PASS' : 'FAIL'}  ${name}${ok ? '' : `\n          got ${got}  want ${want}`}`);
+};
+
+const { installSlowFsProbe } = await import('../src/slowfs.js');
+
+console.log('disabled unless a positive threshold is set');
+const untouched = fs.statSync;
+check('0 disables', installSlowFsProbe({ thresholdMs: 0 }), false);
+check('negative disables', installSlowFsProbe({ thresholdMs: -1 }), false);
+check('NaN disables', installSlowFsProbe({ thresholdMs: Number('nope') }), false);
+check('fs left unwrapped when disabled', fs.statSync, untouched);
+
+// Capture the warnings rather than exporting state from the module purely to
+// be tested: the log line IS the product here.
+const logs = [];
+const realWarn = console.warn;
+console.warn = (...a) => { logs.push(a.join(' ')); };
+
+check('installs', installSlowFsProbe({ thresholdMs: 0.0000001 }), true);
+
+console.log('\ninstalled: every sync call still behaves exactly as before');
+check('readFileSync returns content', fs.readFileSync(FILE, 'utf8'), 'contents\n');
+check('existsSync true', fs.existsSync(FILE), true);
+check('existsSync false', fs.existsSync(path.join(TMP, 'nope')), false);
+check('statSync size', fs.statSync(FILE).size, 9);
+check('readdirSync finds the file', fs.readdirSync(TMP).includes('hello.txt'), true);
+// Wrapping replaces the function object, so properties hanging off it (notably
+// realpathSync.native) would vanish without the Object.assign.
+check('realpathSync.native survives wrapping', typeof fs.realpathSync.native, 'function');
+
+// A wrapper that swallowed errors would be worse than the bug it hunts.
+let threw = null;
+try { fs.readFileSync(path.join(TMP, 'missing'), 'utf8'); } catch (e) { threw = e.code; }
+check('errors still propagate', threw, 'ENOENT');
+
+console.log('\nit attributes the call to the caller, not to itself');
+const joined = logs.join('\n');
+check('something was logged', logs.length > 0, true);
+check('names this test file', joined.includes('slowfs.test.mjs'), true);
+check('does not blame slowfs.js', joined.includes('slowfs.js'), false);
+check('names the method', joined.includes('readFileSync'), true);
+
+console.log('\na stalled mount must not become a log storm');
+logs.length = 0;
+for (let i = 0; i < 5; i++) fs.existsSync(FILE); // one line => one call site
+check('repeats collapse to a single line', logs.filter((l) => l.includes('existsSync')).length, 1);
+
+console.warn = realWarn;
+console.log(`\n${pass} passed, ${fail} failed`);
+fs.rmSync(TMP, { recursive: true, force: true });
+process.exit(fail ? 1 : 0);