From 9474868bb1612f3235e4f171fdc82bc094219c57 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 02:12:11 -0800 Subject: [PATCH 1/7] delphi shadow: a job graph stuck past the job deadline is deferred and holds only its own conversation When a graph's attempt cannot be proven finished (its scope stays held), the scheduler no longer waits on it forever. After the job deadline the graph moves to a deferred list that is polled one per cycle, and the full-pipeline shadow keeps admitting every other conversation; only the conversation whose graph scope is held is skipped. The environments the graph path has used are recorded in the state file before any admission, so a graph admitted under a list that was later emptied is still found and recovered after a restart. The job daemon can bind its retained journal to one host identity file (POLIS_JOBS_JOURNAL_HOST_FILE): a journal written under another host is refused at start-up rather than read as exit evidence. Adds Rust tests for a lost attempt after a restart with the list emptied and for a committed admission whose reply was lost, and two Docker process checks for recovery and for the scheduler under its deployed memory and CPU limits. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- delphi/tests/shadow/graph_recovery.py | 142 ++++++++++++++++++ delphi/tests/shadow/graph_scheduler_limits.py | 78 ++++++++++ queue-rs/src/jobs/claim.rs | 10 +- queue-rs/src/jobs/config.rs | 4 + queue-rs/src/jobs/journal.rs | 48 ++++++ queue-rs/src/shadow/graph.rs | 110 ++++++++++++-- queue-rs/src/shadow/scheduler.rs | 12 ++ queue-rs/src/shadow/state.rs | 6 + queue-rs/tests/delphi_shadow.rs | 81 ++++++++++ 9 files changed, 479 insertions(+), 12 deletions(-) create mode 100644 delphi/tests/shadow/graph_recovery.py create mode 100644 delphi/tests/shadow/graph_scheduler_limits.py diff --git a/delphi/tests/shadow/graph_recovery.py b/delphi/tests/shadow/graph_recovery.py new file mode 100644 index 0000000000..7c8b46b6b1 --- /dev/null +++ b/delphi/tests/shadow/graph_recovery.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Owned Docker process-recovery checks against a migrated throwaway database. + +GRAPH_RECOVERY_IMAGE, GRAPH_RECOVERY_PG_CONTAINER, GRAPH_RECOVERY_DATABASE and +GRAPH_RECOVERY_OUT name caller-owned resources. No votes or external services. +""" +import json +import os +from pathlib import Path +import subprocess as sp +import time +import uuid + +OUT = Path(os.environ['GRAPH_RECOVERY_OUT']).resolve() +OUT.mkdir(parents=True, exist_ok=True) +PG = os.environ['GRAPH_RECOVERY_PG_CONTAINER'] +DB = os.environ['GRAPH_RECOVERY_DATABASE'] +IMAGE = os.environ['GRAPH_RECOVERY_IMAGE'] +PREFIX = 'graph-recovery-' + uuid.uuid4().hex[:12] +owned = [] +results = [] + + +def run(*args, **kwargs): + return sp.run(args, check=True, text=True, capture_output=True, **kwargs).stdout.strip() + + +def sql(query): + return run('docker', 'exec', '-i', PG, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1', + '-U', 'postgres', '-d', DB, input=query) + + +def lit(value): + return "'" + str(value).replace("'", "''") + "'" + + +def until(check): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + value = check() + if value: + return value + time.sleep(.2) + raise AssertionError('condition did not become true') + + +def launch(name, journal): + owned.append(name) + env = dict(POLIS_JOBS_ENABLED='1', QUEUE_ENV='shadow-graph-1', + DELPHI_RESULT_ENV='shadow-graph-1', DELPHI_SHADOW_GRAPH_DAEMON='1', + QUEUE_DATABASE_URL=f'postgresql://graph_recovery@127.0.0.1:5432/{DB}', + POLIS_JOBS_TRANSPORT='loopback', POLIS_JOBS_WORKER_CLASS='delphi', + POLIS_JOBS_STAGES='graph_embed,graph_cluster,graph_topics,graph_narrative', + POLIS_JOBS_CONCURRENCY='1', POLIS_JOBS_JOURNAL_DIR='/journal', + POLIS_JOBS_JOURNAL_HOST_FILE='/proof/host-id', POLIS_JOBS_WORK_DIR='/tmp/work', + DELPHI_APP_PATH='/proof/app', POLIS_JOBS_PYTHON='python3', + POLIS_JOBS_POLL_SECONDS='1', POLIS_JOBS_REAP_SECONDS='1', + POLIS_JOBS_LEASE_SECONDS='15', POLIS_JOBS_HEARTBEAT_SECONDS='3') + args = ['docker', 'run', '-d', '--name', name, '--network', 'container:'+PG, + '--memory', '6g', '--cpus', '2', '--cap-drop', 'ALL', + '-v', str(OUT)+':/proof', '-v', str(journal)+':/journal'] + for key, value in env.items(): + args += ['-e', key+'='+value] + run(*args, '--entrypoint', 'polis-delphi-shadow', IMAGE, 'daemon') + + +def admit(case): + data = {'texts': ['public generated text '+str(i) for i in range(5)]} + sha = sql('SELECT pd_graph_hash('+lit(json.dumps(data))+'::jsonb)') + spec = dict(schema='polis-job-graph/1', nodes=[dict(key='e', stage='graph_embed', + **{'class': 'delphi'}, declared=dict(snapshot=dict(data=data, sha256=sha), + code='a'*64, model='local-token-count/1', runtime='python-test', seed=0, + config={}, mode='full', memory_bytes=1048576, work_units=100), inputs=[], max_attempts=1)]) + reply = json.loads(sql("SELECT pd_graph_admit('shadow-graph-1',1,"+lit(case)+",'first',"+lit(json.dumps(spec))+"::jsonb)")) + return reply['graph_id'] + + +def view(graph): + return json.loads(sql("SELECT pd_graph_view('shadow-graph-1',"+lit(graph)+"::uuid)")) + + +def record(name): + results.append(name) + print('PASS', name, flush=True) + (OUT/'results.json').write_text(json.dumps(results, indent=2)+'\n') + + +def main(): + sql("INSERT INTO users(uid,hname) VALUES(1,'public generated owner'); INSERT INTO conversations(zid,owner,topic) VALUES(1,1,'public generated'); DO $$ BEGIN IF NOT EXISTS(SELECT FROM pg_roles WHERE rolname='graph_recovery') THEN CREATE ROLE graph_recovery LOGIN; END IF; END $$; GRANT polis_queue_executor TO graph_recovery;") + (OUT/'host-id').write_text('public-test-host') + scripts = OUT/'app/scripts' + scripts.mkdir(parents=True) + (scripts/'job_graph_stage.py').write_text("import time\nwhile True: time.sleep(.1)\n") + for case in ['container-replaced', 'same-container-restarted', 'journal-lost']: + journal = OUT/case + journal.mkdir() + graph = admit(case) + first = PREFIX+'-'+case + launch(first, journal) + until(lambda: any(journal.glob('*.json')) and view(graph)['nodes'][0]['readiness']['state'] == 'running') + # Wait for the post-spawn entry, not merely the durable pre-spawn entry. + until(lambda: any(json.loads(p.read_text()).get('pgid') for p in journal.glob('*.json'))) + second = PREFIX+'-second-'+case + launch(second, journal) + until(lambda: run('docker', 'inspect', '-f', '{{.State.Running}}', second) == 'false') + assert run('docker', 'inspect', '-f', '{{.State.ExitCode}}', second) == '2' + assert view(graph)['nodes'][0]['attempts'] == 1 + record(case+': second daemon refused') + run('docker', 'kill', '--signal', 'KILL', first) + until(lambda: run('docker', 'inspect', '-f', '{{.State.Running}}', first) == 'false') + if case == 'same-container-restarted': + run('docker', 'start', first) + else: + next_journal = journal + if case == 'journal-lost': + next_journal = OUT/'empty-journal' + next_journal.mkdir() + launch(PREFIX+'-replacement-'+case, next_journal) + if case == 'journal-lost': + until(lambda: view(graph)['nodes'][0]['readiness']['state'] == 'parked') + assert view(graph)['guard_held'] + assert sql("SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.graph_id="+lit(graph)+"::uuid AND a.process_exit_confirmed_at IS NULL") == '1' + record('lost journal: expiry did not invent exit proof or release scope') + else: + until(lambda: not view(graph)['guard_held']) + assert sql("SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.graph_id="+lit(graph)+"::uuid AND a.process_exit_confirmed_at IS NULL") == '0' + record(case+': retained exact attempt recovered') + for name in owned: + if run('docker', 'inspect', '-f', '{{.State.Running}}', name) == 'true': + run('docker', 'stop', '-t', '3', name) + assert sql('SELECT count(*) FROM votes') == '0' + record('no votes or production data') + + +if __name__ == "__main__": + try: + main() + finally: + for name in owned: + p = sp.run(['docker', 'logs', name], text=True, capture_output=True) + (OUT/(name+'.log')).write_text(p.stdout+p.stderr) + sp.run(['docker', 'rm', '-f', name], check=False, capture_output=True) diff --git a/delphi/tests/shadow/graph_scheduler_limits.py b/delphi/tests/shadow/graph_scheduler_limits.py new file mode 100644 index 0000000000..1e9af00cc9 --- /dev/null +++ b/delphi/tests/shadow/graph_scheduler_limits.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Scheduler restart/list clearing under its deployed 256 MiB, quarter-CPU limits.""" +import json +import os +from pathlib import Path +import subprocess as sp +import time +import uuid + +from graph_recovery import OUT, PG, DB, IMAGE, run, sql, lit, until + +names = [] +prefix = 'graph-scheduler-' + uuid.uuid4().hex[:12] +state = OUT/'scheduler' +state.mkdir(parents=True) + +def start(label, codes): + name = prefix+'-'+label + names.append(name) + env = dict(DELPHI_SHADOW_ENABLED='1', DELPHI_SHADOW_ENV='shadow-delphi-1', + QUEUE_ENV='shadow-delphi-1', POLIS_JOBS_TRANSPORT='loopback', + QUEUE_DATABASE_URL=f'postgresql://graph_recovery@127.0.0.1:5432/{DB}', + DELPHI_SHADOW_READ_DATABASE_URL=f'postgresql://postgres@127.0.0.1:5432/{DB}', + DELPHI_SHADOW_CODE_VERSION='a'*40, DELPHI_SHADOW_STATE_PATH='/state/state.json', + DELPHI_SHADOW_PAUSE_FILE='/state/pause', DELPHI_SHADOW_DISK_FREE_MIN_MB='0', + DELPHI_SHADOW_DISK_USED_MAX_PCT='100', DELPHI_SHADOW_PROBE_MAX_MS='5000', + DELPHI_SHADOW_GRAPH_CONVERSATIONS=codes, DELPHI_SHADOW_GRAPH_ENV='shadow-graph-1', + DELPHI_SHADOW_BENCHMARK='c2,c3', DELPHI_SHADOW_POLL_SECONDS='1', + DELPHI_SHADOW_JOB_POLL_SECONDS='1', DELPHI_SHADOW_JOB_DEADLINE_SECONDS='60', + DELPHI_SHADOW_DUTY_CYCLE='1', + DELPHI_EMBED_MODEL_PATH='/opt/polis/embedding/granite-embedding-311m-multilingual-r2', + DELPHI_EMBED_MODEL_LOCK='/opt/polis/embedding/granite-embedding-311m-multilingual-r2.lock.json') + args = ['docker','run','-d','--name',name,'--network','container:'+PG, + '--memory','256m','--cpus','0.25','-v',str(state)+':/state'] + for key, value in env.items(): args += ['-e',key+'='+value] + run(*args,'--entrypoint','polis-delphi-shadow',IMAGE) + return name + + +def read_state(): + try: return json.loads((state/'state.json').read_text()) + except (FileNotFoundError, json.JSONDecodeError): return {} + + +try: + sql("INSERT INTO users(uid,hname) VALUES(1,'generated owner');") + # The common restricted login may already exist on this owned server. + sql("DO $$ BEGIN IF NOT EXISTS(SELECT FROM pg_roles WHERE rolname='graph_recovery') THEN CREATE ROLE graph_recovery LOGIN; END IF; END $$; GRANT polis_queue_executor TO graph_recovery;") + for zid in [2,3]: + sql(f"INSERT INTO conversations(zid,owner,topic,is_active) VALUES({zid},1,'public generated',true); INSERT INTO zinvites(zid,zinvite) VALUES({zid},'c{zid}'); INSERT INTO participants(zid,pid,uid) VALUES({zid},0,1); INSERT INTO comments(zid,pid,uid,tid,txt,mod,active) SELECT {zid},0,1,i,'Public generated comment '||i,0,true FROM generate_series(0,24) i;") + first = start('before','c2') + flight = until(lambda: read_state().get('graph_flight')) + # A real queue attempt loses its local journal. This test spawns no child; + # the absent receipt must nevertheless remain unproven by the scheduler. + owner, attempt = str(uuid.uuid4()), str(uuid.uuid4()) + claimed = json.loads(sql("SELECT pd_graph_claim('shadow-graph-1',1::smallint,"+lit(owner)+"::uuid,"+lit(attempt)+"::uuid,15,'delphi','graph_embed')")) + assert claimed['outcome'] == 'owned' + run('docker','stop','-t','5',first) + saved = read_state() + saved['graph_flight']['admitted_at_ms'] -= 120000 + (state/'state.json').write_text(json.dumps(saved)) + second = start('after','') + finished = until(lambda: read_state() if read_state().get('in_flight') else None) + assert finished['in_flight']['zid'] == 3 + assert len(finished['graph_deferred']) == 1 + assert finished['graph_deferred'][0]['graph_id'] == flight['graph_id'] + assert sql("SELECT count(*) FROM polis_queue_attempts WHERE attempt_id="+lit(attempt)+"::uuid AND process_exit_confirmed_at IS NULL") == '1' + inspect = json.loads(run('docker','inspect',second))[0] + assert inspect['HostConfig']['Memory'] == 256*1024**2 + assert inspect['HostConfig']['NanoCpus'] == 250000000 + (OUT/'limits-results.json').write_text(json.dumps(dict(checks=4, memory_bytes=256*1024**2, nano_cpus=250000000, + restart=True,list_cleared=True,other_conversation_admitted=True,orphan_scope_retained=True),indent=2)+'\n') + print('PASS 4 scheduler restart/list-clear/exclusion/resource-limit assertions') +finally: + for name in names: + p = sp.run(['docker','logs',name],text=True,capture_output=True) + (OUT/(name+'.log')).write_text(p.stdout+p.stderr) + sp.run(['docker','rm','-f',name],check=False,capture_output=True) diff --git a/queue-rs/src/jobs/claim.rs b/queue-rs/src/jobs/claim.rs index 6f379eff4a..ae867b02da 100644 --- a/queue-rs/src/jobs/claim.rs +++ b/queue-rs/src/jobs/claim.rs @@ -223,7 +223,9 @@ fn recover(ctx: &Ctx, rpc: &mut Rpc) { if entry.env != ctx.cfg.env { continue; } - if !journal::provably_gone(&entry, &ctx.cfg.boot_id, &ctx.cfg.container_id, &ctx.owner) { + let gone = + journal::provably_gone(&entry, &ctx.cfg.boot_id, &ctx.cfg.container_id, &ctx.owner); + if !gone { line(&format!( "journal entry {} not provably gone; left in place", entry.attempt_id @@ -272,6 +274,12 @@ pub fn run(cfg: Config) -> i32 { return EXIT_CONFIG; } }; + if let Some(host_file) = &cfg.journal_host_file + && let Err(error) = journal.bind_host(host_file) + { + line(&format!("config refused: journal host binding ({error})")); + return EXIT_CONFIG; + } let connector = match Connector::new(&cfg) { Ok(c) => Arc::new(c), Err(e) => { diff --git a/queue-rs/src/jobs/config.rs b/queue-rs/src/jobs/config.rs index 90986b299d..0c2543faf4 100644 --- a/queue-rs/src/jobs/config.rs +++ b/queue-rs/src/jobs/config.rs @@ -121,6 +121,7 @@ pub struct Config { pub readiness: Duration, pub identity: String, pub journal_dir: PathBuf, + pub journal_host_file: Option, pub work_dir: PathBuf, pub shutdown_grace: Duration, pub kill_grace: Duration, @@ -322,6 +323,9 @@ pub fn load Option>(get: F) -> Result reap_interval: Duration::from_secs(reap), boot_id, container_id, + journal_host_file: get("POLIS_JOBS_JOURNAL_HOST_FILE") + .filter(|s| !s.is_empty()) + .map(PathBuf::from), }))) } diff --git a/queue-rs/src/jobs/journal.rs b/queue-rs/src/jobs/journal.rs index b91a2c98e6..d477c40342 100644 --- a/queue-rs/src/jobs/journal.rs +++ b/queue-rs/src/jobs/journal.rs @@ -81,6 +81,32 @@ impl Journal { }) } + /// A retained journal belongs to exactly one host. A copied volume is not + /// exit evidence. Bind while holding the directory's exclusive lock. + pub fn bind_host(&self, host_file: &Path) -> anyhow::Result<()> { + let host = fs::read_to_string(host_file)?; + let host = host.trim(); + anyhow::ensure!( + !host.is_empty() && host.len() <= 256, + "journal_host_identity" + ); + let marker = self.dir.join(".host"); + match fs::read_to_string(&marker) { + Ok(previous) => anyhow::ensure!(previous == host, "journal_host_mismatch"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + anyhow::ensure!(self.entries().is_empty(), "journal_host_unrecorded"); + let tmp = self.dir.join(".host.tmp"); + let mut file = File::create(&tmp)?; + file.write_all(host.as_bytes())?; + file.sync_all()?; + fs::rename(tmp, marker)?; + fsync_dir(&self.dir)?; + } + Err(e) => return Err(e.into()), + } + Ok(()) + } + fn path(&self, attempt_id: &str) -> PathBuf { self.dir.join(format!("{attempt_id}.json")) } @@ -231,3 +257,25 @@ mod tests { assert!(!provably_gone(&alive, "boot-a", "box-a", me)); } } + +#[cfg(test)] +mod host_tests { + use super::*; + #[test] + fn retained_journal_refuses_another_host_and_concurrent_daemon() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("journal-host-{}", uuid::Uuid::new_v4())); + let host = dir.join("host-id"); + let journal = Journal::open(&dir.join("journal"))?; + fs::write(&host, "machine-a\n")?; + journal.bind_host(&host)?; + assert!(Journal::open(&dir.join("journal")).is_err()); + drop(journal); + let replacement = Journal::open(&dir.join("journal"))?; + replacement.bind_host(&host)?; + fs::write(&host, "machine-b")?; + assert!(replacement.bind_host(&host).is_err()); + drop(replacement); + fs::remove_dir_all(dir)?; + Ok(()) + } +} diff --git a/queue-rs/src/shadow/graph.rs b/queue-rs/src/shadow/graph.rs index 02c14c1514..639cb58ef2 100644 --- a/queue-rs/src/shadow/graph.rs +++ b/queue-rs/src/shadow/graph.rs @@ -303,28 +303,48 @@ impl Scheduler<'_> { /// Recover the latest unpublished or live graph before another admission. /// Only ids, digests and counts leave SQL; no snapshot text is returned. pub(super) fn recover_graph(&mut self) -> Result<()> { - let Some(cfg) = self.cfg.graph.as_ref() else { - return Ok(()); - }; + if let Some(cfg) = self.cfg.graph.as_ref() + && !self.state.graph_recovery_envs.contains(&cfg.env) + { + self.state.graph_recovery_envs.push(cfg.env.clone()); + // Persist the recovery domain before any admission can commit. + self.state.save(&self.cfg.state_path)?; + } + if let Some(f) = &self.state.graph_flight + && !self.state.graph_recovery_envs.contains(&f.env) + { + self.state.graph_recovery_envs.push(f.env.clone()); + } if self.state.graph_flight.is_some() { return Ok(()); } - let recovered=self.db.read(|tx| { - let rows=tx.query("SELECT DISTINCT ON (zid) zid, coalesce(request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}') FROM delphi_graphs WHERE env=$1 AND scope_key='shadow-graph:'||zid::text ORDER BY zid,created_at DESC", &[&cfg.env])?; + let deferred: Vec = self + .state + .graph_deferred + .iter() + .map(|f| f.graph_id.clone()) + .collect(); + for env in self.state.graph_recovery_envs.clone() { + let recovered=self.db.read(|tx| { + let rows=tx.query("SELECT DISTINCT ON (zid) zid, coalesce(request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}') FROM delphi_graphs WHERE env=$1 AND scope_key='shadow-graph:'||zid::text ORDER BY zid,created_at DESC", &[&env])?; let mut seen=BTreeMap::new(); for r in rows { if let Some(digest)=r.try_get::<_,Option>(1)? { seen.insert(r.try_get::<_,i32>(0)?,digest); } } - let row=tx.query_opt("SELECT g.graph_id::text,g.zid, (extract(epoch FROM g.created_at)*1000)::bigint, coalesce(g.request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}'), (g.request #>> '{nodes,0,declared,config,shadow,comments}')::bigint, (g.request #>> '{nodes,0,declared,config,shadow,expected_generation}')::bigint FROM delphi_graphs g WHERE g.env=$1 AND g.scope_key='shadow-graph:'||g.zid::text AND (g.request #>> '{nodes,0,declared,config,shadow,comments}') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM delphi_graphs newer WHERE newer.env=g.env AND newer.scope_key='shadow-graph:'||newer.zid::text AND newer.created_at>g.created_at) AND (EXISTS (SELECT 1 FROM delphi_job_guards d WHERE d.env=g.env AND d.root_job_id=g.root_job_id) OR (NOT EXISTS (SELECT 1 FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id AND q.state<>'succeeded') AND NOT EXISTS (SELECT 1 FROM delphi_graph_served s JOIN delphi_artifacts a USING(env,artifact_id) JOIN delphi_graph_nodes n USING(env,job_id) WHERE s.env=g.env AND n.graph_id=g.graph_id)))", &[&cfg.env])?; + let row=tx.query_opt("SELECT g.graph_id::text,g.zid, (extract(epoch FROM g.created_at)*1000)::bigint, coalesce(g.request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}'), (g.request #>> '{nodes,0,declared,config,shadow,comments}')::bigint, (g.request #>> '{nodes,0,declared,config,shadow,expected_generation}')::bigint FROM delphi_graphs g WHERE g.env=$1 AND NOT (g.graph_id::text=ANY($2::text[])) AND g.scope_key='shadow-graph:'||g.zid::text AND (g.request #>> '{nodes,0,declared,config,shadow,comments}') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM delphi_graphs newer WHERE newer.env=g.env AND newer.scope_key='shadow-graph:'||newer.zid::text AND newer.created_at>g.created_at) AND (EXISTS (SELECT 1 FROM delphi_job_guards d WHERE d.env=g.env AND d.root_job_id=g.root_job_id) OR (NOT EXISTS (SELECT 1 FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id AND q.state<>'succeeded') AND NOT EXISTS (SELECT 1 FROM delphi_graph_served s JOIN delphi_artifacts a USING(env,artifact_id) JOIN delphi_graph_nodes n USING(env,job_id) WHERE s.env=g.env AND n.graph_id=g.graph_id))) ORDER BY g.created_at,g.graph_id LIMIT 1", &[&env,&deferred])?; let Some(r)=row else { return Ok((seen,None)); }; let graph_id:String=r.try_get(0)?; let mut nodes=Vec::new(); - for n in tx.query("SELECT n.job_id::text,n.run_id::text,q.stage FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid ORDER BY n.node_key", &[&cfg.env,&graph_id])? { + for n in tx.query("SELECT n.job_id::text,n.run_id::text,q.stage FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid ORDER BY n.node_key", &[&env,&graph_id])? { nodes.push(Node {job_id:n.try_get(0)?,run_id:n.try_get(1)?,stage:n.try_get(2)?}); } ensure!(nodes.len()==4 && STAGES.iter().all(|s|nodes.iter().filter(|n|n.stage==*s).count()==1),"graph_recovery_shape"); - Ok((seen,Some(Flight {env:cfg.env.clone(),graph_id,zid:r.try_get(1)?,admitted_at_ms:r.try_get(2)?,digest:r.try_get(3)?,comments:r.try_get(4)?,generation:r.try_get::<_,Option>(5)?.unwrap_or(-1),nodes}))) + Ok((seen,Some(Flight {env:env.clone(),graph_id,zid:r.try_get(1)?,admitted_at_ms:r.try_get(2)?,digest:r.try_get(3)?,comments:r.try_get(4)?,generation:r.try_get::<_,Option>(5)?.unwrap_or(-1),nodes}))) })?; - self.state.graph_seen.extend(recovered.0); - self.state.graph_flight = recovered.1; + self.state.graph_seen.extend(recovered.0); + self.state.graph_flight = recovered.1; + if self.state.graph_flight.is_some() { + break; + } + } Ok(()) } @@ -334,6 +354,9 @@ impl Scheduler<'_> { let Some(cfg) = self.cfg.graph.as_ref() else { return Ok(false); }; + if !self.state.graph_deferred.is_empty() { + return Ok(false); + } let cfg = cfg.clone(); let listed = listed(&self.db, &cfg.codes)?; let counts = (cfg.codes.len(), listed.len()); @@ -531,6 +554,51 @@ impl Scheduler<'_> { Ok(true) } + pub(super) fn graph_scope_held(&self, zid: i32) -> Result { + if self.state.graph_recovery_envs.is_empty() { + return Ok(false); + } + self.db.read(|tx| Ok(tx.query_one( + "SELECT EXISTS (SELECT 1 FROM delphi_graphs g JOIN delphi_job_guards d ON d.env=g.env AND d.root_job_id=g.root_job_id WHERE g.env=ANY($1::text[]) AND g.zid=$2 AND g.scope_key='shadow-graph:'||g.zid::text)", + &[&self.state.graph_recovery_envs, &zid])?.try_get(0)?)) + } + + pub(super) fn poll_deferred_graph(&mut self, now: i64) -> Result<()> { + // One per cycle; rotation keeps recovery polling bounded and fair. + if !self.state.graph_deferred.is_empty() { + let flight = self.state.graph_deferred[0].clone(); + self.state.graph_deferred.rotate_left(1); + self.watch_graph(flight, now)?; + } + Ok(()) + } + + fn defer_graph(&mut self, flight: &Flight, now: i64) { + if now.saturating_sub(flight.admitted_at_ms) < self.cfg.job_deadline.as_millis() as i64 { + return; + } + if !self + .state + .graph_deferred + .iter() + .any(|f| f.graph_id == flight.graph_id) + { + self.state.graph_deferred.push(flight.clone()); + log( + "graph_recovery_pending", + json!({"graph_id":flight.graph_id,"reason":"wait_bound","scope_held":true}), + ); + } + if self + .state + .graph_flight + .as_ref() + .is_some_and(|f| f.graph_id == flight.graph_id) + { + self.state.graph_flight = None; + } + } + fn log_graph_wait( &mut self, flight: &Flight, @@ -594,6 +662,7 @@ impl Scheduler<'_> { // beyond the deadline: otherwise two computations could overlap. if !terminal { self.log_graph_wait(&flight, now, &states, true); + self.defer_graph(&flight, now); return Ok(self.cfg.job_poll); } let mut client = self.db.connect_queue()?; @@ -607,6 +676,7 @@ impl Scheduler<'_> { let view: Value = serde_json::from_str(&view)?; if view["guard_held"] != false { self.log_graph_wait(&flight, now, &states, true); + self.defer_graph(&flight, now); return Ok(self.cfg.job_poll); } drop(client); @@ -703,7 +773,25 @@ impl Scheduler<'_> { "graph_finished", json!({"graph_id":flight.graph_id,"published":published,"bytes":bytes,"graph_row_bytes":graph_bytes}), ); - self.state.graph_flight = None; + let deferred = self + .state + .graph_deferred + .iter() + .any(|f| f.graph_id == flight.graph_id); + self.state + .graph_deferred + .retain(|f| f.graph_id != flight.graph_id); + if self + .state + .graph_flight + .as_ref() + .is_some_and(|f| f.graph_id == flight.graph_id) + { + self.state.graph_flight = None; + } + if deferred { + return Ok(self.cfg.job_poll); + } self.rest_until_ms = now + plan::rest_ms(self.cfg.duty_cycle, now - flight.admitted_at_ms); Ok(Duration::from_millis( (self.rest_until_ms - now).max(1000) as u64 diff --git a/queue-rs/src/shadow/scheduler.rs b/queue-rs/src/shadow/scheduler.rs index 794eaab9d9..0682c00a1c 100644 --- a/queue-rs/src/shadow/scheduler.rs +++ b/queue-rs/src/shadow/scheduler.rs @@ -177,6 +177,8 @@ impl<'a> Scheduler<'a> { self.state.roll_day(now); self.reconcile_storage()?; self.measure_pending(now); + self.poll_deferred_graph(now)?; + self.recover_graph()?; if let Some(flight) = self.state.graph_flight.clone() { return self.watch_graph(flight, now); } @@ -351,6 +353,10 @@ impl<'a> Scheduler<'a> { /// alone holds the rest of those two, and the current set is still /// tried, against the whole budget. fn pick_and_admit(&mut self, now: i64) -> Result> { + self.recover_graph()?; + if self.state.graph_flight.is_some() { + return Ok(None); + } match self.admit_twin(now)? { TwinStep::Admitted => return Ok(None), TwinStep::Held(reason) => return Ok(Some(reason)), @@ -414,6 +420,9 @@ impl<'a> Scheduler<'a> { // A benchmark or featured candidate was passed over for the reserve. let mut reserve_waiting = false; for (candidate, set_name) in candidates { + if self.graph_scope_held(candidate.zid)? { + continue; + } let share = set_name.share(); if reserve_held && share == Share::Shared { reserve_waiting = true; @@ -609,6 +618,9 @@ impl<'a> Scheduler<'a> { if pending.twin_job_id.is_some() { return Ok(TwinStep::Nothing); } + if self.graph_scope_held(pending.zid)? { + return Ok(TwinStep::Nothing); + } let set = self.db.comment_set(pending.zid)?; if set.digest != pending.digest { log( diff --git a/queue-rs/src/shadow/state.rs b/queue-rs/src/shadow/state.rs index a4d438ffc9..244714758b 100644 --- a/queue-rs/src/shadow/state.rs +++ b/queue-rs/src/shadow/state.rs @@ -98,6 +98,10 @@ pub struct State { pub in_flight: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub graph_flight: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub graph_deferred: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub graph_recovery_envs: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub graph_seen: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -126,6 +130,8 @@ impl State { conversations: BTreeMap::new(), in_flight: None, graph_flight: None, + graph_deferred: Vec::new(), + graph_recovery_envs: Vec::new(), graph_seen: BTreeMap::new(), graph_observed: BTreeMap::new(), pending_twin: None, diff --git a/queue-rs/tests/delphi_shadow.rs b/queue-rs/tests/delphi_shadow.rs index 0c1181e7ff..b879abc27e 100644 --- a/queue-rs/tests/delphi_shadow.rs +++ b/queue-rs/tests/delphi_shadow.rs @@ -1680,3 +1680,84 @@ fn graph_provenance_is_cached_until_source_metadata_changes() { 2 ); } + +#[test] +fn lost_graph_attempt_does_not_hold_other_conversations_after_restart_and_list_clear() { + let f = Fixture::new(); + f.quiet(971, 25, 1); + f.quiet(972, 25, 1); + graph_catalog(&f); + let mut cfg = graph_config( + &f, + &[ + ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c971"), + ("DELPHI_SHADOW_BENCHMARK", "c971,c972"), + ], + ); + let mut scheduler = Scheduler::new(&cfg).unwrap(); + scheduler.cycle().unwrap(); + let flight = scheduler.state.graph_flight.clone().unwrap(); + let mut c = f.superuser(); + // A claimed attempt with no exit attestation models a lost journal. No + // child is spawned in this database test; it must still remain unproven. + let owner = Uuid::new_v4().to_string(); + let attempt = Uuid::new_v4().to_string(); + let claim:Value=c.query_one("SELECT pd_graph_claim($1,1::smallint,$2::text::uuid,$3::text::uuid,60,'delphi','graph_embed')",&[&flight.env,&owner,&attempt]).unwrap().get(0); + assert_eq!(claim["outcome"], "owned"); + scheduler.state.save(&cfg.state_path).unwrap(); + drop(scheduler); + cfg.graph = None; + cfg.job_deadline = Duration::ZERO; + let mut scheduler = Scheduler::new(&cfg).unwrap(); + scheduler.cycle().unwrap(); + assert!(scheduler.state.graph_flight.is_none()); + assert_eq!(scheduler.state.graph_deferred.len(), 1); + scheduler.state.save(&cfg.state_path).unwrap(); + drop(scheduler); + let mut scheduler = Scheduler::new(&cfg).unwrap(); + scheduler.cycle().unwrap(); + assert_eq!(scheduler.state.in_flight.as_ref().unwrap().zid, 972); + let view: Value = c + .query_one( + "SELECT pd_graph_view($1,$2::text::uuid)", + &[&flight.env, &flight.graph_id], + ) + .unwrap() + .get(0); + assert_eq!(view["guard_held"], true); + let unconfirmed:i64=c.query_one("SELECT count(*) FROM polis_queue_attempts WHERE env=$1 AND attempt_id=$2::text::uuid AND process_exit_confirmed_at IS NULL",&[&flight.env,&attempt]).unwrap().get(0); + assert_eq!(unconfirmed, 1); +} + +#[test] +fn committed_graph_admission_with_unread_reply_is_recovered_before_full_pipeline() { + let f = Fixture::new(); + f.quiet(981, 25, 1); + graph_catalog(&f); + let cfg = graph_config( + &f, + &[ + ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c981"), + ("DELPHI_SHADOW_BENCHMARK", "c981"), + ], + ); + let mut scheduler = Scheduler::new(&cfg).unwrap(); + scheduler.cycle().unwrap(); + let graph_id = scheduler + .state + .graph_flight + .as_ref() + .unwrap() + .graph_id + .clone(); + // Discard the successful reply as a lost transport reply would do. The + // persisted recovery domain precedes admission, even without flight state. + scheduler.state.graph_flight = None; + scheduler.state.graph_seen.clear(); + scheduler.cycle().unwrap(); + assert_eq!( + scheduler.state.graph_flight.as_ref().unwrap().graph_id, + graph_id + ); + assert!(scheduler.state.in_flight.is_none()); +} From 1608647c861d8dfe05eb0d8efd44f38145a767b7 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 02:12:43 -0800 Subject: [PATCH 2/7] delphi shadow: an optional graph-stage daemon with its own profile, journal volume and deploy check Adds the delphi-shadow-graph Compose service (profile delphi-shadow-graph, off unless DELPHI_SHADOW_GRAPH_ENABLED=1): the job daemon restricted to the four graph stages at concurrency one, with its own journal volume, a read-only host identity file bound in for the journal's host binding, and its own memory and CPU ceilings (6 GiB and 2 CPUs by default). The daemon refuses to start unless it runs as PID 1 of its own container with exactly those stages, concurrency one and a host identity file set. The deploy hook starts it only when the full-pipeline shadow is selected and the setting is on, and first checks the rendered Compose settings with scripts/check-shadow-graph.py (names only); if they are wrong, neither shadow is started. With the setting off, nothing in the deploy changes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- delphi/tests/test_shadow_graph_compose.py | 53 +++++++++++++++ docker-compose.yml | 80 +++++++++++++++++++++++ example.env | 7 ++ queue-rs/src/shadow/daemon.rs | 19 ++++++ scripts/after_install.sh | 23 ++++++- scripts/check-shadow-graph.py | 56 ++++++++++++++++ 6 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 delphi/tests/test_shadow_graph_compose.py create mode 100644 scripts/check-shadow-graph.py diff --git a/delphi/tests/test_shadow_graph_compose.py b/delphi/tests/test_shadow_graph_compose.py new file mode 100644 index 0000000000..325590ded4 --- /dev/null +++ b/delphi/tests/test_shadow_graph_compose.py @@ -0,0 +1,53 @@ +"""The optional executor must retain its local journal and shadow boundary.""" +import importlib.util +from pathlib import Path + +MODULE = Path(__file__).resolve().parents[2] / "scripts/check-shadow-graph.py" +spec = importlib.util.spec_from_file_location("shadow_graph_compose", MODULE) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def valid(): + env = dict(DELPHI_SHADOW_GRAPH_DAEMON="1", POLIS_JOBS_ENABLED="1", + QUEUE_ENV="shadow-graph-1", DELPHI_RESULT_ENV="shadow-graph-1", + POLIS_JOBS_WORKER_CLASS="delphi", + POLIS_JOBS_STAGES="graph_embed,graph_cluster,graph_topics,graph_narrative", + POLIS_JOBS_CONCURRENCY="1", POLIS_JOBS_JOURNAL_DIR="/var/lib/delphi-shadow-graph/journal", + POLIS_JOBS_JOURNAL_HOST_FILE="/run/polis-host-id", DELPHI_EMBEDDINGS_TABLE="1", + HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", ANTHROPIC_TOPIC_MODEL="none") + graph = dict(environment=env, command=["polis-delphi-shadow", "daemon"], + volumes=[dict(type="volume", target="/var/lib/delphi-shadow-graph"), + dict(type="bind", source="/etc/machine-id", target="/run/polis-host-id", read_only=True)], + deploy=dict(resources=dict(limits=dict(memory=6*1024**3, cpus="2"))), + cpu_shares=2, oom_score_adj=1000) + scheduler = dict(environment=dict(DELPHI_SHADOW_ENV="shadow-delphi-1", DELPHI_SHADOW_GRAPH_ENV="shadow-graph-1")) + return dict(services={"delphi-shadow-graph": graph, "delphi-shadow-scheduler": scheduler}) + + +def test_accepts_the_isolated_executor(): + assert module.errors(valid()) == [] + + +def test_refuses_a_served_or_shared_env(): + for env in ["prod", "shadow-delphi-1"]: + config = valid() + config["services"]["delphi-shadow-graph"]["environment"]["QUEUE_ENV"] = env + assert module.errors(config) + + +def test_refuses_a_missing_journal_or_host_identity(): + for index in [0, 1]: + config = valid() + config["services"]["delphi-shadow-graph"]["volumes"].pop(index) + assert module.errors(config) + + +def test_refuses_a_different_process_namespace_or_stage(): + for key, value in [("init", True), ("pid", "host"), ("privileged", True)]: + config = valid() + config["services"]["delphi-shadow-graph"][key] = value + assert module.errors(config) + config = valid() + config["services"]["delphi-shadow-graph"]["environment"]["POLIS_JOBS_STAGES"] = "delphi_full_pipeline" + assert module.errors(config) diff --git a/docker-compose.yml b/docker-compose.yml index 525530238e..ff5a223486 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -543,6 +543,85 @@ services: profiles: - delphi-shadow + # Optional graph execution on this host. Its separate local journal survives + # container replacement. Never copy that volume to a different host. + # Total default ceilings: 16 GiB full pipeline + 6 GiB graph + 256 MiB + # scheduler, before math, Postgres, the web services and the operating system. + delphi-shadow-graph: + image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest + build: + context: ./delphi + additional_contexts: + queue-rs: ./queue-rs # the polis-jobs stage (delphi/Dockerfile) + target: final + labels: + polis_tag: ${TAG:-dev} + command: ["polis-delphi-shadow", "daemon"] + init: false + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + - DELPHI_SHADOW_GRAPH_DAEMON=1 + - POLIS_JOBS_JOURNAL_HOST_FILE=/run/polis-host-id + - POLIS_JOBS_ENABLED=${DELPHI_SHADOW_GRAPH_ENABLED:-0} + - DATABASE_URL=${DATABASE_URL} + - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable} + - QUEUE_DATABASE_URL=${QUEUE_DATABASE_URL:-} + - QUEUE_ENV=${DELPHI_SHADOW_GRAPH_ENV:-shadow-graph-1} + - DELPHI_RESULT_BACKEND=postgres + - DELPHI_RESULT_ENV=${DELPHI_SHADOW_GRAPH_ENV:-shadow-graph-1} + - POLIS_JOBS_TRANSPORT=${POLIS_JOBS_TRANSPORT:-tls} + - POLIS_JOBS_CA_FILE=${POLIS_JOBS_CA_FILE:-} + - POLIS_JOBS_HOST_ALLOWLIST=${POLIS_JOBS_HOST_ALLOWLIST:-} + - POLIS_JOBS_WORKER_CLASS=delphi + - POLIS_JOBS_STAGES=graph_embed,graph_cluster,graph_topics,graph_narrative + - POLIS_JOBS_CONCURRENCY=1 + - POLIS_JOBS_CHILD_TIMEOUT_SECONDS=${DELPHI_SHADOW_CHILD_TIMEOUT_SECONDS:-7200} + - POLIS_JOBS_JOURNAL_DIR=/var/lib/delphi-shadow-graph/journal + - POLIS_JOBS_WORK_DIR=/tmp/delphi-shadow-graph/work + - DELPHI_EMBEDDINGS_TABLE=1 + - MATH_ENV=${MATH_ENV:-prod} + - LOG_LEVEL=${DELPHI_LOG_LEVEL:-INFO} + - DELPHI_DEV_OR_PROD=${DELPHI_DEV_OR_PROD:-prod} + - AWS_REGION=${AWS_REGION:-us-east-1} + - LLM_PROVIDER=anthropic + - ANTHROPIC_TOPIC_MODEL=none + - SENTENCE_TRANSFORMER_MODEL=ibm-granite/granite-embedding-311m-multilingual-r2 + - DELPHI_EMBED_MODEL_PATH=/opt/polis/embedding/granite-embedding-311m-multilingual-r2 + - DELPHI_EMBED_MODEL_LOCK=/opt/polis/embedding/granite-embedding-311m-multilingual-r2.lock.json + - HF_HUB_OFFLINE=1 + - TRANSFORMERS_OFFLINE=1 + - HF_HOME=/tmp/delphi-shadow-graph/hf + - NUMBA_CACHE_DIR=/tmp/delphi-shadow-graph/numba + - TMPDIR=/tmp + - DELPHI_SKIP_DATAMAPPLOT=1 + - DELPHI_LOCAL_OUTPUT_CLEANUP=1 + - DELPHI_STEP_LOG=1 + volumes: + - delphi-shadow-graph-state:/var/lib/delphi-shadow-graph + - type: bind + source: ${DELPHI_SHADOW_GRAPH_HOST_ID_FILE:-/etc/machine-id} + target: /run/polis-host-id + read_only: true + bind: {create_host_path: false} + tmpfs: + - /tmp:size=2g,mode=1777 + - /app/polis_data:size=512m + - /app/visualizations:size=64m + networks: + - "polis-net" + stop_grace_period: 60s + cpu_shares: 2 + oom_score_adj: 1000 + deploy: + resources: + limits: + memory: ${DELPHI_SHADOW_GRAPH_CONTAINER_MEMORY:-6g} + cpus: ${DELPHI_SHADOW_GRAPH_CONTAINER_CPUS:-2} + restart: unless-stopped + profiles: + - delphi-shadow-graph + delphi-shadow-scheduler: image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest build: @@ -1016,6 +1095,7 @@ volumes: labels: polis_tag: ${TAG:-dev} # The Delphi shadow's daemon journal and scheduler state (small files). + delphi-shadow-graph-state: delphi-shadow-state: labels: polis_tag: ${TAG:-dev} diff --git a/example.env b/example.env index 291be194f5..4bc0f3baeb 100644 --- a/example.env +++ b/example.env @@ -371,3 +371,10 @@ ENCRYPTION_PASSWORD_00001= # Requires the admission-function extension and a separate graph-stage daemon. DELPHI_SHADOW_GRAPH_CONVERSATIONS= DELPHI_SHADOW_GRAPH_ENV=shadow-graph-1 + +# Optional same-host graph executor; keep disabled until the host has room for +# both daemon ceilings (16 GiB + 6 GiB by default) and the 256 MiB scheduler. +DELPHI_SHADOW_GRAPH_ENABLED=0 +DELPHI_SHADOW_GRAPH_CONTAINER_MEMORY=6g +DELPHI_SHADOW_GRAPH_CONTAINER_CPUS=2 +DELPHI_SHADOW_GRAPH_HOST_ID_FILE=/etc/machine-id diff --git a/queue-rs/src/shadow/daemon.rs b/queue-rs/src/shadow/daemon.rs index 607c973527..971d8d05d2 100644 --- a/queue-rs/src/shadow/daemon.rs +++ b/queue-rs/src/shadow/daemon.rs @@ -167,6 +167,25 @@ pub fn check Option>(get: F) -> Result = stages.split(',').collect(); + let expected: std::collections::BTreeSet<_> = + crate::jobs::config::GRAPH_STAGES.iter().copied().collect(); + if stages != expected + || get("POLIS_JOBS_CONCURRENCY").as_deref() != Some("1") + || get("POLIS_JOBS_JOURNAL_HOST_FILE").is_none_or(|v| v.is_empty()) + { + return Err(ConfigError( + "graph daemon: stages, concurrency or host journal setting".into(), + )); + } + if std::process::id() != 1 { + return Err(ConfigError( + "graph daemon: requires PID 1 in its private container namespace".into(), + )); + } + } Ok(queue) } diff --git a/scripts/after_install.sh b/scripts/after_install.sh index d489f94862..3cec7511c4 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -302,13 +302,20 @@ if [ "$SERVICE_FROM_FILE" == "delphi" ] && grep -qx 'DELPHI_SHADOW_ENABLED=1' .e DELPHI_SHADOW_SELECTED=1 fi +DELPHI_SHADOW_GRAPH_SELECTED=0 +if [ "$DELPHI_SHADOW_SELECTED" = 1 ] && grep -qx 'DELPHI_SHADOW_GRAPH_ENABLED=1' .env 2>/dev/null; then + DELPHI_SHADOW_GRAPH_SELECTED=1 +fi + # Compose runs the pulled digests: this override sets each service's image, and # every `up` below passes --no-build, so nothing is built on this box. if [ -n "$ROLE_IMAGES" ]; then { echo "services:" for image in $ROLE_IMAGES; do printf ' %s:\n image: %s\n' "$image" "$(image_ref "$image")"; done if [ "$DELPHI_SHADOW_SELECTED" = 1 ]; then - for image in delphi-shadow delphi-shadow-scheduler; do + SHADOW_IMAGES="delphi-shadow delphi-shadow-scheduler" + if [ "$DELPHI_SHADOW_GRAPH_SELECTED" = 1 ]; then SHADOW_IMAGES="$SHADOW_IMAGES delphi-shadow-graph"; fi + for image in $SHADOW_IMAGES; do printf ' %s:\n image: %s\n' "$image" "$(image_ref delphi)" done fi @@ -419,6 +426,16 @@ if [ "$DELPHI_SHADOW_SELECTED" = 1 ]; then fi fi +# A graph executor has its own profile, local journal and resource ceiling. +# Validate only composed settings; never print the environment document. +if [ "$DELPHI_SHADOW_SELECTED" = 1 ] && [ "$DELPHI_SHADOW_GRAPH_SELECTED" = 1 ]; then + if ! sudo /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.images.yml --profile delphi-shadow --profile delphi-shadow-graph config --format json | python3 scripts/check-shadow-graph.py; then + echo "The graph shadow's composed settings are invalid; not starting either shadow" >&2 + DELPHI_SHADOW_SELECTED=0 + DELPHI_SHADOW_GRAPH_SELECTED=0 + fi +fi + if [ -f "/etc/app-info/log_group_name.txt" ]; then LOG_GROUP_NAME=$(cat "/etc/app-info/log_group_name.txt") export AWS_LOG_GROUP_NAME=$LOG_GROUP_NAME @@ -689,6 +706,10 @@ fi # among them) see a Delphi-only block, not a start on the web boxes. if [ "$SERVICE_FROM_FILE" == "delphi" ] && [ "$DELPHI_SHADOW_SELECTED" = 1 ]; then echo "DELPHI_SHADOW_ENABLED=1: starting the free-only Delphi shadow (delphi-shadow, delphi-shadow-scheduler)" + if [ "$DELPHI_SHADOW_GRAPH_SELECTED" = 1 ]; then + "${COMPOSE[@]}" up -d delphi-shadow-graph --no-build --force-recreate \ + || echo "The graph shadow did not start; recovery remains pending" >&2 + fi "${COMPOSE[@]}" up -d delphi-shadow delphi-shadow-scheduler --no-build --force-recreate \ || echo "The Delphi shadow did not start; the deploy continues without it" >&2 fi diff --git a/scripts/check-shadow-graph.py b/scripts/check-shadow-graph.py new file mode 100644 index 0000000000..0e3b40d50e --- /dev/null +++ b/scripts/check-shadow-graph.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Validate the optional graph executor in rendered Compose JSON; names only.""" +import json +import re +import sys + + +def errors(config): + services = config.get("services", {}) + graph = services.get("delphi-shadow-graph", {}) + env = graph.get("environment", {}) + scheduler = services.get("delphi-shadow-scheduler", {}).get("environment", {}) + expected = { + "DELPHI_SHADOW_GRAPH_DAEMON": "1", + "POLIS_JOBS_ENABLED": "1", + "POLIS_JOBS_WORKER_CLASS": "delphi", + "POLIS_JOBS_STAGES": "graph_embed,graph_cluster,graph_topics,graph_narrative", + "POLIS_JOBS_CONCURRENCY": "1", + "POLIS_JOBS_JOURNAL_DIR": "/var/lib/delphi-shadow-graph/journal", + "POLIS_JOBS_JOURNAL_HOST_FILE": "/run/polis-host-id", + "DELPHI_EMBEDDINGS_TABLE": "1", + "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", + "ANTHROPIC_TOPIC_MODEL": "none", + } + bad = [key for key, value in expected.items() if env.get(key) != value] + queue = env.get("QUEUE_ENV", "") + if (not re.fullmatch(r"shadow-[a-z0-9_-]{1,57}", queue) + or queue != env.get("DELPHI_RESULT_ENV") + or queue != scheduler.get("DELPHI_SHADOW_GRAPH_ENV") + or queue == scheduler.get("DELPHI_SHADOW_ENV")): + bad.append("QUEUE_ENV/DELPHI_RESULT_ENV") + if graph.get("command") != ["polis-delphi-shadow", "daemon"]: + bad.append("command") + if graph.get("init") or graph.get("pid") or graph.get("privileged"): + bad.append("process_namespace") + volumes = graph.get("volumes", []) + if not any(v.get("type") == "volume" and v.get("target") == "/var/lib/delphi-shadow-graph" for v in volumes): + bad.append("journal_volume") + if not any(v.get("type") == "bind" and str(v.get("source", "")).startswith("/") and v.get("target") == "/run/polis-host-id" and v.get("read_only") for v in volumes): + bad.append("host_identity") + limits = graph.get("deploy", {}).get("resources", {}).get("limits", {}) + if not limits.get("memory") or not limits.get("cpus"): + bad.append("resource_limits") + if graph.get("cpu_shares") != 2 or graph.get("oom_score_adj") != 1000: + bad.append("resource_priority") + return bad + + +if __name__ == "__main__": + try: + bad = errors(json.load(sys.stdin)) + except (ValueError, TypeError, AttributeError): + bad = ["unreadable"] + if bad: + print("graph shadow settings: " + " ".join(bad), file=sys.stderr) + sys.exit(bool(bad)) From 60e61f4af466ad8d1dede7a46a1f7996887c4941 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 02:13:20 -0800 Subject: [PATCH 3/7] delphi shadow: graph publication is recognised by its served root and retried a bounded number of times; refusals are reconsidered by build A finished graph's publication is now confirmed by reading the served bundle and checking that its root artifact is this graph's leaf at the next generation, not by the publish call's reply alone. A publish whose commit is durable but whose reply was lost is recognised as published; a transient database error before commit (connection, serialization, deadlock, lock or too-many-connections classes) is retried at most three times, each on a fresh connection, and then counted failed. A definite refusal is not retried. If the served bundle cannot be read, the graph stays in flight and is resolved on a later poll. A refused conversation's recorded observation now carries the admission rule and the build's code version, so a later build looks at it again once. A graph admission that errored is followed by a recovery read, so an admission that committed before its reply was lost is picked up before any full-pipeline run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- queue-rs/src/shadow/graph.rs | 170 ++++++++++++++++++++++--------- queue-rs/src/shadow/scheduler.rs | 6 ++ queue-rs/tests/delphi_shadow.rs | 30 ++++++ 3 files changed, 158 insertions(+), 48 deletions(-) diff --git a/queue-rs/src/shadow/graph.rs b/queue-rs/src/shadow/graph.rs index 639cb58ef2..0f8ce7b482 100644 --- a/queue-rs/src/shadow/graph.rs +++ b/queue-rs/src/shadow/graph.rs @@ -25,12 +25,15 @@ const INCLUDED: &str = "NOT EXISTS (SELECT 1 FROM report_comment_selections r WH #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Observation { + #[serde(default)] + pub rule: String, pub comments: i64, pub last_modified: i64, } -impl From<&Live> for Observation { - fn from(c: &Live) -> Self { +impl Observation { + fn new(c: &Live, build: &str) -> Self { Self { + rule: format!("graph-admission/2:{build}"), comments: c.comments, last_modified: c.last_modified, } @@ -365,7 +368,7 @@ impl Scheduler<'_> { self.graph_list_counts = Some(counts); } for candidate in listed { - let observed = Observation::from(&candidate); + let observed = Observation::new(&candidate, &self.cfg.code_version); if self.state.graph_observed.get(&candidate.zid) == Some(&observed) { continue; } @@ -412,17 +415,19 @@ impl Scheduler<'_> { "graph_refused", json!({"reason":reason,"comments":candidate.comments}), ); - self.state - .graph_observed - .insert(candidate.zid, Observation::from(candidate)); + self.state.graph_observed.insert( + candidate.zid, + Observation::new(candidate, &self.cfg.code_version), + ); } fn admit_graph_candidate(&mut self, cfg: &Config, candidate: &Live, now: i64) -> Result { let s = snapshot(&self.db, candidate.zid)?; if self.state.graph_seen.get(&candidate.zid) == Some(&s.digest) { - self.state - .graph_observed - .insert(candidate.zid, Observation::from(candidate)); + self.state.graph_observed.insert( + candidate.zid, + Observation::new(candidate, &self.cfg.code_version), + ); return Ok(false); } // Four jobs plus all three inline copies per node. Charge the @@ -658,8 +663,8 @@ impl Scheduler<'_> { .values() .all(|(s, _)| matches!(s.as_str(), "succeeded" | "dead" | "cancelled")); // A dead producer reconciles its dependent nodes through the daemon. - // Never release the scheduler while a child is still running, even - // beyond the deadline: otherwise two computations could overlap. + // Keep this conversation excluded while execution remains unresolved. + // After the deadline, deferred tracking allows other conversations. if !terminal { self.log_graph_wait(&flight, now, &states, true); self.defer_graph(&flight, now); @@ -683,51 +688,60 @@ impl Scheduler<'_> { let success = states.values().all(|(s, _)| s == "succeeded"); let mut published = false; if success { - let result = (|| -> Result { - let leaf = flight - .nodes - .iter() - .find(|n| n.stage == "graph_narrative") - .ok_or_else(|| anyhow::anyhow!("graph_leaf_missing"))?; - let mut client = self.db.connect_queue()?; - let r: String = client - .query_one( - "SELECT public.pd_graph_publish($1,$2::text::uuid,$3::text::uuid,$4)::text", + let leaf = flight + .nodes + .iter() + .find(|n| n.stage == "graph_narrative") + .ok_or_else(|| anyhow::anyhow!("graph_leaf_missing"))?; + // Each retry and every read use a fresh connection, including after + // a commit whose reply was lost. No retry follows a definite refusal. + for attempt in 0..3 { + let result = (|| -> Result { + let mut client = self.db.connect_queue()?; + let row = client.query_one( + "SELECT public.pd_graph_publish($1,$2::text::uuid,$3::text::uuid,$4)", &[ &flight.env, &flight.graph_id, &leaf.job_id, &flight.generation, ], - )? - .try_get(0)?; - let r: Value = serde_json::from_str(&r)?; - published = r["outcome"] == "published"; - // On restart after a lost publish reply, accept only this exact leaf. - if !published { - let served: String = client - .query_one( - "SELECT public.pd_graph_served($1,$2,$3)::text", - &[&flight.env, &flight.zid, &scope(flight.zid)], - )? - .try_get(0)?; - let served: Value = serde_json::from_str(&served)?; - published = served["generation"].as_i64() == Some(flight.generation + 1) - && served["bundle"]["artifacts"] - .as_array() - .is_some_and(|a| a.iter().any(|x| x["job_id"] == leaf.job_id)); + )?; + Ok(row.try_get(0)?) + })(); + // A publish reply alone does not establish which root is served. + let served = (|| -> Result { + let mut client = self.db.connect_queue()?; + let row = client.query_one( + "SELECT public.pd_graph_served($1,$2,$3)", + &[&flight.env, &flight.zid, &scope(flight.zid)], + )?; + Ok(row.try_get::<_, Option>(0)?.unwrap_or(Value::Null)) + })(); + match served { + Ok(ref value) if exact_leaf(value, &leaf.job_id, flight.generation) => { + published = true; + break; + } + // The outcome is still unknown. Keep the flight and resolve + // it on a later poll; never count an uncertain commit failed. + Err(error) => return Err(error), + _ => {} } - Ok(published) - })(); - match result { - Ok(value) => published = value, - Err(_) => { - log( - "graph_publish_failed", - json!({"graph_id":flight.graph_id,"reason":"publish_error"}), - ); - published = false; + let retry = result.as_ref().err().is_some_and(publish_transient); + if !retry || attempt == 2 { + break; } + log( + "graph_publish_retry", + json!({"graph_id":flight.graph_id,"attempt":attempt+1}), + ); + } + if !published { + log( + "graph_publish_failed", + json!({"graph_id":flight.graph_id,"reason":"publication_not_served"}), + ); } } let mut bytes = 0u64; @@ -799,6 +813,35 @@ impl Scheduler<'_> { } } +fn publish_transient(error: &anyhow::Error) -> bool { + let Some(db) = error.downcast_ref::() else { + return true; + }; + match db.code().map(|c| c.code()) { + None => true, + Some(c) => { + c.starts_with("08") + || matches!( + c, + "40001" | "40P01" | "53300" | "55P03" | "57014" | "57P01" | "57P03" + ) + } + } +} + +fn exact_leaf(served: &Value, job: &str, generation: i64) -> bool { + served["generation"].as_i64() == generation.checked_add(1) + && served["bundle"]["root"].as_str().is_some_and(|root| { + served["bundle"]["artifacts"] + .as_array() + .is_some_and(|artifacts| { + artifacts + .iter() + .any(|a| a["artifact_id"] == root && a["job_id"] == job) + }) + }) +} + /// Additional graph rows, indexed by this job. The root pays the graph request /// once; each consumer pays its edges. Logical row bytes are doubled and a /// per-row allowance covers indexes/page overhead. This remains an estimate, @@ -881,3 +924,34 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod recovery_tests { + use super::*; + #[test] + fn publication_requires_exact_root_and_generation() { + let mut served = json!({"generation": 7,"bundle":{"root":"root","artifacts":[ + {"artifact_id":"ancestor","job_id":"requested"}, + {"artifact_id":"root","job_id":"other"}]}}); + assert!(!exact_leaf(&served, "requested", 6)); + served["bundle"]["root"] = json!("ancestor"); + assert!(exact_leaf(&served, "requested", 6)); + assert!(!exact_leaf(&served, "requested", 5)); + assert!(!exact_leaf(&Value::Null, "requested", 6)); + } + #[test] + fn old_refusal_and_new_code_are_reconsidered() -> Result<()> { + let live = Live { + zid: 1, + comments: 2001, + last_modified: 2, + }; + let old: Observation = serde_json::from_value(json!({"comments":2001,"last_modified":2}))?; + assert_ne!(old, Observation::new(&live, "build-a")); + assert_ne!( + Observation::new(&live, "build-a"), + Observation::new(&live, "build-b") + ); + Ok(()) + } +} diff --git a/queue-rs/src/shadow/scheduler.rs b/queue-rs/src/shadow/scheduler.rs index 0682c00a1c..3eb4f6f5ee 100644 --- a/queue-rs/src/shadow/scheduler.rs +++ b/queue-rs/src/shadow/scheduler.rs @@ -379,6 +379,12 @@ impl<'a> Scheduler<'a> { json!({"reason":"admission_error","retry_seconds":self.cfg.roster_retry.as_secs()}), ); self.graph_failed_at_ms = Some(now); + // Admission may have committed before its reply was lost. + // Recovery failure stops this cycle, including full runs. + self.recover_graph()?; + if self.state.graph_flight.is_some() { + return Ok(None); + } } } } diff --git a/queue-rs/tests/delphi_shadow.rs b/queue-rs/tests/delphi_shadow.rs index b879abc27e..55d647edbc 100644 --- a/queue-rs/tests/delphi_shadow.rs +++ b/queue-rs/tests/delphi_shadow.rs @@ -1562,6 +1562,9 @@ fn graph_scheduler_admits_publishes_and_accepts_only_its_lost_reply() { .unwrap() .get(0); assert_eq!(published["outcome"], "published"); + // The original commit is durable but its reply was lost. A later + // publish call may itself fail; the separate root read still wins. + f.superuser().batch_execute("ALTER FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RENAME TO publish_before_lost_reply; CREATE FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER AS $$ BEGIN RAISE EXCEPTION 'publication reply unavailable' USING ERRCODE='40001'; END $$; GRANT EXECUTE ON FUNCTION pd_graph_publish(text,uuid,uuid,bigint) TO polis_queue_executor;").unwrap(); } s.cycle().unwrap(); assert!(s.state.graph_flight.is_none()); @@ -1761,3 +1764,30 @@ fn committed_graph_admission_with_unread_reply_is_recovered_before_full_pipeline ); assert!(scheduler.state.in_flight.is_none()); } + +#[test] +fn transient_publish_before_commit_retries_and_keeps_one_generation() { + let f = Fixture::new(); + f.quiet(991, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c991")]); + let mut scheduler = Scheduler::new(&cfg).unwrap(); + scheduler.cycle().unwrap(); + let flight = scheduler.state.graph_flight.clone().unwrap(); + finish_graph(&f, &flight); + let mut c = f.superuser(); + // Sequences survive rollback, so the first call refuses before its commit + // and the second exercises the original publication function. + c.batch_execute("CREATE SEQUENCE publish_calls; ALTER FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RENAME TO publish_original; + CREATE FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER AS $$ + BEGIN IF nextval('publish_calls')=1 THEN RAISE EXCEPTION 'retry' USING ERRCODE='40001'; END IF; + RETURN publish_original($1,$2,$3,$4); END $$; + GRANT EXECUTE ON FUNCTION pd_graph_publish(text,uuid,uuid,bigint) TO polis_queue_executor;").unwrap(); + scheduler.cycle().unwrap(); + assert_eq!(scheduler.state.totals.succeeded, 1); + let calls: i64 = c + .query_one("SELECT last_value FROM publish_calls", &[]) + .unwrap() + .get(0); + assert_eq!(calls, 2); +} From 22561d2027b77bd9cdee3e2ebe88a3907f3a201a Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 02:13:20 -0800 Subject: [PATCH 4/7] delphi shadow: graph wait and journal recovery log lines say why, and the recovery runbook is documented A graph waiting past its deadline now logs the reason it is still held (nodes still active, a process exit not yet confirmed, or reconciliation pending) and the count of attempts with no confirmed exit, instead of one fixed reason. A job daemon with a host-bound journal logs one line per retained journal entry saying whether the attempt's exit was proven and why. docs/configuration.md gains a section on the same-host graph executor: its profile, host identity file, journal volume, deadlines, memory ceilings and what to drain before a rollback. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- docs/configuration.md | 46 ++++++++++++++++++++++++++++++++++++ queue-rs/src/jobs/claim.rs | 6 +++++ queue-rs/src/shadow/graph.rs | 24 +++++++++++++++---- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e09eb61881..3d9d7e55e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -388,3 +388,49 @@ translate strings can be found in: `client-participation/js/strings/en_us.js` The current sender uses AWS SES ([senders.ts:1](../server/src/email/senders.ts#L1)). `EMAIL_TRANSPORT_TYPES` and Mailgun names still have config reads, but that alone does not mean the current sender selects a Mailgun transport. Follow the sender implementation and the [service inventory](deployment-configuration.md). Source-only server entry points that skip the build use the root `VERSION` read once at startup. Without generated metadata, `/api/v3/version` reports `+dev.unknown.unknown`, commit `unknown`, distance `null`, channel `development`, and source `unverified-archive`. Normal build, watch, and npm test entry points still generate and embed the build identity first. + +### Same-host graph executor and recovery + +`DELPHI_SHADOW_GRAPH_ENABLED=1` selects the `delphi-shadow-graph` service on +Delphi deployments. For local Compose, enable the `delphi-shadow-graph` profile +as well as `delphi-shadow`. The conversation list still decides new admissions; +clearing it stops new graphs but leaves recovery active for retained state. +The default is disabled. No other service changes when this profile is off. + +The graph executor runs as PID 1 in a private namespace, one graph step at a +time, offline with the same pinned model directory and lock. Its named local +volume contains the attempt journal and a binding to `/etc/machine-id` from +that host. On a host without that file, create a persistent, unique host identity +file once, and name its absolute path in `DELPHI_SHADOW_GRAPH_HOST_ID_FILE`. +The file must already exist; never copy the identity together with the journal. +This is an operator-maintained host identity, not hardware attestation. +Retain the volume across container replacement. Never copy it to +another host or share it between hosts. The daemon refuses a journal bound to +another host, or old entries without that binding. Two daemons cannot hold the +same journal lock. On the same host, replacement of the PID 1 container kills +its namespace's processes; a host reboot kills the prior boot's processes. +A daemon restart in the same namespace needs both its recorded daemon and +child process group gone. A missing process-group record is not exit proof. + +The scheduler cancels overdue graphs at `DELPHI_SHADOW_JOB_DEADLINE_SECONDS` +(default 14400). It keeps their scopes held and polls recovery in the background; +other conversations may resume full-pipeline work. The same conversation stays +excluded while its graph scope is held. A lost host or volume cannot supply exit +proof: `graph_recovery_pending` means just that, never successful recovery. +Database recovery errors stop admissions until the database can be read again. + +The default ceilings are 16 GiB for the full-pipeline daemon, 6 GiB for the graph +executor (`DELPHI_SHADOW_GRAPH_CONTAINER_MEMORY`), and 256 MiB for the scheduler: +22.25 GiB total, plus math, database, web services and the operating system. +Resident memory in both daemons counts even when admissions are serial. Recovery +may overlap an unknown old graph with work for another conversation; provision +for both ceilings. Each daemon has a 2-CPU cap by default; the graph executor's +setting is `DELPHI_SHADOW_GRAPH_CONTAINER_CPUS`. CPU weight and OOM priority make +this optional work yield first; they do not reserve capacity for participation. + +State written by the previous release loads with empty recovery fields. When no +graph has been enabled, state serialization and full-pipeline behavior are +unchanged. Before rolling back, drain every graph, including deferred graphs, +and verify its scope was released. Older schedulers ignore the new deferred +fields and cannot enforce those exclusions. Do not remove scheduler state or a +journal to force progress. diff --git a/queue-rs/src/jobs/claim.rs b/queue-rs/src/jobs/claim.rs index ae867b02da..7cfa596051 100644 --- a/queue-rs/src/jobs/claim.rs +++ b/queue-rs/src/jobs/claim.rs @@ -225,6 +225,12 @@ fn recover(ctx: &Ctx, rpc: &mut Rpc) { } let gone = journal::provably_gone(&entry, &ctx.cfg.boot_id, &ctx.cfg.container_id, &ctx.owner); + if ctx.cfg.journal_host_file.is_some() { + line(&format!( + "graph_recovery {}", + json!({"job_id":entry.job_id,"attempt_id":entry.attempt_id,"owner_id":entry.owner_id,"exit_proven":gone,"reason":if gone { "retained_host_journal" } else { "process_exit_unconfirmed" }}) + )); + } if !gone { line(&format!( "journal entry {} not provably gone; left in place", diff --git a/queue-rs/src/shadow/graph.rs b/queue-rs/src/shadow/graph.rs index 0f8ce7b482..b24f9a0cbd 100644 --- a/queue-rs/src/shadow/graph.rs +++ b/queue-rs/src/shadow/graph.rs @@ -610,17 +610,33 @@ impl Scheduler<'_> { now: i64, states: &BTreeMap, scope_held: bool, - ) { + ) -> Result<()> { let deadline = (self.cfg.job_deadline.as_millis() as i64).max(1); let overdue = (now - flight.admitted_at_ms - deadline).max(0); let bucket = overdue / deadline; if now - flight.admitted_at_ms >= deadline && self.graph_wait_bucket != Some(bucket) { + let unconfirmed: i64 = self.db.read(|tx| Ok(tx.query_one( + "SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid AND a.process_exit_confirmed_at IS NULL", + &[&flight.env, &flight.graph_id])?.try_get(0)?))?; + let reason = if states.values().any(|(s, _)| s == "running") { + "nodes_active" + } else if unconfirmed > 0 { + "exit_unconfirmed" + } else if states + .values() + .any(|(s, _)| !matches!(s.as_str(), "succeeded" | "dead" | "cancelled")) + { + "nodes_active" + } else { + "reconciliation_pending" + }; log( "graph_waiting", - json!({"graph_id":flight.graph_id,"states":states,"scope_held":scope_held,"seconds_overdue":overdue/1000,"reason":"awaiting_exit_proof"}), + json!({"graph_id":flight.graph_id,"states":states,"scope_held":scope_held,"seconds_overdue":overdue/1000,"reason":reason,"unconfirmed_attempts":unconfirmed}), ); self.graph_wait_bucket = Some(bucket); } + Ok(()) } pub(super) fn watch_graph(&mut self, flight: Flight, now: i64) -> Result { @@ -666,7 +682,7 @@ impl Scheduler<'_> { // Keep this conversation excluded while execution remains unresolved. // After the deadline, deferred tracking allows other conversations. if !terminal { - self.log_graph_wait(&flight, now, &states, true); + self.log_graph_wait(&flight, now, &states, true)?; self.defer_graph(&flight, now); return Ok(self.cfg.job_poll); } @@ -680,7 +696,7 @@ impl Scheduler<'_> { .try_get(0)?; let view: Value = serde_json::from_str(&view)?; if view["guard_held"] != false { - self.log_graph_wait(&flight, now, &states, true); + self.log_graph_wait(&flight, now, &states, true)?; self.defer_graph(&flight, now); return Ok(self.cfg.job_poll); } From aa11f15c6c5f2d9220ec356af9298b3fb4c77d98 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 02:32:53 -0800 Subject: [PATCH 5/7] delphi tests: the graph executor's Compose tests run in the Delphi test container The graph executor's validator test located scripts/check-shadow-graph.py by its own path, which inside the Delphi test container resolves to /scripts and does not exist, so every shard of the Delphi Python tests stopped with a collection error. It now reads the validator from the checkout the other Compose tests read ($POLIS_CHECKOUT_DIR), and the deploy-hook runner copies the validator there with its other inputs. The test that the shadow yields CPU and memory to every other service failed once the graph executor was added, because the executor carries the shadow's own lowest weight. Both shadow daemons must now carry exactly that weight and OOM adjustment, and every other service must still outrank them; no assertion about any other service changed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- delphi/tests/test_compose_delphi_shadow.py | 10 ++++++-- delphi/tests/test_shadow_graph_compose.py | 30 +++++++++++++++------- scripts/test-deploy-hooks.sh | 1 + 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/delphi/tests/test_compose_delphi_shadow.py b/delphi/tests/test_compose_delphi_shadow.py index 7d49e01c7d..5a7798543a 100644 --- a/delphi/tests/test_compose_delphi_shadow.py +++ b/delphi/tests/test_compose_delphi_shadow.py @@ -238,11 +238,17 @@ def test_the_shadow_yields_cpu_and_memory_to_every_other_service(): # services included, keeps Docker's default weight and OOM adjustment (or # sets a higher weight and a lower adjustment), so under contention the # shadow gets the smallest share of a CPU, and when the box runs out of - # memory it is killed first. + # memory it is killed first. The optional graph executor is shadow work + # too: it carries exactly the shadow's weight and OOM adjustment, and every + # other service outranks both. services, _ = _services() shadow = services["delphi-shadow"] + shadow_daemons = ("delphi-shadow", "delphi-shadow-graph") + for name in shadow_daemons: + assert services[name]["cpu_shares"] == shadow["cpu_shares"], name + assert services[name]["oom_score_adj"] == shadow["oom_score_adj"], name for name, service in services.items(): - if name == "delphi-shadow": + if name in shadow_daemons: continue assert service.get("cpu_shares", DEFAULT_CPU_SHARES) > shadow["cpu_shares"], name assert service.get("oom_score_adj", DEFAULT_OOM_SCORE_ADJ) < shadow["oom_score_adj"], name diff --git a/delphi/tests/test_shadow_graph_compose.py b/delphi/tests/test_shadow_graph_compose.py index 325590ded4..0e5d476ff4 100644 --- a/delphi/tests/test_shadow_graph_compose.py +++ b/delphi/tests/test_shadow_graph_compose.py @@ -1,11 +1,23 @@ """The optional executor must retain its local journal and shadow boundary.""" import importlib.util -from pathlib import Path -MODULE = Path(__file__).resolve().parents[2] / "scripts/check-shadow-graph.py" -spec = importlib.util.spec_from_file_location("shadow_graph_compose", MODULE) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) +import pytest + +from tests.test_compose_math_env import CHECKOUT, requires_checkout + +# The deploy hook's validator, read from the checkout the Compose tests read +# ($POLIS_CHECKOUT_DIR in the Delphi test container, where the deploy-hook +# runner copies it). A missing file fails these tests; it is never a skip. +pytestmark = requires_checkout + + +@pytest.fixture(scope="module") +def module(): + spec = importlib.util.spec_from_file_location( + "shadow_graph_compose", CHECKOUT / "scripts/check-shadow-graph.py") + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded def valid(): @@ -25,25 +37,25 @@ def valid(): return dict(services={"delphi-shadow-graph": graph, "delphi-shadow-scheduler": scheduler}) -def test_accepts_the_isolated_executor(): +def test_accepts_the_isolated_executor(module): assert module.errors(valid()) == [] -def test_refuses_a_served_or_shared_env(): +def test_refuses_a_served_or_shared_env(module): for env in ["prod", "shadow-delphi-1"]: config = valid() config["services"]["delphi-shadow-graph"]["environment"]["QUEUE_ENV"] = env assert module.errors(config) -def test_refuses_a_missing_journal_or_host_identity(): +def test_refuses_a_missing_journal_or_host_identity(module): for index in [0, 1]: config = valid() config["services"]["delphi-shadow-graph"]["volumes"].pop(index) assert module.errors(config) -def test_refuses_a_different_process_namespace_or_stage(): +def test_refuses_a_different_process_namespace_or_stage(module): for key, value in [("init", True), ("pid", "host"), ("privileged", True)]: config = valid() config["services"]["delphi-shadow-graph"][key] = value diff --git a/scripts/test-deploy-hooks.sh b/scripts/test-deploy-hooks.sh index 7bbf773293..79090c613d 100644 --- a/scripts/test-deploy-hooks.sh +++ b/scripts/test-deploy-hooks.sh @@ -17,6 +17,7 @@ inputs=( scripts/before_install.sh scripts/application_stop.sh scripts/validate_service.sh + scripts/check-shadow-graph.py deploy/build-images.sh deploy/build-on-box.sh docker-compose.yml From 406eaffbcaf965214b09bcc02e686937bf31199d Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 03:19:06 -0800 Subject: [PATCH 6/7] delphi shadow: the graph executor receives the per-environment embedding model setting as the shadow daemon does DELPHI_EMBED_MODEL_BY_ENV is now passed through to delphi-shadow-graph the way the registry passes it to delphi-shadow and the scheduler: listed by name, with no default of its own, so with the setting unset (today) it is absent from the container and nothing changes. Each graph's model is still decided and sealed by the scheduler at admission, and graph children's environment stays the existing allowlist. The deploy check refuses a graph executor whose setting differs from the scheduler's; tests cover the check and that both daemons list the setting the same way. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- delphi/tests/test_shadow_graph_compose.py | 18 ++++++++++++++++++ docker-compose.yml | 1 + docs/configuration.md | 4 +++- scripts/check-shadow-graph.py | 5 +++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/delphi/tests/test_shadow_graph_compose.py b/delphi/tests/test_shadow_graph_compose.py index 0e5d476ff4..4c12c57951 100644 --- a/delphi/tests/test_shadow_graph_compose.py +++ b/delphi/tests/test_shadow_graph_compose.py @@ -2,6 +2,7 @@ import importlib.util import pytest +import yaml from tests.test_compose_math_env import CHECKOUT, requires_checkout @@ -63,3 +64,20 @@ def test_refuses_a_different_process_namespace_or_stage(module): config = valid() config["services"]["delphi-shadow-graph"]["environment"]["POLIS_JOBS_STAGES"] = "delphi_full_pipeline" assert module.errors(config) + + +def test_refuses_an_embedding_model_setting_the_scheduler_does_not_share(module): + config = valid() + config["services"]["delphi-shadow-graph"]["environment"]["DELPHI_EMBED_MODEL_BY_ENV"] = '{"shadow-graph-1": "x"}' + assert module.errors(config) == ["DELPHI_EMBED_MODEL_BY_ENV"] + config["services"]["delphi-shadow-scheduler"]["environment"]["DELPHI_EMBED_MODEL_BY_ENV"] = '{"shadow-graph-1": "x"}' + assert module.errors(config) == [] + + +def test_the_executor_receives_the_embedding_model_setting_as_the_shadow_daemon_does(): + # Passed through unchanged, with no default of its own: unset (today), it + # is absent from both containers. + services = yaml.safe_load((CHECKOUT / "docker-compose.yml").read_text())["services"] + for name in ("delphi-shadow", "delphi-shadow-graph"): + assert services[name]["environment"].count("DELPHI_EMBED_MODEL_BY_ENV") == 1, name + assert not any(e.startswith("DELPHI_EMBED_MODEL_BY_ENV=") for e in services[name]["environment"]), name diff --git a/docker-compose.yml b/docker-compose.yml index 2b92385294..ab503016f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -590,6 +590,7 @@ services: - ANTHROPIC_TOPIC_MODEL=none - SENTENCE_TRANSFORMER_MODEL=ibm-granite/granite-embedding-311m-multilingual-r2 - DELPHI_EMBED_MODEL_PATH=/opt/polis/embedding/granite-embedding-311m-multilingual-r2 + - DELPHI_EMBED_MODEL_BY_ENV - DELPHI_EMBED_MODEL_LOCK=/opt/polis/embedding/granite-embedding-311m-multilingual-r2.lock.json - HF_HUB_OFFLINE=1 - TRANSFORMERS_OFFLINE=1 diff --git a/docs/configuration.md b/docs/configuration.md index c0cf155cc4..a2121bf46d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -416,7 +416,9 @@ clearing it stops new graphs but leaves recovery active for retained state. The default is disabled. No other service changes when this profile is off. The graph executor runs as PID 1 in a private namespace, one graph step at a -time, offline with the same pinned model directory and lock. Its named local +time, offline with the same pinned model directory and lock. It receives +`DELPHI_EMBED_MODEL_BY_ENV` exactly as the scheduler does, which decides each +graph's model at admission; the deploy check refuses the two differing. Its named local volume contains the attempt journal and a binding to `/etc/machine-id` from that host. On a host without that file, create a persistent, unique host identity file once, and name its absolute path in `DELPHI_SHADOW_GRAPH_HOST_ID_FILE`. diff --git a/scripts/check-shadow-graph.py b/scripts/check-shadow-graph.py index 0e3b40d50e..d0c31e71fe 100644 --- a/scripts/check-shadow-graph.py +++ b/scripts/check-shadow-graph.py @@ -29,6 +29,11 @@ def errors(config): or queue != scheduler.get("DELPHI_SHADOW_GRAPH_ENV") or queue == scheduler.get("DELPHI_SHADOW_ENV")): bad.append("QUEUE_ENV/DELPHI_RESULT_ENV") + # The per-environment embedding model setting reaches the executor exactly + # as it reaches the scheduler that seals each graph's model (both unset by + # default), so the two can never disagree. + if env.get("DELPHI_EMBED_MODEL_BY_ENV") != scheduler.get("DELPHI_EMBED_MODEL_BY_ENV"): + bad.append("DELPHI_EMBED_MODEL_BY_ENV") if graph.get("command") != ["polis-delphi-shadow", "daemon"]: bad.append("command") if graph.get("init") or graph.get("pid") or graph.get("privileged"): From e0af0692fd87e6cb946e9d47f25b2ab5f070c117 Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 18 Nov 2026 03:19:19 -0800 Subject: [PATCH 7/7] graph registry check: the daemon recovery proof may admit its test node directly delphi/tests/shadow/graph_recovery.py admits one test-model node through the SQL admission function so that a test-build daemon has a real attempt to lose and recover, as the graph proofs already do. The check that every product admission goes through the shared entry now lists it beside those proofs. The proof's description says it needs a test-build image, since a production daemon refuses to run a test-only model. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KvftUAdasWzxmMYdioZJWs --- ci/check_graph_registry.py | 3 +++ delphi/tests/shadow/graph_recovery.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/ci/check_graph_registry.py b/ci/check_graph_registry.py index 9fd81e60c9..7a25c72f87 100644 --- a/ci/check_graph_registry.py +++ b/ci/check_graph_registry.py @@ -19,6 +19,9 @@ 'ci/test_graph_registry.py', 'delphi/tests/job_graph/adversarial.py', 'delphi/tests/job_graph/prove.py', + # Daemon recovery proof: admits one test-model node straight through SQL + # so a test-build daemon has an attempt to lose and recover. + 'delphi/tests/shadow/graph_recovery.py', 'queue-rs/tests/delphi_shadow.rs', 'queue-rs/tests/jobs_integration.rs', 'queue-rs/polis-contracts/src/delphi_run.rs', diff --git a/delphi/tests/shadow/graph_recovery.py b/delphi/tests/shadow/graph_recovery.py index 7c8b46b6b1..9f72190638 100644 --- a/delphi/tests/shadow/graph_recovery.py +++ b/delphi/tests/shadow/graph_recovery.py @@ -3,6 +3,8 @@ GRAPH_RECOVERY_IMAGE, GRAPH_RECOVERY_PG_CONTAINER, GRAPH_RECOVERY_DATABASE and GRAPH_RECOVERY_OUT name caller-owned resources. No votes or external services. +The image must be a test build (delphi/Dockerfile target test): the node it +admits uses a test-only model, which a production daemon refuses to run. """ import json import os