Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ci/check_graph_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
'ci/test_graph_registry.py',
'delphi/tests/job_graph/adversarial.py',
'delphi/tests/job_graph/prove.py',
# Daemon recovery proof: admits one test-model node straight through SQL
# so a test-build daemon has an attempt to lose and recover.
'delphi/tests/shadow/graph_recovery.py',
'queue-rs/tests/delphi_shadow.rs',
'queue-rs/tests/jobs_integration.rs',
'queue-rs/polis-contracts/src/delphi_run.rs',
Expand Down
144 changes: 144 additions & 0 deletions delphi/tests/shadow/graph_recovery.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Owned Docker process-recovery checks against a migrated throwaway database.

GRAPH_RECOVERY_IMAGE, GRAPH_RECOVERY_PG_CONTAINER, GRAPH_RECOVERY_DATABASE and
GRAPH_RECOVERY_OUT name caller-owned resources. No votes or external services.
The image must be a test build (delphi/Dockerfile target test): the node it
admits uses a test-only model, which a production daemon refuses to run.
"""
import json
import os
from pathlib import Path
import subprocess as sp
import time
import uuid

OUT = Path(os.environ['GRAPH_RECOVERY_OUT']).resolve()
OUT.mkdir(parents=True, exist_ok=True)
PG = os.environ['GRAPH_RECOVERY_PG_CONTAINER']
DB = os.environ['GRAPH_RECOVERY_DATABASE']
IMAGE = os.environ['GRAPH_RECOVERY_IMAGE']
PREFIX = 'graph-recovery-' + uuid.uuid4().hex[:12]
owned = []
results = []


def run(*args, **kwargs):
return sp.run(args, check=True, text=True, capture_output=True, **kwargs).stdout.strip()


def sql(query):
return run('docker', 'exec', '-i', PG, 'psql', '-XqAt', '-v', 'ON_ERROR_STOP=1',
'-U', 'postgres', '-d', DB, input=query)


def lit(value):
return "'" + str(value).replace("'", "''") + "'"


def until(check):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
value = check()
if value:
return value
time.sleep(.2)
raise AssertionError('condition did not become true')


def launch(name, journal):
owned.append(name)
env = dict(POLIS_JOBS_ENABLED='1', QUEUE_ENV='shadow-graph-1',
DELPHI_RESULT_ENV='shadow-graph-1', DELPHI_SHADOW_GRAPH_DAEMON='1',
QUEUE_DATABASE_URL=f'postgresql://graph_recovery@127.0.0.1:5432/{DB}',
POLIS_JOBS_TRANSPORT='loopback', POLIS_JOBS_WORKER_CLASS='delphi',
POLIS_JOBS_STAGES='graph_embed,graph_cluster,graph_topics,graph_narrative',
POLIS_JOBS_CONCURRENCY='1', POLIS_JOBS_JOURNAL_DIR='/journal',
POLIS_JOBS_JOURNAL_HOST_FILE='/proof/host-id', POLIS_JOBS_WORK_DIR='/tmp/work',
DELPHI_APP_PATH='/proof/app', POLIS_JOBS_PYTHON='python3',
POLIS_JOBS_POLL_SECONDS='1', POLIS_JOBS_REAP_SECONDS='1',
POLIS_JOBS_LEASE_SECONDS='15', POLIS_JOBS_HEARTBEAT_SECONDS='3')
args = ['docker', 'run', '-d', '--name', name, '--network', 'container:'+PG,
'--memory', '6g', '--cpus', '2', '--cap-drop', 'ALL',
'-v', str(OUT)+':/proof', '-v', str(journal)+':/journal']
for key, value in env.items():
args += ['-e', key+'='+value]
run(*args, '--entrypoint', 'polis-delphi-shadow', IMAGE, 'daemon')


def admit(case):
data = {'texts': ['public generated text '+str(i) for i in range(5)]}
sha = sql('SELECT pd_graph_hash('+lit(json.dumps(data))+'::jsonb)')
spec = dict(schema='polis-job-graph/1', nodes=[dict(key='e', stage='graph_embed',
**{'class': 'delphi'}, declared=dict(snapshot=dict(data=data, sha256=sha),
code='a'*64, model='local-token-count/1', runtime='python-test', seed=0,
config={}, mode='full', memory_bytes=1048576, work_units=100), inputs=[], max_attempts=1)])
reply = json.loads(sql("SELECT pd_graph_admit('shadow-graph-1',1,"+lit(case)+",'first',"+lit(json.dumps(spec))+"::jsonb)"))
return reply['graph_id']


def view(graph):
return json.loads(sql("SELECT pd_graph_view('shadow-graph-1',"+lit(graph)+"::uuid)"))


def record(name):
results.append(name)
print('PASS', name, flush=True)
(OUT/'results.json').write_text(json.dumps(results, indent=2)+'\n')


def main():
sql("INSERT INTO users(uid,hname) VALUES(1,'public generated owner'); INSERT INTO conversations(zid,owner,topic) VALUES(1,1,'public generated'); DO $$ BEGIN IF NOT EXISTS(SELECT FROM pg_roles WHERE rolname='graph_recovery') THEN CREATE ROLE graph_recovery LOGIN; END IF; END $$; GRANT polis_queue_executor TO graph_recovery;")
(OUT/'host-id').write_text('public-test-host')
scripts = OUT/'app/scripts'
scripts.mkdir(parents=True)
(scripts/'job_graph_stage.py').write_text("import time\nwhile True: time.sleep(.1)\n")
for case in ['container-replaced', 'same-container-restarted', 'journal-lost']:
journal = OUT/case
journal.mkdir()
graph = admit(case)
first = PREFIX+'-'+case
launch(first, journal)
until(lambda: any(journal.glob('*.json')) and view(graph)['nodes'][0]['readiness']['state'] == 'running')
# Wait for the post-spawn entry, not merely the durable pre-spawn entry.
until(lambda: any(json.loads(p.read_text()).get('pgid') for p in journal.glob('*.json')))
second = PREFIX+'-second-'+case
launch(second, journal)
until(lambda: run('docker', 'inspect', '-f', '{{.State.Running}}', second) == 'false')
assert run('docker', 'inspect', '-f', '{{.State.ExitCode}}', second) == '2'
assert view(graph)['nodes'][0]['attempts'] == 1
record(case+': second daemon refused')
run('docker', 'kill', '--signal', 'KILL', first)
until(lambda: run('docker', 'inspect', '-f', '{{.State.Running}}', first) == 'false')
if case == 'same-container-restarted':
run('docker', 'start', first)
else:
next_journal = journal
if case == 'journal-lost':
next_journal = OUT/'empty-journal'
next_journal.mkdir()
launch(PREFIX+'-replacement-'+case, next_journal)
if case == 'journal-lost':
until(lambda: view(graph)['nodes'][0]['readiness']['state'] == 'parked')
assert view(graph)['guard_held']
assert sql("SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.graph_id="+lit(graph)+"::uuid AND a.process_exit_confirmed_at IS NULL") == '1'
record('lost journal: expiry did not invent exit proof or release scope')
else:
until(lambda: not view(graph)['guard_held'])
assert sql("SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.graph_id="+lit(graph)+"::uuid AND a.process_exit_confirmed_at IS NULL") == '0'
record(case+': retained exact attempt recovered')
for name in owned:
if run('docker', 'inspect', '-f', '{{.State.Running}}', name) == 'true':
run('docker', 'stop', '-t', '3', name)
assert sql('SELECT count(*) FROM votes') == '0'
record('no votes or production data')


if __name__ == "__main__":
try:
main()
finally:
for name in owned:
p = sp.run(['docker', 'logs', name], text=True, capture_output=True)
(OUT/(name+'.log')).write_text(p.stdout+p.stderr)
sp.run(['docker', 'rm', '-f', name], check=False, capture_output=True)
78 changes: 78 additions & 0 deletions delphi/tests/shadow/graph_scheduler_limits.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Scheduler restart/list clearing under its deployed 256 MiB, quarter-CPU limits."""
import json
import os
from pathlib import Path
import subprocess as sp
import time
import uuid

from graph_recovery import OUT, PG, DB, IMAGE, run, sql, lit, until

names = []
prefix = 'graph-scheduler-' + uuid.uuid4().hex[:12]
state = OUT/'scheduler'
state.mkdir(parents=True)

def start(label, codes):
name = prefix+'-'+label
names.append(name)
env = dict(DELPHI_SHADOW_ENABLED='1', DELPHI_SHADOW_ENV='shadow-delphi-1',
QUEUE_ENV='shadow-delphi-1', POLIS_JOBS_TRANSPORT='loopback',
QUEUE_DATABASE_URL=f'postgresql://graph_recovery@127.0.0.1:5432/{DB}',
DELPHI_SHADOW_READ_DATABASE_URL=f'postgresql://postgres@127.0.0.1:5432/{DB}',
DELPHI_SHADOW_CODE_VERSION='a'*40, DELPHI_SHADOW_STATE_PATH='/state/state.json',
DELPHI_SHADOW_PAUSE_FILE='/state/pause', DELPHI_SHADOW_DISK_FREE_MIN_MB='0',
DELPHI_SHADOW_DISK_USED_MAX_PCT='100', DELPHI_SHADOW_PROBE_MAX_MS='5000',
DELPHI_SHADOW_GRAPH_CONVERSATIONS=codes, DELPHI_SHADOW_GRAPH_ENV='shadow-graph-1',
DELPHI_SHADOW_BENCHMARK='c2,c3', DELPHI_SHADOW_POLL_SECONDS='1',
DELPHI_SHADOW_JOB_POLL_SECONDS='1', DELPHI_SHADOW_JOB_DEADLINE_SECONDS='60',
DELPHI_SHADOW_DUTY_CYCLE='1',
DELPHI_EMBED_MODEL_PATH='/opt/polis/embedding/granite-embedding-311m-multilingual-r2',
DELPHI_EMBED_MODEL_LOCK='/opt/polis/embedding/granite-embedding-311m-multilingual-r2.lock.json')
args = ['docker','run','-d','--name',name,'--network','container:'+PG,
'--memory','256m','--cpus','0.25','-v',str(state)+':/state']
for key, value in env.items(): args += ['-e',key+'='+value]
run(*args,'--entrypoint','polis-delphi-shadow',IMAGE)
return name


def read_state():
try: return json.loads((state/'state.json').read_text())
except (FileNotFoundError, json.JSONDecodeError): return {}


try:
sql("INSERT INTO users(uid,hname) VALUES(1,'generated owner');")
# The common restricted login may already exist on this owned server.
sql("DO $$ BEGIN IF NOT EXISTS(SELECT FROM pg_roles WHERE rolname='graph_recovery') THEN CREATE ROLE graph_recovery LOGIN; END IF; END $$; GRANT polis_queue_executor TO graph_recovery;")
for zid in [2,3]:
sql(f"INSERT INTO conversations(zid,owner,topic,is_active) VALUES({zid},1,'public generated',true); INSERT INTO zinvites(zid,zinvite) VALUES({zid},'c{zid}'); INSERT INTO participants(zid,pid,uid) VALUES({zid},0,1); INSERT INTO comments(zid,pid,uid,tid,txt,mod,active) SELECT {zid},0,1,i,'Public generated comment '||i,0,true FROM generate_series(0,24) i;")
first = start('before','c2')
flight = until(lambda: read_state().get('graph_flight'))
# A real queue attempt loses its local journal. This test spawns no child;
# the absent receipt must nevertheless remain unproven by the scheduler.
owner, attempt = str(uuid.uuid4()), str(uuid.uuid4())
claimed = json.loads(sql("SELECT pd_graph_claim('shadow-graph-1',1::smallint,"+lit(owner)+"::uuid,"+lit(attempt)+"::uuid,15,'delphi','graph_embed')"))
assert claimed['outcome'] == 'owned'
run('docker','stop','-t','5',first)
saved = read_state()
saved['graph_flight']['admitted_at_ms'] -= 120000
(state/'state.json').write_text(json.dumps(saved))
second = start('after','')
finished = until(lambda: read_state() if read_state().get('in_flight') else None)
assert finished['in_flight']['zid'] == 3
assert len(finished['graph_deferred']) == 1
assert finished['graph_deferred'][0]['graph_id'] == flight['graph_id']
assert sql("SELECT count(*) FROM polis_queue_attempts WHERE attempt_id="+lit(attempt)+"::uuid AND process_exit_confirmed_at IS NULL") == '1'
inspect = json.loads(run('docker','inspect',second))[0]
assert inspect['HostConfig']['Memory'] == 256*1024**2
assert inspect['HostConfig']['NanoCpus'] == 250000000
(OUT/'limits-results.json').write_text(json.dumps(dict(checks=4, memory_bytes=256*1024**2, nano_cpus=250000000,
restart=True,list_cleared=True,other_conversation_admitted=True,orphan_scope_retained=True),indent=2)+'\n')
print('PASS 4 scheduler restart/list-clear/exclusion/resource-limit assertions')
finally:
for name in names:
p = sp.run(['docker','logs',name],text=True,capture_output=True)
(OUT/(name+'.log')).write_text(p.stdout+p.stderr)
sp.run(['docker','rm','-f',name],check=False,capture_output=True)
10 changes: 8 additions & 2 deletions delphi/tests/test_compose_delphi_shadow.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,11 +238,17 @@ def test_the_shadow_yields_cpu_and_memory_to_every_other_service():
# services included, keeps Docker's default weight and OOM adjustment (or
# sets a higher weight and a lower adjustment), so under contention the
# shadow gets the smallest share of a CPU, and when the box runs out of
# memory it is killed first.
# memory it is killed first. The optional graph executor is shadow work
# too: it carries exactly the shadow's weight and OOM adjustment, and every
# other service outranks both.
services, _ = _services()
shadow = services["delphi-shadow"]
shadow_daemons = ("delphi-shadow", "delphi-shadow-graph")
for name in shadow_daemons:
assert services[name]["cpu_shares"] == shadow["cpu_shares"], name
assert services[name]["oom_score_adj"] == shadow["oom_score_adj"], name
for name, service in services.items():
if name == "delphi-shadow":
if name in shadow_daemons:
continue
assert service.get("cpu_shares", DEFAULT_CPU_SHARES) > shadow["cpu_shares"], name
assert service.get("oom_score_adj", DEFAULT_OOM_SCORE_ADJ) < shadow["oom_score_adj"], name
Expand Down
83 changes: 83 additions & 0 deletions delphi/tests/test_shadow_graph_compose.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""The optional executor must retain its local journal and shadow boundary."""
import importlib.util

import pytest
import yaml

from tests.test_compose_math_env import CHECKOUT, requires_checkout

# The deploy hook's validator, read from the checkout the Compose tests read
# ($POLIS_CHECKOUT_DIR in the Delphi test container, where the deploy-hook
# runner copies it). A missing file fails these tests; it is never a skip.
pytestmark = requires_checkout


@pytest.fixture(scope="module")
def module():
spec = importlib.util.spec_from_file_location(
"shadow_graph_compose", CHECKOUT / "scripts/check-shadow-graph.py")
loaded = importlib.util.module_from_spec(spec)
spec.loader.exec_module(loaded)
return loaded


def valid():
env = dict(DELPHI_SHADOW_GRAPH_DAEMON="1", POLIS_JOBS_ENABLED="1",
QUEUE_ENV="shadow-graph-1", DELPHI_RESULT_ENV="shadow-graph-1",
POLIS_JOBS_WORKER_CLASS="delphi",
POLIS_JOBS_STAGES="graph_embed,graph_cluster,graph_topics,graph_narrative",
POLIS_JOBS_CONCURRENCY="1", POLIS_JOBS_JOURNAL_DIR="/var/lib/delphi-shadow-graph/journal",
POLIS_JOBS_JOURNAL_HOST_FILE="/run/polis-host-id", DELPHI_EMBEDDINGS_TABLE="1",
HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", ANTHROPIC_TOPIC_MODEL="none")
graph = dict(environment=env, command=["polis-delphi-shadow", "daemon"],
volumes=[dict(type="volume", target="/var/lib/delphi-shadow-graph"),
dict(type="bind", source="/etc/machine-id", target="/run/polis-host-id", read_only=True)],
deploy=dict(resources=dict(limits=dict(memory=6*1024**3, cpus="2"))),
cpu_shares=2, oom_score_adj=1000)
scheduler = dict(environment=dict(DELPHI_SHADOW_ENV="shadow-delphi-1", DELPHI_SHADOW_GRAPH_ENV="shadow-graph-1"))
return dict(services={"delphi-shadow-graph": graph, "delphi-shadow-scheduler": scheduler})


def test_accepts_the_isolated_executor(module):
assert module.errors(valid()) == []


def test_refuses_a_served_or_shared_env(module):
for env in ["prod", "shadow-delphi-1"]:
config = valid()
config["services"]["delphi-shadow-graph"]["environment"]["QUEUE_ENV"] = env
assert module.errors(config)


def test_refuses_a_missing_journal_or_host_identity(module):
for index in [0, 1]:
config = valid()
config["services"]["delphi-shadow-graph"]["volumes"].pop(index)
assert module.errors(config)


def test_refuses_a_different_process_namespace_or_stage(module):
for key, value in [("init", True), ("pid", "host"), ("privileged", True)]:
config = valid()
config["services"]["delphi-shadow-graph"][key] = value
assert module.errors(config)
config = valid()
config["services"]["delphi-shadow-graph"]["environment"]["POLIS_JOBS_STAGES"] = "delphi_full_pipeline"
assert module.errors(config)


def test_refuses_an_embedding_model_setting_the_scheduler_does_not_share(module):
config = valid()
config["services"]["delphi-shadow-graph"]["environment"]["DELPHI_EMBED_MODEL_BY_ENV"] = '{"shadow-graph-1": "x"}'
assert module.errors(config) == ["DELPHI_EMBED_MODEL_BY_ENV"]
config["services"]["delphi-shadow-scheduler"]["environment"]["DELPHI_EMBED_MODEL_BY_ENV"] = '{"shadow-graph-1": "x"}'
assert module.errors(config) == []


def test_the_executor_receives_the_embedding_model_setting_as_the_shadow_daemon_does():
# Passed through unchanged, with no default of its own: unset (today), it
# is absent from both containers.
services = yaml.safe_load((CHECKOUT / "docker-compose.yml").read_text())["services"]
for name in ("delphi-shadow", "delphi-shadow-graph"):
assert services[name]["environment"].count("DELPHI_EMBED_MODEL_BY_ENV") == 1, name
assert not any(e.startswith("DELPHI_EMBED_MODEL_BY_ENV=") for e in services[name]["environment"]), name
Loading
Loading