diff --git a/VERSION b/VERSION index f607540ac..67e037ddb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.45.2 +2.45.3 diff --git a/delphi/tests/test_after_install_hook.py b/delphi/tests/test_after_install_hook.py index 904a1fb47..729d3565d 100644 --- a/delphi/tests/test_after_install_hook.py +++ b/delphi/tests/test_after_install_hook.py @@ -34,6 +34,7 @@ import os import shutil import subprocess +import sys from pathlib import Path import pytest @@ -89,7 +90,10 @@ def _find(rel): # shadow service's two read-backs: its lanes' switches and its settings. # The Delphi shadow's read-back prints the names of its wrong settings: # none, unless a test says which (or __fail__: jq itself fails). - "jq": RECORD + 'cat >/dev/null\n' + 'case "$*" in *delphi-shadow-scheduler.DELPHI_SHADOW_ENV*) [ "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}" != __fail__ ] || exit 3; printf "%s\\n" "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}";; *polis-api-shadow-reads*POLIS_API_ROLE*) printf "%s\\n" "${FAKE_READS_SETTINGS_WRONG-}";; *polis-api-shadow-reads*_COMMENT_READS*) printf "%s\\n" "${FAKE_READS_SWITCHES_WRONG-}";; *POLIS_API_ROLE*) printf "%s\\n" "${FAKE_SHADOW_SETTINGS_WRONG-}";; *POLIS_API_DB_TRANSPORT*) printf "%s\\n" "${FAKE_POLIS_API_TRANSPORT:-tls}";; *POLIS_API_START*) printf "%s\\n" "${FAKE_POLIS_API_START-0}";; *RUST_API_ROUTES*) printf "%s\\n" "$FAKE_RUST_API_ROUTES";; *POLIS_API_POST_COMMENTS*) if [ -n "${FAKE_SHADOW_COMMENTS_SWITCH+set}" ]; then printf "%s\\n" "$FAKE_SHADOW_COMMENTS_SWITCH"; elif grep -qx " - POLIS_API_POST_COMMENTS=observe" docker-compose.images.yml 2>/dev/null; then echo observe; else echo; fi;; *) echo value;; esac\n', + # The scheduler's graph list, read back after a refused graph executor, + # is answered as Compose would compose it: blank when the hook wrote the + # file that blanks it, unless a test says what the composed list holds. + "jq": RECORD + 'cat >/dev/null\n' + 'case "$*" in *delphi-shadow-scheduler.DELPHI_SHADOW_ENV*) [ "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}" != __fail__ ] || exit 3; printf "%s\\n" "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}";; *polis-api-shadow-reads*POLIS_API_ROLE*) printf "%s\\n" "${FAKE_READS_SETTINGS_WRONG-}";; *polis-api-shadow-reads*_COMMENT_READS*) printf "%s\\n" "${FAKE_READS_SWITCHES_WRONG-}";; *POLIS_API_ROLE*) printf "%s\\n" "${FAKE_SHADOW_SETTINGS_WRONG-}";; *POLIS_API_DB_TRANSPORT*) printf "%s\\n" "${FAKE_POLIS_API_TRANSPORT:-tls}";; *POLIS_API_START*) printf "%s\\n" "${FAKE_POLIS_API_START-0}";; *RUST_API_ROUTES*) printf "%s\\n" "$FAKE_RUST_API_ROUTES";; *POLIS_API_POST_COMMENTS*) if [ -n "${FAKE_SHADOW_COMMENTS_SWITCH+set}" ]; then printf "%s\\n" "$FAKE_SHADOW_COMMENTS_SWITCH"; elif grep -qx " - POLIS_API_POST_COMMENTS=observe" docker-compose.images.yml 2>/dev/null; then echo observe; else echo; fi;; *environment.DELPHI_SHADOW_GRAPH_CONVERSATIONS*) if [ -n "${FAKE_GRAPH_LIST_COMPOSED+set}" ]; then printf "%s\\n" "$FAKE_GRAPH_LIST_COMPOSED"; elif grep -qx " - DELPHI_SHADOW_GRAPH_CONVERSATIONS=" docker-compose.shadow-graph-off.yml 2>/dev/null; then echo; else echo zzgraphlist9; fi;; *) echo value;; esac\n', "git": RECORD + ('case "$1" in clone) mkdir -p "${@: -1}";; rev-parse) echo 0123456789abcdef0123456789abcdef01234567;;\n' ' merge-base) [ -z "$FAKE_NOT_ON_BRANCH" ] || exit 1;; esac\n'), "aws": RECORD + ( @@ -205,10 +209,12 @@ def _deploy(tmp_path, service_type, migration_failure="", revision=REVISION, shadow_start_failure=False, shadow_comments_switch=None, shadow_settings_wrong=None, reads_start_failure=False, reads_switches_wrong=None, reads_settings_wrong=None, env_extra=None, delphi_shadow_settings_wrong=None, delphi_shadow_start_failure=False, - compose_ps="math-python", compose_ps_failure=False, hook_rules=(), **kw): + compose_ps="math-python", compose_ps_failure=False, hook_rules=(), graph_list_composed=None, **kw): root, env = _box(tmp_path, service_type, **kw) if env_extra is not None: env["FAKE_ENV_EXTRA"] = env_extra + if graph_list_composed is not None: + env["FAKE_GRAPH_LIST_COMPOSED"] = graph_list_composed if delphi_shadow_settings_wrong is not None: env["FAKE_DELPHI_SHADOW_SETTINGS_WRONG"] = delphi_shadow_settings_wrong env["FAKE_DELPHI_SHADOW_START_FAILURE"] = "1" if delphi_shadow_start_failure else "0" @@ -1296,6 +1302,118 @@ def test_other_roles_never_start_the_delphi_shadow(tmp_path, service_type): assert DELPHI_SHADOW_UP not in _compose(log) +# The optional graph executor (DELPHI_SHADOW_GRAPH_ENABLED=1 beside the +# shadow's switch) is checked by scripts/check-shadow-graph.py against the +# composed configuration, after the shadow's own read-back. Its refusal is +# about the executor alone: the full-pipeline shadow, already accepted, still +# starts. Here the check is a stand-in that reads the composed configuration +# and accepts or refuses it. +DELPHI_SHADOW_GRAPH_UP = "up -d delphi-shadow-graph --no-build --force-recreate" +DELPHI_SHADOW_GRAPH_CONFIG = COMPOSE_FILES + "--profile delphi-shadow --profile delphi-shadow-graph config --format json" +GRAPH_SWITCHES = "DELPHI_SHADOW_ENABLED=1\nDELPHI_SHADOW_GRAPH_ENABLED=1" + + +def _graph_check(tmp_path, accepts): + # Run by this interpreter: the hook's PATH here holds the fakes and + # /usr/bin:/bin, which need not hold a python3 (the test image's is in + # /usr/local/bin). + check = tmp_path / "graph-check.py" + check.write_text("import sys\nsys.stdin.read()\n" + ("" if accepts else ( + "print('graph shadow settings: QUEUE_ENV/DELPHI_RESULT_ENV', file=sys.stderr)\nsys.exit(1)\n"))) + return [("python3 scripts/check-shadow-graph.py", f"{sys.executable} {check}", 1)] + + +def test_valid_graph_settings_start_the_graph_executor_before_the_shadow(tmp_path): + proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES, hook_rules=_graph_check(tmp_path, True)) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert _calls(log, "docker-compose").count(DELPHI_SHADOW_GRAPH_CONFIG) == 1 + ups = [c for c in _compose(log) if c.startswith("up")] + assert ups == UNCHANGED["delphi"] + [DELPHI_SHADOW_GRAPH_UP, DELPHI_SHADOW_UP] + # The scheduler keeps its conversation list: no file blanks it. + assert not (tmp_path / "root/opt/polis/polis" / GRAPH_OFF).exists() + assert not [c for c in _calls(log, "docker-compose") if GRAPH_OFF in c] + + +# Refused, the executor is not started and the scheduler does no graph work +# for the deploy: its start is given one more Compose file that composes its +# conversation list blank, so it admits no graph that nothing could claim. +# The hook reads the composed list back before starting it, and says so in +# one line naming the setting, never a value. +GRAPH_OFF = "docker-compose.shadow-graph-off.yml" +GRAPH_OFF_TEXT = "services:\n delphi-shadow-scheduler:\n environment:\n - DELPHI_SHADOW_GRAPH_CONVERSATIONS=\n" +GRAPH_OFF_FILES = COMPOSE_FILES + f"-f {GRAPH_OFF} " +GRAPH_REFUSED_LINE = ( + "Error: the graph executor's composed settings are invalid (QUEUE_ENV/DELPHI_RESULT_ENV): not starting it, " + "and the scheduler starts with DELPHI_SHADOW_GRAPH_CONVERSATIONS blank, so no conversation's graph is " + "admitted until a deploy whose graph settings are valid (the full-pipeline shadow still starts)") + + +def test_invalid_graph_settings_refuse_the_graph_executor_only(tmp_path): + proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES + "\nDELPHI_SHADOW_GRAPH_CONVERSATIONS=zzgraphlist9", + compose_ps=SHADOW_LEFT_RUNNING, hook_rules=_graph_check(tmp_path, False)) + # The Delphi box's deploy is not failed, and the full-pipeline shadow + # starts, its scheduler with no graph list. + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "not starting either shadow" not in proc.stderr + # One loud line, naming the refused settings and the blanked list by key. + graph_lines = [line for line in proc.stderr.splitlines() if "graph" in line] + assert graph_lines == [GRAPH_REFUSED_LINE], proc.stderr + off = tmp_path / "root/opt/polis/polis" / GRAPH_OFF + assert off.read_text() == GRAPH_OFF_TEXT + calls = _calls(log, "docker-compose") + assert GRAPH_OFF_FILES + "--profile delphi-shadow config --format json" in calls + assert '-r .services["delphi-shadow-scheduler"].environment.DELPHI_SHADOW_GRAPH_CONVERSATIONS // ""' in _calls(log, "jq") + ups = [c for c in calls if " up " in f" {c} "] + assert ups == [COMPOSE_FILES + u for u in UNCHANGED["delphi"]] + [GRAPH_OFF_FILES + DELPHI_SHADOW_UP] + assert DELPHI_SHADOW_GRAPH_UP not in _compose(log) + # Only the shadow's start carries the file: the engine's `up` and `down` + # are as before. + assert [c for c in calls if GRAPH_OFF in c] == [GRAPH_OFF_FILES + "--profile delphi-shadow config --format json", + GRAPH_OFF_FILES + DELPHI_SHADOW_UP] + # The validator names settings only; nothing of the environment document + # reaches the deploy's output. + for text in ("PUBLIC_SERVICE_URL", "example.invalid", "zzgraphlist9"): + assert text not in proc.stdout + proc.stderr + + +def test_a_graph_list_that_cannot_be_composed_blank_refuses_the_shadow(tmp_path): + # Should Compose not let the file's empty value stand, the scheduler + # would admit graphs nothing claims: the shadow is not started, as when + # its own settings are refused, and the deploy goes on. + proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES, compose_ps=SHADOW_LEFT_RUNNING, + hook_rules=_graph_check(tmp_path, False), graph_list_composed="zzgraphlist9") + assert proc.returncode == 0, proc.stdout + proc.stderr + graph_lines = [line for line in proc.stderr.splitlines() if "graph" in line] + assert graph_lines == [ + "Error: the graph executor's composed settings are invalid (QUEUE_ENV/DELPHI_RESULT_ENV) and the " + "scheduler's DELPHI_SHADOW_GRAPH_CONVERSATIONS is not composed blank; not starting the Delphi shadow"], proc.stderr + assert [c for c in _compose(log) if c.startswith("up")] == UNCHANGED["delphi"] + assert not [c for c in _calls(log, "docker-compose") if c.endswith(DELPHI_SHADOW_UP)] + assert "zzgraphlist9" not in proc.stdout + proc.stderr + + +@pytest.mark.skipif(shutil.which("jq") is None, reason="runs the hook's jq program with the real jq") +@pytest.mark.parametrize("environment,printed", [ + ({"DELPHI_SHADOW_GRAPH_CONVERSATIONS": ""}, ""), + ({}, ""), + ({"DELPHI_SHADOW_GRAPH_CONVERSATIONS": "c1 c2"}, "c1 c2"), + (None, ""), +]) +def test_the_real_jq_reads_the_schedulers_composed_graph_list(environment, printed): + (line,) = [line for line in HOOK_PATH.read_text().splitlines() + if line.lstrip().startswith("DELPHI_SHADOW_GRAPH_LIST=$(")] + command, program, closing = line.split("'") + assert command.lstrip() == ('DELPHI_SHADOW_GRAPH_LIST=$("${DELPHI_SHADOW_COMPOSE[@]}" --profile delphi-shadow ' + 'config --format json | jq -r ') + assert closing.startswith(")") + document = {"services": {"delphi-shadow-scheduler": {"environment": environment}}} + if environment is None: + del document["services"]["delphi-shadow-scheduler"] + proc = subprocess.run(["jq", "-r", program], input=json.dumps(document), text=True, capture_output=True) + assert proc.returncode == 0, proc.stderr + assert proc.stdout == printed + "\n" + + # The Delphi box's `down` (with -f files named) leaves a profile-gated # service's containers running, and the shadow's are: `ps -a --services` # lists them after `down`. The Delphi role removes by name every service it diff --git a/docs/configuration.md b/docs/configuration.md index a2121bf46..8ceccc951 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -431,12 +431,76 @@ same journal lock. On the same host, replacement of the PID 1 container kills its namespace's processes; a host reboot kills the prior boot's processes. A daemon restart in the same namespace needs both its recorded daemon and child process group gone. A missing process-group record is not exit proof. +An entry written under another boot is confirmed only once its attempt's lease +has expired by database time (`graph_recovery` reason +`another_boot_lease_live` while it waits, `another_boot_lease_expired` when it +is confirmed). A copied identity file therefore does not confirm an attempt +whose original still renews its lease. It does confirm one whose original is +alive but has stopped renewing for longer than a lease (a paused container, a +stopped daemon, a frozen machine): writes stay fenced, so the cost is a false +exit record and the step computed twice. An honest reboot waits about one +lease with the defaults (`POLIS_JOBS_LEASE_SECONDS`, default 120), up to one +lease plus one reaper interval (`POLIS_JOBS_REAP_SECONDS`) when that is set +longer. + +The deploy check of the executor's composed settings refuses the executor +alone: with an invalid setting the executor is not started, and the scheduler +is started with `DELPHI_SHADOW_GRAPH_CONVERSATIONS` composed blank (one more +Compose file, `docker-compose.shadow-graph-off.yml`, given to the shadow's +start only), so it admits no graph that nothing could claim; the +full-pipeline shadow still starts. The deploy says so in one line beginning +`Error:` that names the refused settings and that list, never a value. A +graph already in flight from an earlier deploy is still watched, as when the +list is cleared. The next deploy with valid settings admits graphs again. + +A graph none of whose nodes is claimed within ten minutes (no executor +running, unstartable, crash-looping before its first claim) is cancelled and +let go (`graph_unclaimed`, with the reason, the count of graphs let go in a +row in that env and the hold that follows), so the full-pipeline shadow waits +minutes, not the job deadline. A graph found cancelled with no node ever +claimed, or past the job deadline with none claimed, is let go the same way +(reason `cancelled_unclaimed`). The ten minutes are a constant +(`UNCLAIMED_DEADLINE` in `queue-rs/src/shadow/graph.rs`), not a setting: a +running executor claims within one of its polls (`POLIS_JOBS_POLL_SECONDS`, +default 5), and the deploy pulls every image before starting anything and +starts the executor before the shadow, so on a deploy the executor is never +still being pulled when a graph is admitted. Where it is slower than ten +minutes to start (an image still being pulled or built by a local +`docker compose up`, a slow host, a scheduler restarted with an old graph in +flight), that graph is let go and admitted again, as below; nothing is lost. + +After a let-go no graph is admitted in that env for an hour, doubling with +each let-go in a row up to a day (`UNCLAIMED_HOLD`, `UNCLAIMED_HOLD_MAX`), +or until the next code version, whichever is first; the scheduler says so +once per hold (`graph_admission_held`, reason `graph_unclaimed`, with the +count, the seconds until it resumes and the let-go graphs' ids). The +full-pipeline shadow goes on throughout. When the hold ends, the conversation +let go is admitted again, though nothing about it changed, as a new graph: +its admission key names the let-go graph (`graph_admitted` with +`after_let_go`). A claim on a graph admitted after the let-go resets the +count (`graph_unclaimed_cleared`). The hold, the count and the let-go graphs +are kept in the scheduler state (`graph_unclaimed`), so a restart keeps them. +Each graph admitted again adds its rows to the database, as any graph does, +and counts against the shadow's storage budget. + +Publication of a finished graph is tried once per poll, up to three times, +the count kept in the scheduler state; a transient failure keeps the graph +(`graph_publish_retry`). A graph whose publication is given up +(`graph_publish_failed`) is not taken up again by recovery, and its snapshot +is not remembered as seen: when a later build or a change reconsiders the +conversation, the same snapshot's admission answers with that graph if the +graph's specification is unchanged (the same stage code, runtime, models and +expected generation), and then only its publication runs again. A build that +changes any of those admits a new graph, and all its steps run again. The scheduler cancels overdue graphs at `DELPHI_SHADOW_JOB_DEADLINE_SECONDS` (default 14400). It keeps their scopes held and polls recovery in the background; other conversations may resume full-pipeline work. The same conversation stays excluded while its graph scope is held. A lost host or volume cannot supply exit proof: `graph_recovery_pending` means just that, never successful recovery. +While any graph is deferred no new graph is admitted, in any graph env; the +scheduler says so once, with the deferred graphs' ids +(`graph_admission_held`, reason `graph_deferred`). Database recovery errors stop admissions until the database can be read again. The default ceilings are 16 GiB for the full-pipeline daemon, 6 GiB for the graph @@ -452,5 +516,9 @@ State written by the previous release loads with empty recovery fields. When no graph has been enabled, state serialization and full-pipeline behavior are unchanged. Before rolling back, drain every graph, including deferred graphs, and verify its scope was released. Older schedulers ignore the new deferred -fields and cannot enforce those exclusions. Do not remove scheduler state or a +fields and cannot enforce those exclusions. They also ignore the publication +count and the given-up publications, so they may try a given-up graph's +publication again, and they ignore the graphs let go as unclaimed, so they +neither hold admission after one nor admit its unchanged conversation again. +Do not remove scheduler state or a journal to force progress. diff --git a/polis-cli/Cargo.lock b/polis-cli/Cargo.lock index 2dc5cd7f9..4bcf0a811 100644 --- a/polis-cli/Cargo.lock +++ b/polis-cli/Cargo.lock @@ -747,7 +747,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "polis" -version = "2.45.2" +version = "2.45.3" dependencies = [ "base64 0.22.1", "clap", @@ -771,7 +771,7 @@ dependencies = [ [[package]] name = "polis-client" -version = "2.45.2" +version = "2.45.3" dependencies = [ "flate2", "polis-contracts", diff --git a/polis-cli/Cargo.toml b/polis-cli/Cargo.toml index 61a2fd44f..229632ad4 100644 --- a/polis-cli/Cargo.toml +++ b/polis-cli/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/polis-client", "crates/polis", "../queue-rs/polis-contracts"] resolver = "2" [workspace.package] -version = "2.45.2" +version = "2.45.3" edition = "2021" license = "AGPL-3.0-only" rust-version = "1.88" diff --git a/queue-rs/src/jobs/claim.rs b/queue-rs/src/jobs/claim.rs index 7cfa59605..80051c57f 100644 --- a/queue-rs/src/jobs/claim.rs +++ b/queue-rs/src/jobs/claim.rs @@ -216,15 +216,96 @@ fn retry_pending(ctx: &Ctx, rpc: &mut Rpc) { } } +/// A journal entry of a host-bound journal written under another boot. The +/// binding is a file's contents, so a restored snapshot of the volume, or a +/// copied or template-baked identity file, passes it on a second machine +/// while the original may still run the attempt. Its exit is confirmed only +/// once the database shows the attempt's lease over (`lease_over`): a live +/// original keeps renewing it, so it is never confirmed; an honest reboot +/// waits at most one lease. +struct Waiting { + entry: journal::Entry, + /// Database time (seconds since the epoch) at which this daemon first + /// read the job no longer held by this attempt. + released_seen: Option, +} + +fn recovery_line(entry: &journal::Entry, exit_proven: bool, reason: &str) { + line(&format!( + "graph_recovery {}", + json!({"job_id":entry.job_id,"attempt_id":entry.attempt_id,"owner_id":entry.owner_id,"exit_proven":exit_proven,"reason":reason}) + )); +} + +/// Whether the waiting entry's lease is over by database time. While its +/// attempt still holds the job, the lease is the job's `locked_until`. Once +/// the job is no longer held by it (expired and parked by a reaper, +/// cancelled, gone), the end of the lease is not on record, so one whole +/// lease of this daemon's is counted from the moment this daemon first read +/// it so. +fn lease_over(lease_seconds: u32, rpc: &mut Rpc, waiting: &mut Waiting) -> anyhow::Result { + let (status, now, locked_until) = rpc.job_status_at(&waiting.entry.job_id)?; + let e = &waiting.entry; + let held = status["state"] == "running" + && status["owner_id"].as_str() == Some(e.owner_id.as_str()) + && status["attempt_id"].as_str() == Some(e.attempt_id.as_str()) + && status["lease_epoch"].as_str() == Some(e.lease_epoch.as_str()); + if held { + return Ok(locked_until.is_some_and(|until| until <= now)); + } + let since = *waiting.released_seen.get_or_insert(now); + Ok(now - since >= f64::from(lease_seconds)) +} + +/// Confirm each waiting entry whose lease is over (queued as `recover` +/// queues a provably gone one); keep the rest. A read that fails keeps the +/// entry: nothing is confirmed on a guess. +fn confirm_after_lease(ctx: &Ctx, rpc: &mut Rpc, waiting: Vec) -> Vec { + let mut kept = vec![]; + for mut w in waiting { + match lease_over(ctx.cfg.lease_seconds, rpc, &mut w) { + Ok(true) => { + recovery_line(&w.entry, true, "another_boot_lease_expired"); + if let Ok(mut p) = ctx.pending.lock() { + p.push(Pending { + entry: w.entry, + code: Some("daemon_restarted".into()), + }); + } + } + Ok(false) => kept.push(w), + Err(e) => { + line(&format!( + "journal entry {} kept: its lease could not be read ({e})", + w.entry.attempt_id + )); + kept.push(w); + } + } + } + kept +} + /// Recovery: every journaled attempt that is provably gone is confirmed with -/// `pq_fail(env, job, owner, attempt, epoch, false, 'daemon_restarted', true)`. -fn recover(ctx: &Ctx, rpc: &mut Rpc) { +/// `pq_fail(env, job, owner, attempt, epoch, false, 'daemon_restarted', true)`, +/// except that an entry of a host-bound journal from another boot waits for +/// its lease (`Waiting`); those are returned, to be read again at the reaper +/// cadence. +fn recover(ctx: &Ctx, rpc: &mut Rpc) -> Vec { + let mut waiting = vec![]; for entry in ctx.journal.entries() { if entry.env != ctx.cfg.env { continue; } let gone = journal::provably_gone(&entry, &ctx.cfg.boot_id, &ctx.cfg.container_id, &ctx.owner); + if gone && ctx.cfg.journal_host_file.is_some() && entry.boot_id != ctx.cfg.boot_id { + waiting.push(Waiting { + entry, + released_seen: None, + }); + continue; + } if ctx.cfg.journal_host_file.is_some() { line(&format!( "graph_recovery {}", @@ -245,7 +326,12 @@ fn recover(ctx: &Ctx, rpc: &mut Rpc) { }); } } + let waiting = confirm_after_lease(ctx, rpc, waiting); + for w in &waiting { + recovery_line(&w.entry, false, "another_boot_lease_live"); + } retry_pending(ctx, rpc); + waiting } fn snapshot_line(ctx: &Ctx, progress: &str) -> String { @@ -354,7 +440,7 @@ pub fn run(cfg: Config) -> i32 { line("graph stages require polis-queue/5"); return EXIT_CONTRACT; } - recover(&ctx, &mut rpc); + let mut waiting = recover(&ctx, &mut rpc); let mut listener: Option = rpc.listener().ok(); let mut jobs: Vec> = vec![]; let mut reaper = Reaper::new(cfg.lease_seconds, cfg.reap_interval); @@ -411,6 +497,11 @@ pub fn run(cfg: Config) -> i32 { } if Instant::now() >= next_reap { next_reap = Instant::now() + cfg.reap_interval; + // Before the reaper below parks an expired attempt, so that an + // honest reboot's entry is confirmed at the tick its lease ends. + if !waiting.is_empty() { + waiting = confirm_after_lease(&ctx, &mut rpc, waiting); + } retry_pending(&ctx, &mut rpc); if ctx.contract() == "polis-queue/5" && let Err(e) = rpc.committed("pd_graph_reconcile", &[json!(cfg.env)]) diff --git a/queue-rs/src/jobs/rpc.rs b/queue-rs/src/jobs/rpc.rs index 619e3ae2e..4d91380a5 100644 --- a/queue-rs/src/jobs/rpc.rs +++ b/queue-rs/src/jobs/rpc.rs @@ -239,6 +239,44 @@ impl Rpc { result } + /// One job's `pq_job_status` (000019, granted to the executor) read + /// together with the database's own clock, in one read-only statement: + /// the status, then the database time and the status's `locked_until`, + /// both as seconds since the epoch. Nothing is locked or written. + pub fn job_status_at(&mut self, job: &str) -> Result<(Value, f64, Option)> { + let job = Uuid::parse_str(job)?; + let env = self.env.clone(); + let result = (|| -> Result<(Value, f64, Option)> { + let client = self.client()?; + let mut tx = client + .build_transaction() + .isolation_level(IsolationLevel::ReadCommitted) + .read_only(true) + .start()?; + tx.batch_execute( + "SET LOCAL statement_timeout='30s'; SET LOCAL lock_timeout='10s'; \ + SET LOCAL idle_in_transaction_session_timeout='30s'; SET LOCAL TimeZone='UTC'", + )?; + let row = tx + .query_one( + "SELECT s, extract(epoch FROM clock_timestamp())::float8, \ + extract(epoch FROM (s->>'locked_until')::timestamptz)::float8 \ + FROM public.pq_job_status($1::text,$2::uuid) s", + &[&env, &job], + ) + .map_err(classify)?; + let status = row.try_get::<_, Option>(0)?.unwrap_or(Value::Null); + let now: f64 = row.try_get(1)?; + let locked_until: Option = row.try_get(2)?; + tx.commit()?; + Ok((status, now, locked_until)) + })(); + if let Err(error) = &result { + self.drop_on_transport_error(error); + } + result + } + /// `call`, accepting only a committed reply. pub fn committed(&mut self, name: &str, args: &[Value]) -> Result { match self.call(name, args)? { diff --git a/queue-rs/src/shadow/graph.rs b/queue-rs/src/shadow/graph.rs index 9e0e28624..0ad6fb8dd 100644 --- a/queue-rs/src/shadow/graph.rs +++ b/queue-rs/src/shadow/graph.rs @@ -20,6 +20,78 @@ use std::{ }; const MAX_SPEC_BYTES: usize = 1_048_576; +/// How long a graph may wait with none of its nodes ever claimed. A graph +/// executor that runs claims the first node within one of its polls; one +/// that is absent, unstartable or refused at deploy never does. Past this +/// the graph's nodes are cancelled and the graph is let go, so a missing +/// executor costs the full-pipeline shadow minutes per listed conversation, +/// not the whole job deadline. A constant, not a setting +/// (docs/configuration.md, Same-host graph executor and recovery). +pub const UNCLAIMED_DEADLINE: Duration = Duration::from_secs(600); +/// After a graph is let go as unclaimed, no graph is admitted in its env for +/// this long, or until the next code version: the executor that did not +/// claim it would not claim the next one either. Each let-go in a row (no +/// claim seen between) doubles it, up to `UNCLAIMED_HOLD_MAX`. So an absent +/// executor costs the full-pipeline shadow at most one `UNCLAIMED_DEADLINE` +/// per hold, never a spin, and a let-go conversation is always admitted +/// again: at the next deploy at once, otherwise within a day. +pub const UNCLAIMED_HOLD: Duration = Duration::from_secs(3600); +pub const UNCLAIMED_HOLD_MAX: Duration = Duration::from_secs(86_400); +/// How many let-go conversations an env's record keeps (the bound of the +/// state file; one entry per conversation ever let go there). +const LET_GO_KEPT: usize = 256; + +/// The hold after the `count`th graph let go in a row in an env. +pub fn unclaimed_hold(count: u32) -> Duration { + let doublings = count.saturating_sub(1).min(16); + UNCLAIMED_HOLD + .saturating_mul(1 << doublings) + .min(UNCLAIMED_HOLD_MAX) +} + +/// The graphs an env's executor did not claim (`State::graph_unclaimed`). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Unclaimed { + /// Graphs let go in a row in this env, with no claim seen between: it + /// sets the hold, and a claim seen there resets it. + pub count: u32, + /// When the last was let go (ms since the epoch), and the admission rule + /// (its build) then: the hold ends at the next code version. + pub at_ms: i64, + pub rule: String, + /// Per conversation, the last graph let go and its snapshot's digest. + /// Every later admission of that snapshot names that graph in its key, + /// so it is a new graph (and the same one again, should its reply be + /// lost or its publication be retried). + pub graphs: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LetGo { + pub graph_id: String, + pub digest: String, +} + +/// The admission key: the conversation and the specification, and, after a +/// graph of that snapshot was let go as unclaimed, that graph. Without the +/// last part, the admission function would answer the same snapshot's +/// admission with the cancelled graph (`existing`, for the same key). +fn admission_key(zid: i32, specification: &str, after: Option<&str>) -> String { + crate::jobs::sha256_hex( + match after { + None => format!("{zid}:{specification}"), + Some(graph) => format!("{zid}:{specification}:after-unclaimed:{graph}"), + } + .as_bytes(), + ) +} + +/// Publication attempts per graph, one per poll, the count carried in the +/// flight (`Flight::publish_attempts`). +pub const PUBLISH_ATTEMPTS: u32 = 3; +/// How many given-up publications the state file keeps (the newest). Only +/// an env's newest graph can be recovered, so a short list is enough. +const UNPUBLISHED_KEPT: usize = 64; // Count and snapshot deliberately include invalid/blank texts: these refuse the // conversation instead of silently changing its scientific input. const INCLUDED: &str = "NOT EXISTS (SELECT 1 FROM report_comment_selections r WHERE r.zid=c.zid AND r.tid=c.tid AND r.selection=-1)"; @@ -34,12 +106,17 @@ pub struct Observation { impl Observation { fn new(c: &Live, build: &str) -> Self { Self { - rule: format!("graph-admission/2:{build}"), + rule: rule(build), comments: c.comments, last_modified: c.last_modified, } } } +/// The admission rule of a build: a new code version reconsiders every +/// listed conversation once, and ends an unclaimed hold. +fn rule(build: &str) -> String { + format!("graph-admission/2:{build}") +} #[derive(Debug)] struct Refusal(&'static str); impl std::fmt::Display for Refusal { @@ -160,6 +237,12 @@ pub struct Flight { pub admitted_at_ms: i64, pub generation: i64, pub nodes: Vec, + /// Publication attempts already made, one per poll (absent when none). + #[serde(default, skip_serializing_if = "is_zero")] + pub publish_attempts: u32, +} +fn is_zero(n: &u32) -> bool { + *n == 0 } #[derive(Debug)] pub struct Snapshot { @@ -353,15 +436,22 @@ impl Scheduler<'_> { if self.state.graph_flight.is_some() { return Ok(()); } + // Deferred graphs are polled on their own; graphs whose publication + // was given up are not taken up again. let deferred: Vec = self .state .graph_deferred .iter() .map(|f| f.graph_id.clone()) + .chain(self.state.graph_unpublished.iter().cloned()) .collect(); + let unpublished = self.state.graph_unpublished.clone(); for env in self.state.graph_recovery_envs.clone() { let recovered=self.db.read(|tx| { - let rows=tx.query("SELECT DISTINCT ON (zid) zid, coalesce(request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}') FROM delphi_graphs WHERE env=$1 AND scope_key='shadow-graph:'||zid::text ORDER BY zid,created_at DESC", &[&env])?; + // A snapshot whose publication was given up is not remembered as + // seen, nor one whose graph had every node cancelled (it computed + // nothing: a graph let go as unclaimed). + let rows=tx.query("SELECT DISTINCT ON (zid) zid, coalesce(request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}') FROM delphi_graphs g WHERE env=$1 AND scope_key='shadow-graph:'||zid::text AND NOT (graph_id::text=ANY($2::text[])) AND EXISTS (SELECT 1 FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id AND q.state<>'cancelled') ORDER BY zid,created_at DESC", &[&env,&unpublished])?; let mut seen=BTreeMap::new(); for r in rows { if let Some(digest)=r.try_get::<_,Option>(1)? { seen.insert(r.try_get::<_,i32>(0)?,digest); } } let row=tx.query_opt("SELECT g.graph_id::text,g.zid, (extract(epoch FROM g.created_at)*1000)::bigint, coalesce(g.request #>> '{nodes,0,declared,config,shadow,graph_snapshot_sha256}',request #>> '{nodes,0,declared,config,shadow,comment_set_sha256}'), (g.request #>> '{nodes,0,declared,config,shadow,comments}')::bigint, (g.request #>> '{nodes,0,declared,config,shadow,expected_generation}')::bigint FROM delphi_graphs g WHERE g.env=$1 AND NOT (g.graph_id::text=ANY($2::text[])) AND g.scope_key='shadow-graph:'||g.zid::text AND (g.request #>> '{nodes,0,declared,config,shadow,comments}') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM delphi_graphs newer WHERE newer.env=g.env AND newer.scope_key='shadow-graph:'||newer.zid::text AND newer.created_at>g.created_at) AND (EXISTS (SELECT 1 FROM delphi_job_guards d WHERE d.env=g.env AND d.root_job_id=g.root_job_id) OR (NOT EXISTS (SELECT 1 FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=g.env AND n.graph_id=g.graph_id AND q.state<>'succeeded') AND NOT EXISTS (SELECT 1 FROM delphi_graph_served s JOIN delphi_artifacts a USING(env,artifact_id) JOIN delphi_graph_nodes n USING(env,job_id) WHERE s.env=g.env AND n.graph_id=g.graph_id))) ORDER BY g.created_at,g.graph_id LIMIT 1", &[&env,&deferred])?; @@ -372,7 +462,7 @@ impl Scheduler<'_> { nodes.push(Node {job_id:n.try_get(0)?,run_id:n.try_get(1)?,stage:n.try_get(2)?}); } ensure!(nodes.len()==4 && STAGES.iter().all(|s|nodes.iter().filter(|n|n.stage==*s).count()==1),"graph_recovery_shape"); - Ok((seen,Some(Flight {env:env.clone(),graph_id,zid:r.try_get(1)?,admitted_at_ms:r.try_get(2)?,digest:r.try_get(3)?,comments:r.try_get(4)?,generation:r.try_get::<_,Option>(5)?.unwrap_or(-1),nodes}))) + Ok((seen,Some(Flight {env:env.clone(),graph_id,zid:r.try_get(1)?,admitted_at_ms:r.try_get(2)?,digest:r.try_get(3)?,comments:r.try_get(4)?,generation:r.try_get::<_,Option>(5)?.unwrap_or(-1),nodes,publish_attempts:0}))) })?; self.state.graph_seen.extend(recovered.0); self.state.graph_flight = recovered.1; @@ -390,8 +480,55 @@ impl Scheduler<'_> { return Ok(false); }; if !self.state.graph_deferred.is_empty() { + // Said once while the same graphs hold it (they are polled in + // rotation, so the line names them sorted), never every poll. + let mut held: Vec<&str> = self + .state + .graph_deferred + .iter() + .map(|f| f.graph_id.as_str()) + .collect(); + held.sort_unstable(); + let key = held.join(","); + if self.graph_held.as_deref() != Some(key.as_str()) { + log( + "graph_admission_held", + json!({"reason":"graph_deferred","deferred":held.len(),"graph_ids":held}), + ); + self.graph_held = Some(key); + } return Ok(false); } + // A graph let go as unclaimed holds admission in its env until the + // next code version or the hold's end; said once per hold. No query. + if let Some(u) = self.state.graph_unclaimed.get(&cfg.env) + && u.count > 0 + && u.rule == rule(&self.cfg.code_version) + { + let until = u + .at_ms + .saturating_add(unclaimed_hold(u.count).as_millis() as i64); + if now < until { + let key = format!("graph_unclaimed:{}:{}", u.count, u.at_ms); + if self.graph_held.as_deref() != Some(key.as_str()) { + // The let-go graphs whose conversations wait for it. + let mut ids: Vec<&str> = u + .graphs + .iter() + .filter(|(zid, _)| !self.state.graph_observed.contains_key(zid)) + .map(|(_, g)| g.graph_id.as_str()) + .collect(); + ids.sort_unstable(); + log( + "graph_admission_held", + json!({"reason":"graph_unclaimed","let_go":u.count,"resume_seconds":(until - now) / 1000,"graph_ids":ids}), + ); + self.graph_held = Some(key); + } + return Ok(false); + } + } + self.graph_held = None; let cfg = cfg.clone(); let listed = listed(&self.db, &cfg.codes)?; let counts = (cfg.codes.len(), listed.len()); @@ -453,6 +590,72 @@ impl Scheduler<'_> { ); } + /// A graph is let go because nothing claimed it: remember it, and forget + /// that its conversation was seen and observed, so that it is admitted + /// again (as a new graph) at the next code version or when the hold ends; + /// say so with the count. Once per graph (it is decided before its + /// cancel and seen again when it finishes). + fn let_go(&mut self, flight: &Flight, now: i64, reason: &str) { + let rule = rule(&self.cfg.code_version); + if self.state.graph_seen.get(&flight.zid) == Some(&flight.digest) { + self.state.graph_seen.remove(&flight.zid); + } + self.state.graph_observed.remove(&flight.zid); + let u = self + .state + .graph_unclaimed + .entry(flight.env.clone()) + .or_default(); + if u.graphs + .get(&flight.zid) + .is_some_and(|g| g.graph_id == flight.graph_id) + { + return; + } + u.count = u.count.saturating_add(1); + u.at_ms = now; + u.rule = rule; + u.graphs.insert( + flight.zid, + LetGo { + graph_id: flight.graph_id.clone(), + digest: flight.digest.clone(), + }, + ); + // Bounded. A conversation dropped here whose snapshot comes back + // finds the cancelled graph, which is let go again when it finishes. + while u.graphs.len() > LET_GO_KEPT { + let Some(other) = u.graphs.keys().find(|zid| **zid != flight.zid).copied() else { + break; + }; + u.graphs.remove(&other); + } + log( + "graph_unclaimed", + json!({"graph_id":flight.graph_id,"seconds":(now - flight.admitted_at_ms)/1000,"reason":reason,"let_go":u.count,"hold_seconds":unclaimed_hold(u.count).as_secs()}), + ); + } + + /// A node of a graph in the env left the queue: an executor claims + /// there again, and the count starts over. (While the env is held no + /// graph is admitted there, so this is a graph admitted after the hold.) + fn note_claimed(&mut self, flight: &Flight, states: &BTreeMap) { + let Some(u) = self.state.graph_unclaimed.get_mut(&flight.env) else { + return; + }; + if u.count > 0 + && states + .values() + .any(|(s, _)| !matches!(s.as_str(), "queued" | "cancelled")) + { + log( + "graph_unclaimed_cleared", + json!({"graph_id":flight.graph_id,"reason":"executor_claimed","let_go":u.count}), + ); + u.count = 0; + } + } + fn admit_graph_candidate(&mut self, cfg: &Config, candidate: &Live, now: i64) -> Result { let s = snapshot(&self.db, candidate.zid)?; if self.state.graph_seen.get(&candidate.zid) == Some(&s.digest) { @@ -462,6 +665,15 @@ impl Scheduler<'_> { ); return Ok(false); } + // A graph of this very snapshot was let go as unclaimed: the + // admission names that graph in its key. + let after = self + .state + .graph_unclaimed + .get(&cfg.env) + .and_then(|u| u.graphs.get(&candidate.zid)) + .filter(|g| g.digest == s.digest) + .map(|g| g.graph_id.clone()); // Four jobs plus all three inline copies per node. Charge the // conservative request bound before admission, not after it grows. let comments = s.ids.len() as i64; @@ -531,7 +743,7 @@ impl Scheduler<'_> { { return Ok(false); } - let key = crate::jobs::sha256_hex(format!("{}:{specification}", candidate.zid).as_bytes()); + let key = admission_key(candidate.zid, &specification, after.as_deref()); let image: Image = serde_json::from_value(p.clone())?; let reply = crate::jobs::admission::admit( &mut client, @@ -572,13 +784,18 @@ impl Scheduler<'_> { .all(|s| nodes.iter().filter(|n| n.stage == *s).count() == 1), "graph_nodes_mismatch" ); - log( - "graph_admitted", - json!({"graph_id":graph_id,"nodes":nodes,"comments":comments,"env":cfg.env}), - ); + let mut admitted = + json!({"graph_id":graph_id,"nodes":nodes,"comments":comments,"env":cfg.env}); + if let Some(g) = &after { + admitted["after_let_go"] = json!(g); + } + log("graph_admitted", admitted); self.state .graph_seen .insert(candidate.zid, s.digest.clone()); + // The same snapshot's graph, taken up again after its publication + // was given up (the admission answers with the existing graph). + self.state.graph_unpublished.retain(|g| g != &graph_id); self.graph_wait_bucket = None; self.state.graph_flight = Some(Flight { env: cfg.env.clone(), @@ -589,6 +806,7 @@ impl Scheduler<'_> { admitted_at_ms: now, generation, nodes, + publish_attempts: 0, }); self.state.day_runs += 1; self.state.totals.admitted += 1; @@ -640,6 +858,40 @@ impl Scheduler<'_> { } } + /// Record a failed publication attempt in the flight it belongs to (in + /// flight or deferred), so the count survives the poll and a restart. + fn note_publish_attempt(&mut self, graph_id: &str, attempts: u32) { + for flight in self + .state + .graph_flight + .iter_mut() + .chain(self.state.graph_deferred.iter_mut()) + { + if flight.graph_id == graph_id { + flight.publish_attempts = attempts; + } + } + } + + /// The publication was given up: the snapshot is not remembered as seen, + /// so a later build or a change reconsiders the conversation (the same + /// snapshot's admission answers with this graph, and only its + /// publication runs again), and recovery does not take it up again. + fn give_up_publication(&mut self, flight: &Flight) { + if self.state.graph_seen.get(&flight.zid) == Some(&flight.digest) { + self.state.graph_seen.remove(&flight.zid); + } + if !self.state.graph_unpublished.contains(&flight.graph_id) { + self.state.graph_unpublished.push(flight.graph_id.clone()); + } + let excess = self + .state + .graph_unpublished + .len() + .saturating_sub(UNPUBLISHED_KEPT); + self.state.graph_unpublished.drain(..excess); + } + fn log_graph_wait( &mut self, flight: &Flight, @@ -675,6 +927,14 @@ impl Scheduler<'_> { Ok(()) } + /// No node of the graph was ever claimed: no attempt at all, whatever + /// became of it. One read on the scheduler's read connection. + fn never_claimed(&self, flight: &Flight) -> Result { + self.db.read(|tx| Ok(tx.query_one( + "SELECT NOT EXISTS (SELECT 1 FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid)", + &[&flight.env, &flight.graph_id])?.try_get::<_, bool>(0)?)) + } + pub(super) fn watch_graph(&mut self, flight: Flight, now: i64) -> Result { let states=self.db.read(|tx| { let mut out=BTreeMap::new(); @@ -688,7 +948,20 @@ impl Scheduler<'_> { .values() .any(|(s, _)| matches!(s.as_str(), "dead" | "cancelled")); let overdue = now - flight.admitted_at_ms > self.cfg.job_deadline.as_millis() as i64; - if failed || overdue { + // Past the short deadline with every node still queued, one read says + // whether any node was ever claimed (any attempt at all). + let unclaimed = !failed + && !overdue + && now - flight.admitted_at_ms > UNCLAIMED_DEADLINE.as_millis() as i64 + && states.values().all(|(s, _)| s == "queued") + && self.never_claimed(&flight)?; + if unclaimed { + // Recorded before the cancel: a claim racing it leaves an + // attempt, and the graph was still let go as unclaimed. + self.let_go(&flight, now, "no_executor_claim"); + } + self.note_claimed(&flight, &states); + if failed || overdue || unclaimed { let mut client = self.db.connect_queue()?; for n in &flight.nodes { // The queue's fenced management call refuses terminal rewrites. @@ -738,6 +1011,12 @@ impl Scheduler<'_> { } drop(client); let success = states.values().all(|(s, _)| s == "succeeded"); + // Every node cancelled and none ever claimed: let go as unclaimed + // here or, if not here (cancelled before this scheduler's state was + // saved, by a person, or past the job deadline), now. + if states.values().all(|(s, _)| s == "cancelled") && self.never_claimed(&flight)? { + self.let_go(&flight, now, "cancelled_unclaimed"); + } let mut published = false; if success { let leaf = flight @@ -745,55 +1024,57 @@ impl Scheduler<'_> { .iter() .find(|n| n.stage == "graph_narrative") .ok_or_else(|| anyhow::anyhow!("graph_leaf_missing"))?; - // Each retry and every read use a fresh connection, including after - // a commit whose reply was lost. No retry follows a definite refusal. - for attempt in 0..3 { - let result = (|| -> Result { - let mut client = self.db.connect_queue()?; - let row = client.query_one( - "SELECT public.pd_graph_publish($1,$2::text::uuid,$3::text::uuid,$4)", - &[ - &flight.env, - &flight.graph_id, - &leaf.job_id, - &flight.generation, - ], - )?; - Ok(row.try_get(0)?) - })(); - // A publish reply alone does not establish which root is served. - let served = (|| -> Result { - let mut client = self.db.connect_queue()?; - let row = client.query_one( - "SELECT public.pd_graph_served($1,$2,$3)", - &[&flight.env, &flight.zid, &scope(flight.zid)], - )?; - Ok(row.try_get::<_, Option>(0)?.unwrap_or(Value::Null)) - })(); - match served { - Ok(ref value) if exact_leaf(value, &leaf.job_id, flight.generation) => { - published = true; - break; - } - // The outcome is still unknown. Keep the flight and resolve - // it on a later poll; never count an uncertain commit failed. - Err(error) => return Err(error), - _ => {} - } - let retry = result.as_ref().err().is_some_and(publish_transient); - if !retry || attempt == 2 { - break; + // One attempt per poll, on a fresh connection, and every read + // too, including after a commit whose reply was lost. A transient + // failure keeps the flight, the count carried in it, and is + // retried on a later poll; no retry follows a definite refusal. + let result = (|| -> Result { + let mut client = self.db.connect_queue()?; + let row = client.query_one( + "SELECT public.pd_graph_publish($1,$2::text::uuid,$3::text::uuid,$4)", + &[ + &flight.env, + &flight.graph_id, + &leaf.job_id, + &flight.generation, + ], + )?; + Ok(row.try_get(0)?) + })(); + // A publish reply alone does not establish which root is served. + let served = (|| -> Result { + let mut client = self.db.connect_queue()?; + let row = client.query_one( + "SELECT public.pd_graph_served($1,$2,$3)", + &[&flight.env, &flight.zid, &scope(flight.zid)], + )?; + Ok(row.try_get::<_, Option>(0)?.unwrap_or(Value::Null)) + })(); + match served { + Ok(ref value) if exact_leaf(value, &leaf.job_id, flight.generation) => { + published = true; } - log( - "graph_publish_retry", - json!({"graph_id":flight.graph_id,"attempt":attempt+1}), - ); + // The outcome is still unknown. Keep the flight and resolve + // it on a later poll; never count an uncertain commit failed. + Err(error) => return Err(error), + _ => {} } if !published { + let attempts = flight.publish_attempts.saturating_add(1); + let retry = result.as_ref().err().is_some_and(publish_transient); + if retry && attempts < PUBLISH_ATTEMPTS { + self.note_publish_attempt(&flight.graph_id, attempts); + log( + "graph_publish_retry", + json!({"graph_id":flight.graph_id,"attempt":attempts}), + ); + return Ok(self.cfg.poll); + } log( "graph_publish_failed", json!({"graph_id":flight.graph_id,"reason":"publication_not_served"}), ); + self.give_up_publication(&flight); } } let mut bytes = 0u64; @@ -1007,6 +1288,30 @@ mod recovery_tests { assert!(!exact_leaf(&Value::Null, "requested", 6)); } #[test] + fn the_unclaimed_hold_is_an_hour_doubling_to_a_day() { + let hours: Vec = [0, 1, 2, 3, 4, 5, 6, 7, 40, u32::MAX] + .iter() + .map(|n| unclaimed_hold(*n).as_secs() / 3600) + .collect(); + assert_eq!(hours, [1, 1, 2, 4, 8, 16, 24, 24, 24, 24]); + } + #[test] + fn a_let_go_snapshot_gets_a_new_key_and_every_other_keeps_its_own() { + // The key of every admission that follows no let-go is the one the + // scheduler always used, so a graph admitted before this code is + // still answered (`existing`) and only published. + let spec = r#"{"schema":"polis-job-graph/1","nodes":[]}"#; + assert_eq!( + admission_key(7, spec, None), + crate::jobs::sha256_hex(format!("7:{spec}").as_bytes()) + ); + let after = admission_key(7, spec, Some("graph-a")); + assert_ne!(after, admission_key(7, spec, None)); + assert_ne!(after, admission_key(7, spec, Some("graph-b"))); + assert_eq!(after, admission_key(7, spec, Some("graph-a")), "repeatable"); + assert_eq!(after.len(), 64); + } + #[test] fn old_refusal_and_new_code_are_reconsidered() -> Result<()> { let live = Live { zid: 1, diff --git a/queue-rs/src/shadow/scheduler.rs b/queue-rs/src/shadow/scheduler.rs index 4f17c9fb2..471fec8d6 100644 --- a/queue-rs/src/shadow/scheduler.rs +++ b/queue-rs/src/shadow/scheduler.rs @@ -115,6 +115,8 @@ pub struct Scheduler<'a> { pub(super) graph_provenance: Option<(String, Value)>, pub(super) graph_list_counts: Option<(usize, usize)>, pub(super) graph_wait_bucket: Option, + /// The deferred graphs the last `graph_admission_held` line named. + pub(super) graph_held: Option, } impl<'a> Scheduler<'a> { @@ -132,6 +134,7 @@ impl<'a> Scheduler<'a> { graph_provenance: None, graph_list_counts: None, graph_wait_bucket: None, + graph_held: None, }; scheduler.recover_graph()?; Ok(scheduler) diff --git a/queue-rs/src/shadow/state.rs b/queue-rs/src/shadow/state.rs index 13acc0970..38d65f8c2 100644 --- a/queue-rs/src/shadow/state.rs +++ b/queue-rs/src/shadow/state.rs @@ -124,6 +124,15 @@ pub struct State { pub graph_seen: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub graph_observed: BTreeMap, + /// Graphs whose publication this scheduler gave up (`graph::PUBLISH_ATTEMPTS` + /// failed, or a definite refusal), newest last: recovery does not take + /// them up again, and their snapshots are not remembered as seen. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub graph_unpublished: Vec, + /// Per graph env, the graphs let go there because nothing claimed them + /// and the hold on admission that follows (`graph::Unclaimed`). + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub graph_unclaimed: BTreeMap, pub pending_twin: Option, /// The UTC day (days since the epoch) the daily count is for. pub day: i64, @@ -152,6 +161,8 @@ impl State { graph_recovery_envs: Vec::new(), graph_seen: BTreeMap::new(), graph_observed: BTreeMap::new(), + graph_unpublished: Vec::new(), + graph_unclaimed: BTreeMap::new(), pending_twin: None, day: 0, day_runs: 0, @@ -265,6 +276,38 @@ mod tests { Ok(()) } + #[test] + fn with_no_graph_the_state_file_carries_no_graph_field() -> anyhow::Result<()> { + // With the graph list empty nothing graph-shaped is ever written, so + // the file is what the full-pipeline shadow wrote before. + let v = serde_json::to_value(State::new("shadow-delphi-1"))?; + let keys: Vec<&String> = v + .as_object() + .map(|o| o.keys().collect()) + .unwrap_or_default(); + assert!(!keys.iter().any(|k| k.starts_with("graph_")), "{keys:?}"); + // A flight that has made no publication attempt carries no count. + let flight = crate::shadow::graph::Flight { + env: "shadow-graph-1".into(), + graph_id: "g".into(), + zid: 1, + digest: "d".into(), + comments: 5, + admitted_at_ms: 1, + generation: 0, + nodes: vec![], + publish_attempts: 0, + }; + let f = serde_json::to_value(&flight)?; + assert!(f.get("publish_attempts").is_none(), "{f}"); + let earlier: crate::shadow::graph::Flight = serde_json::from_value(f)?; + assert_eq!( + earlier, flight, + "a flight an earlier build wrote loads with no attempts" + ); + Ok(()) + } + #[test] fn the_daily_count_restarts_each_utc_day() { let mut s = State::new("shadow-delphi-1"); diff --git a/queue-rs/tests/delphi_shadow.rs b/queue-rs/tests/delphi_shadow.rs index ae3a320e1..eda9a6a56 100644 --- a/queue-rs/tests/delphi_shadow.rs +++ b/queue-rs/tests/delphi_shadow.rs @@ -82,6 +82,16 @@ impl Fixture { } fn config(&self, extra: &[(&str, &str)]) -> config::Config { + let m = self.environment(extra); + match config::load(|k| m.get(k).cloned()).unwrap() { + config::Load::Ready(c) => *c, + config::Load::Disabled => panic!("disabled"), + } + } + + /// The scheduler's environment, as `config` loads it (and as the + /// scheduler's own process is started with). + fn environment(&self, extra: &[(&str, &str)]) -> HashMap { let mut m: HashMap = [ ("DELPHI_SHADOW_ENABLED", "1".to_owned()), ("QUEUE_DATABASE_URL", url_for(&self.name, &self.login)), @@ -111,10 +121,7 @@ impl Fixture { for (k, v) in extra { m.insert((*k).to_owned(), (*v).to_owned()); } - match config::load(|k| m.get(k).cloned()).unwrap() { - config::Load::Ready(c) => *c, - config::Load::Disabled => panic!("disabled"), - } + m } /// A conversation with `n` comments, its code (`c`), its math row @@ -2108,11 +2115,863 @@ fn transient_publish_before_commit_retries_and_keeps_one_generation() { BEGIN IF nextval('publish_calls')=1 THEN RAISE EXCEPTION 'retry' USING ERRCODE='40001'; END IF; RETURN publish_original($1,$2,$3,$4); END $$; GRANT EXECUTE ON FUNCTION pd_graph_publish(text,uuid,uuid,bigint) TO polis_queue_executor;").unwrap(); + // The retry is on the next poll, not within this one: the flight is + // kept and nothing is counted until then. + scheduler.cycle().unwrap(); + assert!(scheduler.state.graph_flight.is_some()); + assert_eq!( + ( + scheduler.state.totals.succeeded, + scheduler.state.totals.failed + ), + (0, 0) + ); scheduler.cycle().unwrap(); assert_eq!(scheduler.state.totals.succeeded, 1); + assert_eq!(scheduler.state.totals.failed, 0); let calls: i64 = c .query_one("SELECT last_value FROM publish_calls", &[]) .unwrap() .get(0); assert_eq!(calls, 2); + assert_eq!(served(&f, &flight)["generation"], 1, "one generation"); +} + +// --------------------------------------------------------------------------- +// The graph path before its list is set: an absent executor, publication +// retries and a held admission. + +/// Each node's queue state, in node order. +fn graph_states(f: &Fixture, flight: &polis_queue_adapter::shadow::graph::Flight) -> Vec { + f.superuser() + .query( + "SELECT q.state FROM delphi_graph_nodes n JOIN polis_queue_jobs q USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid ORDER BY n.node_key", + &[&flight.env, &flight.graph_id], + ) + .unwrap() + .iter() + .map(|r| r.get(0)) + .collect() +} + +fn served(f: &Fixture, flight: &polis_queue_adapter::shadow::graph::Flight) -> Value { + f.superuser() + .query_one( + "SELECT pd_graph_served($1,$2,$3)", + &[ + &flight.env, + &flight.zid, + &polis_queue_adapter::shadow::graph::scope(flight.zid), + ], + ) + .unwrap() + .get::<_, Option>(0) + .unwrap_or(Value::Null) +} + +#[test] +fn an_unclaimed_graph_is_let_go_after_a_short_deadline_and_the_full_pipeline_goes_on() { + // No graph executor runs (absent, unstartable, or refused at deploy), so + // nothing ever claims the graph's first node. + let f = Fixture::new(); + f.quiet(1011, 25, 1); + f.quiet(1012, 25, 1); + graph_catalog(&f); + let cfg = graph_config( + &f, + &[ + ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1011"), + ("DELPHI_SHADOW_BENCHMARK", "c1012"), + ], + ); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + assert_eq!(flight.zid, 1011); + // Inside the short deadline the graph is waited for, as before: two + // seconds short of ten minutes, nothing of it is cancelled. + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 598_000; + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(flight.graph_id.clone()) + ); + assert_eq!( + graph_states(&f, &flight), + ["queued"; 4], + "nothing is cancelled inside ten minutes" + ); + assert!(s.state.in_flight.is_none()); + // Unclaimed for longer than ten minutes, far inside the job deadline. + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 3_000; + s.cycle().unwrap(); + assert!( + graph_states(&f, &flight).iter().all(|st| st == "cancelled"), + "{:?}", + graph_states(&f, &flight) + ); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert!(s.state.graph_deferred.is_empty(), "let go, not deferred"); + assert_eq!(s.state.totals.failed, 1); + let view: Value = f + .superuser() + .query_one( + "SELECT pd_graph_view($1,$2::text::uuid)", + &[&flight.env, &flight.graph_id], + ) + .unwrap() + .get(0); + assert_eq!(view["guard_held"], false, "its scope is released"); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert_eq!(s.state.in_flight.as_ref().unwrap().zid, 1012); +} + +#[test] +fn a_claimed_graph_is_not_let_go_at_the_short_deadline() { + let f = Fixture::new(); + f.quiet(1016, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1016")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + let (owner, attempt) = (Uuid::new_v4().to_string(), Uuid::new_v4().to_string()); + let claim: Value = f.superuser().query_one("SELECT pd_graph_claim($1,1::smallint,$2::text::uuid,$3::text::uuid,60,'delphi','graph_embed')", &[&flight.env, &owner, &attempt]).unwrap().get(0); + assert_eq!(claim["outcome"], "owned"); + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.cycle().unwrap(); + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(flight.graph_id.clone()) + ); + assert_eq!(s.state.totals.failed, 0); + assert!(!graph_states(&f, &flight).iter().any(|st| st == "cancelled")); +} + +#[test] +fn a_graph_whose_claimed_node_was_put_back_in_the_queue_is_not_let_go() { + // Every node is queued again, but one was claimed once: an executor + // ran, so the short deadline does not apply. Only the read of the + // attempts tells this graph from one nothing ever claimed. + let f = Fixture::new(); + f.quiet(1017, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1017")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + let mut c = f.superuser(); + let (owner, attempt) = (Uuid::new_v4().to_string(), Uuid::new_v4().to_string()); + let claim: Value = c.query_one("SELECT pd_graph_claim($1,1::smallint,$2::text::uuid,$3::text::uuid,60,'delphi','graph_embed')", &[&flight.env, &owner, &attempt]).unwrap().get(0); + assert_eq!(claim["outcome"], "owned"); + let job = claim["job_id"].as_str().unwrap().to_owned(); + let epoch: i64 = claim["lease_epoch"].as_str().unwrap().parse().unwrap(); + // Parked with its exit confirmed (its executor stopped it), then put + // back in the queue by the reaper. + let parked: Value = c.query_one("SELECT pq_end_attempt($1,$2::text::uuid,$3::text::uuid,$4::text::uuid,$5,'park','fixture_parked',true,clock_timestamp())", &[&flight.env, &job, &owner, &attempt, &epoch]).unwrap().get(0); + assert_eq!(parked["outcome"], "parked", "{parked}"); + c.query_one( + "SELECT pq_reap_one($1,$2::text::uuid)", + &[&flight.env, &job], + ) + .unwrap(); + assert_eq!(graph_states(&f, &flight), ["queued"; 4]); + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.cycle().unwrap(); + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(flight.graph_id.clone()) + ); + assert_eq!( + graph_states(&f, &flight), + ["queued"; 4], + "nothing cancelled" + ); + assert_eq!(s.state.totals.failed, 0); + assert!(unclaimed(&s, &flight.env).is_null()); +} + +#[test] +fn a_graph_with_a_node_out_of_the_queue_is_not_let_go_whatever_the_attempts_table_holds() { + // The short deadline is for a graph whose every node is still queued. + // A node in any other state is not let go by it, even with no attempt + // on record: a row only a direct update makes (the queue's functions + // never take a job out of `queued` without an attempt), so that the + // rule is seen to read the nodes' states and not only the attempts. + let f = Fixture::new(); + f.quiet(1018, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1018")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + let embed = flight + .nodes + .iter() + .find(|n| n.stage == "graph_embed") + .unwrap() + .job_id + .clone(); + f.superuser() + .execute( + "UPDATE polis_queue_jobs SET state='retry_wait', eligible_at=clock_timestamp()+interval '1 day' WHERE env=$1 AND job_id=$2::text::uuid", + &[&flight.env, &embed], + ) + .unwrap(); + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.cycle().unwrap(); + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(flight.graph_id.clone()) + ); + let states = graph_states(&f, &flight); + assert!(!states.iter().any(|st| st == "cancelled"), "{states:?}"); + assert_eq!(s.state.totals.failed, 0); + assert!(unclaimed(&s, &flight.env).is_null()); +} + +/// The scheduler's record of the graphs let go as unclaimed in `env`, as +/// its state file holds it (null when there is none). +fn unclaimed(s: &Scheduler<'_>, env: &str) -> Value { + serde_json::to_value(&s.state).unwrap()["graph_unclaimed"][env].clone() +} + +/// Move the env's hold back by `ms`, as if that much more time had passed +/// since the last graph there was let go. +fn age_hold(s: &mut Scheduler<'_>, env: &str, ms: i64) { + let mut v = serde_json::to_value(&s.state).unwrap(); + let at = v["graph_unclaimed"][env]["at_ms"] + .as_i64() + .expect("a graph let go as unclaimed"); + v["graph_unclaimed"][env]["at_ms"] = (at - ms).into(); + s.state = serde_json::from_value(v).unwrap(); +} + +/// The graph in flight, unclaimed past ten minutes: one cycle cancels its +/// nodes, the next finishes it. +fn let_go_unclaimed( + f: &Fixture, + s: &mut Scheduler<'_>, +) -> polis_queue_adapter::shadow::graph::Flight { + let flight = s.state.graph_flight.clone().unwrap(); + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.cycle().unwrap(); + assert_eq!(graph_states(f, &flight), ["cancelled"; 4]); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + flight +} + +/// A conversation let go as unclaimed is neither seen nor observed, so it +/// is admitted again when its env's hold ends. +fn assert_forgotten(s: &Scheduler<'_>, zid: i32) { + assert!(!s.state.graph_seen.contains_key(&zid), "seen: {zid}"); + assert!( + !s.state.graph_observed.contains_key(&zid), + "observed: {zid}" + ); +} + +/// A graph's admission key and specification. +fn graph_request( + f: &Fixture, + flight: &polis_queue_adapter::shadow::graph::Flight, +) -> (String, Value) { + let r = f + .superuser() + .query_one( + "SELECT request_key, request FROM delphi_graphs WHERE env=$1 AND graph_id=$2::text::uuid", + &[&flight.env, &flight.graph_id], + ) + .unwrap(); + (r.get(0), r.get(1)) +} + +#[test] +fn a_let_go_graph_is_admitted_again_as_a_new_graph_after_its_hold_across_a_restart() { + // An unchanged listed conversation (a closed benchmark conversation) + // whose graph nothing claimed. Its next admission is a new graph, not + // the cancelled one its snapshot's own key would answer with, and the + // full pipeline goes on meanwhile. + let f = Fixture::new(); + f.quiet(1051, 25, 1); + f.quiet(1052, 25, 1); + graph_catalog(&f); + let list = ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1051"); + let cfg = graph_config(&f, &[list, ("DELPHI_SHADOW_BENCHMARK", "c1052")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let first = let_go_unclaimed(&f, &mut s); + assert_eq!(s.state.totals.failed, 1); + let held = unclaimed(&s, &first.env); + assert_eq!(held["count"], 1, "{held}"); + assert_eq!(held["graphs"]["1051"]["graph_id"], first.graph_id.as_str()); + assert_eq!(held["graphs"]["1051"]["digest"], first.digest.as_str()); + assert_eq!(held["graphs"]["1051"].as_object().map(|o| o.len()), Some(2)); + assert_forgotten(&s, 1051); + // While the hold lasts no graph is admitted; the full pipeline goes on. + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + let run = s + .state + .in_flight + .clone() + .expect("the full pipeline goes on"); + assert_eq!(run.zid, 1052); + f.cancel(&run.job_id); + s.cycle().unwrap(); + assert!(s.state.in_flight.is_none()); + // A restart of the scheduler keeps the hold and what was let go. + s.state.save(&cfg.state_path).unwrap(); + drop(s); + let mut s = Scheduler::new(&cfg).unwrap(); + assert_eq!(unclaimed(&s, &first.env), held); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none(), "held after the restart"); + // Nor is its snapshot remembered as seen again from the database. + assert_forgotten(&s, 1051); + // One minute short of the hour, the hold stands. + age_hold(&mut s, &first.env, 3_540_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none(), "an hour, not less"); + // Past the hour the same snapshot is admitted again, as a new graph. + age_hold(&mut s, &first.env, 61_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + let second = s.state.graph_flight.clone().expect("admitted again"); + assert_ne!(second.graph_id, first.graph_id, "not the cancelled graph"); + assert_eq!(second.digest, first.digest, "of the same snapshot"); + assert_eq!(graph_states(&f, &second), ["queued"; 4]); + let (first_key, first_request) = graph_request(&f, &first); + let (second_key, second_request) = graph_request(&f, &second); + assert_ne!(first_key, second_key); + assert_eq!(first_request, second_request, "the same specification"); + assert_eq!(s.state.graph_seen.get(&1051), Some(&first.digest)); + // Claimed and finished: the executor is back, and the count is reset. + finish_graph(&f, &second); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert_eq!(s.state.totals.succeeded, 1); + assert_eq!(served(&f, &second)["generation"], 1); + assert_eq!(unclaimed(&s, &first.env)["count"], 0); + // The unchanged snapshot, now published, is not admitted again. + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert_eq!(graphs_of(&f, &first.env, 1051), 2); +} + +#[test] +fn a_let_go_graph_returns_at_the_next_build_its_hold_doubles_and_its_publication_retry_finds_it() { + let f = Fixture::new(); + f.quiet(1061, 25, 1); + graph_catalog(&f); + let list = ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1061"); + let build = |code: &str| graph_config(&f, &[list, ("DELPHI_SHADOW_CODE_VERSION", code)]); + let cfg = build(CODE); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let first = let_go_unclaimed(&f, &mut s); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none(), "held under the same build"); + // The next build (a deploy, which may have fixed the executor) admits + // it at once, as a new graph. + s.state.save(&cfg.state_path).unwrap(); + drop(s); + let next = build("fedcba9876543210fedcba9876543210fedcba98"); + let mut s = Scheduler::new(&next).unwrap(); + s.cycle().unwrap(); + let second = s + .state + .graph_flight + .clone() + .expect("admitted at the next build"); + assert_ne!(second.graph_id, first.graph_id); + // Nothing claims it either: let go again, and the hold is two hours. + let second = let_go_unclaimed(&f, &mut s); + let held = unclaimed(&s, &second.env); + assert_eq!(held["count"], 2, "{held}"); + assert_eq!(held["graphs"]["1061"]["graph_id"], second.graph_id.as_str()); + age_hold(&mut s, &second.env, 3_601_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert!( + s.state.graph_flight.is_none(), + "two hours after a second in a row" + ); + age_hold(&mut s, &second.env, 3_600_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + let third = s + .state + .graph_flight + .clone() + .expect("admitted after two hours"); + assert!(third.graph_id != first.graph_id && third.graph_id != second.graph_id); + // Claimed and finished, but its publication is given up. + finish_graph(&f, &third); + publication_fails_first(&f, 1_000_000); + for _ in 0..3 { + s.cycle().unwrap(); + } + assert!(s.state.graph_flight.is_none()); + assert_eq!(publish_calls(&f), 3); + assert_eq!(s.state.totals.failed, 3); + // At the build after, the same snapshot's admission finds that finished + // graph, and only its publication runs. + publication_restored(&f); + s.state.save(&next.state_path).unwrap(); + drop(s); + let after = build("00112233445566778899aabbccddeeff00112233"); + let mut s = Scheduler::new(&after).unwrap(); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(third.graph_id.clone()), + "the finished graph, not a new one" + ); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert_eq!(s.state.totals.succeeded, 1); + assert_eq!(served(&f, &third)["generation"], 1); + assert_eq!(graphs_of(&f, &third.env, 1061), 3); +} + +#[test] +fn a_graph_found_cancelled_with_no_claim_or_past_its_deadline_with_none_is_let_go_as_unclaimed() { + // A scheduler that stopped after cancelling and before saving its state, + // a person who cancelled the graph before any claim, and a graph past the + // whole job deadline that nothing ever claimed: each is let go as + // unclaimed when it finishes, and admitted again later. + for how in ["cancelled", "overdue"] { + let f = Fixture::new(); + f.quiet(1071, 25, 1); + graph_catalog(&f); + let mut cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1071")]); + if how == "overdue" { + cfg.job_deadline = Duration::from_secs(60); + } + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + if how == "cancelled" { + let mut c = f.superuser(); + for n in &flight.nodes { + let status: Value = c + .query_one( + "SELECT pq_job_status($1,$2::text::uuid)", + &[&flight.env, &n.job_id], + ) + .unwrap() + .get(0); + let version: i64 = status["mgmt_version"].as_str().unwrap().parse().unwrap(); + c.query_one( + "SELECT pq_cancel($1,$2::text::uuid,$3)", + &[&flight.env, &n.job_id, &version], + ) + .unwrap(); + } + } else { + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + } + for _ in 0..4 { + s.cycle().unwrap(); + } + assert!(s.state.graph_flight.is_none(), "{how}"); + assert!(s.state.graph_deferred.is_empty(), "{how}"); + assert_eq!(s.state.totals.failed, 1, "{how}"); + let held = unclaimed(&s, &flight.env); + assert_eq!(held["count"], 1, "{how}: {held}"); + assert_eq!( + held["graphs"]["1071"]["graph_id"], + flight.graph_id.as_str(), + "{how}" + ); + assert_forgotten(&s, 1071); + age_hold(&mut s, &flight.env, 3_601_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + let again = s.state.graph_flight.clone().expect(how); + assert_ne!(again.graph_id, flight.graph_id, "{how}"); + } +} + +#[test] +fn a_claim_racing_the_let_go_does_not_make_it_permanent() { + // The executor's claim can land between the read that finds no attempt + // and the cancel (the cancel then wins): the graph was let go as + // unclaimed, and is admitted again later, though an attempt now exists. + let f = Fixture::new(); + f.quiet(1076, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1076")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.cycle().unwrap(); + assert_eq!(graph_states(&f, &flight), ["cancelled"; 4]); + // The racing claim's attempt, ended by the cancel, as pq_cancel ends a + // claimed job's attempt. + let embed = flight + .nodes + .iter() + .find(|n| n.stage == "graph_embed") + .unwrap() + .job_id + .clone(); + f.superuser() + .execute( + "INSERT INTO polis_queue_attempts(env,attempt_id,job_id,owner_id,lease_epoch,outcome,ended_at,process_exit_confirmed_at) VALUES($1,gen_random_uuid(),$2::text::uuid,gen_random_uuid(),1,'cancelled',clock_timestamp(),clock_timestamp())", + &[&flight.env, &embed], + ) + .unwrap(); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert_eq!( + unclaimed(&s, &flight.env)["graphs"]["1076"]["graph_id"], + flight.graph_id.as_str() + ); + assert_forgotten(&s, 1076); + age_hold(&mut s, &flight.env, 3_601_000); + s.rest_until_ms = 0; + s.cycle().unwrap(); + let again = s.state.graph_flight.clone().expect("admitted again"); + assert_ne!(again.graph_id, flight.graph_id); +} + +#[test] +fn a_let_go_says_so_with_its_count_and_its_hold_once() { + let f = Fixture::new(); + f.quiet(1081, 25, 1); + graph_catalog(&f); + let list = ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1081"); + let cfg = graph_config(&f, &[list]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + // Unclaimed for more than ten minutes when the scheduler's process starts. + s.state.graph_flight.as_mut().unwrap().admitted_at_ms -= 601_000; + s.state.save(&cfg.state_path).unwrap(); + drop(s); + let app = f.dir.display().to_string(); + let python = std::env::var("POLIS_JOBS_TEST_PYTHON").unwrap_or_else(|_| "python3".into()); + let polls_held = |lines: &[Value]| { + lines + .iter() + .skip_while(|l| l["event"] != "graph_admission_held") + .filter(|l| l["event"] == "idle") + .count() + }; + let lines = scheduler_lines( + &f, + &[ + list, + ("DELPHI_SHADOW_POLL_SECONDS", "1"), + ("DELPHI_SHADOW_JOB_POLL_SECONDS", "1"), + ("DELPHI_APP_PATH", &app), + ("POLIS_JOBS_PYTHON", &python), + ], + |lines| polls_held(lines) >= 3, + Duration::from_secs(60), + ); + assert!(polls_held(&lines) >= 3, "{lines:?}"); + let let_go: Vec<&Value> = lines + .iter() + .filter(|l| l["event"] == "graph_unclaimed") + .collect(); + assert_eq!(let_go.len(), 1, "{lines:?}"); + assert_eq!(let_go[0]["graph_id"], flight.graph_id.as_str()); + assert_eq!(let_go[0]["reason"], "no_executor_claim"); + assert_eq!(let_go[0]["let_go"], 1); + assert_eq!(let_go[0]["hold_seconds"], 3600); + let held: Vec<&Value> = lines + .iter() + .filter(|l| l["event"] == "graph_admission_held") + .collect(); + assert_eq!(held.len(), 1, "once, however many polls hold it: {lines:?}"); + assert_eq!(held[0]["reason"], "graph_unclaimed"); + assert_eq!(held[0]["let_go"], 1); + assert_eq!(held[0]["graph_ids"], serde_json::json!([flight.graph_id])); + assert!( + held[0]["resume_seconds"] + .as_i64() + .is_some_and(|n| (3500..=3600).contains(&n)), + "{}", + held[0] + ); + assert!(!lines.iter().any(|l| l["event"] == "graph_admitted")); +} + +/// Replace the disposable database's publication function: its first `fail` +/// calls refuse with a serialization failure before anything commits; the +/// rest run the original. `publish_calls` counts the calls. +fn publication_fails_first(f: &Fixture, fail: i64) { + f.superuser().batch_execute(&format!("CREATE SEQUENCE publish_calls; ALTER FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RENAME TO publish_original; + CREATE FUNCTION pd_graph_publish(text,uuid,uuid,bigint) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER AS $$ + BEGIN IF nextval('publish_calls')<={fail} THEN RAISE EXCEPTION 'retry' USING ERRCODE='40001'; END IF; + RETURN publish_original($1,$2,$3,$4); END $$; + GRANT EXECUTE ON FUNCTION pd_graph_publish(text,uuid,uuid,bigint) TO polis_queue_executor;")).unwrap(); +} + +fn publication_restored(f: &Fixture) { + f.superuser() + .batch_execute( + "DROP FUNCTION pd_graph_publish(text,uuid,uuid,bigint); + ALTER FUNCTION publish_original(text,uuid,uuid,bigint) RENAME TO pd_graph_publish;", + ) + .unwrap(); +} + +fn publish_calls(f: &Fixture) -> i64 { + f.superuser() + .query_one( + "SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM publish_calls", + &[], + ) + .unwrap() + .get(0) +} + +#[test] +fn a_transient_publication_failure_keeps_the_flight_and_is_retried_on_later_polls() { + let f = Fixture::new(); + f.quiet(1021, 25, 1); + graph_catalog(&f); + let cfg = graph_config(&f, &[("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1021")]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + finish_graph(&f, &flight); + publication_fails_first(&f, 2); + for attempt in 1..=2i64 { + s.cycle().unwrap(); + assert_eq!( + publish_calls(&f), + attempt, + "one publication attempt per poll" + ); + let kept = s.state.graph_flight.clone().expect("the flight is kept"); + assert_eq!(kept.graph_id, flight.graph_id); + assert_eq!( + serde_json::to_value(&kept).unwrap()["publish_attempts"], + attempt, + "the count is carried in the flight" + ); + assert_eq!((s.state.totals.succeeded, s.state.totals.failed), (0, 0)); + // And it is carried across a restart of the scheduler. + s.state.save(&cfg.state_path).unwrap(); + s = Scheduler::new(&cfg).unwrap(); + } + s.cycle().unwrap(); + assert_eq!(publish_calls(&f), 3); + assert!(s.state.graph_flight.is_none()); + assert_eq!((s.state.totals.succeeded, s.state.totals.failed), (1, 0)); + assert_eq!(served(&f, &flight)["generation"], 1); +} + +#[test] +fn a_publication_that_keeps_failing_is_counted_once_and_its_snapshot_is_not_remembered_as_seen() { + let f = Fixture::new(); + f.quiet(1031, 25, 1); + f.quiet(1032, 25, 1); + graph_catalog(&f); + let list = ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1031"); + let benchmark = ("DELPHI_SHADOW_BENCHMARK", "c1032"); + let cfg = graph_config(&f, &[list, benchmark]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + finish_graph(&f, &flight); + publication_fails_first(&f, 1_000_000); + for _ in 0..3 { + s.cycle().unwrap(); + } + assert_eq!(publish_calls(&f), 3, "three attempts, one per poll"); + assert!(s.state.graph_flight.is_none()); + assert_eq!((s.state.totals.succeeded, s.state.totals.failed), (0, 1)); + assert!(!s.state.graph_seen.contains_key(&1031)); + // Given up, the graph is not taken up again: no further attempt, no + // second failure, and the full pipeline goes on. + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert_eq!(publish_calls(&f), 3); + assert_eq!(s.state.totals.failed, 1); + let run = s.state.in_flight.clone().unwrap(); + assert_eq!(run.zid, 1032); + assert!( + !s.state.graph_seen.contains_key(&1031), + "nor remembered again from the database" + ); + f.cancel(&run.job_id); + s.cycle().unwrap(); + assert!(s.state.in_flight.is_none()); + // The next build reconsiders the conversation. Its snapshot is unchanged + // and was never published, so the same graph is taken up again and + // published, with nothing computed twice. + publication_restored(&f); + s.state.save(&cfg.state_path).unwrap(); + drop(s); + let next = graph_config( + &f, + &[ + list, + benchmark, + ( + "DELPHI_SHADOW_CODE_VERSION", + "fedcba9876543210fedcba9876543210fedcba98", + ), + ], + ); + let mut s = Scheduler::new(&next).unwrap(); + s.rest_until_ms = 0; + s.cycle().unwrap(); + assert_eq!( + s.state.graph_flight.as_ref().map(|g| g.graph_id.clone()), + Some(flight.graph_id.clone()) + ); + assert!( + !s.state.graph_unpublished.contains(&flight.graph_id), + "taken up again, it is off the given-up list" + ); + s.cycle().unwrap(); + assert!(s.state.graph_flight.is_none()); + assert_eq!(s.state.totals.succeeded, 1); + assert_eq!(served(&f, &flight)["generation"], 1); + // Its nodes ran once: the same four jobs, one attempt each. + let attempts: i64 = f + .superuser() + .query_one( + "SELECT count(*) FROM polis_queue_attempts a JOIN delphi_graph_nodes n USING(env,job_id) WHERE n.env=$1 AND n.graph_id=$2::text::uuid", + &[&flight.env, &flight.graph_id], + ) + .unwrap() + .get(0); + assert_eq!(attempts, 4, "nothing computed twice"); + assert_eq!(graphs_of(&f, &flight.env, 1031), 1, "no second graph"); +} + +/// How many graphs a conversation has in an env. +fn graphs_of(f: &Fixture, env: &str, zid: i32) -> i64 { + f.superuser() + .query_one( + "SELECT count(*) FROM delphi_graphs WHERE env=$1 AND zid=$2", + &[&env, &zid], + ) + .unwrap() + .get(0) +} + +/// The scheduler as its container runs it, until `done` holds of its +/// `delphi_shadow/1` lines so far (or `ceiling` passes), then stopped as a +/// deploy stops it (SIGTERM): its lines. Reading the lines as they come, +/// rather than sleeping a fixed time, keeps a loaded machine's slower polls +/// from deciding the result. +fn scheduler_lines( + f: &Fixture, + extra: &[(&str, &str)], + done: impl Fn(&[Value]) -> bool, + ceiling: Duration, +) -> Vec { + use std::io::BufRead; + let mut c = std::process::Command::new(env!("CARGO_BIN_EXE_polis-delphi-shadow")); + c.env_clear() + .env("PATH", std::env::var("PATH").unwrap_or_default()) + .envs(f.environment(extra)) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()); + let mut child = c.spawn().unwrap(); + let stdout = child.stdout.take().unwrap(); + let (send, recv) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in std::io::BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + let parsed = line + .strip_prefix("delphi_shadow/1 ") + .and_then(|l| serde_json::from_str::(l).ok()); + if let Some(v) = parsed + && send.send(v).is_err() + { + break; + } + } + }); + let start = Instant::now(); + let mut lines = Vec::new(); + while !done(&lines) && start.elapsed() < ceiling { + if let Ok(v) = recv.recv_timeout(Duration::from_millis(200)) { + lines.push(v); + } + } + // SAFETY: a signal to the child this test started. + unsafe { + libc::kill(child.id() as i32, libc::SIGTERM); + } + child.wait().unwrap(); + reader.join().unwrap(); + lines.extend(recv.try_iter()); + lines +} + +fn polls(lines: &[Value]) -> usize { + lines.iter().filter(|l| l["event"] == "idle").count() +} + +#[test] +fn a_held_graph_admission_says_so_once_with_its_reason() { + let f = Fixture::new(); + f.quiet(1041, 25, 1); + f.quiet(1042, 25, 1); + graph_catalog(&f); + let list = ("DELPHI_SHADOW_GRAPH_CONVERSATIONS", "c1041,c1042"); + let mut cfg = graph_config(&f, &[list]); + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + let flight = s.state.graph_flight.clone().unwrap(); + assert_eq!(flight.zid, 1041); + // An attempt with no exit proof, as a lost journal leaves it: past the + // job deadline the graph is deferred, and graph admission is held. + let (owner, attempt) = (Uuid::new_v4().to_string(), Uuid::new_v4().to_string()); + let claim: Value = f.superuser().query_one("SELECT pd_graph_claim($1,1::smallint,$2::text::uuid,$3::text::uuid,60,'delphi','graph_embed')", &[&flight.env, &owner, &attempt]).unwrap().get(0); + assert_eq!(claim["outcome"], "owned"); + s.state.save(&cfg.state_path).unwrap(); + drop(s); + cfg.job_deadline = Duration::ZERO; + let mut s = Scheduler::new(&cfg).unwrap(); + s.cycle().unwrap(); + assert_eq!(s.state.graph_deferred.len(), 1); + s.state.save(&cfg.state_path).unwrap(); + drop(s); + let app = f.dir.display().to_string(); + let python = std::env::var("POLIS_JOBS_TEST_PYTHON").unwrap_or_else(|_| "python3".into()); + let lines = scheduler_lines( + &f, + &[ + list, + ("DELPHI_SHADOW_POLL_SECONDS", "1"), + ("DELPHI_APP_PATH", &app), + ("POLIS_JOBS_PYTHON", &python), + ], + |lines| polls(lines) >= 3, + Duration::from_secs(60), + ); + assert!(polls(&lines) >= 3, "{lines:?}"); + let held: Vec<&Value> = lines + .iter() + .filter(|l| l["event"] == "graph_admission_held") + .collect(); + assert_eq!(held.len(), 1, "once, however many polls hold it: {lines:?}"); + assert_eq!(held[0]["reason"], "graph_deferred"); + assert_eq!(held[0]["graph_ids"], serde_json::json!([flight.graph_id])); + assert_eq!(held[0]["deferred"], 1); + assert!(!lines.iter().any(|l| l["event"] == "graph_admitted")); } diff --git a/queue-rs/tests/jobs_integration.rs b/queue-rs/tests/jobs_integration.rs index 6af246591..63cc7edd6 100644 --- a/queue-rs/tests/jobs_integration.rs +++ b/queue-rs/tests/jobs_integration.rs @@ -902,6 +902,77 @@ fn restart_journal_after_the_lease_gives_fenced_with_proof() { journal_restart(true); } +/// The journal's files as a restored snapshot of its volume would hold them +/// (the entry and the `.host` binding; never the lock, never a temp file). +fn copy_journal(from: &Path, to: &Path) { + fs::create_dir_all(to).unwrap(); + for item in fs::read_dir(from).unwrap().flatten() { + let name = item.file_name().to_string_lossy().into_owned(); + if name == ".lock" || name.ends_with(".tmp") || name.starts_with(".probe-") { + continue; + } + fs::copy(item.path(), to.join(&name)).unwrap(); + } +} + +#[test] +fn a_cloned_host_identity_never_confirms_a_live_attempt_and_confirms_once_its_lease_expires() { + // A bound journal restored on a second machine whose host identity file + // is a copy of the first's (a snapshot, a template-baked machine id): the + // binding passes and the boot differs, but the original is still alive. + let mut db = Db::new("jobs_v2"); + let (job, _) = db.enqueue("delphi_full_pipeline", 1, Some("h1"), full(), 3); + let host = db.scratch.join("machine-id"); + fs::write(&host, "the-same-machine-id\n").unwrap(); + let original = db.scratch.join("journal-original"); + let mut first = Opts::new("clone-a", "sleep") + .set("POLIS_JOBS_JOURNAL_HOST_FILE", host.display().to_string()); + first.journal = Some(original.clone()); + let a = start(&db, first); + db.wait("child pids", 60, |d| pids(d, "clone-a").len() == 2); + let leader = pids(&db, "clone-a")[0]; + let copy = db.scratch.join("journal-copy"); + copy_journal(&original, ©); + assert_eq!(journal_entries(©), 1); + let mut second = Opts::new("clone-b", "success") + .set("POLIS_JOBS_JOURNAL_HOST_FILE", host.display().to_string()); + second.journal = Some(copy.clone()); + second.boot = "boot-on-the-copy".into(); + let b = start(&db, second); + // Longer than one lease (10 s) while the original keeps heartbeating. + std::thread::sleep(Duration::from_secs(14)); + assert!( + !db.attempts(job)[0].5, + "a live attempt's exit is never confirmed from a copy: {}", + b.log() + ); + assert_eq!(db.job(job).0, "running"); + assert!(group_alive(leader), "the original's child still runs"); + assert_eq!(journal_entries(©), 1, "the copy keeps its entry"); + assert!( + b.log().contains("\"reason\":\"another_boot_lease_live\""), + "{}", + b.log() + ); + // The original stops; once its lease has expired by database time the + // copy confirms the exit, and the retry runs there. + a.signal(libc::SIGKILL); + drop(a); + kill_tree(leader); + db.wait_state(job, "succeeded", 90); + let att = db.attempts(job); + assert_eq!(att.len(), 2); + assert!(att[0].5, "the journaled attempt carries exit proof"); + assert!( + b.log() + .contains("\"reason\":\"another_boot_lease_expired\""), + "{}", + b.log() + ); + db.wait("journal empty", 10, |_| journal_entries(©) == 0); + let _ = b.stop(); +} + #[test] fn poison_after_max_attempts() { let mut db = Db::new("jobs_v2"); diff --git a/scripts/after_install.sh b/scripts/after_install.sh index f9f0d3019..9eb99935a 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -349,6 +349,9 @@ else echo "services: {}" | sudo tee docker-compose.images.yml > /dev/null fi COMPOSE=(sudo /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.images.yml) +# The Delphi shadow's start: COMPOSE's files, and one more when the graph +# executor is refused (below). +DELPHI_SHADOW_COMPOSE=(sudo /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.images.yml) # Validate configuration before replacing any running container. Compose replaces # only this Compose project below; never remove unrelated containers or prune here. @@ -428,11 +431,27 @@ fi # A graph executor has its own profile, local journal and resource ceiling. # Validate only composed settings; never print the environment document. +# Refused, the executor is not started and the scheduler does no graph work +# for this deploy: its start is given one more Compose file, which composes +# its conversation list blank, so it admits no graph that nothing could +# claim. The full-pipeline shadow, whose own settings were read back above, +# still starts. The composed list is read back first; should it not be +# blank, the shadow is not started at all. One line says which, naming +# settings (the validator prints names only), never a value. if [ "$DELPHI_SHADOW_SELECTED" = 1 ] && [ "$DELPHI_SHADOW_GRAPH_SELECTED" = 1 ]; then - if ! sudo /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.images.yml --profile delphi-shadow --profile delphi-shadow-graph config --format json | python3 scripts/check-shadow-graph.py; then - echo "The graph shadow's composed settings are invalid; not starting either shadow" >&2 - DELPHI_SHADOW_SELECTED=0 + if ! DELPHI_SHADOW_GRAPH_WRONG=$(sudo /usr/local/bin/docker-compose -f docker-compose.yml -f docker-compose.images.yml --profile delphi-shadow --profile delphi-shadow-graph config --format json | python3 scripts/check-shadow-graph.py 2>&1); then DELPHI_SHADOW_GRAPH_SELECTED=0 + DELPHI_SHADOW_GRAPH_WRONG=$(printf '%s\n' "$DELPHI_SHADOW_GRAPH_WRONG" | sed -n 's#^graph shadow settings: \([A-Za-z0-9_/. -]*\)$#\1#p' | tail -n 1) + printf 'services:\n delphi-shadow-scheduler:\n environment:\n - DELPHI_SHADOW_GRAPH_CONVERSATIONS=\n' | sudo tee docker-compose.shadow-graph-off.yml > /dev/null + DELPHI_SHADOW_COMPOSE+=(-f docker-compose.shadow-graph-off.yml) + DELPHI_SHADOW_GRAPH_LIST=$("${DELPHI_SHADOW_COMPOSE[@]}" --profile delphi-shadow config --format json | jq -r '.services["delphi-shadow-scheduler"].environment.DELPHI_SHADOW_GRAPH_CONVERSATIONS // ""') \ + || DELPHI_SHADOW_GRAPH_LIST=unreadable + if [ -n "${DELPHI_SHADOW_GRAPH_LIST//[,[:space:]]/}" ]; then + echo "Error: the graph executor's composed settings are invalid (${DELPHI_SHADOW_GRAPH_WRONG:-unnamed}) and the scheduler's DELPHI_SHADOW_GRAPH_CONVERSATIONS is not composed blank; not starting the Delphi shadow" >&2 + DELPHI_SHADOW_SELECTED=0 + else + echo "Error: the graph executor's composed settings are invalid (${DELPHI_SHADOW_GRAPH_WRONG:-unnamed}): not starting it, and the scheduler starts with DELPHI_SHADOW_GRAPH_CONVERSATIONS blank, so no conversation's graph is admitted until a deploy whose graph settings are valid (the full-pipeline shadow still starts)" >&2 + fi fi fi @@ -710,7 +729,7 @@ if [ "$SERVICE_FROM_FILE" == "delphi" ] && [ "$DELPHI_SHADOW_SELECTED" = 1 ]; th "${COMPOSE[@]}" up -d delphi-shadow-graph --no-build --force-recreate \ || echo "The graph shadow did not start; recovery remains pending" >&2 fi - "${COMPOSE[@]}" up -d delphi-shadow delphi-shadow-scheduler --no-build --force-recreate \ + "${DELPHI_SHADOW_COMPOSE[@]}" up -d delphi-shadow delphi-shadow-scheduler --no-build --force-recreate \ || echo "The Delphi shadow did not start; the deploy continues without it" >&2 fi