Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2.45.2
2.45.3
122 changes: 120 additions & 2 deletions delphi/tests/test_after_install_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import os
import shutil
import subprocess
import sys
from pathlib import Path

import pytest
Expand Down Expand Up @@ -89,7 +90,10 @@ def _find(rel):
# shadow service's two read-backs: its lanes' switches and its settings.
# The Delphi shadow's read-back prints the names of its wrong settings:
# none, unless a test says which (or __fail__: jq itself fails).
"jq": RECORD + 'cat >/dev/null\n' + 'case "$*" in *delphi-shadow-scheduler.DELPHI_SHADOW_ENV*) [ "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}" != __fail__ ] || exit 3; printf "%s\\n" "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}";; *polis-api-shadow-reads*POLIS_API_ROLE*) printf "%s\\n" "${FAKE_READS_SETTINGS_WRONG-}";; *polis-api-shadow-reads*_COMMENT_READS*) printf "%s\\n" "${FAKE_READS_SWITCHES_WRONG-}";; *POLIS_API_ROLE*) printf "%s\\n" "${FAKE_SHADOW_SETTINGS_WRONG-}";; *POLIS_API_DB_TRANSPORT*) printf "%s\\n" "${FAKE_POLIS_API_TRANSPORT:-tls}";; *POLIS_API_START*) printf "%s\\n" "${FAKE_POLIS_API_START-0}";; *RUST_API_ROUTES*) printf "%s\\n" "$FAKE_RUST_API_ROUTES";; *POLIS_API_POST_COMMENTS*) if [ -n "${FAKE_SHADOW_COMMENTS_SWITCH+set}" ]; then printf "%s\\n" "$FAKE_SHADOW_COMMENTS_SWITCH"; elif grep -qx " - POLIS_API_POST_COMMENTS=observe" docker-compose.images.yml 2>/dev/null; then echo observe; else echo; fi;; *) echo value;; esac\n',
# The scheduler's graph list, read back after a refused graph executor,
# is answered as Compose would compose it: blank when the hook wrote the
# file that blanks it, unless a test says what the composed list holds.
"jq": RECORD + 'cat >/dev/null\n' + 'case "$*" in *delphi-shadow-scheduler.DELPHI_SHADOW_ENV*) [ "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}" != __fail__ ] || exit 3; printf "%s\\n" "${FAKE_DELPHI_SHADOW_SETTINGS_WRONG-}";; *polis-api-shadow-reads*POLIS_API_ROLE*) printf "%s\\n" "${FAKE_READS_SETTINGS_WRONG-}";; *polis-api-shadow-reads*_COMMENT_READS*) printf "%s\\n" "${FAKE_READS_SWITCHES_WRONG-}";; *POLIS_API_ROLE*) printf "%s\\n" "${FAKE_SHADOW_SETTINGS_WRONG-}";; *POLIS_API_DB_TRANSPORT*) printf "%s\\n" "${FAKE_POLIS_API_TRANSPORT:-tls}";; *POLIS_API_START*) printf "%s\\n" "${FAKE_POLIS_API_START-0}";; *RUST_API_ROUTES*) printf "%s\\n" "$FAKE_RUST_API_ROUTES";; *POLIS_API_POST_COMMENTS*) if [ -n "${FAKE_SHADOW_COMMENTS_SWITCH+set}" ]; then printf "%s\\n" "$FAKE_SHADOW_COMMENTS_SWITCH"; elif grep -qx " - POLIS_API_POST_COMMENTS=observe" docker-compose.images.yml 2>/dev/null; then echo observe; else echo; fi;; *environment.DELPHI_SHADOW_GRAPH_CONVERSATIONS*) if [ -n "${FAKE_GRAPH_LIST_COMPOSED+set}" ]; then printf "%s\\n" "$FAKE_GRAPH_LIST_COMPOSED"; elif grep -qx " - DELPHI_SHADOW_GRAPH_CONVERSATIONS=" docker-compose.shadow-graph-off.yml 2>/dev/null; then echo; else echo zzgraphlist9; fi;; *) echo value;; esac\n',
"git": RECORD + ('case "$1" in clone) mkdir -p "${@: -1}";; rev-parse) echo 0123456789abcdef0123456789abcdef01234567;;\n'
' merge-base) [ -z "$FAKE_NOT_ON_BRANCH" ] || exit 1;; esac\n'),
"aws": RECORD + (
Expand Down Expand Up @@ -205,10 +209,12 @@ def _deploy(tmp_path, service_type, migration_failure="", revision=REVISION,
shadow_start_failure=False, shadow_comments_switch=None, shadow_settings_wrong=None,
reads_start_failure=False, reads_switches_wrong=None, reads_settings_wrong=None,
env_extra=None, delphi_shadow_settings_wrong=None, delphi_shadow_start_failure=False,
compose_ps="math-python", compose_ps_failure=False, hook_rules=(), **kw):
compose_ps="math-python", compose_ps_failure=False, hook_rules=(), graph_list_composed=None, **kw):
root, env = _box(tmp_path, service_type, **kw)
if env_extra is not None:
env["FAKE_ENV_EXTRA"] = env_extra
if graph_list_composed is not None:
env["FAKE_GRAPH_LIST_COMPOSED"] = graph_list_composed
if delphi_shadow_settings_wrong is not None:
env["FAKE_DELPHI_SHADOW_SETTINGS_WRONG"] = delphi_shadow_settings_wrong
env["FAKE_DELPHI_SHADOW_START_FAILURE"] = "1" if delphi_shadow_start_failure else "0"
Expand Down Expand Up @@ -1296,6 +1302,118 @@ def test_other_roles_never_start_the_delphi_shadow(tmp_path, service_type):
assert DELPHI_SHADOW_UP not in _compose(log)


# The optional graph executor (DELPHI_SHADOW_GRAPH_ENABLED=1 beside the
# shadow's switch) is checked by scripts/check-shadow-graph.py against the
# composed configuration, after the shadow's own read-back. Its refusal is
# about the executor alone: the full-pipeline shadow, already accepted, still
# starts. Here the check is a stand-in that reads the composed configuration
# and accepts or refuses it.
DELPHI_SHADOW_GRAPH_UP = "up -d delphi-shadow-graph --no-build --force-recreate"
DELPHI_SHADOW_GRAPH_CONFIG = COMPOSE_FILES + "--profile delphi-shadow --profile delphi-shadow-graph config --format json"
GRAPH_SWITCHES = "DELPHI_SHADOW_ENABLED=1\nDELPHI_SHADOW_GRAPH_ENABLED=1"


def _graph_check(tmp_path, accepts):
# Run by this interpreter: the hook's PATH here holds the fakes and
# /usr/bin:/bin, which need not hold a python3 (the test image's is in
# /usr/local/bin).
check = tmp_path / "graph-check.py"
check.write_text("import sys\nsys.stdin.read()\n" + ("" if accepts else (
"print('graph shadow settings: QUEUE_ENV/DELPHI_RESULT_ENV', file=sys.stderr)\nsys.exit(1)\n")))
return [("python3 scripts/check-shadow-graph.py", f"{sys.executable} {check}", 1)]


def test_valid_graph_settings_start_the_graph_executor_before_the_shadow(tmp_path):
proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES, hook_rules=_graph_check(tmp_path, True))
assert proc.returncode == 0, proc.stdout + proc.stderr
assert _calls(log, "docker-compose").count(DELPHI_SHADOW_GRAPH_CONFIG) == 1
ups = [c for c in _compose(log) if c.startswith("up")]
assert ups == UNCHANGED["delphi"] + [DELPHI_SHADOW_GRAPH_UP, DELPHI_SHADOW_UP]
# The scheduler keeps its conversation list: no file blanks it.
assert not (tmp_path / "root/opt/polis/polis" / GRAPH_OFF).exists()
assert not [c for c in _calls(log, "docker-compose") if GRAPH_OFF in c]


# Refused, the executor is not started and the scheduler does no graph work
# for the deploy: its start is given one more Compose file that composes its
# conversation list blank, so it admits no graph that nothing could claim.
# The hook reads the composed list back before starting it, and says so in
# one line naming the setting, never a value.
GRAPH_OFF = "docker-compose.shadow-graph-off.yml"
GRAPH_OFF_TEXT = "services:\n delphi-shadow-scheduler:\n environment:\n - DELPHI_SHADOW_GRAPH_CONVERSATIONS=\n"
GRAPH_OFF_FILES = COMPOSE_FILES + f"-f {GRAPH_OFF} "
GRAPH_REFUSED_LINE = (
"Error: the graph executor's composed settings are invalid (QUEUE_ENV/DELPHI_RESULT_ENV): not starting it, "
"and the scheduler starts with DELPHI_SHADOW_GRAPH_CONVERSATIONS blank, so no conversation's graph is "
"admitted until a deploy whose graph settings are valid (the full-pipeline shadow still starts)")


def test_invalid_graph_settings_refuse_the_graph_executor_only(tmp_path):
proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES + "\nDELPHI_SHADOW_GRAPH_CONVERSATIONS=zzgraphlist9",
compose_ps=SHADOW_LEFT_RUNNING, hook_rules=_graph_check(tmp_path, False))
# The Delphi box's deploy is not failed, and the full-pipeline shadow
# starts, its scheduler with no graph list.
assert proc.returncode == 0, proc.stdout + proc.stderr
assert "not starting either shadow" not in proc.stderr
# One loud line, naming the refused settings and the blanked list by key.
graph_lines = [line for line in proc.stderr.splitlines() if "graph" in line]
assert graph_lines == [GRAPH_REFUSED_LINE], proc.stderr
off = tmp_path / "root/opt/polis/polis" / GRAPH_OFF
assert off.read_text() == GRAPH_OFF_TEXT
calls = _calls(log, "docker-compose")
assert GRAPH_OFF_FILES + "--profile delphi-shadow config --format json" in calls
assert '-r .services["delphi-shadow-scheduler"].environment.DELPHI_SHADOW_GRAPH_CONVERSATIONS // ""' in _calls(log, "jq")
ups = [c for c in calls if " up " in f" {c} "]
assert ups == [COMPOSE_FILES + u for u in UNCHANGED["delphi"]] + [GRAPH_OFF_FILES + DELPHI_SHADOW_UP]
assert DELPHI_SHADOW_GRAPH_UP not in _compose(log)
# Only the shadow's start carries the file: the engine's `up` and `down`
# are as before.
assert [c for c in calls if GRAPH_OFF in c] == [GRAPH_OFF_FILES + "--profile delphi-shadow config --format json",
GRAPH_OFF_FILES + DELPHI_SHADOW_UP]
# The validator names settings only; nothing of the environment document
# reaches the deploy's output.
for text in ("PUBLIC_SERVICE_URL", "example.invalid", "zzgraphlist9"):
assert text not in proc.stdout + proc.stderr


def test_a_graph_list_that_cannot_be_composed_blank_refuses_the_shadow(tmp_path):
# Should Compose not let the file's empty value stand, the scheduler
# would admit graphs nothing claims: the shadow is not started, as when
# its own settings are refused, and the deploy goes on.
proc, log = _deploy(tmp_path, "delphi", env_extra=GRAPH_SWITCHES, compose_ps=SHADOW_LEFT_RUNNING,
hook_rules=_graph_check(tmp_path, False), graph_list_composed="zzgraphlist9")
assert proc.returncode == 0, proc.stdout + proc.stderr
graph_lines = [line for line in proc.stderr.splitlines() if "graph" in line]
assert graph_lines == [
"Error: the graph executor's composed settings are invalid (QUEUE_ENV/DELPHI_RESULT_ENV) and the "
"scheduler's DELPHI_SHADOW_GRAPH_CONVERSATIONS is not composed blank; not starting the Delphi shadow"], proc.stderr
assert [c for c in _compose(log) if c.startswith("up")] == UNCHANGED["delphi"]
assert not [c for c in _calls(log, "docker-compose") if c.endswith(DELPHI_SHADOW_UP)]
assert "zzgraphlist9" not in proc.stdout + proc.stderr


@pytest.mark.skipif(shutil.which("jq") is None, reason="runs the hook's jq program with the real jq")
@pytest.mark.parametrize("environment,printed", [
({"DELPHI_SHADOW_GRAPH_CONVERSATIONS": ""}, ""),
({}, ""),
({"DELPHI_SHADOW_GRAPH_CONVERSATIONS": "c1 c2"}, "c1 c2"),
(None, ""),
])
def test_the_real_jq_reads_the_schedulers_composed_graph_list(environment, printed):
(line,) = [line for line in HOOK_PATH.read_text().splitlines()
if line.lstrip().startswith("DELPHI_SHADOW_GRAPH_LIST=$(")]
command, program, closing = line.split("'")
assert command.lstrip() == ('DELPHI_SHADOW_GRAPH_LIST=$("${DELPHI_SHADOW_COMPOSE[@]}" --profile delphi-shadow '
'config --format json | jq -r ')
assert closing.startswith(")")
document = {"services": {"delphi-shadow-scheduler": {"environment": environment}}}
if environment is None:
del document["services"]["delphi-shadow-scheduler"]
proc = subprocess.run(["jq", "-r", program], input=json.dumps(document), text=True, capture_output=True)
assert proc.returncode == 0, proc.stderr
assert proc.stdout == printed + "\n"


# The Delphi box's `down` (with -f files named) leaves a profile-gated
# service's containers running, and the shadow's are: `ps -a --services`
# lists them after `down`. The Delphi role removes by name every service it
Expand Down
70 changes: 69 additions & 1 deletion docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,12 +431,76 @@ same journal lock. On the same host, replacement of the PID 1 container kills
its namespace's processes; a host reboot kills the prior boot's processes.
A daemon restart in the same namespace needs both its recorded daemon and
child process group gone. A missing process-group record is not exit proof.
An entry written under another boot is confirmed only once its attempt's lease
has expired by database time (`graph_recovery` reason
`another_boot_lease_live` while it waits, `another_boot_lease_expired` when it
is confirmed). A copied identity file therefore does not confirm an attempt
whose original still renews its lease. It does confirm one whose original is
alive but has stopped renewing for longer than a lease (a paused container, a
stopped daemon, a frozen machine): writes stay fenced, so the cost is a false
exit record and the step computed twice. An honest reboot waits about one
lease with the defaults (`POLIS_JOBS_LEASE_SECONDS`, default 120), up to one
lease plus one reaper interval (`POLIS_JOBS_REAP_SECONDS`) when that is set
longer.

The deploy check of the executor's composed settings refuses the executor
alone: with an invalid setting the executor is not started, and the scheduler
is started with `DELPHI_SHADOW_GRAPH_CONVERSATIONS` composed blank (one more
Compose file, `docker-compose.shadow-graph-off.yml`, given to the shadow's
start only), so it admits no graph that nothing could claim; the
full-pipeline shadow still starts. The deploy says so in one line beginning
`Error:` that names the refused settings and that list, never a value. A
graph already in flight from an earlier deploy is still watched, as when the
list is cleared. The next deploy with valid settings admits graphs again.

A graph none of whose nodes is claimed within ten minutes (no executor
running, unstartable, crash-looping before its first claim) is cancelled and
let go (`graph_unclaimed`, with the reason, the count of graphs let go in a
row in that env and the hold that follows), so the full-pipeline shadow waits
minutes, not the job deadline. A graph found cancelled with no node ever
claimed, or past the job deadline with none claimed, is let go the same way
(reason `cancelled_unclaimed`). The ten minutes are a constant
(`UNCLAIMED_DEADLINE` in `queue-rs/src/shadow/graph.rs`), not a setting: a
running executor claims within one of its polls (`POLIS_JOBS_POLL_SECONDS`,
default 5), and the deploy pulls every image before starting anything and
starts the executor before the shadow, so on a deploy the executor is never
still being pulled when a graph is admitted. Where it is slower than ten
minutes to start (an image still being pulled or built by a local
`docker compose up`, a slow host, a scheduler restarted with an old graph in
flight), that graph is let go and admitted again, as below; nothing is lost.

After a let-go no graph is admitted in that env for an hour, doubling with
each let-go in a row up to a day (`UNCLAIMED_HOLD`, `UNCLAIMED_HOLD_MAX`),
or until the next code version, whichever is first; the scheduler says so
once per hold (`graph_admission_held`, reason `graph_unclaimed`, with the
count, the seconds until it resumes and the let-go graphs' ids). The
full-pipeline shadow goes on throughout. When the hold ends, the conversation
let go is admitted again, though nothing about it changed, as a new graph:
its admission key names the let-go graph (`graph_admitted` with
`after_let_go`). A claim on a graph admitted after the let-go resets the
count (`graph_unclaimed_cleared`). The hold, the count and the let-go graphs
are kept in the scheduler state (`graph_unclaimed`), so a restart keeps them.
Each graph admitted again adds its rows to the database, as any graph does,
and counts against the shadow's storage budget.

Publication of a finished graph is tried once per poll, up to three times,
the count kept in the scheduler state; a transient failure keeps the graph
(`graph_publish_retry`). A graph whose publication is given up
(`graph_publish_failed`) is not taken up again by recovery, and its snapshot
is not remembered as seen: when a later build or a change reconsiders the
conversation, the same snapshot's admission answers with that graph if the
graph's specification is unchanged (the same stage code, runtime, models and
expected generation), and then only its publication runs again. A build that
changes any of those admits a new graph, and all its steps run again.

The scheduler cancels overdue graphs at `DELPHI_SHADOW_JOB_DEADLINE_SECONDS`
(default 14400). It keeps their scopes held and polls recovery in the background;
other conversations may resume full-pipeline work. The same conversation stays
excluded while its graph scope is held. A lost host or volume cannot supply exit
proof: `graph_recovery_pending` means just that, never successful recovery.
While any graph is deferred no new graph is admitted, in any graph env; the
scheduler says so once, with the deferred graphs' ids
(`graph_admission_held`, reason `graph_deferred`).
Database recovery errors stop admissions until the database can be read again.

The default ceilings are 16 GiB for the full-pipeline daemon, 6 GiB for the graph
Expand All @@ -452,5 +516,9 @@ State written by the previous release loads with empty recovery fields. When no
graph has been enabled, state serialization and full-pipeline behavior are
unchanged. Before rolling back, drain every graph, including deferred graphs,
and verify its scope was released. Older schedulers ignore the new deferred
fields and cannot enforce those exclusions. Do not remove scheduler state or a
fields and cannot enforce those exclusions. They also ignore the publication
count and the given-up publications, so they may try a given-up graph's
publication again, and they ignore the graphs let go as unclaimed, so they
neither hold admission after one nor admit its unchanged conversation again.
Do not remove scheduler state or a
journal to force progress.
4 changes: 2 additions & 2 deletions polis-cli/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion polis-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ members = ["crates/polis-client", "crates/polis", "../queue-rs/polis-contracts"]
resolver = "2"

[workspace.package]
version = "2.45.2"
version = "2.45.3"
edition = "2021"
license = "AGPL-3.0-only"
rust-version = "1.88"
Expand Down
Loading
Loading