diff --git a/configs/ci/integration/reverse_text_rl_opd/start.toml b/configs/ci/integration/reverse_text_rl_opd/start.toml
index edd23df9c7..abd8c69b60 100644
--- a/configs/ci/integration/reverse_text_rl_opd/start.toml
+++ b/configs/ci/integration/reverse_text_rl_opd/start.toml
@@ -1,10 +1,10 @@
-# Smoke test for the RL-entrypoint OPD (on-policy distillation) training mode.
-# The student inference deployment is launched by the rl entrypoint on GPU 0;
-# the teacher inference server is started externally by the test fixture (see
+# Smoke test for the RL-entrypoint opd algorithm (on-policy distillation).
+# The policy inference deployment is launched by the rl entrypoint on GPU 0;
+# the frozen reference server is started externally by the test fixture (see
 # tests/integration/test_reverse_text_rl_opd.py) on the same GPU. Trainer
 # takes GPU 1. Training config mirrors `reverse_text/start.toml`.
 
-max_steps = 5
+max_steps = 20
 seq_len = 2048
 
 [model]
@@ -15,10 +15,16 @@ project = "reverse-text-ci"
 name = "ci-rl-opd"
 
 [orchestrator]
-training_mode = "opd"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo.advantage]
+type = "opd"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
 [orchestrator.renderer]
 name = "qwen3"
 
@@ -38,12 +44,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 
diff --git a/configs/ci/integration/reverse_text_rl_sft/start.toml b/configs/ci/integration/reverse_text_rl_sft/start.toml
index 6b26bb3335..99b21f3348 100644
--- a/configs/ci/integration/reverse_text_rl_sft/start.toml
+++ b/configs/ci/integration/reverse_text_rl_sft/start.toml
@@ -1,10 +1,10 @@
-# Smoke test for the RL-entrypoint SFT (on-policy hard distillation) training
-# mode. The student inference deployment is launched by the rl entrypoint on
-# GPU 0; the teacher inference server is started externally by the test
-# fixture (see tests/integration/test_reverse_text_rl_sft.py) on the same GPU.
-# Trainer takes GPU 1. Training config mirrors `reverse_text/start.toml`.
+# Smoke test for the RL-entrypoint sft algorithm (hard distillation).
+# The policy inference deployment is launched by the rl entrypoint on GPU 0;
+# the frozen sampling server is started externally by the test fixture (see
+# tests/integration/test_reverse_text_rl_sft.py) on the same GPU. Trainer
+# takes GPU 1. Training config mirrors `reverse_text/start.toml`.
 
-max_steps = 5
+max_steps = 20
 seq_len = 2048
 
 [model]
@@ -15,10 +15,25 @@ project = "reverse-text-ci"
 name = "ci-rl-sft"
 
 [orchestrator]
-training_mode = "sft"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo.advantage]
+type = "sft"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
+# The student's renderer tokenizes the teacher's messages into the student's
+# token space (backfill). Use `default` (wraps the student tokenizer's
+# apply_chat_template) so the CE target is faithful to the student's actual
+# chat template. The stock `qwen3` renderer reimplements the Qwen format and
+# injects an empty `<think></think>` block, which this model's custom template
+# does not emit — corrupting the distillation target.
+[orchestrator.renderer]
+name = "default"
+
 [orchestrator.train.sampling]
 max_completion_tokens = 128
 
@@ -35,12 +50,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 
diff --git a/configs/debug/algorithms/README.md b/configs/debug/algorithms/README.md
new file mode 100644
index 0000000000..a670af2247
--- /dev/null
+++ b/configs/debug/algorithms/README.md
@@ -0,0 +1,65 @@
+# Algorithm — Debug Configs
+
+Minimal end-to-end configs for the algorithms against bundled verifiers envs, using `PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT` as the policy.
+
+| Config | Algorithm | Frozen model | Notes |
+|---|---|---|---|
+| `grpo.toml` | `grpo` | none | |
+| `max_rl.toml` | `max_rl` | none | GRPO with mean-normalized advantages (maximum-likelihood RL) |
+| `opd.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
+| `opd_lora.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
+| `sft_distill.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
+| `sft_distill_lora.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
+| `sft_distill_external.toml` | `sft` | PI inference (`openai/gpt-5-mini`) | external OAI endpoint; no local server |
+| `self_distill.toml` | `opsd` | none (`model = "policy"`) | SDFT against the live policy; demo from reverse-text's `answer` field |
+| `echo.toml` | `echo` | none | multi-turn `alphabet-sort`; CE on observation tokens |
+| `mixed_grpo_opd.toml` | `grpo` + `opd` (per env) | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | two envs, one run; heterogeneous batches (with/without `ref_logprobs`) |
+
+The policy inference server is auto-launched on GPU 0 at `http://localhost:8000/v1` with `gpu_memory_utilization=0.5`. The local frozen model (used by `opd*.toml`, `sft_distill.toml` / `sft_distill_lora.toml`, and `mixed_grpo_opd.toml`) is **not** auto-launched — start it manually on GPU 1.
+
+Frozen models are declared inline on the algorithm — `[orchestrator.algo.teacher]` with `name` + `base_url` — and prime-rl never hosts them; only the trainable policy's server is managed by the `rl` entrypoint.
+
+## Start the local frozen model
+
+Needed for `opd*.toml`, `sft_distill.toml` / `sft_distill_lora.toml`, and `mixed_grpo_opd.toml`:
+
+```bash
+CUDA_VISIBLE_DEVICES=1 uv run inference \
+  --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
+  --server.port 8001 \
+  --gpu-memory-utilization 0.5 \
+  --model.enforce-eager
+```
+
+## Run the debug configs
+
+```bash
+# GRPO (no frozen model)
+uv run rl @ configs/debug/algorithms/grpo.toml
+
+# MaxRL (no frozen model)
+uv run rl @ configs/debug/algorithms/max_rl.toml
+
+# OPD (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/opd.toml
+uv run rl @ configs/debug/algorithms/opd_lora.toml
+
+# SFT distillation (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/sft_distill.toml
+uv run rl @ configs/debug/algorithms/sft_distill_lora.toml
+
+# SFT distillation from openai/gpt-5-mini via PI inference
+# (requires PRIME_API_KEY + PRIME_TEAM_ID in env; no local frozen model needed)
+uv run rl @ configs/debug/algorithms/sft_distill_external.toml
+
+# Self-distillation against the live policy (no frozen model)
+uv run rl @ configs/debug/algorithms/self_distill.toml
+
+# ECHO (no frozen model; multi-turn env)
+uv run rl @ configs/debug/algorithms/echo.toml
+
+# Mixed per-env algorithms: GRPO + OPD in one run (needs the frozen model on port 8001)
+uv run rl @ configs/debug/algorithms/mixed_grpo_opd.toml
+```
+
+See [docs/algorithms.md](../../../docs/algorithms.md) for what each algorithm does and how to compose custom ones.
diff --git a/configs/debug/algorithms/echo.toml b/configs/debug/algorithms/echo.toml
new file mode 100644
index 0000000000..59ffe15e44
--- /dev/null
+++ b/configs/debug/algorithms/echo.toml
@@ -0,0 +1,50 @@
+# ECHO on the multi-turn alphabet-sort env (bundled with verifiers): GRPO on
+# action tokens + weighted CE on the env's observation tokens.
+#   uv run rl @ configs/debug/algorithms/echo.toml
+
+max_steps = 20
+seq_len = 4096
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-echo"
+
+[orchestrator]
+batch_size = 32
+group_size = 4
+
+# alphabet-sort's feedback arrives as user messages, so train the user role
+# instead of echo's tool default.
+[orchestrator.algo.advantage]
+type = "echo"
+
+[orchestrator.algo.advantage.roles.user]
+alpha = 0.1
+
+[[orchestrator.train.env]]
+id = "alphabet-sort"
+args = { min_turns = 3, max_turns = 5, power_per_turn = false }
+
+[orchestrator.train.sampling]
+max_completion_tokens = 512
+
+# ECHO learns from observation tokens even when the GRPO advantage collapses
+# to zero — keep zero-advantage rollouts in the batch.
+[[orchestrator.post_batch_filters]]
+type = "zero_advantage"
+enforce = false
+
+# Qwen3 finetune with the standard PI template patch; always re-emits prior
+# <think> blocks, matched by the qwen3 renderer's preserve_all_thinking.
+[orchestrator.renderer]
+name = "qwen3"
+preserve_all_thinking = true
+
+[trainer.optim]
+lr = 1e-6
+
+[inference]
+gpu_memory_utilization = 0.5
diff --git a/configs/debug/training_modes/rl.toml b/configs/debug/algorithms/grpo.toml
similarity index 88%
rename from configs/debug/training_modes/rl.toml
rename to configs/debug/algorithms/grpo.toml
index 27838809b3..7b95156925 100644
--- a/configs/debug/training_modes/rl.toml
+++ b/configs/debug/algorithms/grpo.toml
@@ -5,14 +5,16 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-rl"
 
 [orchestrator]
-training_mode = "rl"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo.advantage]
+type = "grpo"
+
 [orchestrator.renderer]
 name = "qwen3"
 
diff --git a/configs/debug/algorithms/max_rl.toml b/configs/debug/algorithms/max_rl.toml
new file mode 100644
index 0000000000..2803d564b1
--- /dev/null
+++ b/configs/debug/algorithms/max_rl.toml
@@ -0,0 +1,43 @@
+max_steps = 20
+seq_len = 2048
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-max-rl"
+
+[orchestrator]
+batch_size = 128
+group_size = 16
+
+[orchestrator.algo.advantage]
+type = "max_rl"
+
+[orchestrator.renderer]
+name = "qwen3"
+
+[orchestrator.train.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+
+[orchestrator.eval]
+interval = 1
+num_examples = 128
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.eval.env]]
+id = "reverse-text"
+
+[trainer.optim]
+lr = 3e-6
+
+[ckpt]
+
+[inference]
+gpu_memory_utilization = 0.5
diff --git a/configs/debug/algorithms/mixed_grpo_opd.toml b/configs/debug/algorithms/mixed_grpo_opd.toml
new file mode 100644
index 0000000000..185a297fb8
--- /dev/null
+++ b/configs/debug/algorithms/mixed_grpo_opd.toml
@@ -0,0 +1,63 @@
+# Mixed per-env algorithms in one run: a GRPO env and an OPD env, both on
+# reverse-text. Exercises heterogeneous train batches — OPD samples ship
+# ref_logprobs, GRPO samples don't, and both pack into the same micro batches.
+# Start the frozen reference server first (on a separate GPU):
+#   CUDA_VISIBLE_DEVICES=1 uv run inference \
+#     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
+#     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
+# Then:
+#   uv run rl @ configs/debug/algorithms/mixed_grpo_opd.toml
+
+max_steps = 20
+seq_len = 2048
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-mixed-grpo-opd"
+
+[orchestrator]
+batch_size = 128
+group_size = 16
+
+[orchestrator.algo.advantage]
+type = "grpo"
+
+[orchestrator.renderer]
+name = "qwen3"
+
+[orchestrator.train.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+name = "reverse-text-grpo"
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+name = "reverse-text-opd"
+
+[orchestrator.train.env.algo.advantage]
+type = "opd"
+
+[orchestrator.train.env.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
+[orchestrator.eval]
+interval = 5
+num_examples = 128
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.eval.env]]
+id = "reverse-text"
+
+[trainer.optim]
+lr = 3e-6
+
+[inference]
+gpu_memory_utilization = 0.5
diff --git a/configs/debug/training_modes/opd.toml b/configs/debug/algorithms/opd.toml
similarity index 78%
rename from configs/debug/training_modes/opd.toml
rename to configs/debug/algorithms/opd.toml
index 39cbf6a604..0545ee5bef 100644
--- a/configs/debug/training_modes/opd.toml
+++ b/configs/debug/algorithms/opd.toml
@@ -1,9 +1,9 @@
-# Start the teacher inference server first (on a separate GPU):
+# Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
 #     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
 #     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
 # Then:
-#   uv run rl @ configs/debug/training_modes/opd.toml
+#   uv run rl @ configs/debug/algorithms/opd.toml
 
 max_steps = 20
 seq_len = 2048
@@ -12,14 +12,20 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-opd"
 
 [orchestrator]
-training_mode = "opd"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo.advantage]
+type = "opd"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
 [orchestrator.renderer]
 name = "qwen3"
 
@@ -39,12 +45,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 
diff --git a/configs/debug/training_modes/opd_lora.toml b/configs/debug/algorithms/opd_lora.toml
similarity index 79%
rename from configs/debug/training_modes/opd_lora.toml
rename to configs/debug/algorithms/opd_lora.toml
index ba56ffea5c..20f66156be 100644
--- a/configs/debug/training_modes/opd_lora.toml
+++ b/configs/debug/algorithms/opd_lora.toml
@@ -1,9 +1,9 @@
-# Start the teacher inference server first (on a separate GPU):
+# Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
 #     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
 #     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
 # Then:
-#   uv run rl @ configs/debug/training_modes/opd_lora.toml
+#   uv run rl @ configs/debug/algorithms/opd_lora.toml
 
 max_steps = 20
 seq_len = 2048
@@ -12,14 +12,20 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-opd-lora"
 
 [orchestrator]
-training_mode = "opd"
 batch_size = 128
 group_size = 16
 
+[orchestrator.algo.advantage]
+type = "opd"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
 [orchestrator.renderer]
 name = "qwen3"
 
@@ -39,12 +45,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 1e-4
 
diff --git a/configs/debug/algorithms/self_distill.toml b/configs/debug/algorithms/self_distill.toml
new file mode 100644
index 0000000000..b6eb09cf5e
--- /dev/null
+++ b/configs/debug/algorithms/self_distill.toml
@@ -0,0 +1,50 @@
+# Self-distillation (SDFT, https://arxiv.org/abs/2601.19897) against the live
+# policy itself: the reference for each completion is the current model
+# conditioned on the expert demonstration — no extra deployment needed.
+# reverse-text carries the demonstration in its top-level `answer` field.
+#   uv run rl @ configs/debug/algorithms/self_distill.toml
+
+max_steps = 20
+seq_len = 2048
+
+[model]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
+
+[wandb]
+project = "algorithms-debug"
+name = "debug-self-distill"
+
+[orchestrator]
+batch_size = 32
+group_size = 1
+
+# reverse-text's demo lives in the "answer" column.
+[orchestrator.algo.advantage]
+type = "opsd"
+model = "policy"
+demo_key = "answer"
+
+[orchestrator.renderer]
+name = "qwen3"
+
+[orchestrator.train.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.train.env]]
+id = "reverse-text"
+
+[orchestrator.eval]
+interval = 1
+num_examples = 128
+
+[orchestrator.eval.sampling]
+max_completion_tokens = 128
+
+[[orchestrator.eval.env]]
+id = "reverse-text"
+
+[trainer.optim]
+lr = 3e-6
+
+[inference]
+gpu_memory_utilization = 0.5
diff --git a/configs/debug/training_modes/sft.toml b/configs/debug/algorithms/sft_distill.toml
similarity index 55%
rename from configs/debug/training_modes/sft.toml
rename to configs/debug/algorithms/sft_distill.toml
index aed5b30cb3..fccbe64b69 100644
--- a/configs/debug/training_modes/sft.toml
+++ b/configs/debug/algorithms/sft_distill.toml
@@ -1,9 +1,9 @@
-# Start the teacher inference server first (on a separate GPU):
+# Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
 #     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
 #     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
 # Then:
-#   uv run rl @ configs/debug/training_modes/sft.toml
+#   uv run rl @ configs/debug/algorithms/sft_distill.toml
 
 max_steps = 20
 seq_len = 2048
@@ -12,14 +12,28 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-sft"
 
 [orchestrator]
-training_mode = "sft"
 batch_size = 128
 group_size = 4
 
+[orchestrator.algo.advantage]
+type = "sft"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
+# The student's renderer tokenizes the teacher's messages into the student's
+# token space (backfill). Use `default` (wraps the student tokenizer's
+# apply_chat_template) so the CE target is faithful to the student's actual
+# chat template. The stock `qwen3` renderer injects an empty `<think></think>`
+# block this model's custom template never emits, corrupting the target.
+[orchestrator.renderer]
+name = "default"
+
 [orchestrator.train.sampling]
 max_completion_tokens = 128
 
@@ -36,12 +50,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 3e-6
 
diff --git a/configs/debug/training_modes/sft_external.toml b/configs/debug/algorithms/sft_distill_external.toml
similarity index 51%
rename from configs/debug/training_modes/sft_external.toml
rename to configs/debug/algorithms/sft_distill_external.toml
index cb9ea8d09e..2e57883288 100644
--- a/configs/debug/training_modes/sft_external.toml
+++ b/configs/debug/algorithms/sft_distill_external.toml
@@ -1,8 +1,8 @@
-# SFT from openai/gpt-5-mini via PI inference.
+# SFT distillation from openai/gpt-5-mini via PI inference.
 # Requires PRIME_API_KEY + PRIME_TEAM_ID in the environment.
 #
 # Run with:
-#   uv run rl @ configs/debug/training_modes/sft_external.toml
+#   uv run rl @ configs/debug/algorithms/sft_distill_external.toml
 
 max_steps = 20
 seq_len = 2048
@@ -11,14 +11,30 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-sft-external"
 
 [orchestrator]
-training_mode = "sft"
 batch_size = 128
 group_size = 4
 
+[orchestrator.algo.advantage]
+type = "sft"
+
+[orchestrator.algo.teacher]
+name = "openai/gpt-5-mini"
+base_url = ["https://api.pinference.ai/api/v1"]
+api_key_var = "PRIME_API_KEY"
+headers_from_env."X-Prime-Team-ID" = "PRIME_TEAM_ID"
+
+# The student's renderer tokenizes the teacher's messages into the student's
+# token space (backfill). Use `default` (wraps the student tokenizer's
+# apply_chat_template) so the CE target is faithful to the student's actual
+# chat template. The stock `qwen3` renderer injects an empty `<think></think>`
+# block this model's custom template never emits, corrupting the target.
+[orchestrator.renderer]
+name = "default"
+
 [orchestrator.train.sampling]
 max_completion_tokens = 2048
 extra_body = { reasoning_effort = "minimal" }
@@ -36,16 +52,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "openai/gpt-5-mini"
-
-[orchestrator.teacher.client]
-base_url = ["https://api.pinference.ai/api/v1"]
-api_key_var = "PRIME_API_KEY"
-
-[orchestrator.teacher.client.headers_from_env]
-X-Prime-Team-ID = "PRIME_TEAM_ID"
-
 [trainer.optim]
 lr = 3e-6
 
diff --git a/configs/debug/training_modes/sft_lora.toml b/configs/debug/algorithms/sft_distill_lora.toml
similarity index 57%
rename from configs/debug/training_modes/sft_lora.toml
rename to configs/debug/algorithms/sft_distill_lora.toml
index 687b45bbe3..8a91813c34 100644
--- a/configs/debug/training_modes/sft_lora.toml
+++ b/configs/debug/algorithms/sft_distill_lora.toml
@@ -1,9 +1,9 @@
-# Start the teacher inference server first (on a separate GPU):
+# Start the frozen reference server first (on a separate GPU):
 #   CUDA_VISIBLE_DEVICES=1 uv run inference \
 #     --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
 #     --server.port 8001 --gpu-memory-utilization 0.5 --model.enforce-eager
 # Then:
-#   uv run rl @ configs/debug/training_modes/sft_lora.toml
+#   uv run rl @ configs/debug/algorithms/sft_distill_lora.toml
 
 max_steps = 20
 seq_len = 2048
@@ -12,14 +12,28 @@ seq_len = 2048
 name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT"
 
 [wandb]
-project = "reverse-text-debug"
+project = "algorithms-debug"
 name = "debug-sft-lora"
 
 [orchestrator]
-training_mode = "sft"
 batch_size = 128
 group_size = 4
 
+[orchestrator.algo.advantage]
+type = "sft"
+
+[orchestrator.algo.teacher]
+name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
+base_url = ["http://localhost:8001/v1"]
+
+# The student's renderer tokenizes the teacher's messages into the student's
+# token space (backfill). Use `default` (wraps the student tokenizer's
+# apply_chat_template) so the CE target is faithful to the student's actual
+# chat template. The stock `qwen3` renderer injects an empty `<think></think>`
+# block this model's custom template never emits, corrupting the target.
+[orchestrator.renderer]
+name = "default"
+
 [orchestrator.train.sampling]
 max_completion_tokens = 128
 
@@ -36,12 +50,6 @@ max_completion_tokens = 128
 [[orchestrator.eval.env]]
 id = "reverse-text"
 
-[orchestrator.teacher.model]
-name = "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL"
-
-[orchestrator.teacher.client]
-base_url = ["http://localhost:8001/v1"]
-
 [trainer.optim]
 lr = 1e-4
 
diff --git a/configs/debug/multi_env/reverse_text.toml b/configs/debug/multi_env/reverse_text.toml
index e57f65b1ff..66eaa21506 100644
--- a/configs/debug/multi_env/reverse_text.toml
+++ b/configs/debug/multi_env/reverse_text.toml
@@ -9,7 +9,6 @@ project = "reverse-text-debug"
 name = "debug-multi-env"
 
 [orchestrator]
-training_mode = "rl"
 batch_size = 128
 group_size = 16
 
diff --git a/configs/debug/training_modes/README.md b/configs/debug/training_modes/README.md
deleted file mode 100644
index 96ccebb009..0000000000
--- a/configs/debug/training_modes/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# Training Mode — Debug Configs
-
-Minimal end-to-end configs for the three training modes (`rl` / `opd` / `sft`) against the `reverse-text` env, using `PrimeIntellect/Qwen3-0.6B-Reverse-Text-SFT` as the student.
-
-| Config | Mode | Teacher | Notes |
-|---|---|---|---|
-| `rl.toml` | `rl` | none | |
-| `opd.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
-| `opd_lora.toml` | `opd` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
-| `sft.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | |
-| `sft_lora.toml` | `sft` | local vLLM (`Qwen3-0.6B-Reverse-Text-RL`) | trains a LoRA adapter (rank 8) |
-| `sft_external.toml` | `sft` | PI inference (`openai/gpt-5-mini`) | external OAI endpoint; no local teacher |
-
-The student inference server is auto-launched on GPU 0 at `http://localhost:8000/v1` with `gpu_memory_utilization=0.5`. The local teacher (used by everything except `rl.toml` and `sft_external.toml`) is **not** auto-launched — start it manually on GPU 1.
-
-## Start the local teacher
-
-Needed for `opd*.toml` and `sft.toml` / `sft_lora.toml`:
-
-```bash
-CUDA_VISIBLE_DEVICES=1 uv run inference \
-  --model.name PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL \
-  --server.port 8001 \
-  --gpu-memory-utilization 0.5 \
-  --model.enforce-eager
-```
-
-## Run the debug configs
-
-```bash
-# RL (no teacher)
-uv run rl @ configs/debug/training_modes/rl.toml
-
-# OPD (needs teacher on port 8001)
-uv run rl @ configs/debug/training_modes/opd.toml
-uv run rl @ configs/debug/training_modes/opd_lora.toml
-
-# SFT hard distill (needs teacher on port 8001)
-uv run rl @ configs/debug/training_modes/sft.toml
-uv run rl @ configs/debug/training_modes/sft_lora.toml
-
-# SFT hard distill from openai/gpt-5-mini via PI inference
-# (requires PRIME_API_KEY + PRIME_TEAM_ID in env; no local teacher needed)
-uv run rl @ configs/debug/training_modes/sft_external.toml
-```
-
-See [docs/training.md](../../docs/training.md#training-modes-rl--opd--sft-via-orchestrator) for what each mode does.
diff --git a/configs/elastic/README.md b/configs/elastic/README.md
index 3772520eec..388d8cd26b 100644
--- a/configs/elastic/README.md
+++ b/configs/elastic/README.md
@@ -8,7 +8,7 @@ To test the elastic inference pool without having the `rl` entrypoint start the
    uv run rl @ examples/alphabet_sort/rl_elastic.toml
    ```
 
-   The config uses `[orchestrator.client.elastic]` with `hostname = "localhost"` and `port = 8000`. The orchestrator will wait for at least one ready server before starting rollouts.
+   The config uses `[orchestrator.model.client.elastic]` with `hostname = "localhost"` and `port = 8000`. The orchestrator will wait for at least one ready server before starting rollouts.
 
 2. **Start the inference server manually** (in a separate terminal or on another machine):
 
diff --git a/configs/elastic/rl.toml b/configs/elastic/rl.toml
index 3073dc4537..7ac4bfd275 100644
--- a/configs/elastic/rl.toml
+++ b/configs/elastic/rl.toml
@@ -35,7 +35,7 @@ group_size = 8
 [orchestrator.train.sampling]
 max_completion_tokens = 768
 
-[orchestrator.client.elastic]
+[orchestrator.model.client.elastic]
 hostname = "localhost"
 port = 8000
 sync_interval = 5.0
diff --git a/docs/algorithms.md b/docs/algorithms.md
index 0ffe69edba..d850cab525 100644
--- a/docs/algorithms.md
+++ b/docs/algorithms.md
@@ -1,25 +1,159 @@
 # Algorithms
 
-This page covers the math and the configurable algorithmic components: how off-policy training works, the default loss and advantage functions, how to plug in your own, the filters applied between rollout and training, and how multi-turn rollouts get merged into training samples.
+This page covers the math and the configurable algorithmic components: the algorithm abstraction and its algorithms, how off-policy training works, the loss components and advantage functions, how to plug in your own, the filters applied between rollout and training, and how multi-turn rollouts get merged into training samples.
 
 ## Table of Contents
 
+- [The Algorithm Abstraction](#the-algorithm-abstraction)
+  - [Model References](#model-references)
+  - [The Algorithms](#the-algorithms)
+  - [Customizing Components](#customizing-components)
+  - [Per-Env Algorithms](#per-env-algorithms)
+  - [The Algorithm Classes](#the-algorithm-classes)
 - [Async / Off-Policy Training](#async--off-policy-training)
 - [Loss](#loss)
-  - [Default Loss](#default-loss)
+  - [Loss Components](#loss-components)
+  - [Default RL Loss](#default-rl-loss)
   - [Custom Loss](#custom-loss)
 - [Advantage](#advantage)
   - [Default Advantage](#default-advantage)
   - [Custom Advantage](#custom-advantage)
+  - [Reference Scoring](#reference-scoring)
 - [Filters](#filters)
-- [Difficulty Pools](#difficulty-pools)
-- [Online Difficulty Filtering](#online-difficulty-filtering)
 - [Multi-Turn Trajectories](#multi-turn-trajectories)
   - [Extension Property](#extension-property)
   - [Best-Effort Interleaving](#best-effort-interleaving)
   - [Renderers](#renderers)
   - [Discontinuous Trajectories](#discontinuous-trajectories)
 
+## The Algorithm Abstraction
+
+A training algorithm in `prime-rl` is a bundle of two components, configured under `[orchestrator.algo]`:
+
+1. **Sampling** (`algo.sampling`) — how train rollouts are produced: which model generates them. `source` is a [model reference](#model-references): `"policy"` (the live policy, the default) or an inline frozen hosted model. Group sizing stays on the env config (`group_size`).
+2. **Advantage** (`algo.advantage`) — the per-token training signal: credit assignment and loss routing, fused. One mapping from a finalized rollout to per-token *(loss component, weight)* pairs — the credit a token gets and the loss that consumes it are two coordinates of the same output. Group-relative strategies compute credit on the orchestrator and ship per-token advantage streams; reference-KL strategies query a reference model at batch-ship time (bounded concurrency) and ship its prefill logprobs for the trainer to evaluate against the live policy. The strategy determines which loss component consumes the action tokens (`rl` / `ce` / `ref_kl`) and what happens to env-provided observation tokens in multi-turn rollouts (masked out by default; `echo` trains on them with weighted CE).
+
+The trainer is algorithm-blind: the loss is a sum of three components (rl, ce, ref_kl), each normalized by its own global token count; per-token streams ship on the wire (the `rl_weights` / `ce_weights` / `ref_kl_weights` component weights plus the `advantages` stream on each training sample) and the trainer just executes them. Adding an algorithm never touches the dispatcher, packer, or trainer hot path.
+
+### Model References
+
+`prime-rl` hosts exactly one model: the trainable policy (`[orchestrator.model]`). Every other model an algorithm uses is an external OpenAI-compatible endpoint, declared *inline on the component that uses it*. A model reference is either the string `"policy"` (the live policy) or a frozen hosted model (`name` + `base_url`):
+
+```toml
+[orchestrator.algo.advantage]
+type = "opd"
+
+[orchestrator.algo.teacher]   # alias for `model`; folds into advantage.model
+name = "Qwen/Qwen3-32B"
+base_url = ["http://localhost:8001/v1"]
+```
+
+Model *roles* are labels the algorithm itself declares over these references — the distillation algorithms declare their reference's role as `teacher`, so `[orchestrator.algo.teacher]` parses as an alias for the `model` shorthand and validation errors speak the same language ("advantage 'opd' needs a teacher"). No role exists outside the algorithm that declares it: the dispatcher, sink, and trainer branch on liveness alone, never on what an algorithm calls a model.
+
+`algo.model` (alias: `algo.teacher`) is shorthand for the slot the advantage type declares for its reference — `advantage.model` for `opd` / `opsd`, `sampling.source` for `sft` (its teacher is the sampling source). A slot you didn't set takes the shorthand; an explicit reference that already equals it is accepted, a disagreeing one is an error. Set the component fields directly for multi-model setups.
+
+Liveness is a property of the reference, not of any role: rollouts sampled from `"policy"` get version-salted prefix caches, carry sampling logprobs for importance ratios, and age off-policy as weights update; rollouts and scores from frozen models get a stable prefix cache and never go stale. Frozen models are externally hosted (`base_url` is required) — `prime-rl` never launches or updates them, and each env's algorithm builds its own client pool to the endpoints it declares.
+
+### The Algorithms
+
+The advantage `type` names the algorithm, and each type's class defaults are its vetted setting — picking a type with no other keys IS the algorithm:
+
+```toml
+[orchestrator.algo.advantage]
+type = "grpo"  # the default
+```
+
+| `type` | Sampling | Loss | What it is |
+|---|---|---|---|
+| `grpo` | policy | `rl` on actions | Standard group-relative RL. |
+| `max_rl` | policy | `rl` on actions | MaxRL ([arXiv:2602.02710](https://arxiv.org/abs/2602.02710)): GRPO's centered reward normalized by the group **mean** instead of the standard deviation — the gradient is unbiased for the order-`group_size` truncation of the maximum-likelihood objective, upweighting hard examples like `1/p`. |
+| `opd` | policy | `ref_kl` on actions | On-policy distillation ([Thinking Machines](https://thinkingmachines.ai/blog/on-policy-distillation/)): the policy samples, per-token reverse KL against a reference model as the gradient signal. Needs a `teacher`. |
+| `sft` | *(the teacher)* | `ce` on actions | Hard distillation: a frozen model generates rollouts, the policy trains with CE on its tokens. Needs a `teacher` (folds into `sampling.source`). |
+| `opsd` | policy | `ref_kl` on actions | SDFT ([arXiv:2601.19897](https://arxiv.org/abs/2601.19897)): the model is its own reference, conditioned on an expert demonstration. Defaults to the live policy (the paper's setting, no extra deployment); set an inline `model` to score under a frozen copy instead. |
+| `echo` | policy | `rl` on actions + weighted `ce` on observations | ECHO: standard GRPO plus a cross-entropy loss on env-provided tokens already present in the rollout, selected by message role (needs the renderer's role attribution). Defaults to tool-response bodies at `alpha = 0.1` (ECHO's λ); set `roles` to train other roles, each at its own weight. |
+| `reward` | policy | `rl` on actions | REINFORCE-style: advantage = raw reward, no group baseline. |
+| `custom` | policy | `rl` on actions | Your own advantage function (`import_path`), per-token advantages per rollout — see [Custom Advantage](#custom-advantage). |
+
+### Customizing Components
+
+Every key beyond `type` is visibly your own assembly — there is no preset layer to diverge from. The vetted setting is the class defaults; what you set is what runs:
+
+```toml
+# echo on tool AND user feedback tokens, each at its own weight.
+# Setting any role replaces the whole table.
+[orchestrator.algo.advantage]
+type = "echo"
+
+[orchestrator.algo.advantage.roles.tool]
+alpha = 0.25
+
+[orchestrator.algo.advantage.roles.user]
+alpha = 0.05
+
+# or a custom advantage strategy:
+# [orchestrator.algo.advantage]
+# type = "custom"
+# import_path = "my_module.normalized_advantage"
+```
+
+Echo also takes an optional user-supplied token filter that narrows the role selection per rollout — e.g. dropping warning lines from tool output, or tokens the sampler found unlikely:
+
+```toml
+[orchestrator.algo.advantage.filter]
+import_path = "my_module.drop_warnings"
+kwargs = { patterns = ["WARNING"] }
+```
+
+```python
+# my_module.py — sees the raw rollout (message text, sampling logprobs);
+# returns one keep-mask per trajectory step, spanning that step's
+# prompt_ids + completion_ids. False = never echo-trained.
+def drop_warnings(rollout, *, patterns: list[str]) -> list[list[bool]]: ...
+```
+
+Component compatibility is validated at config time: frozen-model sampling can only feed the `ce` loss component — the `rl` and `ref_kl` components need the live policy's own sampling logprobs for importance ratios — `opd` pointed at `"policy"` is rejected as degenerate (zero KL), `sft` without a frozen source is rejected (CE on the policy's own tokens is not a distillation target), and group-relative advantage with `group_size = 1` warns that every advantage collapses to zero.
+
+### Per-Env Algorithms
+
+Both components resolve per environment. Each env inherits `[orchestrator.algo]` unless it sets its own, so a single run can mix algorithms across envs — e.g. GRPO on math, ECHO on a terminal env:
+
+```toml
+[orchestrator.algo.advantage]
+type = "grpo"
+
+[[orchestrator.train.env]]
+id = "math-env"     # inherits grpo
+
+[[orchestrator.train.env]]
+id = "terminal-env"
+advantage = { type = "echo" }   # shorthand: the env assembles its own algorithm
+```
+
+### The Algorithm Classes
+
+At runtime, each env's resolved config builds two objects: a `Sampler` (`prime_rl.orchestrator.sampler`) from the `sampling` component — the pool rollouts are generated from, and the home of future sampling strategies like replay buffers or branching — and one of the named algorithm classes in `prime_rl.orchestrator.algo` (one module per algorithm: `algo/grpo.py`, `algo/opd.py`, …) from the `advantage` component. Algorithm dispatch is keyed on `advantage.type` — it names the algorithm, and each config class's defaults are its vetted parameterization:
+
+| `advantage.type` | Class | hook(s) — stage |
+|---|---|---|
+| `grpo` | `GRPOAlgorithm` | `score_group`: group-norm credit (optional length penalty) |
+| `echo` | `EchoAlgorithm` | `score_rollout`: weighted ce on observation tokens; `score_group`: group-norm credit (inherited) |
+| `max_rl` | `MaxRLAlgorithm` | `score_group`: mean-normalized group credit |
+| `opd` | `OPDAlgorithm` | `score_batch`: own-context prefill under the teacher |
+| `opsd` | `OPSDAlgorithm` | `score_batch`: demo-conditioned prefill under the teacher |
+| `sft` | `SFTDistillAlgorithm` | `score_group`: group-norm credit (feeds filters) |
+| `reward` | `RewardAlgorithm` | `score_rollout`: raw reward |
+| `custom` | `CustomAlgorithm` | `score_group`: your function |
+
+Each class owns its hooks outright — reading one top to bottom reads the algorithm, and everything on the class is an override point. The three hooks are one scope-and-timing ladder — each wider scope is unlocked by a later barrier, so the two axes coincide. Each is handed a `RolloutView` (a writable handle exposing only what is valid at its stage: `raw`, `samples`, `reward`, and `assign_advantages` — never not-yet-assigned credit or pipeline-internal lifecycle fields):
+
+- `score_rollout(rollout)` — one rollout, **on arrival** (as it's tokenized, before its group is complete): rollout-local credit (`rollout.assign_advantages(...)`, scalar broadcast or per-token) or observation ce weights. No siblings. `reward` writes its raw reward here; `echo` weights observation tokens here, reading interleaving's `obs_spans` provenance (which maps merged completion positions back to trajectory-step coordinates), looking up each source step's role attribution, applying the optional user filter, and writing the `ce_weights` stream.
+- `score_group(group)` — the cohort, **before filtering** (filters read the streams), synchronous: group-relative credit (GRPO/MaxRL baselines). `group` is a list of `RolloutView`.
+- `async score_batch(batch)` — the batch's survivors, **after filtering** (dropped rollouts never cost reference compute), async: the only stage with model access — query the algorithm's reference pool (e.g. `self.teacher_pool`, connected in its `setup()` override via `self.connect(...)` — the live policy pool when the reference is `"policy"`, a freshly connected client pool when frozen) and attach per-token results, or modulate advantages.
+
+The pipeline drives the hooks through three module-level phase functions it never looks inside: `finalize_rollout(algorithm, rollout)` per arrival, `finalize_group(algorithm, rollouts)` per group (scoring + wire stamping; after this the records are frozen — groups die at stamping), and `finalize_batch(train_envs, rollouts)` per batch. Sample construction (interleaving) is pure pipeline — it records the `obs_spans` provenance for any algorithm that trains on env-provided tokens.
+
+Class-level declarations state what the algorithm needs: which loss component its action tokens feed (`action_loss_type`) and what it calls its reference model (`model_role`, e.g. `"teacher"`). Every class is constructed with its advantage config — the component it interprets; the bundle dissolves at construction — plus the two host-owned resources: the policy pool and the policy's renderer. Text → token ids always goes through the renderer, the same path the policy's own prompts take (`opsd` requires one, validated at config time). The pipeline only ever calls the phase functions — writing your own algorithm is subclassing `Algorithm` and overriding the hooks its signal needs. For pure credit assignment, no subclass is needed: `advantage.type = "custom"` imports a plain advantage function (see [Custom Advantage](#custom-advantage)); custom reference scoring means forking one of the named classes. Shared math (group normalization, prefill alignment) lives as plain functions in `prime_rl.orchestrator.algo.advantage`.
+
 ## Async / Off-Policy Training
 
 `prime-rl` is asynchronous by default. The trainer and inference always run one step overlapped: while the trainer is producing $\pi_n$ from rollouts at step $n$, inference is already generating the rollouts for step $n+1$ using $\pi_{n-1}$. With matched trainer and inference step times this produces fully-overlapped pipeline parallelism — neither side ever idles.
@@ -35,7 +169,21 @@ Step indices are 0-indexed so the gap holds at startup — inference is exactly
 
 ## Loss
 
-### Default Loss
+### Loss Components
+
+The training loss is a **sum of three components**, each with its own per-token weight stream and its own normalization:
+
+$$
+\mathcal{L} = \frac{\sum \mathcal{L}_{rl}}{N_{rl}} + \frac{\sum \mathcal{L}_{ce}}{N_{ce}} + \frac{\sum \mathcal{L}_{ref\_kl}}{N_{ref\_kl}}
+$$
+
+- `rl` — the configured RL loss (`[trainer.loss]`): DPPO + KL by default, or a [custom loss](#custom-loss). Fed by the group-relative advantage strategies (`grpo`, `max_rl`, `reward`, `custom`, and `echo`'s action tokens).
+- `ce` — masked NLL. Used for frozen-model tokens (`sft`) and env-observation tokens (`echo`).
+- `ref_kl` — the per-token reverse KL to a reference model ($\log \pi_{\text{ref}} - \log \pi$) as the policy-gradient signal, importance-ratio corrected with a one-sided trust region (`opd`, `opsd`). Requires `ref_logprobs` from a [reference scoring](#reference-scoring); the scoring model must be a vLLM server (it's the only one that exposes `prompt_logprobs`).
+
+The orchestrator stamps each sample's component membership as per-token weight streams (`rl_weights` / `ce_weights` / `ref_kl_weights` on the wire): a weight scales that component's per-token loss, `0.0` leaves the token out of the component entirely (mask *and* denominator), and components may overlap on the same token — their gradients sum. Each $N$ is the global (all-reduced) count of that component's member tokens, so the components don't dilute each other: adding echo observation tokens never changes the rl term's effective per-token learning rate, and an sft env packed next to a GRPO env doesn't soften its gradient. Tokens of different components pack freely into the same micro batch, and a plain GRPO run ships no weight streams at all (absent streams mean rl weight 1.0 on every trainable token — the unchanged hot path). Advantages always ship per token (`advantages` on the wire), assigned as per-token streams from the start — uniform group credit is broadcast over completion tokens at assignment; algorithms with no rl credit (opd, opsd) ship none.
+
+### Default RL Loss
 
 The default RL loss is a DPPO policy-gradient term combined with a KL regularizer similar to Kimi-K2.5. For each prompt $x_j$ we sample a group of $G$ rollouts $\{y_i\}_{i=1}^G$, score them to get $s_i$, then optimize:
 
@@ -66,20 +214,14 @@ The knobs (under `[trainer.loss]` with `type = "default"`):
 | Knob | Default | What it does |
 |---|---|---|
 | `dppo_mask_low` / `dppo_mask_high` | 0.2 / 0.2 | Lower / upper thresholds for DPPO-style token-level masking. |
-| `adv_tau` | 1.0 | Temperature on the advantage term. Set to 0 for pure distillation (no RL signal). |
+| `adv_tau` | 1.0 | Temperature on the advantage term. Set to 0 to drop the policy-gradient term, leaving only the KL regularizer. |
 | `kl_tau` | 1e-3 | Temperature on the KL regularizer. Set to 0 to disable. |
 
-The trainer dispatches automatically based on the batch's training mode (set by the orchestrator via `orchestrator.training_mode`):
-
-- `rl` mode → DPPO + KL with the advantage signal.
-- `opd` mode → KL distillation against the teacher's per-token logprobs. The teacher must be a vLLM server (it's the only one that exposes `prompt_logprobs`).
-- `sft` mode → standard token-level NLL on teacher-generated rollouts.
-
-Set `[trainer.loss] type = "default"` and configure via the knobs above. SFT and OPD modes ignore the policy-gradient–specific fields.
+Set `[trainer.loss] type = "default"` and configure via the knobs above. The `ce` and `ref_kl` components are fixed and unaffected by `[trainer.loss]`.
 
 ### Custom Loss
 
-The loss is computed **per sequence**: you write a function that takes one sequence's tensors and returns a scalar loss. The trainer iterates and aggregates.
+`[trainer.loss] type = "custom"` replaces the `rl` component. The loss is computed **per sequence**: you write a function that takes one sequence's tensors and returns a scalar loss. The trainer iterates and aggregates. `inputs.loss_mask` selects exactly the rl member tokens (for a plain GRPO run, all trainable tokens).
 
 ```python
 # my_module.py
@@ -116,9 +258,10 @@ The dataclasses:
 class LossInputs:
     trainer_logprobs: Float[Tensor, "seq"]      # current policy
     inference_logprobs: Float[Tensor, "seq"]    # rollout-time policy
-    teacher_logprobs: Float[Tensor, "seq"] | None  # only set in OPD mode
+    ref_logprobs: Float[Tensor, "seq"] | None   # set by reference-scoring algorithms
     advantages: Float[Tensor, "seq"]
-    loss_mask: Bool[Tensor, "seq"]
+    loss_mask: Bool[Tensor, "seq"]              # this component's member tokens
+    loss_weights: Float[Tensor, "seq"] | None   # the component's weight stream (None = 1.0)
 
 @dataclass
 class LossOutputs:
@@ -130,32 +273,59 @@ Anything you put in `metrics` is averaged across sequences and logged with the o
 
 ## Advantage
 
+The advantage strategy is the `advantage` component of the [algorithm](#the-algorithm-abstraction) — every training signal is a per-token advantage stream, varying in evaluation site (orchestrator vs. trainer). `[orchestrator.advantage]` (and per-env `advantage = {...}`) is shorthand for `algo.advantage`. Types:
+
+| Type | Component | Effect |
+|---|---|---|
+| `grpo` | `rl` | Group-norm: reward minus per-group baseline, optional length penalty. |
+| `max_rl` | `rl` | Mean-normalized group credit (maximum-likelihood RL). |
+| `echo` | `rl` + `ce` | Group-norm on action tokens, plus weighted CE on env-provided tokens selected by message role (each role's `alpha` is its ECHO λ), optionally narrowed by a user filter. |
+| `reward` | `rl` | Advantage = raw reward, no baseline. |
+| `opd` | `ref_kl` | On-policy distillation: per-token reverse KL to a reference model (`model`, an inline frozen hosted model), evaluated in the trainer from shipped reference logprobs. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream; `group_size` only fans out sampling. |
+| `opsd` | `ref_kl` | SDFT: per-token reverse KL to a demo-conditioned reference. No credit — rollouts keep `advantages = None` (advantage-based filters never fire) and ship no advantage stream. |
+| `sft` | `ce` | Cross-entropy on the sampled tokens. The loss ignores advantages, but group-relative credit is still assigned so reward-based filtering keeps working. |
+| `custom` | `rl` | Your function (below); per-token advantages per rollout. |
+
 ### Default Advantage
 
 The default advantage is per-group reward minus per-group baseline (DR-GRPO without std normalization). For each prompt's group of `group_size` rollouts, every token in rollout $i$ receives advantage $s_i - \bar{s}$ where $\bar{s}$ is the group mean.
 
 This is intentionally simple — it does the right thing for most envs. Switch to a [custom advantage](#custom-advantage) when you need group-aware shaping that depends on trajectory metadata (sub-agent rollouts, relative-rank shaping, …).
 
-Two built-in **length penalties** can be layered on top of any advantage to discourage rambling:
+Three built-in **length penalties** (`length_penalty` on the `grpo`-family strategies) can be layered on top to discourage rambling. `tokens` and `turns` are correctness-gated efficiency shaping: in mixed groups the lower-cost correct rollouts get amplified advantage (`tokens` by weighted token cost, `turns` by turn count). `linear` instead subtracts a `coef * pass_rate * (completion tokens / orchestrator.seq_len)` term from every reward before the baseline subtraction, where `pass_rate` is the group's mean reward — so problems the model already solves reliably get the strongest push toward concise outputs, while rarely-solved problems are barely penalized (a never-solved group, mean reward 0, gets none). Set `gate_by_correctness = true` to apply the linear penalty only to correct rollouts (`reward == 1`):
+
+```toml
+[orchestrator.advantage]
+type = "grpo"
+
+[orchestrator.advantage.length_penalty]
+type = "linear"
+coef = 0.25                 # effective penalty is coef * pass_rate * (completion_tokens / seq_len)
+gate_by_correctness = false # when true, only penalize rollouts with reward == 1
+```
+
+By default the GRPO baseline is the plain group mean reward. Set `length_weighted_baseline = true` to instead use the token-length-weighted mean — `sum(len_i * reward_i) / sum(len_i)` — which centers advantages by per-token expected reward when rollouts vary a lot in length (it applies to the plain and `linear` paths; `tokens` / `turns` keep their own baseline):
 
-- `[orchestrator.length_penalty] type = "tokens"` — penalizes long completions in tokens, with configurable target and slope.
-- `[orchestrator.length_penalty] type = "turns"` — penalizes long multi-turn rollouts by turn count.
+```toml
+[orchestrator.advantage]
+type = "grpo"
+length_weighted_baseline = true
+```
 
 
 ### Custom Advantage
 
-Advantages are computed **per group**. You write a function that takes one group of rollouts and returns one advantage scalar per rollout. The orchestrator handles groups of varying size automatically — partial-group training kicks in when some rollouts in a group errored.
+Advantages are computed **per group**. You write a function that takes one group's `RolloutView`s — the same handles the `score_group` hook sees — and returns one value per rollout: a scalar (broadcast over that rollout's completion tokens) or a per-token list aligned to them (for multi-turn envs the merged completion, including interleaved observation tokens). There is no scalar advantage stored anywhere in the pipeline — the scalar is just a convenience the view broadcasts at write time. The orchestrator handles groups of varying size automatically — partial-group training kicks in when some rollouts in a group errored.
 
 ```python
 # my_module.py
 import statistics
-from prime_rl.orchestrator.advantage import AdvantageInputs, AdvantageOutputs
 
-def normalized_advantage(inputs: AdvantageInputs, eps: float = 1e-8) -> AdvantageOutputs:
-    rewards = [r["reward"] for r in inputs.rollouts]
+def normalized_advantage(group, eps: float = 1e-8) -> list[float]:
+    rewards = [v.reward for v in group]
     mean = statistics.fmean(rewards)
     std = statistics.pstdev(rewards) if len(rewards) > 1 else 0.0
-    return AdvantageOutputs(advantages=[(r - mean) / (std + eps) for r in rewards])
+    return [(r - mean) / (std + eps) for r in rewards]  # one scalar per rollout
 ```
 
 ```toml
@@ -165,24 +335,39 @@ import_path = "my_module.normalized_advantage"
 kwargs = { eps = 1e-8 }
 ```
 
-`AdvantageInputs.rollouts` is a list of `verifiers.RolloutOutput`, so you have access to the full rollout (turns, tool calls, custom metadata) — not just the reward. Use this for anything reward-shaping-like that needs trajectory context.
+Each `RolloutView` exposes `raw` (the env's untouched `verifiers.RolloutOutput`: turns, tool calls, custom metadata), `samples` (the merged token sequences), and `reward` — so you have the full interleaved rollout, not just the reward. Use this for anything reward-shaping-like that needs trajectory context.
+
+Genuinely per-token credit (process rewards, step-level credit assignment) returns shaped lists instead of scalars:
+
+```python
+def step_weighted_advantage(group) -> list[list[float]]:
+    rewards = [v.reward for v in group]
+    baseline = statistics.fmean(rewards)
+    return [
+        [(reward - baseline) * w for w in my_token_weights(view.raw)]  # one float per completion token
+        for reward, view in zip(rewards, group)
+    ]
+```
 
-### Per-Env Advantage
+Each per-token list must match the rollout's completion-token count exactly — validated loudly when the view writes it. Advantage-based filters and metrics derive from the streams (the zero-advantage filter checks for all-zero streams; logged distributions use per-rollout means). Signals that depend on the live policy's weights (like OPD's reverse KL) cannot be precomputed here; those are reference-scoring algorithms, evaluated in the trainer.
 
-`advantage` can be set per training environment. Each env inherits the top-level `[orchestrator.advantage]` when it doesn't set its own, so mixed-env runs can give each env its own advantage computation:
+### Reference Scoring
 
-```toml
-[orchestrator.advantage]
-type = "default"  # the default every env inherits unless it overrides
+`OPDAlgorithm` / `OPSDAlgorithm` have an async ship-time half (`score_batch`): at batch-ship time they query their teacher (`model`, a [model reference](#model-references)) with bounded concurrency (`max_concurrent`, default 32) and attach per-token reference logprobs to each sample:
 
-[[orchestrator.train.env]]
-id = "math-env"   # inherits the default above
+- `opd` — score each sample's own context under the reference model via prefill; fills `ref_logprobs` for the `ref_kl` loss component (on-policy distillation). `model = "policy"` is rejected (the KL would be identically zero).
+- `opsd` — SDFT: rebuild the prompt with an expert demonstration woven into the last user message (`template`, with `{question}` / `{demonstration}` placeholders), score the policy's completion under that demo-conditioned context. `model = "policy"` scores under the live policy itself — the SDFT setting, no extra deployment. The demonstration is read from the example's `info[demo_key]`, falling back to a top-level rollout field of the same name (e.g. `answer`); single-step trajectories only.
 
-[[orchestrator.train.env]]
-id = "agent-env"
-advantage = { type = "custom", import_path = "my_module.normalized_advantage" }
+```toml
+[orchestrator.algo.advantage]
+type = "opsd"
+model = "policy"
+demo_key = "demonstration"
+max_concurrent = 64
 ```
 
+Only batch survivors get scored — rollouts that are filtered or cancelled never cost reference compute. The time shows up as `time/scoring` in the step timing.
+
 ## Filters
 
 Filters drop rollouts between scoring and training. Built-ins (composable):
@@ -193,58 +378,19 @@ Filters drop rollouts between scoring and training. Built-ins (composable):
 | `repetition` | Drops rollouts with high n-gram repetition. |
 | `zero_advantage` | Drops rollouts whose advantage is zero, so the trainer doesn't waste tokens on them. |
 
-The default `[orchestrator]` config already includes all three filters with their defaults. To override, set `filters` explicitly — the list replaces the defaults wholesale:
+The default `[orchestrator]` config registers all three in both filter slots: `post_batch_filters` enforce by default (flagged rollouts are recorded but not shipped to the trainer), while `pre_batch_filters` run in monitor mode (`enforce = false`); flip `enforce = true` there to drop matching rollouts before they consume a slot in the batch. Setting a slot replaces its defaults wholesale:
 
 ```toml
-[[orchestrator.filters]]
+[[orchestrator.post_batch_filters]]
 type = "zero_advantage"
 
-[[orchestrator.filters]]
+[[orchestrator.post_batch_filters]]
 type = "repetition"
 threshold = 0.4
 ```
 
 Filtered rollouts still appear in W&B distributions, just not in the trainer batch — useful for spotting whether filtering is doing its job.
 
-## Difficulty Pools
-
-Difficulty pools gradually retire problems the model has solved or never solves. After each rollout, the average reward across a problem's group is compared to two thresholds:
-
-- `buffer.easy_threshold` — at or above this, the problem moves into the `easy` pool and is no longer sampled.
-- `buffer.hard_threshold` — at or below this, the problem moves into the `hard` pool and is no longer sampled.
-- Otherwise the problem stays in `normal` and remains in the sampling rotation.
-
-Pool assignments persist across checkpoints (`easy_examples.jsonl` / `hard_examples.jsonl` under each step's orchestrator checkpoint). When you resume — or want to broaden the curriculum mid-run — `buffer.easy_fraction` / `buffer.hard_fraction` randomly lift that fraction of pooled problems back into `normal` so they re-enter sampling.
-
-```toml
-[orchestrator.buffer]
-easy_threshold = 0.95
-hard_threshold = 0.05
-easy_fraction = 0.0   # default; bump on resume to bring some easy problems back
-hard_fraction = 0.0   # default; bump on resume to bring some hard problems back
-```
-
-Watch `pool/{env}/{easy,normal,hard}` (current pool ratios) and `evicted_examples/{env}/{easy,hard}` (per-step eviction rate).
-
-## Online Difficulty Filtering
-
-Online difficulty filtering (ODF) drops collapsed-advantage groups on the way *into* the buffer. Set `buffer.online_difficulty_filtering = true` (default `false`) to enable:
-
-- Average reward across the group is **0.0** (every rollout failed) → drop the group, count under `filtered_rollouts/{env}/hard`.
-- Average reward **1.0** (every rollout succeeded) → drop, count under `filtered_rollouts/{env}/easy`.
-- Otherwise → into the buffer.
-
-These are exactly the groups whose within-group advantage collapses to zero — DR-GRPO produces no gradient signal for them, so the trainer would burn step time on tokens it can't learn from.
-
-```toml
-[orchestrator.buffer]
-online_difficulty_filtering = true
-```
-
-**Tradeoff: trainer stability vs. inference speed.** With ODF on, every rollout that reaches the trainer carries non-zero advantage — each trainer step's effective batch is predictable and the gradient signal is denser. The cost is paid on the inference side: rollouts get produced and then thrown away, so the orchestrator has to oversample to keep the trainer fed. If the orchestrator is your bottleneck (`time/wait_for_batch` high on the trainer), ODF can starve the loop. Bump `orchestrator.oversampling_factor` so inference produces enough groups per step to absorb the drops.
-
-ODF is orthogonal to the [pools](#difficulty-pools): ODF reacts to the *current* group's reward distribution, the pools track the *running* per-problem average. Many configs use both — ODF for per-step density, pools for long-horizon curriculum cleanup.
-
 ## Multi-Turn Trajectories
 
 Multi-turn rollouts (tool use, browser environments, long conversations) used to be stitched into a single fake "single-turn" sample, which silently corrupted the importance ratio when chat templates didn't roundtrip. Since [`verifiers` v0.1.8](https://github.com/PrimeIntellect-ai/verifiers/releases/tag/v0.1.8), `prime-rl` records each LLM request/response as an independent **trajectory step** and merges them at training time using best-effort interleaving — with [renderers](#renderers) as the mechanism that keeps the merge safe by construction.
diff --git a/docs/inference.md b/docs/inference.md
index d9b1e9947c..35a1de242c 100644
--- a/docs/inference.md
+++ b/docs/inference.md
@@ -176,12 +176,12 @@ Right now, router handles 2 most important things:
 ### Routing policies
 The 2 policies you might want to configure are:
 - `consistent_hash` - this is the default policy that optimizes for KV cache re-use across turns - this works by hashing a request header to determine where to route the request to. You can configure what to hash by setting
-`orchestrator.student.client.extra_headers_from_state` to the header the `router` expects to be set.
+`orchestrator.model.client.extra_headers_from_state` to the header the `router` expects to be set.
 
 We set it to a sensible default, that works with all verifiers environments.
 
 ```toml
-[orchestrator.student.client.extra_headers_from_state]
+[orchestrator.model.client.extra_headers_from_state]
 X-Session-ID = "trajectory_id" # this is the default - each rollout has a unique trajectory_id and router expects X-Session-ID
 ```
 
diff --git a/docs/scaling.md b/docs/scaling.md
index 84ef0db7df..3e9cc65404 100644
--- a/docs/scaling.md
+++ b/docs/scaling.md
@@ -41,7 +41,7 @@ uv run rl @ rl.toml \
   --inference.parallel.dp 6
 ```
 
-The launcher allocates GPUs in order from `CUDA_VISIBLE_DEVICES` (or all visible GPUs): inference first, trainer next, teacher last. To target a specific physical subset, pin `CUDA_VISIBLE_DEVICES` before launching.
+The launcher allocates GPUs in order from `CUDA_VISIBLE_DEVICES` (or all visible GPUs): inference first, trainer next. To target a specific physical subset, pin `CUDA_VISIBLE_DEVICES` before launching.
 
 For quick A/B ablations on the same node, run two RL instances side-by-side in separate tmux sessions, each pinned to half the GPUs and a separate inference port:
 
@@ -54,7 +54,7 @@ CUDA_VISIBLE_DEVICES=0,1 uv run rl @ rl.toml --output-dir outputs/exp1
 bash scripts/tmux.sh -s exp2 -o outputs/exp2
 CUDA_VISIBLE_DEVICES=2,3 uv run rl @ rl.toml \
   --inference.server.port 8001 \
-  --orchestrator.client.base-url http://localhost:8001/v1 \
+  --orchestrator.model.client.base-url http://localhost:8001/v1 \
   --output-dir outputs/exp2
 ```
 
@@ -237,7 +237,7 @@ gpus_per_node = 8
 
 Full multi-node configs ship in [`examples/multinode/`](https://github.com/PrimeIntellect-ai/prime-rl/tree/main/examples/multinode):
 
-- [`rl.toml`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/examples/multinode/rl.toml) — two-node RL run with NCCL weight broadcast on a 30B MoE student.
+- [`rl.toml`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/examples/multinode/rl.toml) — two-node RL run with NCCL weight broadcast on a 30B MoE policy.
 - [`sft.toml`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/examples/multinode/sft.toml) — two-node SFT against the same model.
 
 For inference-only multi-node, set `[deployment] type = "multi_node"` on an inference TOML — each node runs an independent vLLM replica (TP and DP must fit within one node), and the launcher prints one URL per node. Front the URLs with a router or point clients at any of them.
diff --git a/docs/training.md b/docs/training.md
index 53fc7fa7aa..9c332b7eca 100644
--- a/docs/training.md
+++ b/docs/training.md
@@ -1,6 +1,6 @@
 # Training
 
-This page covers everything you need to launch, observe, checkpoint, and recover a `prime-rl` training run — the RL trainer, the SFT trainer, and the related on-policy distillation mode. For multi-node and cluster layouts, see [Scaling](scaling.md). For the loss math and algorithm knobs, see [Algorithms](algorithms.md).
+This page covers everything you need to launch, observe, checkpoint, and recover a `prime-rl` training run — the RL trainer (and the distillation algorithms that run through it) and the SFT trainer. For multi-node and cluster layouts, see [Scaling](scaling.md). For the loss math and algorithm knobs, see [Algorithms](algorithms.md).
 
 > **AI agents working in this repo:** the equivalent runbooks are at [`skills/training/`](https://github.com/PrimeIntellect-ai/prime-rl/tree/main/skills/training) — top-level routing in [`skills/training/SKILL.md`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/skills/training/SKILL.md), launch details in [`skills/training/start-run/SKILL.md`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/skills/training/start-run/SKILL.md), and check-in / restart procedures in [`skills/training/monitor-run/SKILL.md`](https://github.com/PrimeIntellect-ai/prime-rl/blob/main/skills/training/monitor-run/SKILL.md).
 
@@ -10,7 +10,7 @@ This page covers everything you need to launch, observe, checkpoint, and recover
 - [RL Trainer](#rl-trainer)
   - [Launch](#launch)
   - [Useful Knobs](#useful-knobs)
-  - [Training Modes (RL / OPD / SFT)](#training-modes-rl--opd--sft)
+  - [Algorithms](#algorithms)
   - [Important Metrics](#important-metrics)
 - [SFT Trainer](#sft-trainer)
   - [Dataset Format](#dataset-format)
@@ -59,7 +59,7 @@ A condensed view of the knobs you'll most often tune. For trainer-side paralleli
 | `orchestrator.batch_size` | Tasks per trainer step. |
 | `orchestrator.group_size` | Rollouts generated per task. |
 | `orchestrator.max_off_policy_steps` | How many distinct policies may have contributed to one rollout before it's discarded (default 8). The main off-policy dial on long agentic rollouts — bump for throughput, lower for tighter on-policyness. Watch `errored_rollouts` and `mismatch_kl/all/mean` when tuning. |
-| `orchestrator.training_mode` | `rl` (default), `opd`, or `sft`. See [Training modes](#training-modes-rl--opd--sft). |
+| `[orchestrator.algo]` | Training algorithm — the advantage `type` names it (`grpo` default, `max_rl`, `opd`, `opsd`, `sft`, `echo`, `reward`, `custom`). See [Algorithms](#algorithms). |
 | `[[orchestrator.train.env]]` | Training environments. List multiple tables for multi-env training; weight them via `ratio`. See [Configuration § Environments](configuration.md#environments-orchestratortrainenv). |
 | `[[orchestrator.eval.env]]` + `orchestrator.eval.interval` | Eval environments and cadence (default every 100 steps). |
 
@@ -81,24 +81,29 @@ A condensed view of the knobs you'll most often tune. For trainer-side paralleli
 | `--max-steps N` | Stop after `N` trainer steps. Overrides the config value. |
 | `--dry-run` | Resolve + validate the full config, write per-process TOMLs to `<output_dir>/configs/`, and exit without launching. The fastest way to debug a misbehaving config. |
 
-### Training Modes (RL / OPD / SFT)
+### Algorithms
 
-The RL entrypoint supports three training modes, switched via `orchestrator.training_mode`:
+The RL entrypoint supports several training algorithms, switched via `[orchestrator.algo.advantage]` (see [Algorithms](algorithms.md#the-algorithm-abstraction) for the full reference, model references, and per-component customization):
 
-| Mode | Student | Teacher | Use case |
-|---|---|---|---|
-| `rl` | Required | Forbidden | Standard RL |
-| `opd` | Required | Required, must be vLLM (needs `prompt_logprobs`) | [On-policy distillation](https://thinkingmachines.ai/blog/on-policy-distillation/): student generates rollouts, trainer minimizes KL to teacher logprobs |
-| `sft` | Required | Required, any OpenAI-compatible endpoint | Hard-distill: teacher generates rollouts, student trains on them |
+| `advantage.type` | Frozen model (`algo.teacher`) | Use case |
+|---|---|---|
+| `grpo` (default) | None | Standard group-relative RL |
+| `max_rl` | None | [MaxRL](https://arxiv.org/abs/2602.02710): GRPO with mean-normalized advantages (maximum-likelihood RL) |
+| `opd` | Required, must be vLLM (needs `prompt_logprobs`) | [On-policy distillation](https://thinkingmachines.ai/blog/on-policy-distillation/): the policy generates rollouts, the trainer minimizes per-token reverse KL to a reference model |
+| `sft` | Required, any OpenAI-compatible endpoint | Hard-distill: a frozen model generates rollouts, the policy trains on its tokens |
+| `opsd` | `"policy"` (the default, no deployment) or a vLLM endpoint serving a frozen copy | [SDFT](https://arxiv.org/abs/2601.19897): the model is its own reference conditioned on expert demonstrations |
+| `echo` | None | GRPO plus cross-entropy on env-observation tokens |
+
+`reward` (raw-reward credit, no baseline) and `custom` (your own advantage function) complete the set — see [Algorithms § The Algorithms](algorithms.md#the-algorithms).
 
-The `rl` entrypoint only manages student-policy inference. For OPD and (local-vLLM) SFT, start the teacher inference server manually and point `[orchestrator.teacher.client]` at it:
+Frozen models are declared inline on the algorithm (`[orchestrator.algo.teacher]` with `name` + `base_url`). The `rl` entrypoint only manages policy inference — start frozen-model servers yourself and point `base_url` at them:
 
 ```bash
 CUDA_VISIBLE_DEVICES=1 uv run inference \
-  --model.name <teacher> --server.port 8001
+  --model.name <frozen-model> --server.port 8001
 ```
 
-The standalone `uv run sft` entrypoint is the more traditional SFT path — pure dataset-based, no teacher, no orchestrator. Use `orchestrator.training_mode = "sft"` only when you want a teacher to generate the supervision on the fly.
+The standalone `uv run sft` entrypoint is the more traditional SFT path — pure dataset-based, no orchestrator. Use the `sft` algorithm only when you want a frozen model to generate the supervision on the fly.
 
 ### Important Metrics
 
diff --git a/examples/glm5_pd_disag/rl.toml b/examples/glm5_pd_disag/rl.toml
index 878e79198b..d3a2689f9a 100644
--- a/examples/glm5_pd_disag/rl.toml
+++ b/examples/glm5_pd_disag/rl.toml
@@ -66,7 +66,7 @@ group_size = 16
 oversampling_factor = 3
 max_off_policy_steps = 16
 
-[orchestrator.student.model]
+[orchestrator.model]
 name = "zai-org/GLM-5-FP8"
 
 [orchestrator.eval]
diff --git a/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py
new file mode 100644
index 0000000000..664cfdf63f
--- /dev/null
+++ b/packages/prime-rl-configs/src/prime_rl/configs/algorithm.py
@@ -0,0 +1,441 @@
+"""Algorithm abstraction: sampling and the per-token training signal.
+
+An algorithm is a bundle of two pieces:
+
+1. **Sampling** — which model generates train rollouts. ``source`` is a model
+   reference: ``"policy"`` (the live policy) or an inline frozen hosted model.
+2. **Advantage** — credit assignment and loss routing, fused: one mapping from
+   a finalized rollout to per-token ``(loss component, weight)``.
+   Group-relative strategies compute scalars on the orchestrator and ship
+   numbers; reference-KL strategies ship reference prefill logprobs and the
+   trainer evaluates the per-token signal against the live policy. The
+   strategy determines which loss component consumes the action tokens
+   (``rl`` / ``ce`` / ``ref_kl``) and what happens to env-provided observation
+   tokens (masked out by default; ``echo`` trains on them with weighted CE).
+
+prime-rl only ever hosts the trainable policy. Every other model an algorithm
+uses is an external OpenAI-compatible endpoint, declared inline on the
+component that uses it (a :class:`FrozenModelConfig`). Model roles like
+"teacher" are algorithm-local vocabulary over these references; the pipeline
+branches on liveness alone. The advantage ``type`` names the algorithm and its
+class defaults are the vetted setting — ``type = "opd"`` with nothing else IS
+on-policy distillation; any key you set is visibly your own assembly. The
+trainer is algorithm-blind: the loss is a sum of three components (rl, ce,
+ref_kl), each normalized by its own global token count; per-token component
+weights ship on the wire and the trainer just executes them.
+"""
+
+import warnings
+from typing import Annotated, Any, ClassVar, Literal, TypeAlias
+
+from pydantic import AliasChoices, Field, model_validator
+
+from prime_rl.configs.shared import ClientConfig
+from prime_rl.utils.config import BaseConfig
+
+
+class FrozenModelConfig(ClientConfig):
+    """An externally hosted model behind an OpenAI-compatible endpoint: the
+    client config plus the served model's ``name``.
+
+    prime-rl never launches or updates these — only the trainable policy is
+    ever hosted by prime-rl itself. Frozen models are reachable-but-unmanaged:
+    ``base_url`` is required, their weights never change, and rollouts or
+    scores from them never go stale (stable prefix cache, no off-policy
+    aging)."""
+
+    name: str
+    """Served model name, sent as the ``model`` field of every request."""
+
+    @model_validator(mode="after")
+    def require_explicit_endpoint(self):
+        if "base_url" not in self.model_fields_set and not self.is_elastic:
+            raise ValueError(
+                "a frozen model reference needs base_url — frozen models are externally "
+                "hosted; prime-rl only ever hosts the trainable policy."
+            )
+        return self
+
+
+ModelReference: TypeAlias = Literal["policy"] | FrozenModelConfig
+"""``"policy"`` (the live policy — weight-updated: prefix caches salted per
+version, sampling logprobs carried, rollouts age off-policy) or an inline
+externally-hosted frozen model."""
+
+ActionLossType: TypeAlias = Literal["rl", "ce", "ref_kl"]
+
+
+# ---------------------------------------------------------------------------
+# Component 1: sampling
+# ---------------------------------------------------------------------------
+
+
+class SamplingConfig(BaseConfig):
+    source: ModelReference = "policy"
+    """Model reference for train rollout generation: ``"policy"`` (the live
+    policy — prefix caches salted per version, sampling logprobs requested,
+    rollouts age off-policy) or an inline frozen hosted model (stable prefix
+    cache, no sampling logprobs, rollouts never go stale)."""
+
+
+# ---------------------------------------------------------------------------
+# Component 2: advantage strategies
+# ---------------------------------------------------------------------------
+
+
+class TokensLengthPenaltyConfig(BaseConfig):
+    type: Literal["tokens"] = "tokens"
+
+    completion_weight: float = Field(1.0, ge=0, allow_inf_nan=False)
+    """Weight on model completion tokens. Finite and non-negative."""
+
+    tool_response_weight: float = Field(1.0, ge=0, allow_inf_nan=False)
+    """Weight on tool-response tokens (read from the rollout's ``*_total_tool_response_tokens`` harness metric; 0 if absent). Finite and non-negative."""
+
+
+class TurnsLengthPenaltyConfig(BaseConfig):
+    type: Literal["turns"] = "turns"
+
+
+class LinearLengthPenaltyConfig(BaseConfig):
+    type: Literal["linear"] = "linear"
+
+    coef: float = Field(0.25, ge=0, allow_inf_nan=False)
+    """Scale on the linear length penalty. Each reward is reduced by ``coef * pass_rate * (model completion tokens / orchestrator.seq_len)`` — where ``pass_rate`` is the group's mean reward — before the GRPO baseline subtraction. Finite and non-negative."""
+
+    gate_by_correctness: bool = False
+    """When True, scale each rollout's penalty by its reward (``penalty * reward``), so correct rollouts (``reward == 1``) are penalized and incorrect ones (``reward == 0``) are not. When False, every rollout is penalized equally."""
+
+
+LengthPenaltyConfig: TypeAlias = Annotated[
+    TokensLengthPenaltyConfig | TurnsLengthPenaltyConfig | LinearLengthPenaltyConfig,
+    Field(discriminator="type"),
+]
+
+
+class GRPOAdvantageConfig(BaseConfig):
+    type: Literal["grpo"] = "grpo"
+    """GRPO: scalar advantage = reward minus the per-group mean baseline,
+    consumed by the ``rl`` loss component on the rollout's action tokens."""
+
+    action_loss_type: ClassVar[ActionLossType] = "rl"
+    group_relative: ClassVar[bool] = True
+
+    length_penalty: LengthPenaltyConfig | None = None
+    """Length penalty layered onto the group-relative advantage; None disables it. ``tokens`` / ``turns`` are correctness-gated efficiency shaping over a per-rollout cost — in mixed groups lower-cost correct rollouts get amplified advantage (up to 2x), higher-cost correct rollouts are unchanged, incorrect untouched; in all-correct groups below-average-cost rollouts get advantage in [0, 1], others get 0. ``linear`` instead subtracts a ``coef * pass_rate * (completion tokens / orchestrator.seq_len)`` term from each reward before the baseline subtraction (``pass_rate`` = group mean reward), so solved-often problems get the strongest concision pressure and never-solved groups get none."""
+
+    length_weighted_baseline: bool = False
+    """When True, the GRPO baseline is the token-length-weighted mean reward (``sum(len_i * reward_i) / sum(len_i)``) instead of the plain group mean, centering advantages by per-token expected reward. Applies to the plain and ``linear``-penalty paths; the ``tokens`` / ``turns`` efficiency-shaping paths keep their own baseline."""
+
+
+class EchoRoleConfig(BaseConfig):
+    """Echo CE supervision for one message role."""
+
+    alpha: float = Field(0.1, gt=0)
+    """Per-token ce weight for this role's env-provided tokens (ECHO's lambda)."""
+
+
+class EchoRolesConfig(BaseConfig):
+    """Which env-provided message roles train, each at its own weight.
+    Setting any role replaces the whole table — unset roles stay disabled."""
+
+    system: EchoRoleConfig | None = None
+    user: EchoRoleConfig | None = None
+    assistant: EchoRoleConfig | None = None
+    tool: EchoRoleConfig | None = None
+
+    @model_validator(mode="after")
+    def require_a_role(self):
+        if self.system is None and self.user is None and self.assistant is None and self.tool is None:
+            raise ValueError("echo needs at least one role enabled (system, user, assistant, or tool)")
+        return self
+
+
+class EchoFilterConfig(BaseConfig):
+    """User-supplied per-token filter narrowing the role-selected echo tokens.
+
+    The callable is imported at startup and invoked once per rollout as
+    ``filter_fn(rollout, **kwargs) -> list[list[bool]]`` — one keep-mask per
+    trajectory step, each spanning that step's ``prompt_ids`` +
+    ``completion_ids``. Tokens with ``False`` never receive echo weight; the
+    filter can only narrow the role selection, not widen it. The raw rollout
+    exposes message text and sampling logprobs, so content filters (e.g.
+    dropping tool-output warnings) and sampling-probability filters need no
+    extra framework surface."""
+
+    import_path: str
+    """Import path to the filter callable (e.g. ``my_module.drop_warnings``)."""
+
+    kwargs: dict[str, Any] = Field(default_factory=dict)
+    """Kwargs forwarded to the filter."""
+
+
+class EchoAdvantageConfig(GRPOAdvantageConfig):
+    type: Literal["echo"] = "echo"  # type: ignore[assignment]
+    """ECHO: group-relative advantage on action tokens (GRPO), plus weighted
+    CE on env-provided tokens of later turns (tool output, user feedback),
+    selected by message role via the renderer's per-token attribution
+    (requires ``orchestrator.renderer``; MITO rollouts carry no attribution).
+    Selected tokens feed the ``ce`` loss component at their role's ``alpha``
+    and stay outside the rl mask and its denominator."""
+
+    roles: EchoRolesConfig = EchoRolesConfig(tool=EchoRoleConfig())
+    """The role table. The default — tool-response bodies at ``alpha = 0.1``
+    — is the vetted ECHO setting."""
+
+    filter: EchoFilterConfig | None = None
+    """Optional user-supplied filter narrowing the role-selected tokens."""
+
+
+class MaxRLAdvantageConfig(BaseConfig):
+    type: Literal["max_rl"] = "max_rl"
+    """MaxRL (arXiv:2602.02710): scalar advantage = (reward − group mean) /
+    group mean, consumed by the ``rl`` loss component. Normalizing by the
+    mean instead of GRPO's standard deviation makes the policy gradient
+    unbiased for the order-``group_size`` truncation of the maximum-likelihood
+    objective: low-pass-rate examples get ~1/p weight, and ``group_size`` is
+    the truncation order interpolating REINFORCE (1) → exact maximum
+    likelihood (∞). Designed for non-negative (canonically binary) rewards;
+    a group with mean reward 0 carries zero advantages everywhere (the
+    zero-advantage filter drops it, matching the paper's K=0 convention)."""
+
+    action_loss_type: ClassVar[ActionLossType] = "rl"
+    group_relative: ClassVar[bool] = True
+
+
+class RewardAdvantageConfig(BaseConfig):
+    type: Literal["reward"] = "reward"
+    """Scalar advantage = raw reward, no group baseline. Consumed by the
+    ``rl`` loss component."""
+
+    action_loss_type: ClassVar[ActionLossType] = "rl"
+    group_relative: ClassVar[bool] = False
+
+
+class OPDAdvantageConfig(BaseConfig):
+    type: Literal["opd"] = "opd"
+    """On-policy distillation: the per-token signal is the reverse KL to
+    a reference model, evaluated in the trainer from reference prefill
+    logprobs scored over each sample's own context (``ref_logprobs`` on the
+    wire, ``ref_kl`` loss component). No scalar advantage is assigned —
+    rollouts keep ``advantage=None`` (advantage-based filters never fire) and
+    samples ship a neutral 0.0; rewards still flow to metrics. ``group_size``
+    only fans out sampling."""
+
+    action_loss_type: ClassVar[ActionLossType] = "ref_kl"
+    group_relative: ClassVar[bool] = False
+    model_role: ClassVar[str] = "teacher"
+
+    model: ModelReference | None = None
+    """The teacher — an inline frozen hosted model (``name`` + ``base_url``).
+    Required — set it here or fold via ``algo.model`` / ``algo.teacher``.
+    ``"policy"`` is rejected: scoring the policy under itself yields zero KL
+    signal (use ``opsd`` for demo-conditioned self-teaching)."""
+
+    max_concurrent: int = Field(32, ge=1)
+    """Maximum concurrent prefill requests per batch."""
+
+
+class OPSDAdvantageConfig(BaseConfig):
+    type: Literal["opsd"] = "opsd"
+    """On-policy self-distillation (SDFT, https://arxiv.org/abs/2601.19897):
+    the per-token signal is the reverse KL to a reference model conditioned on
+    an expert demonstration. The scoring prefix is rebuilt from the rollout's
+    first-turn messages with the demonstration woven into the user message via
+    ``template``; completion logprobs are aligned back onto the sample.
+    Requires single-step trajectories. No scalar advantage is assigned —
+    rollouts keep ``advantage=None`` (advantage-based filters never fire) and
+    samples ship a neutral 0.0."""
+
+    action_loss_type: ClassVar[ActionLossType] = "ref_kl"
+    group_relative: ClassVar[bool] = False
+    model_role: ClassVar[str] = "teacher"
+
+    model: ModelReference = "policy"
+    """The teacher. ``"policy"`` (the default) is the SDFT paper's setting —
+    the current model conditioned on the demo *is* the teacher — and needs no
+    extra deployment. Set an inline frozen hosted model to score under a
+    frozen copy instead."""
+
+    demo_key: str = "demonstration"
+    """Key holding the expert demonstration text — looked up in the example's
+    ``info`` dict first, then as a top-level rollout field (e.g. ``answer``)."""
+
+    template: str = (
+        "{question}\n\n"
+        "Here is an example of an expert response:\n"
+        "<demonstration>\n{demonstration}\n</demonstration>\n\n"
+        "Answer with a response of your own."
+    )
+    """Template for the demo-conditioned user message. Receives ``{question}``
+    (the original user message text) and ``{demonstration}``."""
+
+    max_concurrent: int = Field(32, ge=1)
+    """Maximum concurrent prefill requests per batch."""
+
+
+class SFTAdvantageConfig(BaseConfig):
+    type: Literal["sft"] = "sft"
+    """SFT distillation: cross-entropy on the sampled tokens. The ``ce``
+    loss component ignores scalar advantages, but group-relative scalars are still
+    assigned so reward-based filtering keeps working (the zero-advantage
+    filter drops uniform-reward groups)."""
+
+    action_loss_type: ClassVar[ActionLossType] = "ce"
+    group_relative: ClassVar[bool] = True
+    source_role: ClassVar[str] = "teacher"
+    """The sampling source is this algorithm's teacher — the frozen model
+    whose tokens the policy trains on. Required: CE on the policy's own
+    tokens is rejected at validation."""
+
+
+class CustomAdvantageConfig(BaseConfig):
+    type: Literal["custom"] = "custom"
+    """Custom advantage function, consumed by the ``rl`` loss component. Returns
+    one scalar per rollout, optionally with per-token advantages aligned to
+    each rollout's completion tokens."""
+
+    action_loss_type: ClassVar[ActionLossType] = "rl"
+    group_relative: ClassVar[bool] = False
+
+    import_path: str
+    """Import path to the advantage function (e.g. ``my_module.my_advantage``)."""
+
+    kwargs: dict[str, Any] = Field(default_factory=dict)
+    """Kwargs forwarded to the advantage function."""
+
+
+AdvantageConfig: TypeAlias = Annotated[
+    GRPOAdvantageConfig
+    | EchoAdvantageConfig
+    | MaxRLAdvantageConfig
+    | RewardAdvantageConfig
+    | OPDAdvantageConfig
+    | OPSDAdvantageConfig
+    | SFTAdvantageConfig
+    | CustomAdvantageConfig,
+    Field(discriminator="type"),
+]
+
+
+# ---------------------------------------------------------------------------
+# The algorithm bundle
+# ---------------------------------------------------------------------------
+
+
+class AlgorithmConfig(BaseConfig):
+    """The advantage ``type`` names the algorithm, and each type's class
+    defaults are its vetted setting — ``advantage = { type = "opd" }`` with a
+    teacher IS on-policy distillation; any other key you set is visibly your
+    own assembly.
+
+    The algorithms:
+
+    - ``grpo`` — policy group sampling, group-relative advantage, RL loss (the default).
+    - ``max_rl`` — GRPO with mean-normalized advantages (maximum-likelihood RL).
+    - ``opd`` — on-policy distillation: policy samples, per-token reverse KL against a reference model. Needs ``teacher``.
+    - ``opsd`` — SDFT: policy samples, demo-conditioned reverse KL against the live policy by default.
+    - ``sft`` — a frozen model samples, the policy trains with CE on its tokens. Needs ``teacher``.
+    - ``echo`` — GRPO on action tokens + weighted CE on tool-response observation tokens.
+    - ``reward`` / ``custom`` — raw-reward and user-supplied advantage functions.
+    """
+
+    model: ModelReference | None = Field(None, exclude=True, validation_alias=AliasChoices("model", "teacher"))
+    """Model reference shorthand: ``"policy"`` or an inline frozen hosted
+    model. Folds into the slot the advantage type declares for it —
+    ``advantage.model`` when the type has one (opd, opsd), ``sampling.source``
+    when the type's teacher is its sampling source (sft). A slot the user
+    didn't set takes the shorthand; an explicit reference that already equals
+    it is accepted, a disagreeing one is an error. ``teacher`` is an accepted
+    alias — the distillation algorithms declare their reference's role as
+    "teacher", and this is the slot it fills. Write-only input sugar — folded
+    by validation and excluded from dumps so resolved configs round-trip."""
+
+    sampling: SamplingConfig = SamplingConfig()
+    """Sampling component."""
+
+    advantage: AdvantageConfig = GRPOAdvantageConfig()
+    """The per-token training signal: credit assignment and loss routing,
+    fused. The ``type`` selects the algorithm."""
+
+    @property
+    def requires_group_advantage(self) -> bool:
+        """True when the advantage strategy assigns group-relative scalars,
+        i.e. degenerate with ``group_size=1``."""
+        return self.advantage.group_relative
+
+    @model_validator(mode="after")
+    def fold_model(self):
+        """Fold the ``model`` shorthand into the component references.
+
+        Fill-or-agree: the slot the advantage type declares (``model`` field,
+        or ``sampling.source`` for source-role types) takes the shorthand
+        when the user didn't set it; an explicit reference that already
+        equals it is redundant-but-consistent; if no slot accepts it, that's
+        an error."""
+        if self.model is None:
+            return self
+        matched = False
+        advantage = self.advantage
+        if "model" in type(advantage).model_fields:
+            if advantage.model is None or "model" not in advantage.model_fields_set:
+                advantage.model = self.model
+                matched = True
+            elif advantage.model == self.model:
+                matched = True
+        if getattr(advantage, "source_role", None) is not None:
+            if "source" not in self.sampling.model_fields_set:
+                self.sampling.source = self.model
+                matched = True
+            elif self.sampling.source == self.model:
+                matched = True
+        if not matched:
+            raise ValueError(
+                f"advantage '{self.advantage.type}': 'model' is set but no component reference accepts it — "
+                "every reference is already explicitly set to a different value, or the algorithm "
+                "references no model. Set advantage.model / sampling.source directly instead."
+            )
+        return self
+
+    @model_validator(mode="after")
+    def validate_component_compatibility(self):
+        source_role = getattr(self.advantage, "source_role", None)
+        if source_role is not None and self.sampling.source == "policy":
+            raise ValueError(
+                f"advantage '{self.advantage.type}' needs a {source_role} to sample rollouts from — "
+                f"CE on the policy's own tokens is not a distillation target. Set '{source_role}' on "
+                "the algorithm (an inline hosted model: name + base_url), or sampling.source explicitly."
+            )
+        if getattr(self.advantage, "model", "<absent>") is None:
+            role = getattr(self.advantage, "model_role", "reference model")
+            raise ValueError(
+                f"advantage '{self.advantage.type}' needs a {role} — "
+                f"set '{role}' on the algorithm (an inline hosted model: name + base_url), "
+                "or advantage.model explicitly."
+            )
+        if isinstance(self.advantage, OPDAdvantageConfig) and self.advantage.model == "policy":
+            raise ValueError(
+                "advantage 'opd' with model='policy' is degenerate — the reference distribution "
+                "equals the policy, so the KL signal is zero. Point at a frozen hosted model, or "
+                "use 'opsd' for demo-conditioned self-teaching."
+            )
+        if self.advantage.action_loss_type in ("rl", "ref_kl") and self.sampling.source != "policy":
+            raise ValueError(
+                f"advantage '{self.advantage.type}' trains with the "
+                f"{self.advantage.action_loss_type} loss type but sampling.source is a frozen model — "
+                "the importance ratio and trust region need the live policy's own sampling logprobs. "
+                "Use the 'sft' advantage to distill frozen-model tokens."
+            )
+        return self
+
+    def warn_group_size(self, group_size: int, env_name: str) -> None:
+        """Group-relative scoring with a single rollout per example collapses
+        every advantage to zero. Warn loudly — this is the classic footgun."""
+        if self.requires_group_advantage and group_size == 1:
+            warnings.warn(
+                f"Env '{env_name}' uses group-relative advantage ('{self.advantage.type}') with "
+                "group_size=1 — every advantage is 0 and (with the default zero-advantage filter) "
+                "no rollout will train. Set group_size >= 2 or a non-group-relative advantage "
+                "(e.g. advantage.type='reward').",
+                stacklevel=2,
+            )
diff --git a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py
index 55a2210abe..fbed7d8cd4 100644
--- a/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py
+++ b/packages/prime-rl-configs/src/prime_rl/configs/orchestrator.py
@@ -1,3 +1,4 @@
+import copy
 import math
 import warnings
 from pathlib import Path
@@ -7,6 +8,10 @@
 from pydantic_core.core_schema import SerializerFunctionWrapHandler
 from renderers import AutoRendererConfig, RendererConfig
 
+from prime_rl.configs.algorithm import (
+    AdvantageConfig,
+    AlgorithmConfig,
+)
 from prime_rl.configs.shared import (
     BaseModelConfig,
     ClientConfig,
@@ -143,49 +148,6 @@ def _deprecate_max_tokens(cls, data: Any) -> Any:
         return data
 
 
-class TokensLengthPenaltyConfig(BaseConfig):
-    type: Literal["tokens"] = "tokens"
-
-    completion_weight: float = Field(1.0, ge=0, allow_inf_nan=False)
-    """Weight on model completion tokens. Finite and non-negative."""
-
-    tool_response_weight: float = Field(1.0, ge=0, allow_inf_nan=False)
-    """Weight on tool-response tokens (read from the rollout's ``*_total_tool_response_tokens`` harness metric; 0 if absent). Finite and non-negative."""
-
-
-class TurnsLengthPenaltyConfig(BaseConfig):
-    type: Literal["turns"] = "turns"
-
-
-LengthPenaltyConfig: TypeAlias = Annotated[
-    TokensLengthPenaltyConfig | TurnsLengthPenaltyConfig,
-    Field(discriminator="type"),
-]
-
-
-class DefaultAdvantageConfig(BaseConfig):
-    type: Literal["default"] = "default"
-
-    length_penalty: LengthPenaltyConfig | None = None
-    """Correctness-gated length penalty. ``tokens`` shapes by weighted token cost; ``turns`` shapes by trajectory turn count; None disables shaping. In mixed groups, lower-cost correct rollouts get amplified advantage (up to 2x), higher-cost correct rollouts are unchanged, incorrect untouched. In all-correct groups, below-average-cost rollouts get advantage in [0, 1], others get 0."""
-
-
-class CustomAdvantageConfig(BaseConfig):
-    type: Literal["custom"] = "custom"
-
-    import_path: str
-    """Import path to the advantage function (e.g. ``my_module.my_advantage``)."""
-
-    kwargs: dict[str, Any] = Field(default_factory=dict)
-    """Kwargs forwarded to the advantage function."""
-
-
-AdvantageConfig: TypeAlias = Annotated[
-    DefaultAdvantageConfig | CustomAdvantageConfig,
-    Field(discriminator="type"),
-]
-
-
 class EnvConfig(BaseConfig):
     id: str = "reverse-text"
     """Registered verifiers environment ID (e.g. ``math-env``, ``primeintellect/math-env``). May include an ``@version`` suffix for installation."""
@@ -257,10 +219,17 @@ class TrainEnvConfig(EnvConfig):
     """Rollouts generated per example for GRPO group-relative advantages.
     Inherits from ``orchestrator.group_size`` when unset."""
 
-    advantage: AdvantageConfig | None = None
-    """Advantage strategy for this env's GRPO groups. Inherits from the top-level
-    ``orchestrator.advantage`` when unset; set a different ``default``/``custom``
-    config to give this env its own advantage computation."""
+    algo: AlgorithmConfig | None = None
+    """Training algorithm for this env. Inherits from the top-level
+    ``orchestrator.algo`` when unset; set its components to give this env its
+    own algorithm."""
+
+    advantage: AdvantageConfig | None = Field(None, exclude=True)
+    """Shorthand for ``algo.advantage`` — the env assembles its own
+    algorithm around it instead of inheriting the top-level one. Setting both
+    this and an explicit ``algo.advantage`` to different values is an error.
+    Write-only input sugar — folded on raw input and excluded from dumps so
+    resolved configs round-trip."""
 
 
 class EvalEnvConfig(EnvConfig):
@@ -498,21 +467,25 @@ class OrchestratorExperimentalConfig(BaseConfig):
     pass
 
 
-class RolloutModelConfig(BaseConfig):
-    model: ModelConfig = ModelConfig()
+class HostedModelConfig(ModelConfig):
+    """A served model reachable through an OpenAI-compatible endpoint: the
+    model fields plus the client of the live deployment."""
 
     client: ClientConfig = ClientConfig()
 
 
 class OrchestratorConfig(BaseConfig):
-    training_mode: Literal["rl", "opd", "sft"] = "rl"
-    """Training mode. ``rl``: student generates rollouts, no teacher. ``opd``: student generates rollouts, teacher computes logprobs (teacher_tau > 0). ``sft``: teacher generates rollouts, student inference pool used for evals and weight sync."""
-
-    student: RolloutModelConfig = Field(RolloutModelConfig(), validation_alias=AliasChoices("student", "model"))
-    """Student rollout participant (model + client) — the model being trained."""
-
-    teacher: RolloutModelConfig | None = Field(None, validation_alias=AliasChoices("teacher", "teacher_model"))
-    """Teacher rollout participant (model + client). Role depends on ``training_mode``: ``opd`` — teacher computes logprobs; ``sft`` — teacher generates rollouts."""
+    algo: AlgorithmConfig = AlgorithmConfig()
+    """Training algorithm: sampling plus the advantage (credit assignment
+    and loss routing, fused — its ``type`` names the algorithm). Defaults to
+    ``grpo``. Override per env via ``[[orchestrator.train.env]]``'s
+    ``algo``."""
+
+    model: HostedModelConfig = HostedModelConfig()
+    """The model being trained: its model fields plus the client of the live
+    vLLM deployment (``[orchestrator.model] name = ...`` with
+    ``[orchestrator.model.client]``). Algorithm components reference it as
+    ``"policy"``."""
 
     train: TrainConfig = TrainConfig()
 
@@ -522,7 +495,8 @@ class OrchestratorConfig(BaseConfig):
     """Typed renderer config (``renderers.RendererConfig`` discriminated
     union). Defaults to ``"auto"``, which resolves from
     ``tokenizer.name_or_path`` via ``MODEL_RENDERER_MAP``. ``None``
-    opts into MITO (``openai_chat_completions``)."""
+    opts into MITO (``openai_chat_completions``); forced when no train env
+    samples from the policy."""
 
     pool_size: int | None = Field(None, ge=1)
     """Number of renderer slots shared across concurrent rollouts. Bump
@@ -549,7 +523,10 @@ def _preserve_mito_renderer(self, handler: SerializerFunctionWrapHandler) -> dic
     eval: EvalConfig | None = None
     """Evaluation configuration."""
 
-    advantage: AdvantageConfig | None = DefaultAdvantageConfig()
+    advantage: AdvantageConfig | None = Field(None, exclude=True)
+    """Shorthand for ``algo.advantage`` (and, through inheritance, for envs
+    without their own algorithm). Write-only input sugar — folded on raw
+    input and excluded from dumps so resolved configs round-trip."""
 
     pre_batch_filters: list[FilterConfig] = [
         GibberishFilterConfig(enforce=False),
@@ -579,7 +556,7 @@ def _preserve_mito_renderer(self, handler: SerializerFunctionWrapHandler) -> dic
     """Collect inference-server metrics (requires wandb)."""
 
     inference_metrics_roles: list[Literal["prefill", "decode"]] | None = None
-    """Role for each student admin client when collecting P/D inference metrics."""
+    """Role for each policy admin client when collecting P/D inference metrics."""
 
     ckpt: CheckpointConfig | None = None
     """Checkpoint configuration."""
@@ -637,63 +614,49 @@ def _preserve_mito_renderer(self, handler: SerializerFunctionWrapHandler) -> dic
 
     @model_validator(mode="before")
     @classmethod
-    def fold_student_shortcuts(cls, data: Any) -> Any:
-        """Accept top-level ``[orchestrator.model]`` / ``[orchestrator.client]``
-        as shorthand for the student sub-config. Useful for ergonomic rl configs
-        where ``[orchestrator.student.*]`` is overkill, and required for
-        pre-refactor configs that used the flat layout to keep parsing:
-
-        - [orchestrator.client.*]     -> [orchestrator.student.client.*]
-        - [orchestrator.model.<k>]    -> [orchestrator.student.model.<k>]
-          (where <k> is any ModelConfig field)
-
-        Teacher must always be configured under [orchestrator.teacher.*]
-        (no equivalent shortcut), because rl mode forbids a teacher and we
-        don't want the same shortcut to silently route to two different roles.
+    def fold_advantage_shortcuts(cls, data: Any) -> Any:
+        """Fold the ``advantage`` shorthands into ``algo.advantage`` on raw
+        input, before any ``AlgorithmConfig`` is built — each algorithm then
+        validates exactly once with everything in place. Defined before
+        ``_env_to_train`` (before-validators run in reverse definition order)
+        so the legacy ``[[env]]`` layout is already translated.
+
+        ``advantage = None`` (or the string ``"None"``) selects the raw-reward advantage.
         """
         if not isinstance(data, dict):
             return data
 
-        def deep_merge(dst: dict, src: dict) -> None:
-            """In-place recursive merge of ``src`` into ``dst``. ``src`` wins at the leaf."""
-            for k, v in src.items():
-                if isinstance(v, dict) and isinstance(dst.get(k), dict):
-                    deep_merge(dst[k], v)
-                else:
-                    dst[k] = v
-
-        # 1. Re-nest top-level [orchestrator.client] under student.client.
-        legacy_client = data.pop("client", None)
-        if isinstance(legacy_client, dict):
-            student = data.setdefault("student", {})
-            if isinstance(student, dict):
-                deep_merge(student.setdefault("client", {}), legacy_client)
-            else:
-                # Mismatched types - put it back and let pydantic surface the error.
-                data["client"] = legacy_client
-
-        # 2. Consolidate the legacy `model` alias into `student` so the
-        # flat-layout fix-up below sees a single target. Deep-merge with the
-        # legacy keys winning so a CLI `--model.<k>` overrides TOML `student.model.<k>`.
-        legacy_model = data.pop("model", None)
-        if legacy_model is not None:
-            existing = data.get("student")
-            if existing is None:
-                data["student"] = legacy_model
-            elif isinstance(existing, dict) and isinstance(legacy_model, dict):
-                deep_merge(existing, legacy_model)
-            else:
-                # Mismatched types - put it back and let pydantic surface the error.
-                data["model"] = legacy_model
+        def fold(algo: Any, shorthand: Any, owner: str) -> None:
+            if not isinstance(algo, dict):
+                raise ValueError(
+                    f"{owner}: the 'advantage' shorthand needs 'algo' as plain config data — "
+                    "set 'algo.advantage' directly instead."
+                )
+            if shorthand is None or shorthand == "None":
+                shorthand = {"type": "reward"}
+            existing = algo.get("advantage")
+            if existing is not None and existing != shorthand:
+                raise ValueError(
+                    f"{owner}: 'advantage' shorthand conflicts with the explicit 'algo.advantage'. Set one."
+                )
+            algo["advantage"] = copy.deepcopy(shorthand)
 
-        # 3. Re-nest flat ModelConfig keys under student.model.
-        model_only_keys = set(ModelConfig.model_fields)
-        student = data.get("student")
-        if isinstance(student, dict):
-            flat = {k: student.pop(k) for k in list(student) if k in model_only_keys}
-            if flat:
-                student.setdefault("model", {}).update(flat)
+        if "advantage" in data:
+            fold(data.setdefault("algo", {}), data["advantage"], "orchestrator")
 
+        train = data.get("train")
+        envs = train.get("env") if isinstance(train, dict) else None
+        if not isinstance(envs, list):
+            return data
+        for env in envs:
+            if not isinstance(env, dict) or "advantage" not in env:
+                continue
+            # The shorthand makes the env assemble its own algorithm instead
+            # of inheriting-and-modifying the top-level one.
+            if env.get("algo") is None:
+                env["algo"] = {}
+            name = env.get("name") or str(env.get("id", "?")).split("@")[0]
+            fold(env["algo"], env["advantage"], f"env '{name}'")
         return data
 
     @model_validator(mode="before")
@@ -726,15 +689,15 @@ def _env_to_train(cls, data: Any) -> Any:
     @model_validator(mode="after")
     def auto_setup_tokenizer(self):
         if self.tokenizer.name is None:
-            self.tokenizer.name = self.student.model.name
+            self.tokenizer.name = self.model.name
         if self.tokenizer.trust_remote_code is None:
-            self.tokenizer.trust_remote_code = self.student.model.trust_remote_code
+            self.tokenizer.trust_remote_code = self.model.trust_remote_code
         return self
 
     @model_validator(mode="after")
     def auto_setup_session_headers(self):
         """Ensure X-Session-ID header is always set for sticky DP-aware routing at the inference router."""
-        self.student.client.extra_headers_from_state.setdefault("X-Session-ID", "trajectory_id")
+        self.model.client.extra_headers_from_state.setdefault("X-Session-ID", "trajectory_id")
         return self
 
     @model_validator(mode="after")
@@ -758,35 +721,62 @@ def validate_unique_filter_types(self):
         return self
 
     @model_validator(mode="after")
-    def _force_no_renderer_for_sft(self):
-        """Teacher-backed SFT rolls out via the teacher's plain chat-completions
-        endpoint; the renderer client doesn't apply. When no teacher is
-        configured, SFT uses the student rollout path and keeps the renderer."""
-        if self.training_mode == "sft" and self.teacher is not None:
-            self.renderer = None
+    def inherit_env_algorithms(self):
+        """Envs without their own algorithm inherit the top-level one (the
+        ``advantage`` shorthands are already folded in on raw input by
+        ``fold_advantage_shortcuts``). Declared before any validator that
+        reads ``algo``."""
+        for env_cfg in self.train.env:
+            if env_cfg.algo is None:
+                env_cfg.algo = self.algo.model_copy(deep=True)
         return self
 
+    @property
+    def any_policy_sourced(self) -> bool:
+        """True when at least one train env samples rollouts from the live policy."""
+        return any(env.algo is not None and env.algo.sampling.source == "policy" for env in self.train.env)
+
     @model_validator(mode="after")
-    def validate_training_mode(self):
-        """Enforce training mode invariants that involve only orchestrator fields."""
-        has_teacher = self.teacher is not None
-        if self.training_mode == "rl" and has_teacher:
-            raise ValueError("orchestrator.teacher must not be set when training_mode = 'rl'.")
-        if self.training_mode == "opd" and not has_teacher:
-            raise ValueError("orchestrator.teacher must be configured when training_mode = 'opd'.")
+    def validate_renderer_for_demo_scoring(self):
+        """``opsd`` rebuilds its demo-conditioned scoring prefix
+        client-side, which requires the policy's renderer (the canonical
+        messages → token ids path)."""
+        if self.renderer is not None:
+            return self
+        for env in self.train.env:
+            if env.algo is not None and env.algo.advantage.type == "opsd":
+                raise ValueError(
+                    f"env '{env.resolved_name}' uses opsd, which renders its demo-conditioned "
+                    "scoring prefix client-side and requires orchestrator.renderer — remove "
+                    "'renderer = \"None\"'."
+                )
+            if env.algo is not None and env.algo.advantage.type == "echo":
+                raise ValueError(
+                    f"env '{env.resolved_name}' trains env-provided tokens by message role (echo), "
+                    "which needs the renderer's per-token attribution — set orchestrator.renderer."
+                )
         return self
 
     @model_validator(mode="after")
     def validate_pool_size(self):
-        """``pool_size`` is only meaningful when the renderer is enabled
-        (``renderer is not None``). Reject otherwise so callers don't
-        silently pass it and wonder why it's ignored."""
-        if self.renderer is None and self.pool_size is not None:
+        """``pool_size`` sizes the renderer-client pool for policy-sourced
+        sampling. Reject it when that path never runs — no renderer, or no
+        train env samples from the policy — so callers don't silently pass
+        it and wonder why it's ignored."""
+        if self.pool_size is None:
+            return self
+        if self.renderer is None:
             raise ValueError(
                 f"orchestrator.pool_size={self.pool_size!r} is set but "
                 "orchestrator.renderer is None (MITO mode). Either configure a renderer "
                 "or remove pool_size."
             )
+        if not self.any_policy_sourced:
+            raise ValueError(
+                f"orchestrator.pool_size={self.pool_size!r} is set but no train env samples "
+                "from the policy — the renderer-client sampling pool never runs (the renderer "
+                "is still used for client-side tokenization). Remove pool_size."
+            )
         return self
 
     @model_validator(mode="after")
@@ -797,7 +787,7 @@ def vlm_requires_renderer(self):
         tokens, and ships generic ``mm_kwargs`` keyed by whatever the
         model's forward signature expects.
         """
-        if self.student.model.vlm is not None and self.renderer is None:
+        if self.model.vlm is not None and self.renderer is None:
             raise ValueError(
                 "orchestrator.renderer must be set when model.vlm is set. "
                 "VLMs must go through a renderer (e.g. Qwen3VLRenderer) that owns the processor."
@@ -821,7 +811,7 @@ def validate_renderer_auto_resolves(self):
             return self
         from renderers.base import MODEL_RENDERER_MAP
 
-        model_id = self.tokenizer.name or self.student.model.name
+        model_id = self.tokenizer.name or self.model.name
         if model_id in MODEL_RENDERER_MAP:
             return self
         raise ValueError(
@@ -879,11 +869,8 @@ def resolve_batching(self):
         for env_cfg in self.train.env:
             if "group_size" not in env_cfg.model_fields_set:
                 env_cfg.group_size = self.group_size
-
-        # Propagate the top-level ``advantage`` into each train env that didn't set its own.
-        for env_cfg in self.train.env:
-            if "advantage" not in env_cfg.model_fields_set:
-                env_cfg.advantage = self.advantage
+            assert env_cfg.algo is not None  # materialized by inherit_env_algorithms
+            env_cfg.algo.warn_group_size(env_cfg.group_size, env_cfg.resolved_name)
 
         # Resolve train env num_workers from max_inflight_rollouts
         for env_cfg in self.train.env:
@@ -910,10 +897,12 @@ def auto_setup_bench(self):
     @model_validator(mode="after")
     def resolve_env_config(self):
         """Populate extra_env_kwargs and vLLM sampling defaults from top-level fields."""
-        is_vllm = self.training_mode != "sft"
         for env in self.train.env:
             env.extra_env_kwargs.update(max_seq_len=self.seq_len)
-            if is_vllm:
+            # Policy-sourced rollouts hit our vLLM server; frozen-sourced
+            # rollouts may hit external OAI endpoints that reject these knobs.
+            assert env.algo is not None
+            if env.algo.sampling.source == "policy":
                 env.sampling.extra_body.setdefault("top_k", -1)
                 env.sampling.extra_body.setdefault("min_p", 0.0)
                 env.sampling.extra_body.setdefault("return_token_ids", True)
diff --git a/packages/prime-rl-configs/src/prime_rl/configs/rl.py b/packages/prime-rl-configs/src/prime_rl/configs/rl.py
index 832234eea9..353266c791 100644
--- a/packages/prime-rl-configs/src/prime_rl/configs/rl.py
+++ b/packages/prime-rl-configs/src/prime_rl/configs/rl.py
@@ -373,40 +373,40 @@ def auto_setup_lora(self):
             if self.trainer.weight_broadcast.type == "nccl":
                 raise ValueError("NCCL weight broadcast does not support LoRA yet.")
 
-            if self.orchestrator.student.model.lora is None:
+            if self.orchestrator.model.lora is None:
                 from prime_rl.configs.orchestrator import LoRAConfig
 
-                self.orchestrator.student.model.lora = LoRAConfig()
+                self.orchestrator.model.lora = LoRAConfig()
 
             if (
-                self.orchestrator.student.model.lora.rank is not None
-                and self.orchestrator.student.model.lora.rank != self.trainer.model.lora.rank
+                self.orchestrator.model.lora.rank is not None
+                and self.orchestrator.model.lora.rank != self.trainer.model.lora.rank
             ):
                 raise ValueError(
-                    f"orchestrator.student.model.lora.rank ({self.orchestrator.student.model.lora.rank}) conflicts with "
+                    f"orchestrator.model.lora.rank ({self.orchestrator.model.lora.rank}) conflicts with "
                     f"trainer.model.lora.rank ({self.trainer.model.lora.rank}). "
-                    f"Remove orchestrator.student.model.lora.rank to inherit from trainer, or update trainer.model.lora.rank to match."
+                    f"Remove orchestrator.model.lora.rank to inherit from trainer, or update trainer.model.lora.rank to match."
                 )
 
             if (
-                self.orchestrator.student.model.lora.alpha is not None
-                and self.orchestrator.student.model.lora.alpha != self.trainer.model.lora.alpha
+                self.orchestrator.model.lora.alpha is not None
+                and self.orchestrator.model.lora.alpha != self.trainer.model.lora.alpha
             ):
                 raise ValueError(
-                    f"orchestrator.student.model.lora.alpha ({self.orchestrator.student.model.lora.alpha}) conflicts with "
+                    f"orchestrator.model.lora.alpha ({self.orchestrator.model.lora.alpha}) conflicts with "
                     f"trainer.model.lora.alpha ({self.trainer.model.lora.alpha}). "
-                    f"Remove orchestrator.student.model.lora.alpha to inherit from trainer, or update trainer.model.lora.alpha to match."
+                    f"Remove orchestrator.model.lora.alpha to inherit from trainer, or update trainer.model.lora.alpha to match."
                 )
 
-            if self.orchestrator.student.model.lora.rank is None:
-                self.orchestrator.student.model.lora.rank = self.trainer.model.lora.rank
+            if self.orchestrator.model.lora.rank is None:
+                self.orchestrator.model.lora.rank = self.trainer.model.lora.rank
 
-            if self.orchestrator.student.model.lora.alpha is None:
-                self.orchestrator.student.model.lora.alpha = self.trainer.model.lora.alpha
+            if self.orchestrator.model.lora.alpha is None:
+                self.orchestrator.model.lora.alpha = self.trainer.model.lora.alpha
 
-            if self.orchestrator.student.model.lora.name is None:
-                self.orchestrator.student.model.lora.name = (
-                    f"r{self.orchestrator.student.model.lora.rank}-a{self.orchestrator.student.model.lora.alpha}"
+            if self.orchestrator.model.lora.name is None:
+                self.orchestrator.model.lora.name = (
+                    f"r{self.orchestrator.model.lora.rank}-a{self.orchestrator.model.lora.alpha}"
                 )
 
             if self.inference is not None:
@@ -604,19 +604,20 @@ def auto_setup_disaggregated_inference(self):
 
     @model_validator(mode="after")
     def auto_setup_inference_client(self):
-        """Auto-configure orchestrator student client from the inference server config.
+        """Auto-configure the orchestrator policy client from the inference server config.
 
-        For all modes, sets dp_rank_count from inference DP size. For SFT mode,
-        also sets base_url - rl/opd rely on the ClientConfig default
+        Always sets dp_rank_count from inference DP size. When no train env
+        samples from the policy (e.g. sft_distill), also sets base_url —
+        policy-sourced algorithms rely on the ClientConfig default
         (``["http://localhost:8000/v1"]``) which already matches the auto-launched
-        student vLLM at inference.server.port = 8000.
+        policy vLLM at inference.server.port = 8000.
         """
         if self.inference is None:
             return self
-        client = self.orchestrator.student.client
+        client = self.orchestrator.model.client
         if "dp_rank_count" not in client.model_fields_set:
             client.dp_rank_count = self.inference.data_parallel_size_local or self.inference.parallel.dp
-        if self.orchestrator.training_mode == "sft" and "base_url" not in client.model_fields_set:
+        if not self.orchestrator.any_policy_sourced and "base_url" not in client.model_fields_set:
             host = self.inference.server.host or "localhost"
             port = self.inference.server.port
             client.base_url = [f"http://{host}:{port}/v1"]
diff --git a/packages/prime-rl-configs/src/prime_rl/configs/trainer.py b/packages/prime-rl-configs/src/prime_rl/configs/trainer.py
index f1522d4efe..14ac89b342 100644
--- a/packages/prime-rl-configs/src/prime_rl/configs/trainer.py
+++ b/packages/prime-rl-configs/src/prime_rl/configs/trainer.py
@@ -510,7 +510,7 @@ class TrainerConfig(BaseConfig):
     data: DataLoaderConfig = DataLoaderConfig()
 
     loss: LossConfig = DefaultLossConfig()
-    """Loss config for rl-mode batches. opd and sft batches dispatch to their own loss fns unconditionally and do not read this."""
+    """Loss config for the rl loss component (see ``setup_rl_loss_fn``). The ce / ref_kl components are fixed and do not read this."""
 
     optim: OptimizerConfig = AdamWConfig()
 
diff --git a/packages/prime-rl-configs/src/prime_rl/utils/validation.py b/packages/prime-rl-configs/src/prime_rl/utils/validation.py
index 1249061bd0..aa34f8c3e7 100644
--- a/packages/prime-rl-configs/src/prime_rl/utils/validation.py
+++ b/packages/prime-rl-configs/src/prime_rl/utils/validation.py
@@ -22,8 +22,9 @@ def propagate_shared_fields(data: Any) -> Any:
         The original footgun the mutex was designed to catch — a sub-config
         value silently winning over a later CLI shared override — is still
         caught because that scenario produces *different* values.
-      - **Aliased sub-paths**: ``orchestrator.model.*`` is checked against its
-        ``orchestrator.student.model.*`` alias (and vice versa), so the
+      - **Aliased sub-paths**: ``orchestrator.model.*`` (flat) is checked
+        against the nested ``orchestrator.model.*`` spelling and the
+        ``orchestrator.policy.*`` / ``orchestrator.student.*`` aliases, so the
         conflict fires regardless of which spelling the user wrote.
     """
     if not isinstance(data, dict):
@@ -67,20 +68,29 @@ def propagate(shared_path: str, *targets: str, aliases: tuple[str, ...] = ()) ->
         for target in targets:
             fill(target, value)
 
-    # [model] → trainer / orchestrator (student, via AliasChoices) / inference.
+    # [model] → trainer / orchestrator (flat spelling, re-nested by
+    # fold_policy_shortcuts) / inference.
     propagate(
         "model.name",
         "trainer.model.name",
         "inference.model.name",
         "orchestrator.model.name",
-        aliases=("orchestrator.student.model.name",),
+        aliases=(
+            "orchestrator.model.name",
+            "orchestrator.policy.model.name",
+            "orchestrator.student.model.name",
+        ),
     )
     propagate(
         "model.vlm",
         "trainer.model.vlm",
         "inference.model.vlm",
         "orchestrator.model.vlm",
-        aliases=("orchestrator.student.model.vlm",),
+        aliases=(
+            "orchestrator.model.vlm",
+            "orchestrator.policy.model.vlm",
+            "orchestrator.student.model.vlm",
+        ),
     )
 
     # [log]
@@ -224,18 +234,18 @@ def validate_shared_model_name(
 ) -> None:
     # Orchestrator must match inference (it queries the inference server)
     if inference is not None:
-        if inference.model.name != orchestrator.student.model.name:
+        if inference.model.name != orchestrator.model.name:
             raise ValueError(
-                f"Inference model name ({inference.model.name}) and orchestrator model name ({orchestrator.student.model.name}) are not the same. "
+                f"Inference model name ({inference.model.name}) and orchestrator model name ({orchestrator.model.name}) are not the same. "
                 "The orchestrator queries the inference server and must use the same model name."
             )
         return
 
     if trainer.model.name.startswith("Jackmin108/"):  # The TT MoE models will have a different name on the orchestrator
         return
-    if trainer.model.name != orchestrator.student.model.name:
+    if trainer.model.name != orchestrator.model.name:
         raise ValueError(
-            f"Trainer model name ({trainer.model.name}) and orchestrator model name ({orchestrator.student.model.name}) are not the same. Please specify the same model name for both."
+            f"Trainer model name ({trainer.model.name}) and orchestrator model name ({orchestrator.model.name}) are not the same. Please specify the same model name for both."
         )
 
 
diff --git a/skills/configs/SKILL.md b/skills/configs/SKILL.md
index 264cf97df7..0321033da9 100644
--- a/skills/configs/SKILL.md
+++ b/skills/configs/SKILL.md
@@ -47,9 +47,11 @@ id = "reverse-text"
 
 CLI: `--env.0.id reverse-text --env.1.id math-env`.
 
-**Dicts** — TOML uses a section; CLI takes a JSON string: `--vllm-extra '{"key1": "value1"}'`.
+**Dicts** — TOML uses a section; CLI takes a JSON string: `--vllm-extra '{"key1": "value1"}'`. This works for plain `dict` fields only — nested pydantic-model fields (e.g. `advantage`) reject JSON strings; use dotted keys (`--orchestrator.algo.advantage.type custom`) or a TOML overlay file.
 
-**Discriminated unions** — set the `type` field to pick the variant (`[trainer.loss] type = "sft"`). Omit `type` to keep the default variant.
+**Discriminated unions** — set the `type` field to pick the variant (`[orchestrator.advantage] type = "max_rl"`). Omit `type` to keep the default variant.
+
+**Algorithms** — `[orchestrator.algo.advantage] type = "grpo" | "max_rl" | "opd" | "opsd" | "sft" | "echo" | "reward" | "custom"` — the advantage type names the algorithm (credit assignment + loss routing, fused), and each type's class defaults are its vetted setting; any other key you set is your own assembly (e.g. `[orchestrator.algo.advantage.roles.user] alpha = 0.1` for echo — setting any echo role replaces the whole role table). There is no preset layer. Per-env override: `[[orchestrator.train.env]]` `advantage = { type = "echo" }` (the env assembles its own algorithm). prime-rl only hosts the trainable policy; frozen models are inline external endpoints on the algorithm — `[orchestrator.algo.teacher]` (alias for `model`) with `name` + `base_url` folds into the slot the type declares (`advantage.model` for opd/opsd, `sampling.source` for sft). `model = "policy"` points a component at the live policy (opsd's default). See `docs/algorithms.md`.
 
 **`BaseModel | None` fields** — bare flag enables defaults; nested override enables and sets:
 
@@ -62,7 +64,7 @@ In TOML, an empty section header (`[ckpt]`) does the same.
 
 ## RL trainer token exports
 
-For rollout debugging, enable trainer-side token export with `trainer.enable_token_export = true` (or `--enable-token-export` when running the trainer entrypoint directly). It writes one JSONL record per exported sequence. Single-run/fallback exports go under `output_dir/token_exports/step_<step>/rank_<rank>.jsonl`; multi-run trainer exports with packer metadata go under the owning run directory, `output_dir/<run_id>/token_exports/step_<run_step>/rank_<rank>.jsonl`. Each record stores aligned per-token arrays for token ids, loss mask, advantage, reward, entropy, mismatch KL, inference/trainer logprobs, importance ratios, probability deltas, and masking diagnostics. It does not decode token text in the trainer.
+For rollout debugging, enable trainer-side token export with `trainer.enable_token_export = true` (or `--enable-token-export` when running the trainer entrypoint directly). It writes one JSONL record per exported sequence. Single-run/fallback exports go under `output_dir/token_exports/step_<step>/rank_<rank>.jsonl`; multi-run trainer exports with packer metadata go under the owning run directory, `output_dir/<run_id>/token_exports/step_<run_step>/rank_<rank>.jsonl`. Each record stores aligned per-token arrays for token ids, loss mask, component weight streams (rl/ce/ref_kl), advantage, reward, entropy, mismatch KL, inference/trainer logprobs, importance ratios, probability deltas, and masking diagnostics. It does not decode token text in the trainer.
 
 ```toml
 enable_token_export = true
diff --git a/src/prime_rl/entrypoints/rl.py b/src/prime_rl/entrypoints/rl.py
index db1e27b995..a7b85621ca 100644
--- a/src/prime_rl/entrypoints/rl.py
+++ b/src/prime_rl/entrypoints/rl.py
@@ -12,6 +12,7 @@
 import pynvml
 import tomli_w
 
+from prime_rl.configs.algorithm import FrozenModelConfig
 from prime_rl.configs.rl import RLConfig
 from prime_rl.utils.config import cli
 from prime_rl.utils.logger import get_logger, setup_logger
@@ -116,16 +117,16 @@ def rl_local(config: RLConfig):
     }
 
     # Validate client port matches inference server port
-    if config.inference is not None and not config.orchestrator.student.client.is_elastic:
+    if config.inference is not None and not config.orchestrator.model.client.is_elastic:
         from urllib.parse import urlparse
 
-        base_url = config.orchestrator.student.client.base_url[0]
+        base_url = config.orchestrator.model.client.base_url[0]
         parsed = urlparse(base_url)
         client_port = parsed.port
         expected_port = config.inference.server.port
         if client_port != expected_port:
             raise ValueError(
-                f"orchestrator.student.client.base_url port ({client_port}) does not match "
+                f"orchestrator.model.client.base_url port ({client_port}) does not match "
                 f"inference.server.port ({expected_port}). "
                 f"Update the base_url to use port {expected_port} to match the inference server."
             )
@@ -179,19 +180,27 @@ def sigterm_handler(signum, frame):
             monitor_threads.append(monitor_thread)
         else:
             logger.warning(
-                "No [inference] block configured - the student inference server will not be started here. "
-                "All training modes (rl/opd/sft) require a student inference pool for evals + weight sync; "
-                "make sure one is running at orchestrator.student.client.base_url "
-                f"({', '.join(config.orchestrator.student.client.base_url)}), otherwise the orchestrator "
+                "No [inference] block configured - the policy inference server will not be started here. "
+                "Every algorithm requires a policy inference pool for evals + weight sync; "
+                "make sure one is running at orchestrator.model.client.base_url "
+                f"({', '.join(config.orchestrator.model.client.base_url)}), otherwise the orchestrator "
                 "will hang waiting for it."
             )
 
-        if config.orchestrator.teacher:
+        frozen_endpoints: list[str] = []
+        for env in config.orchestrator.train.env:
+            algo = env.algo
+            if algo is None:
+                continue
+            for ref in (algo.sampling.source, getattr(algo.advantage, "model", None)):
+                if isinstance(ref, FrozenModelConfig):
+                    frozen_endpoints.append(f"{ref.name} ({', '.join(ref.base_url)})")
+        if frozen_endpoints:
+            endpoints = ", ".join(dict.fromkeys(frozen_endpoints))
             logger.info(
-                "orchestrator.teacher is configured - the rl entrypoint does not start teacher inference "
-                "servers. Make sure your teacher endpoint at "
-                f"{', '.join(config.orchestrator.teacher.client.base_url)} is running before the "
-                "orchestrator starts, otherwise rollouts will hang."
+                "Frozen model references are configured - the rl entrypoint does not start them. "
+                f"Make sure these endpoints are serving before the orchestrator starts: {endpoints}; "
+                "otherwise rollouts will hang."
             )
 
         orchestrator_cmd = ["orchestrator", "@", (config_dir / ORCHESTRATOR_TOML).as_posix()]
diff --git a/src/prime_rl/orchestrator/advantage.py b/src/prime_rl/orchestrator/advantage.py
deleted file mode 100644
index b58a410326..0000000000
--- a/src/prime_rl/orchestrator/advantage.py
+++ /dev/null
@@ -1,147 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Callable
-
-import torch
-import verifiers as vf
-from jaxtyping import Float
-from torch import Tensor
-
-if TYPE_CHECKING:
-    from prime_rl.orchestrator.types import TrainRollout
-
-from prime_rl.configs.orchestrator import (
-    AdvantageConfig,
-    CustomAdvantageConfig,
-    LengthPenaltyConfig,
-    TokensLengthPenaltyConfig,
-    TurnsLengthPenaltyConfig,
-)
-from prime_rl.orchestrator.utils import get_model_completion_len, get_tool_response_len
-from prime_rl.utils.utils import import_object
-
-
-@dataclass
-class AdvantageInputs:
-    """Inputs for advantage computation of a single group (one example × N rollouts)."""
-
-    rollouts: list[vf.RolloutOutput]
-
-
-@dataclass
-class AdvantageOutputs:
-    """Outputs from advantage computation of a single group."""
-
-    advantages: list[float]
-
-
-AdvantageFn = Callable[..., AdvantageOutputs]
-"""Type for an advantage function.
-
-Expected signature:
-    def my_advantage(inputs: AdvantageInputs, **kwargs) -> AdvantageOutputs:
-        ...
-
-The function receives a single group and returns a list of advantages with one
-entry per rollout. `assign_advantages` calls it on one already-grouped cohort.
-"""
-
-
-def default_advantage_fn(
-    inputs: AdvantageInputs,
-    length_penalty: LengthPenaltyConfig | None = None,
-) -> AdvantageOutputs:
-    """Default GRPO advantage for a single group: reward minus per-group baseline.
-
-    `length_penalty` enables correctness-gated efficiency shaping over a per-rollout
-    cost: tokens (weighted completion + tool-response) or trajectory turn count.
-    """
-    rewards = torch.tensor([r["reward"] for r in inputs.rollouts], dtype=torch.float32)
-
-    if isinstance(length_penalty, TokensLengthPenaltyConfig):
-        w_c = length_penalty.completion_weight
-        w_t = length_penalty.tool_response_weight
-        costs = torch.tensor(
-            [w_c * get_model_completion_len(r) + w_t * get_tool_response_len(r) for r in inputs.rollouts],
-            dtype=rewards.dtype,
-        )
-        return AdvantageOutputs(advantages=_efficiency_shaping(rewards, costs).tolist())
-    if isinstance(length_penalty, TurnsLengthPenaltyConfig):
-        costs = torch.tensor([len(r["trajectory"]) for r in inputs.rollouts], dtype=rewards.dtype)
-        return AdvantageOutputs(advantages=_efficiency_shaping(rewards, costs).tolist())
-
-    return AdvantageOutputs(advantages=(rewards - rewards.mean()).tolist())
-
-
-def _efficiency_shaping(
-    rewards: Float[Tensor, "group_size"],
-    costs: Float[Tensor, "group_size"],
-) -> Float[Tensor, "group_size"]:
-    """Correctness-gated efficiency shaping with bounded advantages.
-
-    Shapes rewards with a bounded efficiency bonus before standard GRPO subtraction,
-    preserving zero-mean advantages within the group. `costs` is a per-rollout cost
-    (e.g., completion length in tokens or number of turns).
-
-    Correct rollouts get reward amplified by up to 2x based on relative efficiency.
-    Incorrect rollouts are untouched. Lower-cost correct rollouts get higher advantage.
-    """
-    max_reward = rewards.max()
-    correct_mask = rewards >= max_reward
-    num_correct = correct_mask.sum()
-
-    # No shaping when max reward is 0 — no correct rollouts to differentiate
-    if max_reward <= 0:
-        return rewards - rewards.mean()
-
-    # Mean cost of correct rollouts
-    mean_correct_cost = (costs * correct_mask).sum() / num_correct.clamp(min=1)
-
-    # Bounded efficiency bonus: [0, 1], positive for below-average cost, zero for above.
-    # When mean_correct_cost is 0 (e.g. tool-only shaping with no harness metric, or
-    # all-zero turn counts), no rollouts can be differentiated — fall back to no bonus.
-    if mean_correct_cost <= 0:
-        return rewards - rewards.mean()
-
-    bonus = (1 - costs / mean_correct_cost).clamp(0, 1)
-
-    # Shape rewards: correct rollouts amplified by up to 2x, incorrect untouched
-    shaped_rewards = rewards * (1 + bonus * correct_mask)
-    return shaped_rewards - shaped_rewards.mean()
-
-
-def setup_advantage_fn(config: AdvantageConfig) -> AdvantageFn:
-    """Setup advantage function from config."""
-    if isinstance(config, CustomAdvantageConfig):
-        custom_fn = import_object(config.import_path)
-        kwargs = config.kwargs
-
-        def advantage_fn(inputs: AdvantageInputs) -> AdvantageOutputs:
-            return custom_fn(inputs, **kwargs)
-
-        return advantage_fn
-
-    def advantage_fn(inputs: AdvantageInputs) -> AdvantageOutputs:
-        return default_advantage_fn(inputs, length_penalty=config.length_penalty)
-
-    return advantage_fn
-
-
-def assign_advantages(
-    rollouts: list["TrainRollout"],  # noqa: F821 (forward ref)
-    advantage_fn: AdvantageFn | None,
-) -> None:
-    """Compute and assign advantages for one finished group of rollouts
-    (``TrainSink.process_group`` hands in a single group's surviving rollouts).
-    ``advantage_fn=None`` is the trivial case (advantage = reward); a custom
-    ``advantage_fn`` receives the raw ``vf.RolloutOutput``\\ s via
-    ``AdvantageInputs.rollouts``.
-    """
-    if advantage_fn is None:
-        for rollout in rollouts:
-            rollout.advantage = rollout.reward
-        return
-    result = advantage_fn(AdvantageInputs(rollouts=[r.raw for r in rollouts]))
-    for rollout, advantage in zip(rollouts, result.advantages):
-        rollout.advantage = advantage
diff --git a/src/prime_rl/orchestrator/algo/__init__.py b/src/prime_rl/orchestrator/algo/__init__.py
new file mode 100644
index 0000000000..0c799dae98
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/__init__.py
@@ -0,0 +1,120 @@
+"""Orchestrator-side algorithm runtime.
+
+The config side (``prime_rl.configs.algorithm``) defines *what* an algorithm
+is — a bundle of sampling and the per-token training signal. This package
+turns the signal half into runtime objects (the sampling half is the env's
+:class:`~prime_rl.orchestrator.sampler.Sampler`):
+
+- one module per algorithm (``grpo``, ``echo``, ``max_rl``, ``opd``,
+  ``opsd``, ``sft``, ``reward``, ``custom``) — each named class owns its
+  scoring hooks (``score_rollout`` / ``score_group`` / ``score_batch``) and
+  declares what it needs (loss component, a "teacher", ...). One instance per
+  env, built by :func:`build_algorithm`. Custom credit assignment plugs in
+  through the ``custom`` advantage type (:class:`CustomAlgorithm` imports a
+  user function by path).
+- ``base`` — the :class:`Algorithm` base class and the pipeline phase
+  functions (:func:`finalize_rollout` / :func:`finalize_group` /
+  :func:`finalize_batch`).
+- ``advantage`` — pure advantage math (default group-norm + the
+  custom-function interface). Advantages are per-token everywhere they are
+  stored or shipped — there is no scalar advantage in the pipeline. A
+  function takes ``RolloutView`` objects and returns one value per rollout: a
+  scalar that the view *broadcasts* over the rollout's completion tokens
+  (uniform credit, the common case), or an explicit per-token list.
+- ``routing`` — wire-field stamping: per-token component weight streams
+  (rl / ce / ref_kl) and the per-token advantage stream.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.orchestrator.algo.advantage import (
+    AdvantageFn,
+    apply_advantage_fn,
+    efficiency_shaping_advantage,
+    grpo_advantage,
+    length_penalty_advantage,
+    max_rl_advantage_fn,
+)
+from prime_rl.orchestrator.algo.base import (
+    Algorithm,
+    connect_frozen_pool,
+    finalize_batch,
+    finalize_group,
+    finalize_rollout,
+)
+from prime_rl.orchestrator.algo.custom import CustomAlgorithm
+from prime_rl.orchestrator.algo.echo import EchoAlgorithm
+from prime_rl.orchestrator.algo.grpo import GRPOAlgorithm
+from prime_rl.orchestrator.algo.max_rl import MaxRLAlgorithm
+from prime_rl.orchestrator.algo.opd import OPDAlgorithm
+from prime_rl.orchestrator.algo.opsd import OPSDAlgorithm
+from prime_rl.orchestrator.algo.reward import RewardAlgorithm
+from prime_rl.orchestrator.algo.routing import stamp_advantages, stamp_loss_routing
+from prime_rl.orchestrator.algo.sft import SFTDistillAlgorithm
+from prime_rl.orchestrator.types import RolloutView
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.configs.algorithm import AlgorithmConfig
+    from prime_rl.utils.client import InferencePool
+
+# Runtime dispatch is keyed on the advantage type — it names the algorithm,
+# and each config class's defaults are its vetted parameterization.
+ALGORITHM_CLASSES: dict[str, type[Algorithm]] = {
+    "grpo": GRPOAlgorithm,
+    "echo": EchoAlgorithm,
+    "max_rl": MaxRLAlgorithm,
+    "opd": OPDAlgorithm,
+    "opsd": OPSDAlgorithm,
+    "sft": SFTDistillAlgorithm,
+    "reward": RewardAlgorithm,
+    "custom": CustomAlgorithm,
+}
+
+
+def build_algorithm(
+    config: AlgorithmConfig,
+    policy_pool: InferencePool,
+    renderer: Renderer | None,
+    max_seq_len: int | None = None,
+) -> Algorithm:
+    cls = ALGORITHM_CLASSES[config.advantage.type]
+    assert cls.action_loss_type == config.advantage.action_loss_type  # config and runtime declare in two places
+    # The bundle dissolves at construction: the Algorithm is the advantage
+    # component's runtime (its sibling Sampler interprets the sampling half).
+    algorithm = cls(config.advantage, policy_pool, renderer)
+    # Host resource the constructor contract doesn't carry — only the GRPO
+    # linear length penalty reads it, so it's injected rather than threaded
+    # through every algorithm's __init__.
+    algorithm.max_seq_len = max_seq_len
+    return algorithm
+
+
+__all__ = [
+    "AdvantageFn",
+    "Algorithm",
+    "CustomAlgorithm",
+    "EchoAlgorithm",
+    "GRPOAlgorithm",
+    "MaxRLAlgorithm",
+    "OPDAlgorithm",
+    "OPSDAlgorithm",
+    "RewardAlgorithm",
+    "RolloutView",
+    "SFTDistillAlgorithm",
+    "apply_advantage_fn",
+    "build_algorithm",
+    "connect_frozen_pool",
+    "efficiency_shaping_advantage",
+    "finalize_batch",
+    "finalize_group",
+    "finalize_rollout",
+    "grpo_advantage",
+    "length_penalty_advantage",
+    "max_rl_advantage_fn",
+    "stamp_advantages",
+    "stamp_loss_routing",
+]
diff --git a/src/prime_rl/orchestrator/algo/advantage.py b/src/prime_rl/orchestrator/algo/advantage.py
new file mode 100644
index 0000000000..2039f416e9
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/advantage.py
@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable
+
+import torch
+from jaxtyping import Float
+from torch import Tensor
+
+if TYPE_CHECKING:
+    from prime_rl.orchestrator.types import RolloutView
+
+from prime_rl.configs.algorithm import (
+    LinearLengthPenaltyConfig,
+    TokensLengthPenaltyConfig,
+    TurnsLengthPenaltyConfig,
+)
+from prime_rl.orchestrator.utils import get_model_completion_len, get_tool_response_len
+
+AdvantageFn = Callable[..., list[float | list[float]]]
+"""Type for an advantage function.
+
+Expected signature:
+    def my_advantage(group: list[RolloutView], **kwargs) -> list[float | list[float]]:
+        ...
+
+The function receives one finalized group — the same ``RolloutView``\\ s the
+``score_group`` hook sees (``raw`` in step coordinates, ``samples`` in merged
+token coordinates) — and returns one value per rollout: a scalar (broadcast
+over the rollout's completion tokens) or a per-token list aligned to them.
+`apply_advantage_fn` writes each through ``RolloutView.assign_advantages``.
+"""
+
+
+def grpo_advantage(group: list["RolloutView"], length_weighted_baseline: bool = False) -> list[float]:
+    """Plain GRPO advantage for a single group: reward minus the per-group
+    baseline (DR-GRPO without std normalization).
+
+    ``length_weighted_baseline`` uses the token-length-weighted mean reward
+    (``sum(len_i * reward_i) / sum(len_i)``) as the baseline instead of the plain
+    mean, centering advantages by per-token expected reward.
+    """
+    rewards = torch.tensor([v.reward for v in group], dtype=torch.float32)
+    if length_weighted_baseline:
+        lengths = torch.tensor([get_model_completion_len(v.raw) for v in group], dtype=rewards.dtype)
+        baseline = (lengths * rewards).sum() / lengths.sum()
+    else:
+        baseline = rewards.mean()
+    return (rewards - baseline).tolist()
+
+
+def length_penalty_advantage(
+    group: list["RolloutView"],
+    config: LinearLengthPenaltyConfig,
+    max_seq_len: int | None,
+    length_weighted_baseline: bool = False,
+) -> list[float]:
+    """The linear length penalty as a standalone additive advantage term.
+
+    Each rollout's penalty is ``coef * pass_rate * (completion tokens / max_seq_len)``
+    (``pass_rate`` = group mean reward; optionally gated to correct rollouts), and
+    this returns the group-centered negative penalty ``-(penalty_i - baseline)``.
+    Summed onto :func:`grpo_advantage` it is *identical* to subtracting the penalty
+    from each reward before centering — centering is linear, so
+    ``center(reward - penalty) = center(reward) + center(-penalty)`` — provided both
+    terms use the same baseline operator, hence ``length_weighted_baseline`` is
+    threaded here too (it picks the plain vs token-length-weighted mean, matching
+    :func:`grpo_advantage`).
+    """
+    if max_seq_len is None:
+        raise ValueError("max_seq_len is required when the linear length penalty is enabled")
+    rewards = torch.tensor([v.reward for v in group], dtype=torch.float32)
+    lengths = torch.tensor([get_model_completion_len(v.raw) for v in group], dtype=rewards.dtype)
+    penalty = config.coef * rewards.mean() * (lengths / max_seq_len)
+    if config.gate_by_correctness:
+        penalty = penalty * rewards
+    baseline = (lengths * penalty).sum() / lengths.sum() if length_weighted_baseline else penalty.mean()
+    return (baseline - penalty).tolist()
+
+
+def efficiency_shaping_advantage(
+    group: list["RolloutView"], config: TokensLengthPenaltyConfig | TurnsLengthPenaltyConfig
+) -> list[float]:
+    """Correctness-gated efficiency shaping (the ``tokens`` / ``turns`` length
+    penalties) over a per-rollout cost: weighted completion + tool-response tokens,
+    or trajectory turn count. Unlike :func:`length_penalty_advantage` this is not an
+    additive term — it amplifies correct rewards (see :func:`_efficiency_shaping`) and
+    returns the full advantage, so it replaces the GRPO baseline rather than summing
+    with it.
+    """
+    rewards = torch.tensor([v.reward for v in group], dtype=torch.float32)
+    if isinstance(config, TokensLengthPenaltyConfig):
+        w_c, w_t = config.completion_weight, config.tool_response_weight
+        costs = torch.tensor(
+            [w_c * get_model_completion_len(v.raw) + w_t * get_tool_response_len(v.raw) for v in group],
+            dtype=rewards.dtype,
+        )
+    else:
+        costs = torch.tensor([len(v.raw["trajectory"]) for v in group], dtype=rewards.dtype)
+    return _efficiency_shaping(rewards, costs).tolist()
+
+
+def max_rl_advantage_fn(group: list["RolloutView"]) -> list[float]:
+    """MaxRL advantage for a single group (arXiv:2602.02710): reward minus the
+    per-group mean, divided by that mean — equivalent to averaging score
+    functions over successful rollouts only, which makes the policy gradient
+    unbiased for the order-``group_size`` truncation of the maximum-likelihood
+    objective instead of pass@1. Assumes non-negative (canonically binary)
+    rewards; a group with mean reward <= 0 carries no signal and gets zero
+    advantages (the zero-advantage filter drops it, matching the paper's
+    no-success convention)."""
+    rewards = torch.tensor([v.reward for v in group], dtype=torch.float32)
+    mean = rewards.mean()
+    if mean <= 0:
+        return torch.zeros_like(rewards).tolist()
+    return ((rewards - mean) / mean).tolist()
+
+
+def _efficiency_shaping(
+    rewards: Float[Tensor, "group_size"],
+    costs: Float[Tensor, "group_size"],
+) -> Float[Tensor, "group_size"]:
+    """Correctness-gated efficiency shaping with bounded advantages.
+
+    Shapes rewards with a bounded efficiency bonus before standard GRPO subtraction,
+    preserving zero-mean advantages within the group. `costs` is a per-rollout cost
+    (e.g., completion length in tokens or number of turns).
+
+    Correct rollouts get reward amplified by up to 2x based on relative efficiency.
+    Incorrect rollouts are untouched. Lower-cost correct rollouts get higher advantage.
+    """
+    max_reward = rewards.max()
+    correct_mask = rewards >= max_reward
+    num_correct = correct_mask.sum()
+
+    # No shaping when max reward is 0 — no correct rollouts to differentiate
+    if max_reward <= 0:
+        return rewards - rewards.mean()
+
+    # Mean cost of correct rollouts
+    mean_correct_cost = (costs * correct_mask).sum() / num_correct.clamp(min=1)
+
+    # Bounded efficiency bonus: [0, 1], positive for below-average cost, zero for above.
+    # When mean_correct_cost is 0 (e.g. tool-only shaping with no harness metric, or
+    # all-zero turn counts), no rollouts can be differentiated — fall back to no bonus.
+    if mean_correct_cost <= 0:
+        return rewards - rewards.mean()
+
+    bonus = (1 - costs / mean_correct_cost).clamp(0, 1)
+
+    # Shape rewards: correct rollouts amplified by up to 2x, incorrect untouched
+    shaped_rewards = rewards * (1 + bonus * correct_mask)
+    return shaped_rewards - shaped_rewards.mean()
+
+
+def apply_advantage_fn(group: list["RolloutView"], advantage_fn: AdvantageFn) -> None:
+    """Run an advantage function over one finished group and write each
+    rollout's result through :meth:`RolloutView.assign_advantages` (scalar
+    broadcast or per-token list). The group-relative algorithms' ``score_group``
+    hook delegates here."""
+    for view, advs in zip(group, advantage_fn(group), strict=True):
+        view.assign_advantages(advs)
diff --git a/src/prime_rl/orchestrator/algo/base.py b/src/prime_rl/orchestrator/algo/base.py
new file mode 100644
index 0000000000..df57018719
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/base.py
@@ -0,0 +1,187 @@
+"""The per-env algorithm runtime: base class and pipeline phase functions.
+
+Each named class in this package *is* one training algorithm, one module per
+algorithm: it owns the algorithm's three scoring hooks directly —
+``score_rollout`` (per arrival), ``score_group`` (per group), ``score_batch``
+(per batch) — and declares what it needs (``action_loss_type``, a
+``model_role`` like "teacher"). Reading a module top to bottom reads the
+algorithm; writing your own is subclassing :class:`Algorithm` and overriding
+the hooks its signal needs. Shared math (group normalization, prefill
+alignment) lives as plain functions in ``advantage.py``; duplication of
+orchestration between similar algorithms (e.g. OPD and OPSD) is accepted so
+each module stays self-contained.
+
+The three hooks are one scope-and-timing ladder — each wider scope is
+unlocked by a later barrier, so the two axes coincide. All three are
+``async`` (any stage may do I/O); a hook that only does advantage math never
+awaits:
+
+- ``score_rollout(rollout)`` — one rollout, on arrival: rollout-local signals
+  (raw reward, process rewards, echo's observation weighting). No siblings.
+- ``score_group(group)`` — the cohort, on group completion, *before* filtering
+  (filters read the streams): group-relative credit (GRPO/MaxRL baselines).
+- ``score_batch(batch)`` — the batch's survivors, *after* filtering: the home
+  for reference I/O (``self.teacher_pool``), where queries are batched for
+  concurrency and — running after filtering — dropped rollouts cost nothing.
+
+How rollouts are *produced* is not the algorithm's concern: that is the env's
+:class:`~prime_rl.orchestrator.sampler.Sampler`, and sample construction
+(interleaving, with observation-token provenance recorded as ``obs_spans``)
+is pure pipeline.
+
+The pipeline (dispatcher, train sink, orchestrator) calls the module-level
+phase functions (:func:`finalize_rollout`, :func:`finalize_group`,
+:func:`finalize_batch`) and reads the class declarations; it never branches on
+algorithm config fields or model roles — liveness of a reference is the only
+runtime distinction. prime-rl hosts exactly one model — the trainable policy,
+whose pool is passed in; every frozen model reference is an external endpoint
+the algorithm *connects to* (never launches) in :meth:`Algorithm.setup`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict
+from typing import TYPE_CHECKING, ClassVar
+
+from prime_rl.configs.algorithm import ActionLossType, AdvantageConfig, FrozenModelConfig, ModelReference
+from prime_rl.orchestrator.algo.routing import stamp_advantages, stamp_loss_routing
+from prime_rl.orchestrator.types import RolloutView
+from prime_rl.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.envs import TrainEnvs
+    from prime_rl.orchestrator.types import TrainRollout
+    from prime_rl.utils.client import InferencePool
+
+
+async def connect_frozen_pool(config: FrozenModelConfig) -> InferencePool:
+    """Connect a client pool to an inline frozen model and wait for it to be
+    ready. The endpoint is externally hosted — prime-rl connects and waits,
+    never launches."""
+    from prime_rl.utils.client import setup_inference_pool
+
+    get_logger().info(f"Initializing frozen model pool (model={config.name}, base_url={', '.join(config.base_url)})")
+    pool = await setup_inference_pool(config, model_name=config.name)
+    await pool.wait_for_ready(config.name)
+    return pool
+
+
+class Algorithm:
+    """Base class for one env's training algorithm — the runtime of the
+    bundle's ``advantage`` component (its sibling :class:`Sampler` interprets
+    ``sampling``).
+
+    Everything on this class is yours to override; the pipeline drives the
+    compilation through the module-level phase functions below
+    (:func:`finalize_rollout` / :func:`finalize_group` / :func:`finalize_batch`)
+    and never calls anything else. The surface is:
+
+    - declarations — which loss component the action tokens feed
+      (``action_loss_type``) and what the algorithm calls its reference
+      model, if it has one (``model_role``, e.g. "teacher");
+    - lifecycle — :meth:`setup` connects client pools to the frozen models
+      the algorithm declares, resolving each reference via :meth:`connect`;
+    - the three scoring hooks, each ``async`` and given a :class:`RolloutView`
+      (a writable handle exposing only what is valid at its stage). They are
+      async so any stage may do I/O — e.g. a process-reward model at arrival,
+      or a judge at group time whose signal a pre-batch filter then reads; a
+      hook that only does advantage math simply never awaits.
+
+      - :meth:`score_rollout` — one rollout, on arrival: rollout-local credit
+        or observation ce weights. Default: nothing.
+      - :meth:`score_group` — the cohort, *before* filtering (filters read the
+        streams): group-relative credit. Default: nothing — rollouts keep
+        ``advantages=None``, so advantage-based filters skip them.
+      - :meth:`score_batch` — the batch's survivors, *after* filtering:
+        query the algorithm's reference pool (e.g. ``self.teacher_pool``) and
+        attach per-token results, or modulate advantages. Default: nothing.
+
+    ``score_batch`` is the home for reference I/O: it runs after filtering, so
+    only survivors cost reference compute. I/O in ``score_rollout`` /
+    ``score_group`` runs *before* the pre-batch filters — do it when a filter
+    must read the result, accepting that it pays compute on rollouts that may
+    then be filtered out.
+
+    Constructed with the advantage component it interprets plus the two
+    host-owned resources: the policy pool and the policy's renderer (the
+    canonical messages → token ids path; ``None`` under MITO)."""
+
+    action_loss_type: ClassVar[ActionLossType] = "rl"
+    model_role: ClassVar[str | None] = None
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        self.advantage = advantage
+        self.policy_pool = policy_pool
+        self.renderer = renderer
+        self.connected_pools: list[InferencePool] = []  # client pools connected in setup(); closed at shutdown
+        # Training sequence length, injected by build_algorithm — the denominator
+        # of the GRPO linear length penalty. None when the host doesn't set it.
+        self.max_seq_len: int | None = None
+
+    async def setup(self) -> None:
+        """Connect client pools to the algorithm's frozen models — override
+        and resolve each reference via :meth:`connect`. The base has nothing
+        to connect."""
+
+    async def connect(self, reference: ModelReference) -> InferencePool:
+        """Resolve a model reference to a client pool: the live policy's own
+        pool, or a freshly connected pool to a frozen endpoint. Only the
+        latter is tracked in ``connected_pools`` — the host closes what the
+        algorithm opened, and nothing else, at shutdown."""
+        if reference == "policy":
+            return self.policy_pool
+        pool = await connect_frozen_pool(reference)
+        self.connected_pools.append(pool)
+        return pool
+
+    async def score_rollout(self, rollout: RolloutView) -> None:
+        """Arrival phase, one rollout, before its group is complete: write
+        rollout-local credit (``rollout.assign_advantages``) or observation ce
+        weights (echo). No siblings, no group stats."""
+
+    async def score_group(self, group: list[RolloutView]) -> None:
+        """Group phase, the finalized cohort, before filtering: write
+        group-relative credit."""
+
+    async def score_batch(self, batch: list[RolloutView]) -> None:
+        """Ship phase, survivors only, after filtering, async: query the
+        algorithm's reference models and attach per-token results, or modulate
+        advantages."""
+
+
+async def finalize_rollout(algorithm: Algorithm, rollout: TrainRollout) -> None:
+    """Arrival phase: rollout-local scoring as each rollout is tokenized."""
+    if rollout.samples:
+        await algorithm.score_rollout(RolloutView(rollout))
+
+
+async def finalize_group(algorithm: Algorithm, rollouts: list[TrainRollout]) -> None:
+    """Group phase: group-relative scoring, then stamp each sample's wire
+    fields (the advantage stream + loss routing). After this the records are
+    frozen — groups die at stamping."""
+    await algorithm.score_group([RolloutView(rollout) for rollout in rollouts])
+    for rollout in rollouts:
+        stamp_advantages(rollout)
+        for sample in rollout.samples:
+            sample.reward = rollout.reward
+            sample.env_name = rollout.env_name
+            stamp_loss_routing(sample, algorithm.action_loss_type)
+
+
+async def finalize_batch(train_envs: TrainEnvs, rollouts: list[TrainRollout]) -> None:
+    """Ship phase: run each env's ``score_batch`` over its unfiltered rollouts
+    (survivors), concurrently across envs. Per-env concurrency is bounded by
+    the algorithm's own config; envs without references return immediately."""
+    by_env: dict[str, list[TrainRollout]] = defaultdict(list)
+    for rollout in rollouts:
+        if not rollout.is_filtered:
+            by_env[rollout.env_name].append(rollout)
+    await asyncio.gather(
+        *(
+            train_envs.get(env_name).algorithm.score_batch([RolloutView(r) for r in env_rollouts])
+            for env_name, env_rollouts in by_env.items()
+        )
+    )
diff --git a/src/prime_rl/orchestrator/algo/custom.py b/src/prime_rl/orchestrator/algo/custom.py
new file mode 100644
index 0000000000..6036e0e208
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/custom.py
@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import AdvantageConfig, CustomAdvantageConfig
+from prime_rl.orchestrator.algo.advantage import apply_advantage_fn
+from prime_rl.orchestrator.algo.base import Algorithm
+from prime_rl.utils.utils import import_object
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.types import RolloutView
+    from prime_rl.utils.client import InferencePool
+
+
+class CustomAlgorithm(Algorithm):
+    """User-supplied advantage function — the ``score_group`` hook body without
+    the class: receives the group's ``RolloutView``\\ s, returns one value per
+    rollout (a scalar broadcast over its completion tokens, or a per-token
+    list)."""
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        super().__init__(advantage, policy_pool, renderer)
+        assert isinstance(advantage, CustomAdvantageConfig)
+        custom_fn = import_object(advantage.import_path)
+        kwargs = advantage.kwargs
+
+        def advantage_fn(group: list[RolloutView]) -> list[float | list[float]]:
+            return custom_fn(group, **kwargs)
+
+        self.advantage_fn = advantage_fn
+
+    async def score_group(self, group: list[RolloutView]) -> None:
+        apply_advantage_fn(group, self.advantage_fn)
diff --git a/src/prime_rl/orchestrator/algo/echo.py b/src/prime_rl/orchestrator/algo/echo.py
new file mode 100644
index 0000000000..24c954ebba
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/echo.py
@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from functools import partial
+from typing import TYPE_CHECKING, Any, Callable
+
+from prime_rl.configs.algorithm import AdvantageConfig, EchoAdvantageConfig
+from prime_rl.orchestrator.algo.grpo import GRPOAlgorithm
+from prime_rl.utils.utils import import_object
+
+if TYPE_CHECKING:
+    import verifiers as vf
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.types import RolloutView
+    from prime_rl.utils.client import InferencePool
+
+
+def _prompt_role_weights(tokens: dict[str, Any], role_weights: dict[str, float]) -> list[float]:
+    """Per-token echo weights over one step's prompt tokens.
+
+    Each token gets its message role's weight (0.0 for unselected roles), via
+    the renderer's per-token attribution — message content bodies when the
+    renderer provides ``is_content``, whole messages otherwise."""
+    attribution = tokens.get("prompt_attribution")
+    if attribution is None:
+        raise ValueError(
+            "echo selects env-provided tokens by message role, which needs the renderer's "
+            "per-token attribution — MITO rollouts don't carry it; set orchestrator.renderer."
+        )
+
+    # Serialized steps carry the attribution as a dict of RenderedTokens
+    # fields; in-process steps may carry the dataclass itself.
+    def field(key: str) -> Any:
+        return attribution.get(key) if isinstance(attribution, dict) else getattr(attribution, key, None)
+
+    indices = field("message_indices")
+    roles = field("message_roles")
+    is_content = field("is_content") or []
+    weights = []
+    for k in range(len(tokens["prompt_ids"])):
+        idx = indices[k]
+        selected = idx >= 0 and roles[idx] in role_weights
+        if selected and is_content:
+            selected = bool(is_content[k])
+        weights.append(role_weights[roles[idx]] if selected else 0.0)
+    return weights
+
+
+class EchoAlgorithm(GRPOAlgorithm):
+    """GRPO on action tokens, plus weighted CE on env-provided tokens of
+    later turns (tool output, user feedback), selected by message role —
+    tool-response bodies at the vetted default. Selected tokens feed the
+    ``ce`` loss component at their role's ``alpha`` and stay outside the rl
+    mask and its denominator. An optional user filter narrows the selection
+    per rollout (e.g. dropping tool-output warnings)."""
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        super().__init__(advantage, policy_pool, renderer)
+        assert isinstance(advantage, EchoAdvantageConfig)
+        self.role_weights = {
+            role: role_config.alpha
+            for role in ("system", "user", "assistant", "tool")
+            if (role_config := getattr(advantage.roles, role)) is not None
+        }
+        self.filter_fn: Callable[..., list[list[bool]]] | None = None
+        if advantage.filter is not None:
+            self.filter_fn = partial(import_object(advantage.filter.import_path), **advantage.filter.kwargs)
+
+    async def score_rollout(self, rollout: RolloutView) -> None:
+        # Observation weighting is rollout-local; the group-relative GRPO
+        # baseline is inherited unchanged as ``score_group``.
+        self._weight_observations(rollout)
+
+    def _weight_observations(self, rollout: RolloutView) -> None:
+        """Write each sample's ``ce_weights`` stream for the env-provided
+        observation spans interleaving recorded (``obs_spans``): each token
+        gets its message role's weight, narrowed by the optional user filter.
+        The selected tokens stay outside ``completion_mask``, so ce is the
+        only component that trains them. Step attribution is looked up
+        lazily — only steps whose prompt tokens actually landed as
+        observations are computed; samples where nothing is selected ship no
+        ce stream at all."""
+        trajectory = rollout.raw["trajectory"]
+        filter_masks = self._filter_masks(rollout.raw) if self.filter_fn is not None else None
+        step_weights: dict[int, list[float]] = {}
+        for sample in rollout.samples:
+            if not sample.obs_spans:
+                continue
+            weights = [0.0] * len(sample.completion_ids)
+            for start, step_idx, step_start, length in sample.obs_spans:
+                if step_idx not in step_weights:
+                    prompt_weights = _prompt_role_weights(trajectory[step_idx]["tokens"], self.role_weights)
+                    if filter_masks is not None:
+                        # Masks span the step's prompt+completion; obs spans
+                        # only ever come from the prompt part.
+                        prompt_weights = [w if keep else 0.0 for w, keep in zip(prompt_weights, filter_masks[step_idx])]
+                    step_weights[step_idx] = prompt_weights
+                weights[start : start + length] = step_weights[step_idx][step_start : step_start + length]
+            if any(weights):
+                sample.ce_weights = [0.0] * len(sample.prompt_ids) + weights
+
+    def _filter_masks(self, output: vf.RolloutOutput) -> list[list[bool]]:
+        """Invoke the user echo filter and validate its shape: one keep-mask
+        per trajectory step, each spanning that step's ``prompt_ids`` +
+        ``completion_ids``."""
+        assert self.filter_fn is not None
+        trajectory = output["trajectory"]
+        masks = self.filter_fn(output)
+        if not isinstance(masks, list) or len(masks) != len(trajectory):
+            got = len(masks) if isinstance(masks, list) else type(masks).__name__
+            raise ValueError(
+                f"echo filter must return one keep-mask per trajectory step: got {got}, expected {len(trajectory)}"
+            )
+        for step_idx, (step, mask) in enumerate(zip(trajectory, masks)):
+            tokens = step["tokens"]
+            expected = len(tokens["prompt_ids"]) + len(tokens["completion_ids"])
+            if not isinstance(mask, list) or len(mask) != expected:
+                got = len(mask) if isinstance(mask, list) else type(mask).__name__
+                raise ValueError(
+                    f"echo filter mask for step {step_idx} must span the step's prompt+completion "
+                    f"tokens: got {got}, expected {expected}"
+                )
+        return masks
diff --git a/src/prime_rl/orchestrator/algo/grpo.py b/src/prime_rl/orchestrator/algo/grpo.py
new file mode 100644
index 0000000000..7fdb524d39
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/grpo.py
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import (
+    AdvantageConfig,
+    GRPOAdvantageConfig,
+    LinearLengthPenaltyConfig,
+    TokensLengthPenaltyConfig,
+    TurnsLengthPenaltyConfig,
+)
+from prime_rl.orchestrator.algo.advantage import (
+    efficiency_shaping_advantage,
+    grpo_advantage,
+    length_penalty_advantage,
+)
+from prime_rl.orchestrator.algo.base import Algorithm
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.types import RolloutView
+    from prime_rl.utils.client import InferencePool
+
+
+class GRPOAlgorithm(Algorithm):
+    """Group Relative Policy Optimization: sample a group of rollouts from the
+    policy per example; credit = reward minus the group mean (optionally
+    length-shaped); action tokens feed the ``rl`` loss."""
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        super().__init__(advantage, policy_pool, renderer)
+        assert isinstance(advantage, GRPOAdvantageConfig)
+        self.length_penalty = advantage.length_penalty
+        self.length_weighted_baseline = advantage.length_weighted_baseline
+
+    async def score_group(self, group: list[RolloutView]) -> None:
+        length_penalty = self.length_penalty
+        # tokens/turns are non-additive reward shaping — they replace the baseline.
+        if isinstance(length_penalty, (TokensLengthPenaltyConfig, TurnsLengthPenaltyConfig)):
+            advantages = efficiency_shaping_advantage(group, length_penalty)
+        else:
+            # The linear length penalty is a separate advantage that sums onto GRPO's.
+            advantages = grpo_advantage(group, self.length_weighted_baseline)
+            if isinstance(length_penalty, LinearLengthPenaltyConfig):
+                penalty = length_penalty_advantage(
+                    group, length_penalty, self.max_seq_len, self.length_weighted_baseline
+                )
+                advantages = [a + p for a, p in zip(advantages, penalty, strict=True)]
+        for view, advantage in zip(group, advantages, strict=True):
+            view.assign_advantages(advantage)
diff --git a/src/prime_rl/orchestrator/algo/max_rl.py b/src/prime_rl/orchestrator/algo/max_rl.py
new file mode 100644
index 0000000000..039f44be50
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/max_rl.py
@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.orchestrator.algo.advantage import apply_advantage_fn, max_rl_advantage_fn
+from prime_rl.orchestrator.algo.base import Algorithm
+
+if TYPE_CHECKING:
+    from prime_rl.orchestrator.types import RolloutView
+
+
+class MaxRLAlgorithm(Algorithm):
+    """Maximum-likelihood RL (arXiv:2602.02710): the GRPO pipeline with
+    mean-normalized advantages — ``(reward − group mean) / group mean``
+    instead of plain centering. The mean normalization upweights low-pass-rate
+    examples like the maximum-likelihood gradient does, and ``group_size``
+    doubles as the truncation order of the likelihood expansion the gradient
+    is unbiased for (REINFORCE at 1 → exact maximum likelihood as it grows)."""
+
+    async def score_group(self, group: list[RolloutView]) -> None:
+        apply_advantage_fn(group, max_rl_advantage_fn)
diff --git a/src/prime_rl/orchestrator/algo/opd.py b/src/prime_rl/orchestrator/algo/opd.py
new file mode 100644
index 0000000000..b41e9645e4
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/opd.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import asyncio
+from itertools import cycle
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import AdvantageConfig, OPDAdvantageConfig
+from prime_rl.orchestrator.algo.base import Algorithm
+from prime_rl.orchestrator.utils import compute_prefill_logprobs
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.types import RolloutView
+    from prime_rl.transport import TrainingSample
+    from prime_rl.utils.client import InferencePool
+
+
+class OPDAlgorithm(Algorithm):
+    """On-policy distillation. Needs a teacher: the frozen reference model the
+    per-token reverse KL is computed against.
+
+    The policy samples its own rollouts; at ship time each sample's full
+    context is prefill-scored under the teacher (``ref_logprobs`` on the
+    wire), and the trainer evaluates the KL against the live policy. No
+    credit is assigned — rollouts keep ``advantages=None`` (advantage-based
+    filters never fire) and samples ship no advantage stream; ``group_size``
+    only fans out sampling."""
+
+    action_loss_type = "ref_kl"
+    model_role = "teacher"
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        super().__init__(advantage, policy_pool, renderer)
+        assert isinstance(advantage, OPDAdvantageConfig)
+        self.max_concurrent = advantage.max_concurrent
+        self.teacher = advantage.model
+        self.teacher_pool: InferencePool | None = None  # connected in setup()
+
+    async def setup(self) -> None:
+        self.teacher_pool = await self.connect(self.teacher)
+
+    async def score_batch(self, batch: list[RolloutView]) -> None:
+        pool = self.teacher_pool
+        assert pool is not None, "teacher pool not connected — Algorithm.setup() must run first"
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+        samples = [sample for view in batch for sample in view.samples]
+
+        async def score_sample(client, sample: TrainingSample) -> None:
+            async with semaphore:
+                token_ids = list(sample.prompt_ids) + list(sample.completion_ids)
+                sample.ref_logprobs = await compute_prefill_logprobs(client, pool.model_name, token_ids)
+
+        await asyncio.gather(
+            *[score_sample(client, sample) for client, sample in zip(cycle(pool.train_clients), samples)]
+        )
diff --git a/src/prime_rl/orchestrator/algo/opsd.py b/src/prime_rl/orchestrator/algo/opsd.py
new file mode 100644
index 0000000000..e8cddb1ff8
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/opsd.py
@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import asyncio
+from itertools import cycle
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import AdvantageConfig, OPSDAdvantageConfig
+from prime_rl.orchestrator.algo.base import Algorithm
+from prime_rl.orchestrator.utils import compute_prefill_logprobs
+
+if TYPE_CHECKING:
+    from renderers.base import Renderer
+
+    from prime_rl.orchestrator.types import RolloutView
+    from prime_rl.utils.client import InferencePool
+
+
+class OPSDAlgorithm(Algorithm):
+    """On-policy self-distillation (SDFT). The teacher defaults to the policy
+    itself, conditioned on an expert demonstration — no extra deployment.
+
+    The scoring prefix is rebuilt from the rollout's first-turn prompt
+    messages with the demonstration woven into the last user message; the
+    returned completion logprobs are aligned back onto the sample (the
+    sample's prompt positions are 0.0 and stay outside the loss mask). No
+    scalar advantage is assigned."""
+
+    action_loss_type = "ref_kl"
+    model_role = "teacher"
+
+    def __init__(self, advantage: AdvantageConfig, policy_pool: InferencePool, renderer: Renderer | None):
+        super().__init__(advantage, policy_pool, renderer)
+        assert isinstance(advantage, OPSDAdvantageConfig)
+        assert renderer is not None, "opsd requires the renderer (validated at config time)"
+        self.demo_key = advantage.demo_key
+        self.template = advantage.template
+        self.max_concurrent = advantage.max_concurrent
+        self.teacher = advantage.model
+        self.teacher_pool: InferencePool | None = None  # connected in setup()
+
+    async def setup(self) -> None:
+        self.teacher_pool = await self.connect(self.teacher)
+
+    def _ref_prefix_ids(self, rollout: RolloutView) -> list[int]:
+        trajectory = rollout.raw.get("trajectory") or []
+        if len(trajectory) != 1:
+            raise ValueError(
+                f"opsd supports single-step trajectories only; "
+                f"env '{rollout.env_name}' produced {len(trajectory)} steps."
+            )
+        info = rollout.raw.get("info") or {}
+        demonstration = info.get(self.demo_key) if isinstance(info, dict) else None
+        if demonstration is None:
+            demonstration = rollout.raw.get(self.demo_key)
+        if demonstration is None:
+            raise ValueError(
+                f"opsd requires '{self.demo_key}' in the example's info dict or as a "
+                f"top-level rollout field (env '{rollout.env_name}', example {rollout.example_id})."
+            )
+
+        messages = [dict(m) for m in trajectory[0]["prompt"]]
+        user_indices = [i for i, m in enumerate(messages) if m.get("role") == "user"]
+        if not user_indices:
+            raise ValueError(f"opsd found no user message to condition (env '{rollout.env_name}').")
+        last_user = messages[user_indices[-1]]
+        question = last_user.get("content")
+        if not isinstance(question, str):
+            raise ValueError("opsd supports text-only prompts (user content must be a string).")
+        last_user["content"] = self.template.format(question=question, demonstration=demonstration)
+
+        # Render through the policy's renderer — the same messages → token ids
+        # path the policy's own prompts take, so the scoring prefix matches
+        # the prompt distribution the teacher conditions on.
+        assert self.renderer is not None
+        return self.renderer.render_ids(messages, add_generation_prompt=True)
+
+    async def score_batch(self, batch: list[RolloutView]) -> None:
+        pool = self.teacher_pool
+        assert pool is not None, "teacher pool not connected — Algorithm.setup() must run first"
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def score_one(client, rollout: RolloutView) -> None:
+            prefix_ids = self._ref_prefix_ids(rollout)
+            assert len(rollout.samples) == 1  # single-step trajectory → one sample
+            sample = rollout.samples[0]
+            async with semaphore:
+                full_logprobs = await compute_prefill_logprobs(
+                    client, pool.model_name, prefix_ids + list(sample.completion_ids)
+                )
+            completion_logprobs = full_logprobs[-len(sample.completion_ids) :]
+            sample.ref_logprobs = [0.0] * len(sample.prompt_ids) + completion_logprobs
+
+        await asyncio.gather(*[score_one(client, rollout) for client, rollout in zip(cycle(pool.train_clients), batch)])
diff --git a/src/prime_rl/orchestrator/algo/reward.py b/src/prime_rl/orchestrator/algo/reward.py
new file mode 100644
index 0000000000..64bc040943
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/reward.py
@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.orchestrator.algo.base import Algorithm
+
+if TYPE_CHECKING:
+    from prime_rl.orchestrator.types import RolloutView
+
+
+class RewardAlgorithm(Algorithm):
+    """REINFORCE-style: credit = raw reward, no group baseline. Purely
+    rollout-local — no siblings needed — so it scores on arrival; action
+    tokens feed the ``rl`` loss."""
+
+    async def score_rollout(self, rollout: RolloutView) -> None:
+        rollout.assign_advantages(rollout.reward)
diff --git a/src/prime_rl/orchestrator/algo/routing.py b/src/prime_rl/orchestrator/algo/routing.py
new file mode 100644
index 0000000000..643b5bb6bb
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/routing.py
@@ -0,0 +1,76 @@
+"""Wire-field stamping for the per-token streams.
+
+The training loss is a sum of three components — ``rl`` (importance-weighted
+PG + KL), ``ce`` (masked NLL), and ``ref_kl`` (reverse KL to a reference model
+as the PG signal) — each normalized by its own global token count in the
+trainer. The advantage strategy decides which component the action tokens feed
+and the per-token advantages the rl component consumes; these helpers write
+the component weight streams and the advantage stream onto the
+``TrainingSample`` wire fields at group finalization.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import ActionLossType
+from prime_rl.transport import TrainingSample
+
+if TYPE_CHECKING:
+    from prime_rl.orchestrator.types import TrainRollout
+
+
+def stamp_loss_routing(sample: TrainingSample, action_loss_type: ActionLossType) -> None:
+    """Stamp the algorithm's loss routing onto one sample's component weight
+    streams: action tokens (the trainable completion tokens, per the loss
+    mask) feed the algorithm's declared component.
+
+    ``rl`` is the default and ships nothing (absent streams mean rl weight
+    1.0 on the loss mask — the hot path); ``ce``/``ref_kl`` weight the action
+    tokens into that component's stream and zero the rl stream. Streams an
+    algorithm wrote directly (echo's observation ce weights) are merged, not
+    clobbered — env-provided tokens stay out of ``completion_mask``, so the
+    component an algorithm weights them into is the only one that trains
+    them.
+    """
+    sample.obs_spans = None  # orchestrator-internal provenance, never ships
+    if action_loss_type == "rl":
+        return
+
+    prompt_len = len(sample.prompt_ids)
+    seq_len = prompt_len + len(sample.completion_ids)
+    sample.rl_weights = [0.0] * seq_len
+    action_weights = (
+        sample.ce_weights if action_loss_type == "ce" and sample.ce_weights is not None else [0.0] * seq_len
+    )
+    for i, trains in enumerate(sample.completion_mask):
+        if trains:
+            action_weights[prompt_len + i] = 1.0
+    if action_loss_type == "ce":
+        sample.ce_weights = action_weights
+    else:
+        assert action_loss_type == "ref_kl"
+        sample.ref_kl_weights = action_weights
+
+
+def stamp_advantages(rollout: TrainRollout) -> None:
+    """Stamp the rollout's per-token advantage stream onto its samples' wire
+    fields, padded with 0.0 over prompt positions (never trained). The stream
+    is aligned to the samples' completion tokens (concatenated in step order)
+    and sliced across them. Rollouts with no credit assigned
+    (``advantages=None``, e.g. opd/opsd) ship no advantage stream.
+    """
+    advantages = rollout.advantages
+    if advantages is None:
+        return
+    total = sum(len(sample.completion_ids) for sample in rollout.samples)
+    if len(advantages) != total:
+        raise ValueError(
+            f"advantage stream must align with the rollout's completion tokens: "
+            f"got {len(advantages)}, expected {total} (env '{rollout.env_name}')."
+        )
+    offset = 0
+    for sample in rollout.samples:
+        num_completion = len(sample.completion_ids)
+        sample.advantages = [0.0] * len(sample.prompt_ids) + advantages[offset : offset + num_completion]
+        offset += num_completion
diff --git a/src/prime_rl/orchestrator/algo/sft.py b/src/prime_rl/orchestrator/algo/sft.py
new file mode 100644
index 0000000000..b1fb49e9d2
--- /dev/null
+++ b/src/prime_rl/orchestrator/algo/sft.py
@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.orchestrator.algo.advantage import apply_advantage_fn, grpo_advantage
+from prime_rl.orchestrator.algo.base import Algorithm
+
+if TYPE_CHECKING:
+    from prime_rl.orchestrator.types import RolloutView
+
+
+class SFTDistillAlgorithm(Algorithm):
+    """Hard distillation. Needs a teacher: the frozen model that generates the
+    rollouts (``sampling.source``); the policy trains with CE on its tokens.
+
+    The ``ce`` loss ignores credit, but group-relative advantages are still
+    assigned so reward-based filtering keeps working."""
+
+    action_loss_type = "ce"
+
+    async def score_group(self, group: list[RolloutView]) -> None:
+        apply_advantage_fn(group, grpo_advantage)
diff --git a/src/prime_rl/orchestrator/dispatcher.py b/src/prime_rl/orchestrator/dispatcher.py
index 7ebb0149d2..36a8a0fd10 100644
--- a/src/prime_rl/orchestrator/dispatcher.py
+++ b/src/prime_rl/orchestrator/dispatcher.py
@@ -14,7 +14,9 @@
 - ``on_new_version`` (called by the watcher) bumps ``off_policy_steps`` on
   in-flight train rollouts and drops groups past ``max_off_policy_steps``.
   Eval rollouts are measurements for the policy version they started with,
-  so they are allowed to finish even if training advances.
+  so they are allowed to finish even if training advances. Train rollouts
+  sampled from a frozen model never age — their sampler doesn't change
+  with policy updates.
   Cancellations surface as synthetic ``Cancelled`` markers so the sink's
   count-to-``group_size`` finalization still fires.
 """
@@ -126,26 +128,20 @@ def __init__(
         eval_envs: EvalEnvs | None,
         train_source: TrainSource,
         eval_source: EvalSource | None,
-        inference: InferencePool,
-        eval_inference: InferencePool,
+        policy_pool: InferencePool,
         policy: Policy,
         max_inflight_rollouts: int,
         tasks_per_minute: float | None,
         max_off_policy_steps: int,
-        training_mode: Literal["rl", "opd", "sft"],
-        use_cache_salt: bool = True,
     ) -> None:
         self.policy = policy
         self.train_envs = train_envs
         self.eval_envs = eval_envs
-        # Train rollouts go to ``inference`` (the teacher in SFT mode);
-        # eval always evaluates the student, so it uses ``eval_inference``.
-        self.inference = inference
-        self.eval_inference = eval_inference
+        # Train rollouts go to the env sampler's pool; eval always
+        # evaluates the policy.
+        self.policy_pool = policy_pool
         self.train_source = train_source
         self.eval_source = eval_source
-        self.training_mode = training_mode
-        self.use_cache_salt = use_cache_salt
         self.max_off_policy_steps = max_off_policy_steps
 
         self.max_inflight = max_inflight_rollouts
@@ -175,14 +171,13 @@ def __init__(
         self.stopped = asyncio.Event()
         self.task: asyncio.Task | None = None
 
-    @property
-    def train_model_name(self) -> str:
-        """Model name for *train* rollouts. In SFT mode train data comes from
-        the teacher pool, so use its model name; otherwise the live student
-        policy. (Eval always uses ``policy.model_name`` — the student.)"""
-        if self.training_mode == "sft":
-            return self.inference.model_name
-        return self.policy.model_name
+    def _train_pool_for(self, env_name: str) -> tuple[InferencePool, str, bool]:
+        """``(pool, model_name, is_live)`` for *train* rollouts of this env —
+        the env sampler's pool. (Eval always uses the policy.)"""
+        sampler = self.train_envs.get(env_name).sampler
+        if sampler.samples_from_live_policy:
+            return sampler.pool, self.policy.model_name, True
+        return sampler.pool, sampler.pool.model_name, False
 
     @property
     def inflight_train_count(self) -> int:
@@ -273,6 +268,10 @@ async def on_new_version(self, step: int) -> None:
         for meta in self.inflight.values():
             if meta.kind != "train":
                 continue
+            # Frozen-sourced rollouts never go stale — their sampler doesn't
+            # change with policy updates.
+            if not self.train_envs.get(meta.env_name).sampler.samples_from_live_policy:
+                continue
             meta.off_policy_steps += 1
             if meta.off_policy_steps > self.max_off_policy_steps:
                 stale_groups.add(meta.group_id)
@@ -387,14 +386,15 @@ async def schedule_group_rollout(self, group_id: uuid.UUID, group: GroupState) -
         ready, no permits). Returns True after issuing one task — the caller
         loops to keep scheduling.
         """
-        # Train rollouts use the rollout pool (teacher in SFT) via the
-        # renderer/token train client. Eval always evaluates the student and
+        # Train rollouts use the env sampler's pool via the
+        # renderer/token train client. Eval always evaluates the policy and
         # goes through the eval client (chat-completions) — the same path the
         # legacy orchestrator used, so eval scores stay comparable.
         if group.kind == "eval":
-            pool, model_name = self.eval_inference, self.policy.model_name
+            pool, model_name = self.policy_pool, self.policy.model_name
+            live_sourced = True
         else:
-            pool, model_name = self.inference, self.train_model_name
+            pool, model_name, live_sourced = self._train_pool_for(group.env_name)
 
         # Pin a single client per group to keep prefix-cache hits
         if group.pinned_client is None:
@@ -415,7 +415,10 @@ async def schedule_group_rollout(self, group_id: uuid.UUID, group: GroupState) -
         if env_collection is None:
             return False
         env = env_collection.get(group.env_name)
-        if group.kind == "eval" or self.use_cache_salt:
+        # Frozen-sourced train rollouts hit a frozen pool; salting per policy
+        # version would invalidate its prefix cache every weight update for
+        # no reason.
+        if live_sourced:
             cache_salt = str(group.policy_version_at_start)
         else:
             cache_salt = None
diff --git a/src/prime_rl/orchestrator/envs.py b/src/prime_rl/orchestrator/envs.py
index 34e12aa63f..2891e19cff 100644
--- a/src/prime_rl/orchestrator/envs.py
+++ b/src/prime_rl/orchestrator/envs.py
@@ -12,7 +12,8 @@
 from verifiers.utils.serve_utils import get_free_port
 
 from prime_rl.configs.orchestrator import EnvConfig, EvalEnvConfig, TrainEnvConfig
-from prime_rl.orchestrator.advantage import AdvantageFn, setup_advantage_fn
+from prime_rl.orchestrator.algo import Algorithm, build_algorithm
+from prime_rl.orchestrator.sampler import Sampler
 from prime_rl.utils.logger import get_logger
 
 REQUIRED_STATE_COLUMNS = ["trajectory"]
@@ -162,14 +163,11 @@ def shutdown(self) -> None:
 class TrainEnv(Env):
     config: TrainEnvConfig
 
-    def __init__(self, config: TrainEnvConfig):
+    def __init__(self, config: TrainEnvConfig, sampler: Sampler, algorithm: Algorithm):
         super().__init__(config)
-        self.sampling_args = config.sampling.to_sampling_args()
-        # Built once — custom advantage funcs do an ``import_object`` we don't
-        # want to pay per group. ``None`` = reward-only path.
-        self.advantage_fn: AdvantageFn | None = (
-            setup_advantage_fn(config.advantage) if config.advantage is not None else None
-        )
+        self.sampler = sampler
+        self.algorithm = algorithm
+        self.sampling_args = sampler.sampling_args(config.sampling.to_sampling_args())
 
     def get_dataset(self, seed: int | None = None):
         return self.env.get_dataset(seed=seed)
@@ -244,12 +242,19 @@ def shutdown(self) -> None:
 
 
 class TrainEnvs(Envs[TrainEnv]):
-    """Collection of training environments."""
+    """Collection of training environments, each paired with its rollout
+    :class:`Sampler` and runtime :class:`Algorithm`, built from the env's
+    resolved algorithm config."""
 
-    def __init__(self, configs: Sequence[TrainEnvConfig]):
+    def __init__(self, configs: Sequence[TrainEnvConfig], *, policy_pool, renderer, max_seq_len: int):
         self._envs: dict[str, TrainEnv] = {}
         for config in configs:
-            env = TrainEnv(config)
+            assert config.algo is not None, "TrainEnvConfig.algo must be resolved before env construction"
+            env = TrainEnv(
+                config,
+                Sampler(config.algo.sampling, policy_pool),
+                build_algorithm(config.algo, policy_pool, renderer, max_seq_len=max_seq_len),
+            )
             self._envs[env.name] = env
 
 
diff --git a/src/prime_rl/orchestrator/filters.py b/src/prime_rl/orchestrator/filters.py
index f8deda1230..782438baa5 100644
--- a/src/prime_rl/orchestrator/filters.py
+++ b/src/prime_rl/orchestrator/filters.py
@@ -99,14 +99,14 @@ def check(self, rollout: "TrainRollout") -> FilterResult:
 
 @dataclass
 class ZeroAdvantageFilter:
-    """Flags rollouts whose computed advantage is zero (e.g. all rollouts in a
-    GRPO group earned the same reward, so the centered advantage collapses)."""
+    """Flags rollouts whose advantage stream is all zero (e.g. all rollouts in
+    a GRPO group earned the same reward, so the centered advantage collapses)."""
 
     name: str
     enforce: bool = True
 
     def check(self, rollout: "TrainRollout") -> FilterResult:
-        if rollout.advantage is not None and rollout.advantage == 0.0:
+        if rollout.advantages is not None and all(a == 0.0 for a in rollout.advantages):
             return FilterResult(detected=True)
         return FilterResult(detected=False)
 
diff --git a/src/prime_rl/orchestrator/metrics.py b/src/prime_rl/orchestrator/metrics.py
index d32dbc02de..5dca63be56 100644
--- a/src/prime_rl/orchestrator/metrics.py
+++ b/src/prime_rl/orchestrator/metrics.py
@@ -32,7 +32,7 @@ def build(
         progress: Progress,
         step_time: float,
         save_ckpt_time: float,
-        teacher_logprobs_time: float,
+        scoring_time: float,
         pre_filter_seen: int,
         pre_filter_dropped: int,
         pre_filter_dropped_by_name: dict[str, int],
@@ -132,7 +132,7 @@ def compute_solve_rates(df):
             "effective_batch_size/all": effective_batch_size,
             **{f"batch/{env}": r for env, r in results_df.env_name.value_counts(normalize=True).items()},
             "time/step": step_time,
-            "time/teacher_logprobs": teacher_logprobs_time,
+            "time/scoring": scoring_time,
             "time/save_ckpt": save_ckpt_time,
             "filters/all/is_filtered": results_df.is_filtered.astype(float).mean(),
             **{f"filters/all/{name}": filter_df[name].astype(float).mean() for name in filter_df.columns},
diff --git a/src/prime_rl/orchestrator/orchestrator.py b/src/prime_rl/orchestrator/orchestrator.py
index cb8bbcf852..75616c2c93 100644
--- a/src/prime_rl/orchestrator/orchestrator.py
+++ b/src/prime_rl/orchestrator/orchestrator.py
@@ -41,6 +41,7 @@
 
 import prime_rl._compat  # noqa: F401 — patch ring_flash_attn compat before transitive imports
 from prime_rl.configs.orchestrator import OrchestratorConfig
+from prime_rl.orchestrator.algo import finalize_batch
 from prime_rl.orchestrator.ckpt import setup_ckpt_manager
 from prime_rl.orchestrator.dispatcher import DispatcherMetrics, DispatcherMode, RolloutDispatcher
 from prime_rl.orchestrator.envs import EvalEnvs, TrainEnvs
@@ -66,18 +67,17 @@
     TrainRollout,
 )
 from prime_rl.orchestrator.utils import (
-    compute_teacher_logprobs,
     get_weight_dir,
     intercept_vf_logging,
     save_rollouts,
     set_default_executor,
-    setup_student_inference_pool,
+    setup_policy_inference_pool,
 )
 from prime_rl.orchestrator.watcher import WeightWatcher
 from prime_rl.trainer.model import setup_tokenizer
 from prime_rl.transport import TrainingBatch, setup_training_batch_sender
 from prime_rl.utils.async_utils import safe_cancel
-from prime_rl.utils.client import init_nccl_broadcast, setup_inference_pool
+from prime_rl.utils.client import init_nccl_broadcast
 from prime_rl.utils.heartbeat import Heartbeat
 from prime_rl.utils.logger import format_time, get_logger, setup_logger
 from prime_rl.utils.monitor import setup_monitor
@@ -129,7 +129,7 @@ class Orchestrator:
 
     # Always set by ``setup()``
     tokenizer: PreTrainedTokenizer
-    student_inference: InferencePool
+    policy_inference: InferencePool
     monitor: Monitor
     sender: TrainingBatchSender
     train_envs: TrainEnvs
@@ -144,7 +144,6 @@ class Orchestrator:
     # Set by ``setup()`` only when relevant config is present
     renderer: Renderer | None
     mm_token_type_ids_mapping: dict[int, int] | None
-    teacher_inference: InferencePool | None
     heart: Heartbeat | None
     usage_reporter: UsageReporter | None
     inference_metrics: InferenceMetricsCollector | None
@@ -162,7 +161,8 @@ def __init__(self, config: OrchestratorConfig) -> None:
         # ``verifiers.serve`` (env-server lifecycle) through our handler
         logging.getLogger("verifiers").setLevel(logging.CRITICAL + 1)
         intercept_vf_logging(logger="verifiers.serve", level="WARN")
-        get_logger().info(f"Starting orchestrator ({config.training_mode})")
+        algorithms = sorted({env.algo.advantage.type for env in config.train.env if env.algo is not None})
+        get_logger().info(f"Starting orchestrator (algorithm: {', '.join(algorithms)})")
 
         if config.bench:
             get_logger().warning(f"Running in benchmark mode (max_steps={config.max_steps})")
@@ -186,7 +186,6 @@ def __init__(self, config: OrchestratorConfig) -> None:
         # config is present
         self.renderer = None
         self.mm_token_type_ids_mapping = None
-        self.teacher_inference = None
         self.heart = None
         self.usage_reporter = None
         self.inference_metrics = None
@@ -220,12 +219,14 @@ async def setup(self) -> None:
         get_logger().info(f"Initializing tokenizer ({config.tokenizer})")
         self.tokenizer = setup_tokenizer(config.tokenizer)
 
-        # Student inference pool
+        # The one model prime-rl hosts: the live policy. Frozen model
+        # references are external endpoints — each env's Algorithm builds its
+        # own pools in ``setup()`` below.
         get_logger().info(
-            f"Initializing student inference pool (base_url={', '.join(config.student.client.base_url)}, "
-            f"model={config.student.model.name})"
+            f"Initializing policy inference pool (base_url={', '.join(config.model.client.base_url)}, "
+            f"model={config.model.name})"
         )
-        self.renderer, self.student_inference = await setup_student_inference_pool(
+        self.renderer, self.policy_inference = await setup_policy_inference_pool(
             config=config, tokenizer=self.tokenizer
         )
         self.mm_token_type_ids_mapping = (
@@ -234,17 +235,6 @@ async def setup(self) -> None:
         if self.mm_token_type_ids_mapping == {}:
             self.mm_token_type_ids_mapping = None
 
-        if config.teacher is not None:
-            get_logger().info(
-                f"Initializing teacher inference pool (base_url={', '.join(config.teacher.client.base_url)}, "
-                f"model={config.teacher.model.name})"
-            )
-            self.teacher_inference = await setup_inference_pool(
-                config.teacher.client,
-                model_name=config.teacher.model.name,
-                train_client_type="openai_chat_completions",
-            )
-
         get_logger().info(f"Initializing monitor (wandb={config.wandb}, prime_monitor={config.prime_monitor})")
         self.monitor = setup_monitor(
             wandb_config=config.wandb,
@@ -268,10 +258,9 @@ async def setup(self) -> None:
         post_filters = setup_filters(config.post_batch_filters, vocab_size=self.tokenizer.vocab_size, kind="post-batch")
 
         get_logger().info("Loading training environments")
-        self.train_envs = TrainEnvs(config.train.env)
-        if config.training_mode == "sft":
-            for env in self.train_envs:
-                env.sampling_args.pop("logprobs", None)
+        self.train_envs = TrainEnvs(
+            config.train.env, policy_pool=self.policy_inference, renderer=self.renderer, max_seq_len=config.seq_len
+        )
         get_logger().debug(
             f"Loaded {len(self.train_envs)} training environment(s) ({', '.join(self.train_envs.names)})"
         )
@@ -300,20 +289,21 @@ async def setup(self) -> None:
                 self.resume_step = config.ckpt.resume_step
 
         # Resume below may bump ``policy.version`` and the LoRA model name
-        self.policy.model_name = self.student_inference.model_name
-
-        get_logger().info("Waiting for student inference pool to be ready")
-        await self.student_inference.wait_for_ready(config.student.model.name)
-        get_logger().success("Student inference pool ready")
-        if self.teacher_inference is not None:
-            assert config.teacher is not None
-            get_logger().info("Waiting for teacher inference pool to be ready")
-            await self.teacher_inference.wait_for_ready(config.teacher.model.name)
-            get_logger().success("Teacher inference pool ready")
+        self.policy.model_name = self.policy_inference.model_name
+
+        get_logger().info("Waiting for policy inference pool to be ready")
+        await self.policy_inference.wait_for_ready(config.model.name)
+        get_logger().success("Policy inference pool ready")
+        # Build + ready pools for each env's frozen sampling source and the
+        # algorithm's frozen reference model
+        await asyncio.gather(
+            *(env.sampler.setup() for env in self.train_envs),
+            *(env.algorithm.setup() for env in self.train_envs),
+        )
 
         if config.wandb is not None and config.collect_inference_metrics:
             self.inference_metrics = InferenceMetricsCollector(
-                self.student_inference.admin_clients,
+                self.policy_inference.admin_clients,
                 roles=config.inference_metrics_roles,
             )
             await self.inference_metrics.start()
@@ -321,7 +311,7 @@ async def setup(self) -> None:
         get_logger().info(f"Initializing weight broadcast ({config.weight_broadcast})")
         if config.weight_broadcast.type == "nccl":
             await init_nccl_broadcast(
-                self.student_inference.admin_clients,
+                self.policy_inference.admin_clients,
                 config.weight_broadcast.host,
                 config.weight_broadcast.port,
                 config.weight_broadcast.timeout,
@@ -332,7 +322,7 @@ async def setup(self) -> None:
         get_logger().info(f"Initializing training batch sender ({config.rollout_transport})")
         self.sender = setup_training_batch_sender(config.output_dir, config.rollout_transport)
 
-        self.lora_name = config.student.model.lora.name if config.student.model.lora else None
+        self.lora_name = config.model.lora.name if config.model.lora else None
 
         if self.resume_step is not None and self.ckpt_manager is not None:
             self.ckpt_manager.load(self.progress, step=self.resume_step)
@@ -342,23 +332,14 @@ async def setup(self) -> None:
             weights_path = get_weight_dir(
                 config.output_dir, self.progress.step, check_exists=check_exists, wait_timeout=wait_timeout
             )
-            await self.student_inference.update_weights(weights_path, lora_name=self.lora_name, step=self.progress.step)
+            await self.policy_inference.update_weights(weights_path, lora_name=self.lora_name, step=self.progress.step)
             if self.lora_name is not None:
-                self.student_inference.update_model_name(self.lora_name)
+                self.policy_inference.update_model_name(self.lora_name)
                 self.policy.model_name = self.lora_name
             self.policy.version = self.progress.step
         else:
             get_logger().info("Training from scratch")
 
-        # SFT train rollouts come from the teacher when configured; otherwise
-        # they use the existing student rollout pool.
-        if config.training_mode == "sft" and self.teacher_inference is not None:
-            rollout_inference = self.teacher_inference
-            use_cache_salt = False
-        else:
-            rollout_inference = self.student_inference
-            use_cache_salt = True
-
         self.train_source = TrainSource(self.train_envs, seed=42)
         self.eval_source: EvalSource | None = (
             EvalSource(
@@ -378,14 +359,11 @@ async def setup(self) -> None:
             eval_envs=self.eval_envs,
             train_source=self.train_source,
             eval_source=self.eval_source,
-            inference=rollout_inference,
-            eval_inference=self.student_inference,
+            policy_pool=self.policy_inference,
             policy=self.policy,
             max_inflight_rollouts=config.max_inflight_rollouts,
             tasks_per_minute=config.tasks_per_minute,
             max_off_policy_steps=config.max_off_policy_steps,
-            training_mode=config.training_mode,
-            use_cache_salt=use_cache_salt,
         )
         self.metrics = MetricsBuilder(config, start_step=self.progress.step)
         self.train_sink = TrainSink(
@@ -403,7 +381,7 @@ async def setup(self) -> None:
         self.watcher = WeightWatcher(
             config,
             policy=self.policy,
-            inference=self.student_inference,
+            inference=self.policy_inference,
             observers=[self.dispatcher, self],
             lora_name=self.lora_name,
             ckpt_step=self.progress.step,
@@ -547,7 +525,7 @@ async def _drain_token_export_metrics(self) -> None:
 
     async def finalize_train_batch(self, batch: TrainBatch) -> None:
         """Ship one ``TrainBatch`` out to the trainer and handle the I/O
-        side-effects (ckpt, save_rollouts, teacher logprobs, sender.send,
+        side-effects (ckpt, save_rollouts, reference scoring, sender.send,
         metrics, heartbeat, progress, eval trigger). The sink has already
         done all data-transformation work."""
         config = self.config
@@ -599,18 +577,11 @@ async def finalize_train_batch(self, batch: TrainBatch) -> None:
             save_rollouts, rollout_dicts, step_path / "train_rollouts.jsonl", exclude_keys={"trajectory"}
         )
 
-        teacher_logprobs_time = 0.0  # opd only
-        if config.training_mode == "opd" and self.teacher_inference is not None:
-            assert config.teacher is not None
-            t = time.perf_counter()
-            teacher_logprobs_list = await compute_teacher_logprobs(
-                clients=self.teacher_inference.train_clients,
-                model_name=config.teacher.model.name,
-                samples=batch.samples,
-            )
-            for ex, lp in zip(batch.samples, teacher_logprobs_list):
-                ex.teacher_logprobs = lp
-            teacher_logprobs_time = time.perf_counter() - t
+        # Per-env reference scoring runs at the batch boundary; envs without a
+        # reference are a no-op, so this is unconditional.
+        t = time.perf_counter()
+        await finalize_batch(self.train_envs, batch.rollouts)
+        scoring_time = time.perf_counter() - t
 
         await self.sender.send(TrainingBatch(examples=batch.samples, step=step))
         self.update_dispatch_gate()
@@ -622,7 +593,7 @@ async def finalize_train_batch(self, batch: TrainBatch) -> None:
             progress=self.progress,
             step_time=step_time,
             save_ckpt_time=save_ckpt_time,
-            teacher_logprobs_time=teacher_logprobs_time,
+            scoring_time=scoring_time,
             pre_filter_seen=self.train_sink.pre_filter_seen,
             pre_filter_dropped=self.train_sink.pre_filter_dropped,
             pre_filter_dropped_by_name=dict(self.train_sink.pre_filter_dropped_by_name),
@@ -632,7 +603,8 @@ async def finalize_train_batch(self, batch: TrainBatch) -> None:
         self.monitor.log_distributions(
             distributions={
                 "rewards": [r.reward for r in batch.rollouts],
-                "advantages": [r.advantage for r in batch.rollouts if r.advantage is not None],
+                # Scalar view of the per-token streams (exact for uniform streams)
+                "advantages": [sum(r.advantages) / len(r.advantages) for r in batch.rollouts if r.advantages],
             },
             step=step,
         )
@@ -903,11 +875,12 @@ async def teardown() -> None:
             self.component_tasks.clear()
             if self.inference_metrics is not None:
                 await self.inference_metrics.stop()
-            if self.student_inference is not None:
-                await self.student_inference.stop()
-            if self.teacher_inference is not None:
-                await self.teacher_inference.stop()
+            if getattr(self, "policy_inference", None) is not None:
+                await self.policy_inference.stop()
             if self.train_envs is not None:
+                for env in self.train_envs:
+                    for pool in (*env.sampler.connected_pools, *env.algorithm.connected_pools):
+                        await pool.stop()
                 self.train_envs.shutdown()
             if self.eval_envs is not None:
                 self.eval_envs.shutdown()
diff --git a/src/prime_rl/orchestrator/sampler.py b/src/prime_rl/orchestrator/sampler.py
new file mode 100644
index 0000000000..e57e7c3cbf
--- /dev/null
+++ b/src/prime_rl/orchestrator/sampler.py
@@ -0,0 +1,52 @@
+"""How an env's train rollouts are produced — the sample strategy.
+
+The algorithm (``algo/``) consumes finalized rollouts and compiles them into
+per-token loss-component weights; the sampler owns where those rollouts come
+from. Today that is one question — which model generates them — and its
+consequences (sampling logprobs, prefix-cache salting, and off-policy
+staleness are all liveness questions about the source). Future sampling
+strategies (replay buffers, branching) extend here, not the algorithm.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from prime_rl.configs.algorithm import FrozenModelConfig, SamplingConfig
+from prime_rl.orchestrator.algo import connect_frozen_pool
+
+if TYPE_CHECKING:
+    from prime_rl.utils.client import InferencePool
+
+
+class Sampler:
+    """One env's rollout source.
+
+    ``pool`` is the pool train rollouts are generated from: the policy pool,
+    swapped for a connected frozen pool in :meth:`setup` when the source is an
+    inline frozen model."""
+
+    def __init__(self, config: SamplingConfig, policy_pool: InferencePool):
+        assert config.source is not None, "sampling.source must be resolved by config validation"
+        self.config = config
+        self.pool: InferencePool = policy_pool
+        self.connected_pools: list[InferencePool] = []  # client pools connected in setup(); closed at shutdown
+
+    async def setup(self) -> None:
+        """Connect a client pool to a frozen sampling source and wait for
+        readiness. Must run before dispatching."""
+        if isinstance(self.config.source, FrozenModelConfig):
+            self.pool = await connect_frozen_pool(self.config.source)
+            self.connected_pools.append(self.pool)
+
+    @property
+    def samples_from_live_policy(self) -> bool:
+        return self.config.source == "policy"
+
+    def sampling_args(self, args: dict) -> dict:
+        """Source-specific sampling-arg overrides. Sampling logprobs are only
+        needed for importance ratios on policy-sampled tokens — frozen
+        endpoints may reject the knob."""
+        if not self.samples_from_live_policy:
+            args.pop("logprobs", None)
+        return args
diff --git a/src/prime_rl/orchestrator/train_sink.py b/src/prime_rl/orchestrator/train_sink.py
index f79a0d5eff..66926ad51b 100644
--- a/src/prime_rl/orchestrator/train_sink.py
+++ b/src/prime_rl/orchestrator/train_sink.py
@@ -2,13 +2,14 @@
 
 1. ``process_rollout`` — eager per-rollout tokenization (overlaps with
    dispatcher producing more rollouts). Errored rollouts skip this.
-2. ``process_group`` — filters errored rollouts, computes advantages over
-   survivors, runs the pre-batch filter pass.
+2. ``process_group`` — filters errored rollouts, hands survivors to the env
+   algorithm's ``finalize_group`` (advantages + per-sample wire stamping),
+   runs the pre-batch filter pass.
 3. ``process_batch`` — applies post-batch filter annotations and assembles
    the trainer-bound ``TrainingSample`` list. Returns a ``TrainBatch``.
 
 ``add()`` returns ``TrainBatch | None``. I/O concerns (ship to trainer,
-save_rollouts, monitor.log, teacher logprobs) live on the orchestrator.
+save_rollouts, monitor.log, reference scoring) live on the orchestrator.
 """
 
 from __future__ import annotations
@@ -18,7 +19,7 @@
 from collections import defaultdict
 
 from prime_rl.configs.orchestrator import OrchestratorConfig
-from prime_rl.orchestrator.advantage import assign_advantages
+from prime_rl.orchestrator.algo import finalize_group, finalize_rollout
 from prime_rl.orchestrator.envs import TrainEnvs
 from prime_rl.orchestrator.filters import RolloutFilter, apply_filters
 from prime_rl.orchestrator.trajectories import (
@@ -132,7 +133,7 @@ async def add(self, rollout: TrainRollout) -> TrainBatch | None:
             self.errors_by_env[env_name] += 1
         self.pending_groups[rollout.group_id].append(rollout)
         if len(self.pending_groups[rollout.group_id]) >= self.group_size_for(env_name):
-            self.process_group(rollout.group_id)
+            await self.process_group(rollout.group_id)
         ready = (
             len(self.pending_batch) >= self.batch_size
             if self.batch_size is not None
@@ -148,7 +149,7 @@ async def add(self, rollout: TrainRollout) -> TrainBatch | None:
 
     async def process_rollout(self, rollout: TrainRollout) -> None:
         """Tokenize the rollout eagerly. Backfills tokens if the env didn't
-        return them (SFT against external teacher APIs); errored rollouts
+        return them (frozen-sourced rollouts from external APIs); errored rollouts
         skip tokenization and get dropped at the group level."""
         if rollout.error is not None:
             return
@@ -157,18 +158,23 @@ async def process_rollout(self, rollout: TrainRollout) -> None:
         if needs_backfill:
             await asyncio.to_thread(backfill_rollout_tokens, raw, self.tokenizer, renderer=self.renderer)
         samples = await asyncio.to_thread(
-            interleave_rollout,
-            raw,
-            mm_token_type_ids_mapping=self.mm_token_type_ids_mapping,
-            env_name=rollout.env_name,
+            lambda: interleave_rollout(
+                raw,
+                env_name=rollout.env_name,
+                mm_token_type_ids_mapping=self.mm_token_type_ids_mapping,
+            )
         )
         rollout.samples = samples or []
+        # Arrival phase: rollout-local scoring (raw reward, echo observation
+        # weighting) runs as soon as the rollout is tokenized — before its
+        # group is complete.
+        await finalize_rollout(self.train_envs.get(rollout.env_name).algorithm, rollout)
         # Offload base64 image bytes to disk as soon as the rollout is
         # tokenized, so memory stays flat instead of holding every buffered
         # rollout's images until the batch ships (no-op for text-only).
         await asyncio.to_thread(offload_images_to_disk, [raw], self.config.output_dir)
 
-    def process_group(self, group_id: uuid.UUID) -> None:
+    async def process_group(self, group_id: uuid.UUID) -> None:
         """Finalize one GRPO group: drop errored rollouts (the whole group
         when ``requires_group_scoring`` and any failed), assign advantages,
         run pre-batch filters, append survivors to ``pending_batch``."""
@@ -196,19 +202,16 @@ def process_group(self, group_id: uuid.UUID) -> None:
             )
             return
 
-        assign_advantages(survivors, self.train_envs.get(env_name).advantage_fn)
+        # Advantages + per-sample wire stamping (advantage stream, loss
+        # routing) are the algorithm's job; the sink only owns the grouping
+        # mechanics.
+        await finalize_group(env.algorithm, survivors)
 
-        # Propagate to the pre-tokenized samples so the orchestrator can
-        # collect samples at ship time without re-walking rollouts. The env
-        # has a single sampling temperature; fan it out across each sample's
-        # completion tokens here (interleave leaves it empty).
+        # The env has a single sampling temperature; fan it out across each
+        # sample's completion tokens (interleave leaves it empty).
         temperature = env.sampling_args["temperature"]
         for r in survivors:
             for sample in r.samples:
-                sample.advantage = r.advantage
-                sample.reward = r.reward
-                sample.env_name = r.env_name
-                sample.training_mode = self.config.training_mode
                 sample.completion_temperatures = [temperature] * len(sample.completion_ids)
 
         if self.pre_filters:
@@ -266,7 +269,7 @@ def process_batch(self) -> TrainBatch:
             apply_filters(self.post_filters, cohort)
 
         # Samples are pre-built by ``process_rollout``; ``process_group``
-        # already set advantage/reward on each sample
+        # already set advantages/reward on each sample
         samples: list[TrainingSample] = []
         prefill_lens: list[int] = []
         decode_lens: list[int] = []
diff --git a/src/prime_rl/orchestrator/trajectories.py b/src/prime_rl/orchestrator/trajectories.py
index 3e8431c12a..5b62c26b89 100644
--- a/src/prime_rl/orchestrator/trajectories.py
+++ b/src/prime_rl/orchestrator/trajectories.py
@@ -222,6 +222,13 @@ def interleave_rollout(
     Returns a list of samples - could be 1 (extension always held) or up to T
     (extension never held).
 
+    Env-provided observation tokens (the spans that land as later-turn prompt
+    extensions) are recorded as provenance on each sample: ``obs_spans``
+    entries map sample completion positions back to trajectory-step
+    coordinates (``[completion_start, step_idx, step_prompt_start, length]``).
+    Algorithms that train on observations (ECHO) consume the spans at group
+    time to write per-token ce weights; everything else ignores them.
+
     For VLM models, each renderer-produced trajectory step carries its
     per-image processed tensors inline on ``multi_modal_data``; the last
     merged step's sidecar covers every image in the sample.
@@ -265,6 +272,9 @@ def prepare_step_tokens(step: vf.TrajectoryStep, step_idx: int) -> dict[str, Any
                 # a multimodal-aware renderer (e.g. Qwen3VLRenderer); absent
                 # for text-only rollouts.
                 "multi_modal_data": tokens.get("multi_modal_data"),
+                # Renderer per-token attribution (message_indices / roles /
+                # is_content); absent on MITO rollouts.
+                "prompt_attribution": tokens.get("prompt_attribution"),
             }
 
         logger.warning(f"Missing rollout tokens for example {output['example_id']} step {step_idx}.")
@@ -303,8 +313,7 @@ def make_sample(tokens: dict[str, Any], step_idx: int) -> TrainingSample:
             completion_mask=completion_mask,
             completion_logprobs=list(tokens["completion_logprobs"]),
             completion_temperatures=[],
-            teacher_logprobs=None,
-            advantage=None,
+            ref_logprobs=None,
             env_name=env_name,
             mm_token_type_ids=None,
             routed_experts=None,  # deferred — finalized at end of interleave_rollout
@@ -370,8 +379,15 @@ def extend_sample(
         """Extend an existing sample with a new trajectory step (extension property holds)."""
         tokens = prepared_steps[step_idx]
 
-        # Extend with new prompt tokens (mask=False, no gradient)
+        # Extend with new prompt tokens (mask=False, no gradient). These are
+        # the env's response to the previous action — observation tokens;
+        # record where they came from so group-time observation weighting
+        # (echo) can look up the step's attribution.
         new_prompt_ids = tokens["prompt_ids"][prefix_len:]
+        if new_prompt_ids:
+            if sample.obs_spans is None:
+                sample.obs_spans = []
+            sample.obs_spans.append([len(sample.completion_ids), step_idx, prefix_len, len(new_prompt_ids)])
         sample.completion_ids.extend(new_prompt_ids)
         sample.completion_mask.extend([False] * len(new_prompt_ids))
         sample.completion_logprobs.extend([0.0] * len(new_prompt_ids))
diff --git a/src/prime_rl/orchestrator/types.py b/src/prime_rl/orchestrator/types.py
index c2a3f5de79..ad9b694bda 100644
--- a/src/prime_rl/orchestrator/types.py
+++ b/src/prime_rl/orchestrator/types.py
@@ -97,7 +97,8 @@ def to_dict(self) -> vf.RolloutOutput:
         ``monitor.log_samples``). Shallow copy; never mutates ``self.raw``."""
         out: vf.RolloutOutput = dict(self.raw)  # type: ignore[assignment]
         for f in fields(self):
-            if f.name in ("raw", "samples"):
+            # advantages is per-token bulk data like samples — skip it
+            if f.name in ("raw", "samples", "advantages"):
                 continue
             val = getattr(self, f.name)
             if f.name == "filter_results":
@@ -110,10 +111,74 @@ def to_dict(self) -> vf.RolloutOutput:
 @dataclass
 class TrainRollout(FinishedRollout):
     samples: list[TrainingSample] = field(default_factory=list)
-    advantage: float | None = None
+    # Per-token advantages from the advantage strategy, aligned to the
+    # samples' completion tokens (concatenated in step order). None = no
+    # credit assigned (advantage-based filters skip it; the wire ships no
+    # advantage stream).
+    advantages: list[float] | None = None
     is_filtered: bool = False
     filter_results: dict[str, bool] = field(default_factory=dict)
 
+    def to_dict(self) -> vf.RolloutOutput:
+        out = super().to_dict()
+        # ``advantages`` is skipped as bulk; dumps keep a scalar view (exact
+        # for uniform streams, the mean otherwise).
+        if self.advantages:
+            out["advantage"] = sum(self.advantages) / len(self.advantages)
+        return out
+
+
+@dataclass(frozen=True)
+class RolloutView:
+    """A finalized rollout as a writable handle — the single currency the
+    scoring hooks operate on. Exposes what the env produced (``raw``), the
+    samples interleaving built (``samples``, carrying ``obs_spans``), and the
+    rollout's identity/reward; credit is written through
+    :meth:`assign_advantages`, which spreads over the samples' completion
+    tokens. Deliberately does *not* expose pipeline-internal lifecycle fields
+    (``is_filtered``, ``filter_results``, ``group_id``) or not-yet-assigned
+    credit (``advantages``) — a hook can only touch what is valid at its
+    stage."""
+
+    _rollout: TrainRollout
+
+    @property
+    def raw(self) -> vf.RolloutOutput:
+        return self._rollout.raw
+
+    @property
+    def samples(self) -> list[TrainingSample]:
+        return self._rollout.samples
+
+    @property
+    def reward(self) -> float:
+        return self._rollout.reward
+
+    @property
+    def env_name(self) -> str:
+        return self._rollout.env_name
+
+    @property
+    def example_id(self) -> int | str:
+        return self._rollout.example_id
+
+    def assign_advantages(self, values: float | list[float]) -> None:
+        """Write the rl advantage stream: a scalar broadcast over the
+        rollout's completion tokens, or a per-token list aligned to them
+        (concatenated across samples in step order). Prompt positions are
+        padded at stamping; a rollout never assigned ships no advantage
+        stream."""
+        total = sum(len(sample.completion_ids) for sample in self._rollout.samples)
+        if isinstance(values, (int, float)):
+            self._rollout.advantages = [float(values)] * total
+            return
+        if len(values) != total:
+            raise ValueError(
+                f"per-token advantages must align with the rollout's completion tokens: "
+                f"got {len(values)}, expected {total} (env '{self._rollout.env_name}')."
+            )
+        self._rollout.advantages = [float(v) for v in values]
+
 
 @dataclass
 class EvalRollout(FinishedRollout):
diff --git a/src/prime_rl/orchestrator/utils.py b/src/prime_rl/orchestrator/utils.py
index 5675ba3f34..810787b423 100644
--- a/src/prime_rl/orchestrator/utils.py
+++ b/src/prime_rl/orchestrator/utils.py
@@ -2,7 +2,6 @@
 import logging
 import time
 from concurrent.futures import ThreadPoolExecutor
-from itertools import cycle
 from pathlib import Path
 
 import orjson
@@ -11,7 +10,6 @@
 from verifiers.utils.save_utils import make_serializable
 
 from prime_rl.configs.orchestrator import OrchestratorConfig
-from prime_rl.transport import TrainingSample
 from prime_rl.utils.client import setup_inference_pool
 from prime_rl.utils.logger import InterceptHandler, get_logger
 from prime_rl.utils.utils import (
@@ -21,18 +19,29 @@
 )
 
 
-async def setup_student_inference_pool(*, config: OrchestratorConfig, tokenizer):
-    """Build the student inference pool + matching renderer. Returns
-    ``(renderer | None, inference_pool)``; ``renderer`` is ``None`` on the
-    MITO path (``config.renderer is None``)."""
+async def setup_policy_inference_pool(*, config: OrchestratorConfig, tokenizer):
+    """Build the live policy inference pool + matching renderer. Returns
+    ``(renderer | None, inference_pool)``; ``renderer`` is ``None`` only when
+    the run opts out (``config.renderer is None``, MITO).
+
+    The renderer object and the renderer *client* are decoupled: the object
+    is the canonical messages → token ids path (sft backfill, hinted scoring
+    prefixes, role attribution) and exists whenever configured; the
+    renderer-client sampling path is wired onto the pool only when a train
+    env actually samples from the policy. Frozen-sourced runs keep the
+    renderer for tokenization while the policy pool serves plain
+    chat-completions (evals)."""
     from renderers.base import create_renderer
 
-    client_config = config.student.client
-    model_name = config.student.model.name
+    client_config = config.model.client
+    model_name = config.model.name
 
+    renderer = None
     if config.renderer is not None:
         renderer = create_renderer(tokenizer, config.renderer)
         get_logger().info(f"Initialized {type(renderer).__name__} for {model_name}")
+
+    if renderer is not None and config.any_policy_sourced:
         inference_pool = await setup_inference_pool(
             client_config,
             model_name=model_name,
@@ -44,14 +53,17 @@ async def setup_student_inference_pool(*, config: OrchestratorConfig, tokenizer)
         get_logger().info("Using direct renderer rollout client")
         return renderer, inference_pool
 
-    get_logger().info("Using MITO (openai_chat_completions) for rollouts")
+    if renderer is not None:
+        get_logger().info("No policy-sourced train env — renderer kept for client-side tokenization only")
+    else:
+        get_logger().info("Using MITO (openai_chat_completions) for rollouts")
     inference_pool = await setup_inference_pool(
         client_config,
         model_name=model_name,
         train_client_type="openai_chat_completions",
         eval_client_type="openai_chat_completions",
     )
-    return None, inference_pool
+    return renderer, inference_pool
 
 
 def get_model_completion_len(output: vf.RolloutOutput) -> int:
@@ -96,60 +108,58 @@ def set_default_executor(max_workers: int = 64) -> None:
     asyncio.get_event_loop().set_default_executor(ThreadPoolExecutor(max_workers=max_workers))
 
 
-async def compute_teacher_logprobs(
-    clients: list[vf.ClientConfig],
+async def compute_prefill_logprobs(
+    client_config: vf.ClientConfig,
     model_name: str,
-    samples: list[TrainingSample],
-) -> list[list[float]]:
-    """Compute teacher model logprobs for a batch of training samples via prefill."""
+    token_ids: list[int],
+) -> list[float]:
+    """Score ``token_ids`` under ``model_name`` via prefill; returns one
+    logprob per token (0.0 for the leading token, which has no context)."""
     import httpx
     from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
 
-    async def _compute_single(client_config: vf.ClientConfig, sample: TrainingSample) -> list[float]:
-        client = setup_openai_client(client_config)
-
-        # Two escape hatches from ``AsyncOpenAI.post``:
-        #   1. URL — ``/inference/v1/generate`` is mounted at server root, not
-        #      under ``/v1``. Pass an absolute URL so the SDK's
-        #      ``_prepare_url`` skips the base-url merge (it short-circuits
-        #      when the path passes ``httpx.URL.is_relative_url`` as False).
-        #   2. Parse — vLLM's ``GenerateResponse`` is a plain
-        #      ``pydantic.BaseModel`` and the SDK's parse layer rejects any
-        #      ``cast_to`` that doesn't subclass ``openai.BaseModel``. Use
-        #      ``cast_to=httpx.Response`` so the SDK still builds the request
-        #      (preserving ``auth_headers``, retries, timeouts, idempotency
-        #      keys) and just hands us the raw response to validate ourselves.
-        base = str(client.base_url).rstrip("/").removesuffix("/v1")
-        http_response = await client.post(
-            f"{base}/inference/v1/generate",
-            cast_to=httpx.Response,
-            body={
-                "model": model_name,
-                "token_ids": list(sample.prompt_ids) + list(sample.completion_ids),
-                "sampling_params": {
-                    "max_tokens": 1,
-                    "temperature": 1.0,
-                    "top_p": 1.0,
-                    "prompt_logprobs": 1,
-                },
+    client = setup_openai_client(client_config)
+
+    # Two escape hatches from ``AsyncOpenAI.post``:
+    #   1. URL — ``/inference/v1/generate`` is mounted at server root, not
+    #      under ``/v1``. Pass an absolute URL so the SDK's
+    #      ``_prepare_url`` skips the base-url merge (it short-circuits
+    #      when the path passes ``httpx.URL.is_relative_url`` as False).
+    #   2. Parse — vLLM's ``GenerateResponse`` is a plain
+    #      ``pydantic.BaseModel`` and the SDK's parse layer rejects any
+    #      ``cast_to`` that doesn't subclass ``openai.BaseModel``. Use
+    #      ``cast_to=httpx.Response`` so the SDK still builds the request
+    #      (preserving ``auth_headers``, retries, timeouts, idempotency
+    #      keys) and just hands us the raw response to validate ourselves.
+    base = str(client.base_url).rstrip("/").removesuffix("/v1")
+    http_response = await client.post(
+        f"{base}/inference/v1/generate",
+        cast_to=httpx.Response,
+        body={
+            "model": model_name,
+            "token_ids": token_ids,
+            "sampling_params": {
+                "max_tokens": 1,
+                "temperature": 1.0,
+                "top_p": 1.0,
+                "prompt_logprobs": 1,
             },
-        )
-        response = GenerateResponse.model_validate_json(http_response.content)
-        # ``prompt_logprobs[i]`` is a ``{token_id: Logprob}`` dict for tokens
-        # the engine could score, or ``None`` for the leading token which has
-        # no preceding context. Flatten to ``list[float]`` with 0.0 in the
-        # unscored slot.
-        flat: list[float] = []
-        for entry in response.prompt_logprobs or []:
-            if not entry:
-                flat.append(0.0)
-                continue
-            first = next(iter(entry.values()))
-            lp = first.logprob if hasattr(first, "logprob") else first.get("logprob")
-            flat.append(float(lp) if lp is not None else 0.0)
-        return flat
-
-    return await asyncio.gather(*[_compute_single(client, sample) for client, sample in zip(cycle(clients), samples)])
+        },
+    )
+    response = GenerateResponse.model_validate_json(http_response.content)
+    # ``prompt_logprobs[i]`` is a ``{token_id: Logprob}`` dict for tokens
+    # the engine could score, or ``None`` for the leading token which has
+    # no preceding context. Flatten to ``list[float]`` with 0.0 in the
+    # unscored slot.
+    flat: list[float] = []
+    for entry in response.prompt_logprobs or []:
+        if not entry:
+            flat.append(0.0)
+            continue
+        first = next(iter(entry.values()))
+        lp = first.logprob if hasattr(first, "logprob") else first.get("logprob")
+        flat.append(float(lp) if lp is not None else 0.0)
+    return flat
 
 
 def get_weight_dir(output_dir: Path, step: int, check_exists: bool = True, wait_timeout: int | None = None) -> Path:
diff --git a/src/prime_rl/trainer/batch.py b/src/prime_rl/trainer/batch.py
index 2c39a83e4c..059d850fad 100644
--- a/src/prime_rl/trainer/batch.py
+++ b/src/prime_rl/trainer/batch.py
@@ -11,6 +11,11 @@
     "int32": 4,
 }
 
+# Backfill value per component weight stream when a packed sample doesn't
+# carry it: absent rl means weight 1.0 on the loss mask, absent ce/ref_kl
+# means no component (weight 0.0).
+STREAM_FILL = {"rl_weights": 1.0, "ce_weights": 0.0, "ref_kl_weights": 0.0}
+
 
 def _copy_routed_experts(routed_experts: RoutedExperts) -> RoutedExperts:
     return RoutedExperts(
@@ -49,7 +54,22 @@ def prepare_sample(training_example: TrainingSample, seq_len: int) -> MicroBatch
     input_ids = training_example.prompt_ids + training_example.completion_ids
     loss_mask = training_example.prompt_mask + training_example.completion_mask
     inference_logprobs = [0.0] * len(training_example.prompt_ids) + training_example.completion_logprobs
-    advantages = [training_example.advantage] * len(input_ids)
+    if training_example.advantages is not None:
+        advantages = list(training_example.advantages)
+    else:
+        rl_w = training_example.rl_weights
+        has_rl_members = any(loss_mask) if rl_w is None else any(m and w != 0 for m, w in zip(loss_mask, rl_w))
+        if has_rl_members:
+            raise ValueError(
+                f"sample from env '{training_example.env_name}' has rl member tokens but no advantages — "
+                "the producer must stamp the advantage stream (the orchestrator broadcasts the rollout scalar)"
+            )
+        advantages = [0.0] * len(input_ids)
+    # Component weight streams: keep absent streams None (rl weight 1.0 on the
+    # loss mask, no ce/ref_kl component) so the packed batch stays as small as before.
+    rl_weights = list(training_example.rl_weights) if training_example.rl_weights is not None else None
+    ce_weights = list(training_example.ce_weights) if training_example.ce_weights is not None else None
+    ref_kl_weights = list(training_example.ref_kl_weights) if training_example.ref_kl_weights is not None else None
     reward = training_example.reward if training_example.reward is not None else float("nan")
     rewards = [reward] * len(input_ids)
     position_ids = list(range(len(input_ids)))
@@ -62,9 +82,9 @@ def prepare_sample(training_example: TrainingSample, seq_len: int) -> MicroBatch
     prompt_temp = training_example.completion_temperatures[0] if training_example.completion_temperatures else 1.0
     temperatures = [prompt_temp] * len(training_example.prompt_ids) + training_example.completion_temperatures
 
-    # Teacher logprobs already cover the full sequence (prompt + completion),
-    # computed via prefill in the orchestrator when a teacher model is configured
-    teacher_logprobs = training_example.teacher_logprobs
+    # Ref logprobs already cover the full sequence (prompt + completion),
+    # computed via prefill in the orchestrator when the algorithm scores against a reference
+    ref_logprobs = training_example.ref_logprobs
     routed_experts = (
         _copy_routed_experts(training_example.routed_experts) if training_example.routed_experts is not None else None
     )
@@ -77,8 +97,14 @@ def prepare_sample(training_example: TrainingSample, seq_len: int) -> MicroBatch
         advantages = advantages[:seq_len]
         rewards = rewards[:seq_len]
         temperatures = temperatures[:seq_len]
-        if teacher_logprobs is not None:
-            teacher_logprobs = teacher_logprobs[:seq_len]
+        if ref_logprobs is not None:
+            ref_logprobs = ref_logprobs[:seq_len]
+        if rl_weights is not None:
+            rl_weights = rl_weights[:seq_len]
+        if ce_weights is not None:
+            ce_weights = ce_weights[:seq_len]
+        if ref_kl_weights is not None:
+            ref_kl_weights = ref_kl_weights[:seq_len]
         if routed_experts is not None:
             routed_experts = _slice_routed_experts(routed_experts, seq_len)
         if mm_token_type_ids is not None:
@@ -96,8 +122,15 @@ def prepare_sample(training_example: TrainingSample, seq_len: int) -> MicroBatch
     ), (
         f"input_ids: {len(input_ids)}, advantages: {len(advantages)}, loss_mask: {len(loss_mask)}, position_ids: {len(position_ids)}, inference_logprobs: {len(inference_logprobs)}, rewards: {len(rewards)}, temperatures: {len(temperatures)}"
     )
-    if teacher_logprobs is not None:
-        assert len(teacher_logprobs) == len(input_ids), f"teacher_logprobs: {len(teacher_logprobs)}"
+    if ref_logprobs is not None:
+        assert len(ref_logprobs) == len(input_ids), f"ref_logprobs: {len(ref_logprobs)}"
+    for stream_name, stream in (
+        ("rl_weights", rl_weights),
+        ("ce_weights", ce_weights),
+        ("ref_kl_weights", ref_kl_weights),
+    ):
+        if stream is not None:
+            assert len(stream) == len(input_ids), f"{stream_name}: {len(stream)}"
 
     if routed_experts is not None:
         assert routed_experts.shape[0] == len(input_ids), (
@@ -118,14 +151,16 @@ def prepare_sample(training_example: TrainingSample, seq_len: int) -> MicroBatch
         position_ids=position_ids,
         inference_logprobs=inference_logprobs,
         sequence_lengths=[len(input_ids)],
-        teacher_logprobs=teacher_logprobs,
+        ref_logprobs=ref_logprobs,
         temperatures=temperatures,
         rewards=rewards,
         routed_experts=routed_experts,
         mm_token_type_ids=mm_token_type_ids,
         env_names=env_names,
         mm_kwargs=training_example.mm_kwargs,
-        training_mode=training_example.training_mode,
+        rl_weights=rl_weights,
+        ce_weights=ce_weights,
+        ref_kl_weights=ref_kl_weights,
     )
 
 
@@ -148,12 +183,14 @@ def first_sample(self) -> MicroBatch:
         return self.samples[0][1]
 
     def can_add(self, sample: MicroBatch, max_seq_len: int) -> bool:
+        # Loss routing is per token (component weight streams), so samples of
+        # different loss types pack together freely — only modality, length and
+        # routed-experts presence constrain packing.
         first_sample = self.first_sample
         return (
             not _is_multimodal_sample(first_sample)
             and not _is_multimodal_sample(sample)
             and self.length + len(sample.input_ids) <= max_seq_len
-            and first_sample.training_mode == sample.training_mode
             and (first_sample.routed_experts is None) == (sample.routed_experts is None)
         )
 
@@ -186,8 +223,11 @@ def split_by_workload(self, bin_cost: Callable[[Sequence[int]], int]) -> tuple["
 
 def _materialize_bin(bin_content: _MicroBatchBin, num_loras: int) -> MicroBatch:
     has_rewards = any(sample.rewards is not None for _, sample in bin_content.samples)
-    has_teacher_logprobs = any(sample.teacher_logprobs is not None for _, sample in bin_content.samples)
+    has_ref_logprobs = any(sample.ref_logprobs is not None for _, sample in bin_content.samples)
     has_mm_token_type_ids = any(sample.mm_token_type_ids is not None for _, sample in bin_content.samples)
+    # A weight stream materializes as soon as one packed sample carries it; the
+    # samples that lack it get the stream's identity fill (STREAM_FILL).
+    has_stream = {name: any(getattr(s, name) is not None for _, s in bin_content.samples) for name in STREAM_FILL}
 
     input_ids: list[int] = []
     loss_mask: list[bool] = []
@@ -197,8 +237,9 @@ def _materialize_bin(bin_content: _MicroBatchBin, num_loras: int) -> MicroBatch:
     temperatures: list[float] = []
     env_names: list[str] = []
     rewards: list[float] | None = [] if has_rewards else None
-    teacher_logprobs: list[float] | None = [] if has_teacher_logprobs else None
+    ref_logprobs: list[float] | None = [] if has_ref_logprobs else None
     mm_token_type_ids: list[int] | None = [] if has_mm_token_type_ids else None
+    streams: dict[str, list[float] | None] = {name: ([] if has_stream[name] else None) for name in STREAM_FILL}
     routed_experts: RoutedExperts | None = None
     lora_num_tokens = [0] * num_loras
 
@@ -213,10 +254,13 @@ def _materialize_bin(bin_content: _MicroBatchBin, num_loras: int) -> MicroBatch:
         env_names.extend(sample.env_names)
         if rewards is not None:
             rewards.extend(sample.rewards if sample.rewards is not None else [float("nan")] * sample_len)
-        if teacher_logprobs is not None:
-            teacher_logprobs.extend(
-                sample.teacher_logprobs if sample.teacher_logprobs is not None else [0.0] * sample_len
-            )
+        if ref_logprobs is not None:
+            ref_logprobs.extend(sample.ref_logprobs if sample.ref_logprobs is not None else [0.0] * sample_len)
+        for name, fill in STREAM_FILL.items():
+            stream = streams[name]
+            if stream is not None:
+                sample_stream = getattr(sample, name)
+                stream.extend(sample_stream if sample_stream is not None else [fill] * sample_len)
         if mm_token_type_ids is not None:
             mm_token_type_ids.extend(
                 sample.mm_token_type_ids if sample.mm_token_type_ids is not None else [0] * sample_len
@@ -242,7 +286,7 @@ def _materialize_bin(bin_content: _MicroBatchBin, num_loras: int) -> MicroBatch:
         position_ids=position_ids,
         inference_logprobs=inference_logprobs,
         sequence_lengths=sequence_lengths,
-        teacher_logprobs=teacher_logprobs,
+        ref_logprobs=ref_logprobs,
         temperatures=temperatures,
         rewards=rewards,
         lora_num_tokens=lora_num_tokens,
@@ -250,7 +294,9 @@ def _materialize_bin(bin_content: _MicroBatchBin, num_loras: int) -> MicroBatch:
         mm_token_type_ids=mm_token_type_ids,
         env_names=env_names,
         mm_kwargs=first_sample.mm_kwargs if _is_multimodal_sample(first_sample) else None,
-        training_mode=first_sample.training_mode,
+        rl_weights=streams["rl_weights"],
+        ce_weights=streams["ce_weights"],
+        ref_kl_weights=streams["ref_kl_weights"],
     )
 
 
@@ -357,8 +403,15 @@ def pad_micro_batch(micro_batch: MicroBatch, pad_to_multiple_of: int) -> MicroBa
     micro_batch.inference_logprobs.extend([0.0] * padding_size)
     # Use temperature 1.0 for padding tokens (doesn't matter since loss_mask is False)
     micro_batch.temperatures.extend([1.0] * padding_size)
-    if micro_batch.teacher_logprobs is not None:
-        micro_batch.teacher_logprobs.extend([0.0] * padding_size)
+    if micro_batch.ref_logprobs is not None:
+        micro_batch.ref_logprobs.extend([0.0] * padding_size)
+    # Padding is loss-masked, so no component trains it; fill every stream
+    # with 0.0 (not the pack-boundary defaults) so a padded pure-ce batch
+    # still reads as rl-empty in token export, which keys off nonzero weights.
+    for stream_name in STREAM_FILL:
+        stream = getattr(micro_batch, stream_name)
+        if stream is not None:
+            stream.extend([0.0] * padding_size)
     if micro_batch.lora_num_tokens is not None:
         micro_batch.lora_num_tokens[-1] += (
             padding_size  # We send padding to the last lora so that tokens have ascending lora idx
@@ -372,11 +425,49 @@ def pad_micro_batch(micro_batch: MicroBatch, pad_to_multiple_of: int) -> MicroBa
     return micro_batch
 
 
+def _assert_token_arrays_aligned(micro_batch: MicroBatch) -> None:
+    """Every per-token array must stay position-aligned with ``input_ids``
+    through packing and padding — a field extended without backfill would
+    corrupt training silently."""
+    num_tokens = len(micro_batch.input_ids)
+    per_token_fields = (
+        "loss_mask",
+        "advantages",
+        "inference_logprobs",
+        "position_ids",
+        "temperatures",
+        "env_names",
+        "ref_logprobs",
+        "rl_weights",
+        "ce_weights",
+        "ref_kl_weights",
+        "rewards",
+        "mm_token_type_ids",
+    )
+    for name in per_token_fields:
+        values = getattr(micro_batch, name)
+        assert values is None or len(values) == num_tokens, (
+            f"{name} misaligned after packing: {len(values)} != {num_tokens} tokens"
+        )
+    assert sum(micro_batch.sequence_lengths) == num_tokens, (
+        f"sequence_lengths sum {sum(micro_batch.sequence_lengths)} != {num_tokens} tokens"
+    )
+    if micro_batch.routed_experts is not None:
+        assert micro_batch.routed_experts.shape[0] == num_tokens, (
+            f"routed_experts misaligned after packing: {micro_batch.routed_experts.shape[0]} != {num_tokens} tokens"
+        )
+
+
 def _make_dummy_batch(source: MicroBatch) -> MicroBatch:
     """Create a zero-loss dummy batch from an existing batch, preserving its modality."""
     dummy = copy.deepcopy(source)
     dummy.advantages = [0.0] * len(dummy.input_ids)
     dummy.loss_mask = [False] * len(dummy.input_ids)
+    # ce/ref_kl membership is weight != 0 (independent of loss_mask), so the
+    # streams must go too or the dummy would still train those tokens.
+    dummy.rl_weights = None
+    dummy.ce_weights = None
+    dummy.ref_kl_weights = None
     return dummy
 
 
@@ -411,6 +502,8 @@ def prepare_batch(
 
     micro_batches = packed_samples_into_micro_bs(all_samples, seq_len, num_loras, num_train_workers, bin_cost)
     micro_batches = [pad_micro_batch(micro_batch, pad_to_multiple_of) for micro_batch in micro_batches]
+    for micro_batch in micro_batches:
+        _assert_token_arrays_aligned(micro_batch)
 
     # Separate by modality so each step index has uniform modality across all ranks
     mm_batches = [b for b in micro_batches if _is_multimodal_sample(b)]
diff --git a/src/prime_rl/trainer/rl/broadcast/filesystem.py b/src/prime_rl/trainer/rl/broadcast/filesystem.py
index e8e2d68db9..49110352f9 100644
--- a/src/prime_rl/trainer/rl/broadcast/filesystem.py
+++ b/src/prime_rl/trainer/rl/broadcast/filesystem.py
@@ -77,7 +77,7 @@ def broadcast_weights(self, model: nn.Module, step: int) -> None:
                     self.logger.debug(f"Saving weights for run {idx} to {save_dir}")
                     save_state_dict(state_dict, save_dir, self.save_format, self.save_sharded, adapter=adapter_only)
                     if adapter_only:
-                        orch_lora = self.multi_run_manager.config[idx].student.model.lora
+                        orch_lora = self.multi_run_manager.config[idx].model.lora
                         save_lora_config(
                             model,
                             save_dir,
diff --git a/src/prime_rl/trainer/rl/data.py b/src/prime_rl/trainer/rl/data.py
index 7c3189bc4d..0aaaa52125 100644
--- a/src/prime_rl/trainer/rl/data.py
+++ b/src/prime_rl/trainer/rl/data.py
@@ -28,7 +28,7 @@ class TensorMicroBatch(TypedDict):
     advantages: Float[Tensor, "batch seq"]
     rewards: Float[Tensor, "batch seq"] | None
     inference_logprobs: Float[Tensor, "batch seq"]
-    teacher_logprobs: Float[Tensor, "batch seq"] | None
+    ref_logprobs: Float[Tensor, "batch seq"] | None
     loss_mask: Bool[Tensor, "batch seq"]
     temperatures: Float[Tensor, "batch seq"]  # Per-token temperatures
     env_names: list[str]
@@ -49,9 +49,11 @@ class TensorMicroBatch(TypedDict):
     # mm_token_type_ids: token type per token [batch seq], int64 (0=text, 1=image, 2=video)
     mm_token_type_ids: Int[Tensor, "batch seq"] | None
 
-    # Selects loss dispatch (rl/opd → default loss with mode-specific taus,
-    # sft → sft loss). All samples in a micro batch share the same mode.
-    training_mode: str
+    # Per-token component weight streams. ``None`` means absent: no ce/ref_kl
+    # component, rl weight 1.0 on every loss-masked token.
+    rl_weights: Float[Tensor, "batch seq"] | None
+    ce_weights: Float[Tensor, "batch seq"] | None
+    ref_kl_weights: Float[Tensor, "batch seq"] | None
 
     # Packer-derived metadata used for run-local debug exports.
     run_id: str | None
@@ -124,7 +126,7 @@ def _get_sample_micro_batch(self, generator: torch.Generator) -> TensorMicroBatc
             "advantages": advantages.unsqueeze(0),
             "rewards": None,
             "inference_logprobs": inference_logprobs.unsqueeze(0),
-            "teacher_logprobs": None,
+            "ref_logprobs": None,
             "temperatures": torch.ones(input_ids.shape[0]).unsqueeze(0),
             "env_names": ["fake"] * input_ids.shape[0],
             "sequence_lengths": sequence_lengths,
@@ -133,7 +135,9 @@ def _get_sample_micro_batch(self, generator: torch.Generator) -> TensorMicroBatc
             "routed_experts": None,
             "mm_kwargs": None,
             "mm_token_type_ids": None,
-            "training_mode": "rl",
+            "rl_weights": None,
+            "ce_weights": None,
+            "ref_kl_weights": None,
             "run_id": None,
             "run_step": None,
         }
@@ -155,7 +159,7 @@ def _get_micro_batch(self, generator: torch.Generator) -> TensorMicroBatch:
             "advantages": torch.randn(self.seq_len, generator=generator).unsqueeze(0),
             "rewards": None,
             "inference_logprobs": torch.randn(self.seq_len, generator=generator).unsqueeze(0),
-            "teacher_logprobs": None,
+            "ref_logprobs": None,
             "temperatures": torch.ones(self.seq_len).unsqueeze(0),
             "env_names": ["fake"] * self.seq_len,
             "sequence_lengths": [self.seq_len],
@@ -164,7 +168,9 @@ def _get_micro_batch(self, generator: torch.Generator) -> TensorMicroBatch:
             "routed_experts": None,
             "mm_kwargs": None,
             "mm_token_type_ids": None,
-            "training_mode": "rl",
+            "rl_weights": None,
+            "ce_weights": None,
+            "ref_kl_weights": None,
             "run_id": None,
             "run_step": None,
         }
@@ -251,8 +257,8 @@ def _micro_batch_to_tensor(self, micro_batch: MicroBatch) -> TensorMicroBatch:
             if micro_batch.rewards is not None
             else None,
             inference_logprobs=torch.tensor(micro_batch.inference_logprobs, dtype=torch.float).unsqueeze(0),
-            teacher_logprobs=torch.tensor(micro_batch.teacher_logprobs, dtype=torch.float).unsqueeze(0)
-            if micro_batch.teacher_logprobs is not None
+            ref_logprobs=torch.tensor(micro_batch.ref_logprobs, dtype=torch.float).unsqueeze(0)
+            if micro_batch.ref_logprobs is not None
             else None,
             loss_mask=torch.tensor(micro_batch.loss_mask, dtype=torch.bool).unsqueeze(0),
             temperatures=torch.tensor(micro_batch.temperatures, dtype=torch.float).unsqueeze(0),
@@ -264,7 +270,15 @@ def _micro_batch_to_tensor(self, micro_batch: MicroBatch) -> TensorMicroBatch:
             if micro_batch.mm_token_type_ids is not None
             else None,
             routed_experts=routed_experts,
-            training_mode=micro_batch.training_mode,
+            rl_weights=torch.tensor(micro_batch.rl_weights, dtype=torch.float).unsqueeze(0)
+            if micro_batch.rl_weights is not None
+            else None,
+            ce_weights=torch.tensor(micro_batch.ce_weights, dtype=torch.float).unsqueeze(0)
+            if micro_batch.ce_weights is not None
+            else None,
+            ref_kl_weights=torch.tensor(micro_batch.ref_kl_weights, dtype=torch.float).unsqueeze(0)
+            if micro_batch.ref_kl_weights is not None
+            else None,
             run_id=micro_batch.run_id,
             run_step=micro_batch.run_step,
         )
diff --git a/src/prime_rl/trainer/rl/loss.py b/src/prime_rl/trainer/rl/loss.py
index 7cd5e53bb2..3de7dbebe5 100644
--- a/src/prime_rl/trainer/rl/loss.py
+++ b/src/prime_rl/trainer/rl/loss.py
@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Callable
 
 import torch
@@ -12,13 +12,20 @@
 
 @dataclass
 class LossInputs:
-    """Inputs for computing loss on a single sample."""
+    """Inputs for computing loss on a single sample.
+
+    ``loss_mask`` already selects the tokens that belong to the receiving
+    component — the component loss functions never re-derive eligibility.
+    ``loss_weights`` is the component's per-token weight stream (None means
+    1.0 everywhere).
+    """
 
     trainer_logprobs: Float[Tensor, " seq"]
     inference_logprobs: Float[Tensor, " seq"]
-    teacher_logprobs: Float[Tensor, " seq"] | None
+    ref_logprobs: Float[Tensor, " seq"] | None
     advantages: Float[Tensor, " seq"]
     loss_mask: Bool[Tensor, " seq"]
+    loss_weights: Float[Tensor, " seq"] | None = field(default=None)
 
 
 @dataclass
@@ -150,7 +157,10 @@ def default_loss_fn(inputs: LossInputs, loss_config: DefaultLossConfig) -> LossO
     advantages = loss_config.adv_tau * advantages
     pg_loss = keep_mask * advantages * importance_ratio
     kl_loss = loss_mask * log_importance_ratio**2
-    loss = (-pg_loss + loss_config.kl_tau * kl_loss).sum()
+    per_token_loss = -pg_loss + loss_config.kl_tau * kl_loss
+    if inputs.loss_weights is not None:
+        per_token_loss = per_token_loss * inputs.loss_weights
+    loss = per_token_loss.sum()
 
     metrics = {
         "masked_mismatch_kl": _safe_mean(mismatch_kl, loss_mask & is_masked),  # all trainable, masked tokens
@@ -166,6 +176,9 @@ def default_loss_fn(inputs: LossInputs, loss_config: DefaultLossConfig) -> LossO
 
 
 def ipo_loss_fn(inputs: LossInputs, loss_config: IPOLossConfig) -> LossOutputs:
+    """IPO loss type: a symmetric trust region (mask tokens whose probability
+    moved more than ``ipo_threshold`` in absolute terms), policy gradient via
+    the importance ratio, and a squared-log-ratio KL regularizer."""
     trainer_logprobs = inputs.trainer_logprobs
     inference_logprobs = inputs.inference_logprobs
     advantages = inputs.advantages
@@ -183,7 +196,10 @@ def ipo_loss_fn(inputs: LossInputs, loss_config: IPOLossConfig) -> LossOutputs:
     advantages = loss_config.adv_tau * advantages
     pg_loss = keep_mask * advantages * importance_ratio
     kl_loss = loss_mask * log_importance_ratio**2
-    loss = (-pg_loss + loss_config.kl_tau * kl_loss).sum()
+    per_token_loss = -pg_loss + loss_config.kl_tau * kl_loss
+    if inputs.loss_weights is not None:
+        per_token_loss = per_token_loss * inputs.loss_weights
+    loss = per_token_loss.sum()
 
     metrics = {
         "masked_mismatch_kl": _safe_mean(mismatch_kl, loss_mask & is_masked),  # all trainable, masked tokens
@@ -194,86 +210,74 @@ def ipo_loss_fn(inputs: LossInputs, loss_config: IPOLossConfig) -> LossOutputs:
     return LossOutputs(loss=loss, metrics=metrics)
 
 
-def opd_loss_fn(inputs: LossInputs) -> LossOutputs:
+def ref_kl_loss_fn(inputs: LossInputs) -> LossOutputs:
     """
-    On-policy distillation loss: the default DPPO+KL math with the tau knobs
-    hardcoded to drop the reward signal and use the teacher KL as the
-    per-token policy-gradient signal.
+    Ref-KL loss type (on-policy distillation): the reverse KL to the reference
+    model is the per-token policy-gradient signal, with the importance ratio
+    correcting trainer/inference mismatch and staleness. A one-sided trust
+    region drops tokens whose trainer probability fell more than 0.2 below the
+    inference probability; a squared-log-ratio term regularizes drift. Scalar
+    advantages are not read — ref_kl algorithms ship none.
     """
     trainer_logprobs = inputs.trainer_logprobs
     inference_logprobs = inputs.inference_logprobs
-    teacher_logprobs = inputs.teacher_logprobs
-    advantages = inputs.advantages
+    ref_logprobs = inputs.ref_logprobs
     loss_mask = inputs.loss_mask
 
-    if teacher_logprobs is None:
-        raise ValueError("opd_loss_fn requires teacher_logprobs - configure a teacher for opd mode.")
+    if ref_logprobs is None:
+        raise ValueError("ref_kl loss type requires ref_logprobs — use an 'opd' or 'opsd' advantage strategy.")
 
     log_importance_ratio, importance_ratio, mismatch_kl = compute_importance_ratio_and_mismatch_kl(
         trainer_logprobs, inference_logprobs
     )
 
     probs_diff = torch.exp(trainer_logprobs) - torch.exp(inference_logprobs)
-    dppo_invalid_mask_high = probs_diff > 0.2
-    dppo_invalid_mask_low = probs_diff < -0.2
-    positive_advantages = advantages > 0
-    negative_advantages = advantages < 0
-    dppo_invalid_mask = torch.where(positive_advantages, dppo_invalid_mask_high, dppo_invalid_mask_low)
-
-    is_masked = dppo_invalid_mask
-    is_masked_high = positive_advantages & dppo_invalid_mask_high
-    is_masked_low = negative_advantages & dppo_invalid_mask_low
+    is_masked = probs_diff < -0.2
     drop_mask = loss_mask & is_masked
     keep_mask = loss_mask & ~is_masked
 
-    teacher_kl = teacher_logprobs - trainer_logprobs
-    advantages = 0.0 * advantages + 1.0 * teacher_kl.detach()
+    ref_kl = ref_logprobs - trainer_logprobs
 
-    pg_loss = keep_mask * advantages * importance_ratio
+    pg_loss = keep_mask * ref_kl.detach() * importance_ratio
     kl_loss = loss_mask * log_importance_ratio**2
-    loss = (-pg_loss + 1e-3 * kl_loss).sum()
+    per_token_loss = -pg_loss + 1e-3 * kl_loss
+    if inputs.loss_weights is not None:
+        per_token_loss = per_token_loss * inputs.loss_weights
+    loss = per_token_loss.sum()
 
+    # Namespaced: the rl loss fn emits same-named trust-region metrics with a
+    # different definition, and mixed batches run both fns in one step.
     metrics = {
-        "masked_mismatch_kl": _safe_mean(mismatch_kl, loss_mask & is_masked),
-        "unmasked_mismatch_kl": _safe_mean(mismatch_kl, keep_mask),
-        "is_masked": _safe_mean(is_masked, loss_mask),
-        "is_masked_low": _safe_mean(is_masked_low, loss_mask),
-        "is_masked_high": _safe_mean(is_masked_high, loss_mask),
-        "masked_advantage_positive": _safe_mean(positive_advantages, drop_mask),
-        "masked_advantage_negative": _safe_mean(negative_advantages, drop_mask),
-        "teacher_kl": _safe_mean(teacher_kl, loss_mask),
+        "ref_kl/masked_mismatch_kl": _safe_mean(mismatch_kl, drop_mask),
+        "ref_kl/unmasked_mismatch_kl": _safe_mean(mismatch_kl, keep_mask),
+        "ref_kl/is_masked": _safe_mean(is_masked, loss_mask),
+        "ref_kl": _safe_mean(ref_kl, loss_mask),
     }
 
     return LossOutputs(loss=loss, metrics=metrics)
 
 
-def sft_loss_fn(inputs: LossInputs) -> LossOutputs:
-    """SFT-style masked negative log-likelihood over trainable tokens."""
+def ce_loss_fn(inputs: LossInputs) -> LossOutputs:
+    """Cross-entropy loss type: masked negative log-likelihood (SFT / ECHO
+    observation prediction)."""
     trainer_logprobs = inputs.trainer_logprobs
     loss_mask = inputs.loss_mask
 
-    loss = -(trainer_logprobs[loss_mask]).sum()
+    nll = -trainer_logprobs
+    if inputs.loss_weights is not None:
+        nll = nll * inputs.loss_weights
+    loss = nll[loss_mask].sum()
     metrics = {
         "nll": _safe_mean(-trainer_logprobs, loss_mask),
     }
     return LossOutputs(loss=loss, metrics=metrics)
 
 
-def setup_loss_fns(loss_config: LossConfig) -> dict[str, LossFn]:
-    """Build the per-training-mode loss fn dispatch table.
-
-    Always returns all three modes - the trainer is mode-agnostic and routes
-    per batch from ``TrainingSample.training_mode``:
-
-    - ``"sft"`` → ``sft_loss_fn`` (masked NLL on teacher tokens)
-    - ``"opd"`` → ``opd_loss_fn`` (teacher KL as gradient signal, hardcoded
-      DPPO + KL knobs)
-    - ``"rl"``  → ``default_loss_fn(loss_config)`` for ``DefaultLossConfig``,
-      ``ipo_loss_fn(loss_config)`` for ``IPOLossConfig``, or the imported
-      function for ``CustomLossConfig``.
-
-    ``trainer.loss`` only affects the rl path - opd and sft are independent.
-    """
+def setup_rl_loss_fn(loss_config: LossConfig) -> LossFn:
+    """Build the loss fn for the rl component from ``trainer.loss``:
+    ``default_loss_fn`` (``DefaultLossConfig``), ``ipo_loss_fn``
+    (``IPOLossConfig``), or the imported function (``CustomLossConfig``).
+    The ce / ref_kl loss types are fixed and unaffected by ``trainer.loss``."""
     if isinstance(loss_config, CustomLossConfig):
         custom_fn = import_object(loss_config.import_path)
         kwargs = loss_config.kwargs
@@ -289,78 +293,119 @@ def rl_fn(inputs: LossInputs) -> LossOutputs:
         def rl_fn(inputs: LossInputs) -> LossOutputs:
             return default_loss_fn(inputs, loss_config)
 
-    return {"sft": sft_loss_fn, "opd": opd_loss_fn, "rl": rl_fn}
+    return rl_fn
 
 
 def compute_loss(
     trainer_logprobs: list[Float[Tensor, " seq_i"]],
     inference_logprobs: list[Float[Tensor, " seq_i"]],
-    teacher_logprobs: list[Float[Tensor, " seq_i"]] | None,
+    ref_logprobs: list[Float[Tensor, " seq_i"]] | None,
     advantages: list[Float[Tensor, " seq_i"]],
     loss_mask: list[Bool[Tensor, " seq_i"]],
-    loss_fns: dict[str, LossFn],
-    loss_scale: int,
-    training_mode: str = "rl",
+    rl_weights: list[Float[Tensor, " seq_i"]] | None,
+    ce_weights: list[Float[Tensor, " seq_i"]] | None,
+    ref_kl_weights: list[Float[Tensor, " seq_i"]] | None,
+    rl_loss_fn: LossFn,
+    rl_scale: int,
+    ce_scale: int,
+    ref_kl_scale: int,
 ) -> tuple[Float[Tensor, ""], dict[str, Any]]:
     """
     Compute loss for packed sequences (batch size = 1, multiple sequences packed along sequence dimension).
 
-    Loss dispatch is batch-driven: ``training_mode`` selects the loss fn from
-    ``loss_fns`` (built by ``setup_loss_fns``). sft → sft_loss_fn, opd →
-    opd_loss_fn, rl → the configured default/custom loss.
+    The loss is a sum of three components, each running over its own per-token
+    weight stream and normalized by its own global token count:
+
+    - rl → ``rl_loss_fn`` (built by ``setup_rl_loss_fn``) on
+      ``loss_mask & (rl_weights != 0)``; an absent stream means weight 1.0 on
+      the full loss mask (the hot path — no extra device syncs).
+    - ce → ``ce_loss_fn`` (masked NLL) on ``ce_weights != 0``.
+    - ref_kl → ``ref_kl_loss_fn`` on ``ref_kl_weights != 0``.
+
+    A weight scales its component's per-token loss; 0.0 removes the token from
+    the component's mask and denominator. Per-component normalization keeps the
+    components from diluting each other: a token only enters the denominator of
+    the components it belongs to.
 
     Args:
         trainer_logprobs: Log probabilities for each sequence
-        inference_logprobs: Reference log probabilities for each sequence
-        teacher_logprobs: Teacher log probabilities for each sequence, or None
+        inference_logprobs: Sampling-policy log probabilities for each sequence
+        ref_logprobs: Reference-model log probabilities for each sequence, or None
         advantages: Advantages for each sequence
         loss_mask: Loss mask for each sequence
-        loss_fns: Per-mode loss fn dispatch table from setup_loss_fns()
-        loss_scale: Scale factor to normalize the loss
-        training_mode: Selects which loss fn to apply
+        rl_weights: Per-token rl weights for each sequence, or None (1.0 on the loss mask)
+        ce_weights: Per-token ce weights for each sequence, or None (no ce component)
+        ref_kl_weights: Per-token ref_kl weights for each sequence, or None (no ref_kl component)
+        rl_loss_fn: Loss fn for the rl component from setup_rl_loss_fn()
+        rl_scale: Global rl-token count normalizing the rl component
+        ce_scale: Global ce-token count normalizing the ce component
+        ref_kl_scale: Global ref_kl-token count normalizing the ref_kl component
 
     Returns:
         Tuple of (scaled_loss, aggregated_metrics)
     """
-    try:
-        effective_loss_fn = loss_fns[training_mode]
-    except KeyError:
-        raise ValueError(
-            f"No loss fn available for training_mode={training_mode!r} "
-            f"(available: {sorted(loss_fns)}). Check trainer.loss.type."
-        )
-
-    total_loss = 0.0
     all_metrics: dict[str, list[Tensor]] = {}
 
-    if teacher_logprobs is None:
-        teacher_logprobs = [None] * len(trainer_logprobs)
-
-    for t_logp, i_logp, teach_logp, adv, mask in zip(
+    n = len(trainer_logprobs)
+    if ref_logprobs is None:
+        ref_logprobs = [None] * n
+    if rl_weights is None:
+        rl_weights = [None] * n
+    if ce_weights is None:
+        ce_weights = [None] * n
+    if ref_kl_weights is None:
+        ref_kl_weights = [None] * n
+
+    def run_loss_fn(loss_fn: LossFn, inputs: LossInputs) -> Tensor:
+        result = loss_fn(inputs)
+        for k, v in result.metrics.items():
+            all_metrics.setdefault(k, []).append(v)
+        return result.loss
+
+    # Graph anchor: a micro batch whose components are all empty (e.g. a fully
+    # truncated distillation sample, whose stamped streams survive as all-zero
+    # prefixes) must still return a backward-able loss so every rank runs
+    # backward and FSDP collectives stay in sync.
+    rl_loss = trainer_logprobs[0].sum() * 0.0
+    ce_loss = 0.0
+    ref_kl_loss = 0.0
+    for t_logp, i_logp, ref_logp, adv, mask, rl_w, ce_w, ref_kl_w in zip(
         trainer_logprobs,
         inference_logprobs,
-        teacher_logprobs,
+        ref_logprobs,
         advantages,
         loss_mask,
+        rl_weights,
+        ce_weights,
+        ref_kl_weights,
     ):
-        inputs = LossInputs(
-            trainer_logprobs=t_logp,
-            inference_logprobs=i_logp,
-            teacher_logprobs=teach_logp,
-            advantages=adv,
-            loss_mask=mask,
-        )
-
-        result = effective_loss_fn(inputs)
-
-        total_loss = total_loss + result.loss
 
-        for k, v in result.metrics.items():
-            if k not in all_metrics:
-                all_metrics[k] = []
-            all_metrics[k].append(v)
-
-    scaled_loss = total_loss / loss_scale
+        def make_inputs(component_mask: Bool[Tensor, " seq"], weights: Float[Tensor, " seq"] | None) -> LossInputs:
+            return LossInputs(
+                trainer_logprobs=t_logp,
+                inference_logprobs=i_logp,
+                ref_logprobs=ref_logp,
+                advantages=adv,
+                loss_mask=component_mask,
+                loss_weights=weights,
+            )
+
+        if rl_w is None:
+            rl_loss = rl_loss + run_loss_fn(rl_loss_fn, make_inputs(mask, None))
+        else:
+            rl_mask = mask & (rl_w != 0)
+            if bool(rl_mask.any()):
+                rl_loss = rl_loss + run_loss_fn(rl_loss_fn, make_inputs(rl_mask, rl_w))
+        if ce_w is not None:
+            ce_mask = ce_w != 0
+            if bool(ce_mask.any()):
+                ce_loss = ce_loss + run_loss_fn(ce_loss_fn, make_inputs(ce_mask, ce_w))
+        if ref_kl_w is not None:
+            ref_kl_mask = ref_kl_w != 0
+            if bool(ref_kl_mask.any()):
+                ref_kl_loss = ref_kl_loss + run_loss_fn(ref_kl_loss_fn, make_inputs(ref_kl_mask, ref_kl_w))
+
+    scaled_loss = rl_loss / rl_scale + ce_loss / ce_scale + ref_kl_loss / ref_kl_scale
 
     aggregated: dict[str, Any] = {}
     for k, v in all_metrics.items():
diff --git a/src/prime_rl/trainer/rl/packer.py b/src/prime_rl/trainer/rl/packer.py
index ae0d9863e1..b0daa48db9 100644
--- a/src/prime_rl/trainer/rl/packer.py
+++ b/src/prime_rl/trainer/rl/packer.py
@@ -188,10 +188,10 @@ def _validate_sample(self, sample: TrainingSample) -> tuple[bool, str | None]:
                 False,
                 f"Run wrote a sample with length {sample_length} which exceeds max sequence length {self.seq_len}",
             )
-        if sample.teacher_logprobs is not None and len(sample.teacher_logprobs) != sample_length:
+        if sample.ref_logprobs is not None and len(sample.ref_logprobs) != sample_length:
             return (
                 False,
-                f"Run wrote a sample with teacher logprobs length != sample length ({len(sample.teacher_logprobs)} != {sample_length})",
+                f"Run wrote a sample with ref logprobs length != sample length ({len(sample.ref_logprobs)} != {sample_length})",
             )
         return True, None
 
diff --git a/src/prime_rl/trainer/rl/token_export.py b/src/prime_rl/trainer/rl/token_export.py
index 29d490302f..efc17e1084 100644
--- a/src/prime_rl/trainer/rl/token_export.py
+++ b/src/prime_rl/trainer/rl/token_export.py
@@ -70,7 +70,6 @@ def export(
                         "export_sequence_idx": self._sequences_by_file.get(file_key, 0),
                         "run_id": run_id,
                         "env_name": _first_non_empty(columns["env_names"][start:end]),
-                        "training_mode": str(micro_batch["training_mode"]),
                         **_slice_columns(columns, start, end),
                     },
                     run_id,
@@ -163,6 +162,11 @@ def _export_columns(
         "is_masked": _optional_tensor_to_bools(export_tensors["is_masked"], seq_len),
         "is_masked_high": _optional_tensor_to_bools(export_tensors["is_masked_high"], seq_len),
         "is_masked_low": _optional_tensor_to_bools(export_tensors["is_masked_low"], seq_len),
+        # Component weight streams; ``None`` columns mean the defaults (rl 1.0
+        # on the loss mask, no ce/ref_kl component).
+        "rl_weights": _optional_tensor_to_floats(micro_batch.get("rl_weights"), seq_len),
+        "ce_weights": _optional_tensor_to_floats(micro_batch.get("ce_weights"), seq_len),
+        "ref_kl_weights": _optional_tensor_to_floats(micro_batch.get("ref_kl_weights"), seq_len),
         "env_names": list(micro_batch["env_names"]),
     }
 
@@ -179,7 +183,14 @@ def _compute_export_tensors(
         "is_masked_high": None,
         "is_masked_low": None,
     }
-    if micro_batch["training_mode"] == "sft":
+    # Ratio-based fields are meaningless when no token has sampling logprobs
+    # (e.g. pure CE batches distilling frozen-model tokens): no rl member
+    # (stream present but all-zero) and no ref_kl member.
+    rl_weights = micro_batch.get("rl_weights")
+    ref_kl_weights = micro_batch.get("ref_kl_weights")
+    no_rl = rl_weights is not None and not bool((rl_weights != 0).any())
+    no_ref_kl = ref_kl_weights is None or not bool((ref_kl_weights != 0).any())
+    if no_rl and no_ref_kl:
         return fields
 
     inference_logprobs = micro_batch["inference_logprobs"].to(trainer_logprobs.device)
diff --git a/src/prime_rl/trainer/rl/train.py b/src/prime_rl/trainer/rl/train.py
index 3e4aa34a0f..512c2d491d 100644
--- a/src/prime_rl/trainer/rl/train.py
+++ b/src/prime_rl/trainer/rl/train.py
@@ -31,7 +31,7 @@
     compute_loss,
     compute_importance_ratio_and_mismatch_kl,
     selective_log_softmax,
-    setup_loss_fns,
+    setup_rl_loss_fn,
     shift_tensor_left,
     shift_tensor_right,
 )
@@ -149,9 +149,9 @@ def train(config: TrainerConfig):
     logger.info(f"Initializing tokenizer ({config.tokenizer})")
     tokenizer = setup_tokenizer(config.tokenizer)
 
-    # Set up the loss function
+    # Set up the loss function for the RL loss type (ce / ref_kl are fixed)
     logger.info(f"Setting up loss function ({config.loss})")
-    loss_fns = setup_loss_fns(config.loss)
+    rl_loss_fn = setup_rl_loss_fn(config.loss)
 
     # Set up the optimizer
     logger.info(f"Initializing optimizer ({config.optim})")
@@ -351,15 +351,30 @@ def load_run_checkpoint(_optimizer, idx: int) -> None:
         forward_backward_start_time = time.perf_counter()
         seq_len = micro_batches[0]["input_ids"].shape[1]
 
-        # Normalize by the global (dp_cp) number of unmasked tokens in the batch, so every rank
-        # divides by the same denominator. With a per-rank denominator, ranks with fewer loss
-        # tokens implicitly upweight their per-token gradient contribution after FSDP averaging.
-        # FSDP's per-rank divide is undone after the microbatch loop via fsdp_gradient_divide_factor.
-        local_loss_scale = sum(micro_batch["loss_mask"].sum().item() for micro_batch in micro_batches)
-        global_loss_scale = torch.tensor(local_loss_scale, dtype=torch.int64, device="cuda")
+        # Normalize each loss component by its own global (dp_cp) token count, so every rank
+        # divides by the same denominator and the components don't dilute each other — a token
+        # only enters the denominator of the components it belongs to. With a per-rank
+        # denominator, ranks with fewer loss tokens implicitly upweight their per-token gradient
+        # contribution after FSDP averaging. FSDP's per-rank divide is undone after the
+        # microbatch loop via fsdp_gradient_divide_factor. One batched collective keeps every
+        # rank issuing the same op regardless of which components its samples carry.
+        local_rl_scale = 0
+        local_ce_scale = 0
+        local_ref_kl_scale = 0
+        for micro_batch in micro_batches:
+            mask = micro_batch["loss_mask"]
+            rl_w = micro_batch["rl_weights"]
+            local_rl_scale += int((mask & (rl_w != 0)).sum()) if rl_w is not None else int(mask.sum())
+            if micro_batch["ce_weights"] is not None:
+                local_ce_scale += int((micro_batch["ce_weights"] != 0).sum())
+            if micro_batch["ref_kl_weights"] is not None:
+                local_ref_kl_scale += int((micro_batch["ref_kl_weights"] != 0).sum())
+        global_scales = torch.tensor(
+            [local_rl_scale, local_ce_scale, local_ref_kl_scale], dtype=torch.int64, device="cuda"
+        )
         dp_cp_group = parallel_dims.get_mesh("dp_cp").get_group()
-        dist.all_reduce(global_loss_scale, op=dist.ReduceOp.SUM, group=dp_cp_group)
-        loss_scale = max(global_loss_scale.item(), 1)
+        dist.all_reduce(global_scales, op=dist.ReduceOp.SUM, group=dp_cp_group)
+        rl_scale, ce_scale, ref_kl_scale = (max(scale, 1) for scale in global_scales.tolist())
 
         logger.debug(f"Starting forward and backward pass ({batch_size=})")
         tensors = Tensors()  # Used to accumulate tensor statistics across micro-batches and ranks for logging
@@ -374,8 +389,11 @@ def load_run_checkpoint(_optimizer, idx: int) -> None:
             advantages = micro_batch["advantages"].to("cuda")
             loss_mask = micro_batch["loss_mask"].to("cuda")
             inference_logprobs = micro_batch["inference_logprobs"].to("cuda")
-            teacher_logprobs = (
-                micro_batch["teacher_logprobs"].to("cuda") if micro_batch["teacher_logprobs"] is not None else None
+            ref_logprobs = micro_batch["ref_logprobs"].to("cuda") if micro_batch["ref_logprobs"] is not None else None
+            rl_weights = micro_batch["rl_weights"].to("cuda") if micro_batch["rl_weights"] is not None else None
+            ce_weights = micro_batch["ce_weights"].to("cuda") if micro_batch["ce_weights"] is not None else None
+            ref_kl_weights = (
+                micro_batch["ref_kl_weights"].to("cuda") if micro_batch["ref_kl_weights"] is not None else None
             )
             routed_experts = (
                 micro_batch["routed_experts"].to("cuda") if micro_batch["routed_experts"] is not None else None
@@ -477,14 +495,16 @@ def load_run_checkpoint(_optimizer, idx: int) -> None:
             loss, loss_tensors = compute_loss(
                 trainer_logprobs=out["logprobs"].squeeze().split(sequence_lengths),
                 inference_logprobs=inference_logprobs.squeeze().split(sequence_lengths),
-                teacher_logprobs=teacher_logprobs.squeeze().split(sequence_lengths)
-                if teacher_logprobs is not None
-                else None,
+                ref_logprobs=ref_logprobs.squeeze().split(sequence_lengths) if ref_logprobs is not None else None,
                 advantages=advantages.squeeze().split(sequence_lengths),
                 loss_mask=loss_mask.squeeze().split(sequence_lengths),
-                loss_fns=loss_fns,
-                loss_scale=loss_scale,
-                training_mode=micro_batch["training_mode"],
+                rl_weights=rl_weights.squeeze().split(sequence_lengths) if rl_weights is not None else None,
+                ce_weights=ce_weights.squeeze().split(sequence_lengths) if ce_weights is not None else None,
+                ref_kl_weights=ref_kl_weights.squeeze().split(sequence_lengths) if ref_kl_weights is not None else None,
+                rl_loss_fn=rl_loss_fn,
+                rl_scale=rl_scale,
+                ce_scale=ce_scale,
+                ref_kl_scale=ref_kl_scale,
             )
 
             # Backward pass
@@ -505,12 +525,30 @@ def load_run_checkpoint(_optimizer, idx: int) -> None:
             for env_name, indices in env_to_indices.items():
                 tensors[f"entropy/{env_name}"].append(entropy[indices])
 
-            if micro_batch["training_mode"] != "sft":
+            # Mismatch KL is only meaningful where sampling logprobs exist —
+            # keep rl/ref_kl member tokens (policy-sampled), exclude tokens
+            # whose action component is ce (frozen-model tokens).
+            if rl_weights is None and ref_kl_weights is None:
+                mismatch_mask = loss_mask
+                has_mismatch_tokens = True
+            else:
+                sampled_mask = (rl_weights != 0) if rl_weights is not None else loss_mask
+                if ref_kl_weights is not None:
+                    sampled_mask = sampled_mask | (ref_kl_weights != 0)
+                mismatch_mask = loss_mask & sampled_mask
+                has_mismatch_tokens = bool(mismatch_mask.any())
+            if has_mismatch_tokens:
                 with torch.no_grad():
                     _, _, mismatch_kl = compute_importance_ratio_and_mismatch_kl(out["logprobs"], inference_logprobs)
-                mismatch_kl = mismatch_kl[loss_mask].detach().to("cpu")
+                mismatch_kl = mismatch_kl[mismatch_mask].detach().to("cpu")
                 tensors["mismatch_kl/all"].append(mismatch_kl)
-                for env_name, indices in env_to_indices.items():
+                mismatch_env_names = [
+                    env_name for env_name, keep in zip(env_names, mismatch_mask.flatten().tolist()) if keep
+                ]
+                mismatch_env_to_indices: dict[str, list[int]] = {}
+                for idx, env_name in enumerate(mismatch_env_names):
+                    mismatch_env_to_indices.setdefault(env_name, []).append(idx)
+                for env_name, indices in mismatch_env_to_indices.items():
                     tensors[f"mismatch_kl/{env_name}"].append(mismatch_kl[indices])
 
             token_exporter.export(
@@ -534,7 +572,7 @@ def load_run_checkpoint(_optimizer, idx: int) -> None:
 
             # Debug log with *local, micro step* stats
             micro_step_message = f"Micro Step {micro_step}/{len(micro_batches)} | Loss {tensors['loss'][-1].mean().item():.4f} | Entropy {tensors['entropy/all'][-1].mean().item():.4f}"
-            if micro_batch["training_mode"] != "sft":
+            if has_mismatch_tokens:
                 micro_step_message += f" | Mismatch KL {tensors['mismatch_kl/all'][-1].mean().item():.4f}"
             if "max_vio" in tensors:
                 micro_step_message += f" | Max Vio {tensors['max_vio'][-1].mean().item():.4f}"
diff --git a/src/prime_rl/trainer/runs.py b/src/prime_rl/trainer/runs.py
index 1bdb7e0f50..bc9dd9cb08 100644
--- a/src/prime_rl/trainer/runs.py
+++ b/src/prime_rl/trainer/runs.py
@@ -515,24 +515,22 @@ def setup_multi_run_manager(
         trainer_lora = lora_config
 
         def validate_lora_rank(orch_config: "OrchestratorConfig") -> tuple[bool, str]:
-            if orch_config.student.model.lora is None:
-                return False, "student.model.lora is required when trainer is configured with LoRA"
+            if orch_config.model.lora is None:
+                return False, "orchestrator.model.lora is required when trainer is configured with LoRA"
             # Default to trainer's rank/alpha if not specified
-            if orch_config.student.model.lora.rank is None:
-                orch_config.student.model.lora.rank = trainer_lora.rank
-            if orch_config.student.model.lora.alpha is None:
-                orch_config.student.model.lora.alpha = trainer_lora.alpha
-            if orch_config.student.model.lora.rank > trainer_lora.rank:
+            if orch_config.model.lora.rank is None:
+                orch_config.model.lora.rank = trainer_lora.rank
+            if orch_config.model.lora.alpha is None:
+                orch_config.model.lora.alpha = trainer_lora.alpha
+            if orch_config.model.lora.rank > trainer_lora.rank:
                 return (
                     False,
-                    f"student.model.lora.rank ({orch_config.student.model.lora.rank}) exceeds trainer max rank ({trainer_lora.rank})",
+                    f"orchestrator.model.lora.rank ({orch_config.model.lora.rank}) exceeds trainer max rank ({trainer_lora.rank})",
                 )
             return True, ""
 
         def on_run_discovered(idx: int, run_id: str, orch_config: "OrchestratorConfig") -> None:
-            _MULTI_RUN_MANAGER.scaling_factors[idx] = (
-                orch_config.student.model.lora.alpha / orch_config.student.model.lora.rank
-            )
+            _MULTI_RUN_MANAGER.scaling_factors[idx] = orch_config.model.lora.alpha / orch_config.model.lora.rank
 
         _MULTI_RUN_MANAGER.register_config_validation_hook(validate_lora_rank)
         _MULTI_RUN_MANAGER.register_discovered_hook(on_run_discovered)
diff --git a/src/prime_rl/trainer/utils.py b/src/prime_rl/trainer/utils.py
index 2d278a1225..bc0f2e8c93 100644
--- a/src/prime_rl/trainer/utils.py
+++ b/src/prime_rl/trainer/utils.py
@@ -606,6 +606,9 @@ def filter_rl_trainer_tensor_stats_for_wandb(metrics: dict[str, float | int]) ->
         "mismatch_kl/",
         "masked_mismatch_kl/",
         "unmasked_mismatch_kl/",
+        "ref_kl/is_masked/",
+        "ref_kl/masked_mismatch_kl/",
+        "ref_kl/unmasked_mismatch_kl/",
     )
     out: dict[str, float | int] = {}
     for k, v in metrics.items():
diff --git a/src/prime_rl/transport/types.py b/src/prime_rl/transport/types.py
index 1666c51092..afacb13cf1 100644
--- a/src/prime_rl/transport/types.py
+++ b/src/prime_rl/transport/types.py
@@ -1,9 +1,5 @@
-from typing import Literal
-
 import msgspec
 
-TrainingMode = Literal["rl", "opd", "sft"]
-
 
 # Encoded tensor: {dtype: "float32", shape: [...], data: <bytes>}.
 # Mirrors verifiers.utils.serve_utils.msgpack_encoder so the same wire
@@ -33,8 +29,7 @@ class TrainingSample(msgspec.Struct, array_like=True, gc=False, omit_defaults=Tr
     completion_logprobs: list[float]
     completion_temperatures: list[float]  # Per-token temperatures used during generation
     env_name: str
-    teacher_logprobs: list[float] | None = None
-    advantage: float | None = None
+    ref_logprobs: list[float] | None = None  # reference-model logprobs (ref_kl component)
     reward: float | None = None
 
     # Generic multimodal kwargs: flat dict keyed by the kwarg names the
@@ -52,9 +47,33 @@ class TrainingSample(msgspec.Struct, array_like=True, gc=False, omit_defaults=Tr
     # mm_token_type_ids: token type ids per token [batch seq], int64 (0=text, 1=image, 2=video)
     mm_token_type_ids: list[int] | None = None
 
-    # Loss dispatch is batch-driven: rl/opd use default_loss_fn (with mode-specific
-    # taus), sft uses sft_loss_fn. Stamped by the orchestrator from training_mode.
-    training_mode: TrainingMode = "rl"
+    # Per-token component weight streams (full prompt+completion length),
+    # stamped by the orchestrator from the env's algorithm. The training loss
+    # is a sum of three components, each normalized by its own global token
+    # count: rl (importance-weighted PG + KL), ce (masked NLL), and ref_kl
+    # (reverse KL to a reference model as the PG signal). A weight scales that
+    # component's per-token loss; 0.0 leaves the token out of the component
+    # (mask and denominator). ``None`` means absent: no ce/ref_kl component,
+    # and an rl weight of 1.0 on every trainable token — so the plain GRPO
+    # wire stays as small as before.
+    rl_weights: list[float] | None = None
+    ce_weights: list[float] | None = None
+    ref_kl_weights: list[float] | None = None
+
+    # Per-token advantages (full prompt+completion length), the fourth stream:
+    # the orchestrator broadcasts the rollout's scalar over the completion for
+    # scalar algorithms. ``None`` means no rl credit assigned — legal only for
+    # samples without live rl member tokens (the trainer raises otherwise).
+    advantages: list[float] | None = None
+
+    # Orchestrator-internal, cleared before transport: interleaving's
+    # provenance record for env-provided observation tokens — one
+    # ``[completion_start, step_idx, step_prompt_start, length]`` entry per
+    # span that landed as a later-turn prompt extension, mapping sample
+    # positions back to trajectory-step coordinates. Algorithms that train
+    # on observations (echo) consume it at group time and write the
+    # ``ce_weights`` stream directly.
+    obs_spans: list[list[int]] | None = None
 
 
 class TrainingBatch(msgspec.Struct, array_like=True, gc=False, omit_defaults=True):
@@ -76,8 +95,10 @@ class MicroBatch(msgspec.Struct, array_like=True, gc=False, omit_defaults=True):
     position_ids: list[int]
     temperatures: list[float]  # Per-token temperatures used during generation
     env_names: list[str]
+    # Per-sample token counts within the packed batch (one entry per packed
+    # sample); the loss splits the packed sequence back into samples by these.
     sequence_lengths: list[int]
-    teacher_logprobs: list[float] | None = None
+    ref_logprobs: list[float] | None = None
     lora_num_tokens: list[int] | None = None
     routed_experts: RoutedExperts | None = None
 
@@ -86,9 +107,12 @@ class MicroBatch(msgspec.Struct, array_like=True, gc=False, omit_defaults=True):
     # mm_token_type_ids: token type ids per token [batch seq], int64 (0=text, 1=image, 2=video)
     mm_token_type_ids: list[int] | None = None
 
-    # Loss dispatch is batch-driven (rl/opd → default loss with mode-specific taus,
-    # sft → sft loss). All samples packed into a micro batch share the same mode.
-    training_mode: TrainingMode = "rl"
+    # Per-token component weight streams (see TrainingSample). ``None`` means
+    # absent: no ce/ref_kl component, rl weight 1.0 everywhere — packing
+    # materializes a stream as soon as one packed sample carries it.
+    rl_weights: list[float] | None = None
+    ce_weights: list[float] | None = None
+    ref_kl_weights: list[float] | None = None
     rewards: list[float] | None = None
 
     # Packer-derived metadata used for run-local token exports.
diff --git a/tests/integration/test_reverse_text_rl_opd.py b/tests/integration/test_reverse_text_rl_opd.py
index 2ca1d7fdfe..69266b48e7 100644
--- a/tests/integration/test_reverse_text_rl_opd.py
+++ b/tests/integration/test_reverse_text_rl_opd.py
@@ -10,17 +10,17 @@
 
 from prime_rl.utils.process import cleanup_process
 from tests.conftest import ProcessResult
-from tests.utils import check_eval_avg_goes_up, check_no_error, strip_escape_codes
+from tests.utils import check_final_eval_reward_above, check_no_error, strip_escape_codes
 
 pytestmark = [pytest.mark.gpu, pytest.mark.slow]
 
 TIMEOUT = 600  # 10 minutes
-TEACHER_PORT = 8001
-TEACHER_READY_TIMEOUT_S = 300
+REF_PORT = 8001
+REF_READY_TIMEOUT_S = 300
 
 
-def _wait_for_teacher(port: int, timeout_s: int) -> None:
-    """Block until the teacher inference server's /v1/models endpoint is reachable."""
+def _wait_for_ref_server(port: int, timeout_s: int) -> None:
+    """Block until the frozen reference server's /v1/models endpoint is reachable."""
     url = f"http://localhost:{port}/v1/models"
     deadline = time.time() + timeout_s
     while time.time() < deadline:
@@ -35,15 +35,15 @@ def _wait_for_teacher(port: int, timeout_s: int) -> None:
 
 
 @pytest.fixture(scope="module")
-def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, None]:
-    """Spawn a `uv run inference` teacher on GPU 0 (shared with the rl-launched
-    student) at 40% gpu_memory_utilization. Tears down at module scope.
+def ref_inference(output_dir: Path) -> Generator[subprocess.Popen, None, None]:
+    """Spawn a `uv run inference` frozen reference server on GPU 0 (shared with the rl-launched
+    policy) at 40% gpu_memory_utilization. Tears down at module scope.
     """
     # The rl entrypoint's --clean-output-dir wipes the rl output_dir on start,
-    # so park the teacher log next to it instead of inside it.
-    teacher_log_dir = output_dir.parent / f"{output_dir.name}_teacher"
-    teacher_log_dir.mkdir(parents=True, exist_ok=True)
-    log_path = teacher_log_dir / "teacher_inference.log"
+    # so park the reference-server log next to it instead of inside it.
+    ref_log_dir = output_dir.parent / f"{output_dir.name}_ref"
+    ref_log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = ref_log_dir / "ref_inference.log"
     cmd = [
         "uv",
         "run",
@@ -51,7 +51,7 @@ def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, Non
         "--model.name",
         "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL",
         "--server.port",
-        str(TEACHER_PORT),
+        str(REF_PORT),
         "--gpu-memory-utilization",
         "0.4",
     ]
@@ -59,7 +59,7 @@ def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, Non
     with open(log_path, "w") as log_file:
         proc = subprocess.Popen(cmd, env=env, stdout=log_file, stderr=log_file)
     try:
-        _wait_for_teacher(TEACHER_PORT, TEACHER_READY_TIMEOUT_S)
+        _wait_for_ref_server(REF_PORT, REF_READY_TIMEOUT_S)
         yield proc
     finally:
         cleanup_process(proc.pid, signal.SIGTERM)
@@ -77,13 +77,13 @@ def wandb_name(branch_name: str) -> str:
 
 @pytest.fixture(scope="module")
 def rl_opd_process(
-    teacher_inference,
+    ref_inference,
     run_process: Callable[..., ProcessResult],
     output_dir: Path,
     wandb_project: str,
     wandb_name: str,
 ) -> ProcessResult:
-    """Run the RL entrypoint with training_mode = "opd"; teacher_inference is
+    """Run the RL entrypoint with the opd algorithm; ref_inference is
     a fixture-managed external vLLM at http://localhost:8001/v1."""
     cmd = [
         "uv",
@@ -107,7 +107,7 @@ def test_no_error(rl_opd_process: ProcessResult, output_dir: Path):
     check_no_error(rl_opd_process, output_dir)
 
 
-def test_eval_avg_goes_up(rl_opd_process: ProcessResult, test_no_error, output_dir: Path):
+def test_eval_reward_converges(rl_opd_process: ProcessResult, test_no_error, output_dir: Path):
     with open(output_dir / "logs" / "orchestrator.log", "r") as f:
         orchestrator_stdout = strip_escape_codes(f.read()).splitlines()
-    check_eval_avg_goes_up(orchestrator_stdout, env_name="reverse-text")
+    check_final_eval_reward_above(orchestrator_stdout, env_name="reverse-text", min_threshold=0.5)
diff --git a/tests/integration/test_reverse_text_rl_sft.py b/tests/integration/test_reverse_text_rl_sft.py
index df8b3f4786..f56a0d683e 100644
--- a/tests/integration/test_reverse_text_rl_sft.py
+++ b/tests/integration/test_reverse_text_rl_sft.py
@@ -10,17 +10,17 @@
 
 from prime_rl.utils.process import cleanup_process
 from tests.conftest import ProcessResult
-from tests.utils import check_eval_avg_goes_up, check_no_error, strip_escape_codes
+from tests.utils import check_final_eval_reward_above, check_no_error, strip_escape_codes
 
 pytestmark = [pytest.mark.gpu, pytest.mark.slow]
 
 TIMEOUT = 600  # 10 minutes
-TEACHER_PORT = 8001
-TEACHER_READY_TIMEOUT_S = 300
+REF_PORT = 8001
+REF_READY_TIMEOUT_S = 300
 
 
-def _wait_for_teacher(port: int, timeout_s: int) -> None:
-    """Block until the teacher inference server's /v1/models endpoint is reachable."""
+def _wait_for_ref_server(port: int, timeout_s: int) -> None:
+    """Block until the frozen reference server's /v1/models endpoint is reachable."""
     url = f"http://localhost:{port}/v1/models"
     deadline = time.time() + timeout_s
     while time.time() < deadline:
@@ -35,15 +35,15 @@ def _wait_for_teacher(port: int, timeout_s: int) -> None:
 
 
 @pytest.fixture(scope="module")
-def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, None]:
-    """Spawn a `uv run inference` teacher on GPU 0 (shared with the rl-launched
-    student) at 40% gpu_memory_utilization. Tears down at module scope.
+def ref_inference(output_dir: Path) -> Generator[subprocess.Popen, None, None]:
+    """Spawn a `uv run inference` frozen reference server on GPU 0 (shared with the rl-launched
+    policy) at 40% gpu_memory_utilization. Tears down at module scope.
     """
     # The rl entrypoint's --clean-output-dir wipes the rl output_dir on start,
-    # so park the teacher log next to it instead of inside it.
-    teacher_log_dir = output_dir.parent / f"{output_dir.name}_teacher"
-    teacher_log_dir.mkdir(parents=True, exist_ok=True)
-    log_path = teacher_log_dir / "teacher_inference.log"
+    # so park the reference-server log next to it instead of inside it.
+    ref_log_dir = output_dir.parent / f"{output_dir.name}_ref"
+    ref_log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = ref_log_dir / "ref_inference.log"
     cmd = [
         "uv",
         "run",
@@ -51,7 +51,7 @@ def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, Non
         "--model.name",
         "PrimeIntellect/Qwen3-0.6B-Reverse-Text-RL",
         "--server.port",
-        str(TEACHER_PORT),
+        str(REF_PORT),
         "--gpu-memory-utilization",
         "0.4",
     ]
@@ -59,7 +59,7 @@ def teacher_inference(output_dir: Path) -> Generator[subprocess.Popen, None, Non
     with open(log_path, "w") as log_file:
         proc = subprocess.Popen(cmd, env=env, stdout=log_file, stderr=log_file)
     try:
-        _wait_for_teacher(TEACHER_PORT, TEACHER_READY_TIMEOUT_S)
+        _wait_for_ref_server(REF_PORT, REF_READY_TIMEOUT_S)
         yield proc
     finally:
         cleanup_process(proc.pid, signal.SIGTERM)
@@ -77,13 +77,13 @@ def wandb_name(branch_name: str) -> str:
 
 @pytest.fixture(scope="module")
 def rl_sft_process(
-    teacher_inference,
+    ref_inference,
     run_process: Callable[..., ProcessResult],
     output_dir: Path,
     wandb_project: str,
     wandb_name: str,
 ) -> ProcessResult:
-    """Run the RL entrypoint with training_mode = "sft"; teacher_inference is
+    """Run the RL entrypoint with the sft algorithm; ref_inference is
     a fixture-managed external vLLM at http://localhost:8001/v1."""
     cmd = [
         "uv",
@@ -107,7 +107,7 @@ def test_no_error(rl_sft_process: ProcessResult, output_dir: Path):
     check_no_error(rl_sft_process, output_dir)
 
 
-def test_eval_avg_goes_up(rl_sft_process: ProcessResult, test_no_error, output_dir: Path):
+def test_eval_reward_converges(rl_sft_process: ProcessResult, test_no_error, output_dir: Path):
     with open(output_dir / "logs" / "orchestrator.log", "r") as f:
         orchestrator_stdout = strip_escape_codes(f.read()).splitlines()
-    check_eval_avg_goes_up(orchestrator_stdout, env_name="reverse-text")
+    check_final_eval_reward_above(orchestrator_stdout, env_name="reverse-text", min_threshold=0.5)
diff --git a/tests/unit/orchestrator/test_advantage.py b/tests/unit/orchestrator/test_advantage.py
index 89022c8428..5b8c06a50c 100644
--- a/tests/unit/orchestrator/test_advantage.py
+++ b/tests/unit/orchestrator/test_advantage.py
@@ -1,22 +1,25 @@
+import asyncio
 import math
 import uuid
 
 import pytest
 
-from prime_rl.configs.orchestrator import (
-    CustomAdvantageConfig,
-    DefaultAdvantageConfig,
+from prime_rl.configs.algorithm import (
+    AlgorithmConfig,
+    LinearLengthPenaltyConfig,
     TokensLengthPenaltyConfig,
     TurnsLengthPenaltyConfig,
 )
-from prime_rl.orchestrator.advantage import (
-    AdvantageInputs,
-    AdvantageOutputs,
-    assign_advantages,
-    default_advantage_fn,
-    setup_advantage_fn,
+from prime_rl.orchestrator.algo import CustomAlgorithm, build_algorithm
+from prime_rl.orchestrator.algo.advantage import (
+    apply_advantage_fn,
+    efficiency_shaping_advantage,
+    grpo_advantage,
+    length_penalty_advantage,
+    max_rl_advantage_fn,
 )
-from prime_rl.orchestrator.types import TrainRollout
+from prime_rl.orchestrator.types import RolloutView, TrainRollout
+from prime_rl.transport.types import TrainingSample
 
 
 def _make_rollout(
@@ -28,7 +31,9 @@ def _make_rollout(
 ) -> dict:
     """Create a minimal rollout dict for advantage testing.
 
-    `completion_len` tokens are split across `num_turns` trajectory steps.
+    `completion_len` tokens are split across `num_turns` trajectory steps —
+    they feed the length penalty's cost computation (read from `raw`), not the
+    sample's advantage length.
     """
     per_turn, rem = divmod(completion_len, max(num_turns, 1))
     trajectory = [
@@ -43,14 +48,45 @@ def _make_rollout(
     }
 
 
-def _make_group(rewards, completion_lengths=None, num_turns=None) -> AdvantageInputs:
-    """Build single-group AdvantageInputs from 1D arrays of rewards/lengths/turns."""
-    rollouts = []
+def _train_rollout(raw: dict, completion_ids: tuple[int, ...] = (2,)) -> TrainRollout:
+    """One ``TrainRollout`` carrying a single training sample with the given
+    completion tokens (default length 1, so a group-norm scalar is its
+    rollout's whole advantage)."""
+    return TrainRollout(
+        raw=raw,
+        env_name="test",
+        example_id=0,
+        group_id=uuid.uuid4(),
+        policy_version=0,
+        off_policy_steps=0,
+        samples=[
+            TrainingSample(
+                prompt_ids=[1],
+                prompt_mask=[False],
+                completion_ids=list(completion_ids),
+                completion_mask=[True] * len(completion_ids),
+                completion_logprobs=[-0.1] * len(completion_ids),
+                completion_temperatures=[],
+                env_name="test",
+            )
+        ],
+    )
+
+
+def _views(raw_rollouts: list[dict]) -> list[RolloutView]:
+    """Wrap raw rollout dicts into ``RolloutView``\\ s over single-token
+    samples — the advantage fns see exactly what ``score_group`` sees."""
+    return [RolloutView(_train_rollout(raw)) for raw in raw_rollouts]
+
+
+def _make_group(rewards, completion_lengths=None, num_turns=None) -> list[RolloutView]:
+    """Build one group of ``RolloutView``\\ s from 1D arrays of rewards/lengths/turns."""
+    raw_rollouts = []
     for i, reward in enumerate(rewards):
         cl = int(completion_lengths[i]) if completion_lengths is not None else 0
         nt = int(num_turns[i]) if num_turns is not None else 1
-        rollouts.append(_make_rollout(float(reward), cl, nt))
-    return AdvantageInputs(rollouts=rollouts)
+        raw_rollouts.append(_make_rollout(float(reward), cl, nt))
+    return _views(raw_rollouts)
 
 
 # Helper aliases for readability — completion-only and tool-only token shaping.
@@ -58,39 +94,51 @@ def _make_group(rewards, completion_lengths=None, num_turns=None) -> AdvantageIn
 _TOKENS_TOOL_ONLY = TokensLengthPenaltyConfig(completion_weight=0.0, tool_response_weight=1.0)
 
 
-def test_default_advantage_fn_simple_mean():
-    inputs = _make_group(rewards=[1.0, 0.5, 0.8], completion_lengths=[10, 12, 8])
-    result = default_advantage_fn(inputs)
+def test_grpo_advantage_simple_mean():
+    result = grpo_advantage(_make_group(rewards=[1.0, 0.5, 0.8], completion_lengths=[10, 12, 8]))
+
+    assert len(result) == 3
+    assert sum(result) == pytest.approx(0.0, abs=1e-6)
+
+
+def test_max_rl_advantage_fn_mean_normalized():
+    # mean 0.25: the success gets (1 - 0.25)/0.25 = 3, failures (0 - 0.25)/0.25 = -1
+    result = max_rl_advantage_fn(_make_group(rewards=[1.0, 0.0, 0.0, 0.0]))
+    assert result == pytest.approx([3.0, -1.0, -1.0, -1.0])
 
-    assert len(result.advantages) == 3
-    assert sum(result.advantages) == pytest.approx(0.0, abs=1e-6)
+    # no-success groups carry no signal (the paper's K=0 convention) ...
+    assert max_rl_advantage_fn(_make_group(rewards=[0.0, 0.0])) == [0.0, 0.0]
+    # ... and all-success groups center to zero like GRPO
+    assert max_rl_advantage_fn(_make_group(rewards=[1.0, 1.0])) == pytest.approx([0.0, 0.0])
 
 
 def test_efficiency_mixed_group():
     """Mixed group: reward shaping preserves zero-mean, shorter correct gets higher advantage."""
-    inputs = _make_group(rewards=[1.0, 1.0, 0.0, 1.0], completion_lengths=[10, 30, 20, 20])
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
+    group = _make_group(rewards=[1.0, 1.0, 0.0, 1.0], completion_lengths=[10, 30, 20, 20])
+    result = efficiency_shaping_advantage(group, _TOKENS_COMPLETION)
 
     # mean_correct_len = (10+30+20)/3 = 20
     # bonus = clamp(1 - [10,30,20,20]/20, 0, 1) = [0.5, 0, 0, 0]
     # shaped_rewards = R * (1 + bonus * correct_mask) = [1.5, 1, 0, 1]
     # baseline = mean(shaped_rewards) = 0.875
     # A = shaped_rewards - baseline = [0.625, 0.125, -0.875, 0.125]
-    assert result.advantages == pytest.approx([0.625, 0.125, -0.875, 0.125], abs=1e-6)
+    assert result == pytest.approx([0.625, 0.125, -0.875, 0.125], abs=1e-6)
 
     # Zero-mean per group
-    assert sum(result.advantages) == pytest.approx(0.0, abs=1e-6)
+    assert sum(result) == pytest.approx(0.0, abs=1e-6)
 
     # All correct rollouts have positive advantage
-    for rollout, adv in zip(inputs.rollouts, result.advantages):
-        if rollout["reward"] >= 1.0:
+    for view, adv in zip(group, result):
+        if view.reward >= 1.0:
             assert adv > 0
 
 
 def test_efficiency_all_correct_group():
     """All-correct group: zero-mean, shorter gets higher advantage."""
-    inputs = _make_group(rewards=[1.0, 1.0, 1.0], completion_lengths=[10, 20, 40])
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
+    result = efficiency_shaping_advantage(
+        _make_group(rewards=[1.0, 1.0, 1.0], completion_lengths=[10, 20, 40]),
+        _TOKENS_COMPLETION,
+    )
 
     # mean_len = 70/3 ≈ 23.33
     # bonus = clamp(1 - [10, 20, 40] / (70/3), 0, 1) = [4/7, 1/7, 0]
@@ -98,67 +146,70 @@ def test_efficiency_all_correct_group():
     shaped = [11.0 / 7, 8.0 / 7, 1.0]
     mean_shaped = sum(shaped) / len(shaped)
     expected = [s - mean_shaped for s in shaped]
-    assert result.advantages == pytest.approx(expected, abs=1e-6)
+    assert result == pytest.approx(expected, abs=1e-6)
 
     # Zero-mean
-    assert sum(result.advantages) == pytest.approx(0.0, abs=1e-6)
+    assert sum(result) == pytest.approx(0.0, abs=1e-6)
 
     # Shortest has highest advantage
-    assert result.advantages[0] > result.advantages[1] > result.advantages[2]
+    assert result[0] > result[1] > result[2]
 
 
 def test_efficiency_all_zero_rewards():
     """When all rewards are 0, no length shaping — falls back to standard GRPO."""
-    inputs = _make_group(rewards=[0.0, 0.0, 0.0], completion_lengths=[10, 20, 15])
-    result_with = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
-    result_without = default_advantage_fn(inputs)
+    group = _make_group(rewards=[0.0, 0.0, 0.0], completion_lengths=[10, 20, 15])
+    result_with = efficiency_shaping_advantage(group, _TOKENS_COMPLETION)
+    result_without = grpo_advantage(group)
 
-    assert result_with.advantages == pytest.approx(result_without.advantages, abs=1e-6)
+    assert result_with == pytest.approx(result_without, abs=1e-6)
 
 
 def test_efficiency_single_correct():
     """Single correct rollout: bonus=0 (at its own mean), same as standard GRPO."""
-    inputs = _make_group(rewards=[1.0, 0.0, 0.0, 0.0], completion_lengths=[100, 50, 200, 150])
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
+    result = efficiency_shaping_advantage(
+        _make_group(rewards=[1.0, 0.0, 0.0, 0.0], completion_lengths=[100, 50, 200, 150]),
+        _TOKENS_COMPLETION,
+    )
 
-    assert result.advantages == pytest.approx([0.75, -0.25, -0.25, -0.25], abs=1e-6)
+    assert result == pytest.approx([0.75, -0.25, -0.25, -0.25], abs=1e-6)
 
 
 def test_efficiency_shorter_correct_higher_advantage():
     """Among correct rollouts in a mixed group, shorter always gets higher advantage."""
-    inputs = _make_group(rewards=[1.0, 1.0, 1.0, 0.0, 0.0], completion_lengths=[50, 100, 200, 80, 120])
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
+    result = efficiency_shaping_advantage(
+        _make_group(rewards=[1.0, 1.0, 1.0, 0.0, 0.0], completion_lengths=[50, 100, 200, 80, 120]),
+        _TOKENS_COMPLETION,
+    )
 
-    advs = result.advantages
-    assert advs[0] > advs[1] > advs[2]
-    assert all(a > 0 for a in advs[:3])
-    assert all(a < 0 for a in advs[3:])
+    assert result[0] > result[1] > result[2]
+    assert all(a > 0 for a in result[:3])
+    assert all(a < 0 for a in result[3:])
 
 
 def test_efficiency_zero_mean_per_group():
     """Reward shaping preserves zero-mean advantages within each group."""
-    mixed = default_advantage_fn(
+    mixed = efficiency_shaping_advantage(
         _make_group(rewards=[1.0, 1.0, 0.0, 1.0], completion_lengths=[10, 30, 20, 20]),
-        length_penalty=_TOKENS_COMPLETION,
+        _TOKENS_COMPLETION,
     )
-    all_correct = default_advantage_fn(
+    all_correct = efficiency_shaping_advantage(
         _make_group(rewards=[1.0, 1.0, 1.0, 1.0], completion_lengths=[10, 20, 40, 80]),
-        length_penalty=_TOKENS_COMPLETION,
+        _TOKENS_COMPLETION,
     )
 
-    assert sum(mixed.advantages) == pytest.approx(0.0, abs=1e-6)
-    assert sum(all_correct.advantages) == pytest.approx(0.0, abs=1e-6)
+    assert sum(mixed) == pytest.approx(0.0, abs=1e-6)
+    assert sum(all_correct) == pytest.approx(0.0, abs=1e-6)
 
 
 def test_efficiency_amplification_bounded():
     """Even with extreme length outliers, reward amplification is capped at 2x."""
-    inputs = _make_group(rewards=[1.0, 1.0, 0.0], completion_lengths=[1, 10000, 5000])
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
+    result = efficiency_shaping_advantage(
+        _make_group(rewards=[1.0, 1.0, 0.0], completion_lengths=[1, 10000, 5000]),
+        _TOKENS_COMPLETION,
+    )
 
-    # Shortest correct gets bonus ≈ 1, so shaped_reward ≈ 2
-    # Standard reward = 1, so amplification ≈ 2x
-    # shaped_rewards ≈ [2, 1, 0], baseline ≈ 1, max advantage ≈ 1
-    assert result.advantages[0] < 1.0 + 1e-3
+    # Shortest correct gets bonus ≈ 1, so shaped_reward ≈ 2; baseline ≈ 1, max advantage ≈ 1
+    assert result[0] < 1.0 + 1e-3
 
 
 def test_efficiency_tokens_with_tool_response_weight():
@@ -180,16 +231,15 @@ def test_efficiency_tokens_with_tool_response_weight():
             "metrics": {"rlm_total_tool_response_tokens": 100},
         },
     ]
-    inputs = AdvantageInputs(rollouts=rollouts)
+    group = _views(rollouts)
 
     # completion tokens identical (10 each) → completion-only shaping is a no-op
-    result_completion_only = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
-    assert result_completion_only.advantages == pytest.approx([0.0, 0.0, 0.0], abs=1e-6)
+    result_completion_only = efficiency_shaping_advantage(group, _TOKENS_COMPLETION)
+    assert result_completion_only == pytest.approx([0.0, 0.0, 0.0], abs=1e-6)
 
     # tool-response only: costs are [200, 0, 100], mean=100, bonus is one-sided
     # so only the below-mean rollout (idx 1) gets amplified; the at/above-mean tie.
-    result_tool_only = default_advantage_fn(inputs, length_penalty=_TOKENS_TOOL_ONLY)
-    advs = result_tool_only.advantages
+    advs = efficiency_shaping_advantage(group, _TOKENS_TOOL_ONLY)
     assert advs[1] > advs[0]
     assert advs[1] > advs[2]
     assert advs[0] == pytest.approx(advs[2], abs=1e-6)
@@ -206,9 +256,9 @@ def test_efficiency_fractional_weight_with_int_rewards():
     rollouts_float = [{**r, "reward": float(r["reward"])} for r in rollouts_int]
 
     fractional = TokensLengthPenaltyConfig(completion_weight=0.3, tool_response_weight=0.0)
-    int_result = default_advantage_fn(AdvantageInputs(rollouts=rollouts_int), length_penalty=fractional)
-    float_result = default_advantage_fn(AdvantageInputs(rollouts=rollouts_float), length_penalty=fractional)
-    assert int_result.advantages == pytest.approx(float_result.advantages, abs=1e-6)
+    int_result = efficiency_shaping_advantage(_views(rollouts_int), fractional)
+    float_result = efficiency_shaping_advantage(_views(rollouts_float), fractional)
+    assert int_result == pytest.approx(float_result, abs=1e-6)
 
 
 def test_efficiency_zero_costs_falls_back_to_plain_grpo():
@@ -219,91 +269,199 @@ def test_efficiency_zero_costs_falls_back_to_plain_grpo():
         {"reward": 1.0, "trajectory": [{"tokens": {"prompt_ids": [0], "completion_ids": list(range(10))}}]},
         {"reward": 0.0, "trajectory": [{"tokens": {"prompt_ids": [0], "completion_ids": list(range(10))}}]},
     ]
-    inputs = AdvantageInputs(rollouts=rollouts)
-    result = default_advantage_fn(inputs, length_penalty=_TOKENS_TOOL_ONLY)
-    expected = default_advantage_fn(inputs)  # plain GRPO
-    assert not any(math.isnan(a) for a in result.advantages)
-    assert result.advantages == pytest.approx(expected.advantages, abs=1e-6)
+    group = _views(rollouts)
+    result = efficiency_shaping_advantage(group, _TOKENS_TOOL_ONLY)
+    expected = grpo_advantage(group)  # plain GRPO
+    assert not any(math.isnan(a) for a in result)
+    assert result == pytest.approx(expected, abs=1e-6)
 
 
 def test_efficiency_tokens_default_weights_match_completion_when_no_metric():
     """Default TokensLengthPenaltyConfig (1,1) reduces to completion-only when rollouts lack the metric."""
-    inputs = _make_group(rewards=[1.0, 1.0, 0.0, 1.0], completion_lengths=[10, 30, 20, 20])
-    result_default = default_advantage_fn(inputs, length_penalty=TokensLengthPenaltyConfig())
-    result_completion = default_advantage_fn(inputs, length_penalty=_TOKENS_COMPLETION)
-    assert result_default.advantages == pytest.approx(result_completion.advantages, abs=1e-6)
+    group = _make_group(rewards=[1.0, 1.0, 0.0, 1.0], completion_lengths=[10, 30, 20, 20])
+    result_default = efficiency_shaping_advantage(group, TokensLengthPenaltyConfig())
+    result_completion = efficiency_shaping_advantage(group, _TOKENS_COMPLETION)
+    assert result_default == pytest.approx(result_completion, abs=1e-6)
 
 
 def test_efficiency_turns_penalty():
     """`TurnsLengthPenaltyConfig` shapes by trajectory turn count rather than token count."""
-    inputs = _make_group(
-        rewards=[1.0, 1.0, 0.0, 1.0],
-        # token counts identical, but turns differ — turns penalty should still differentiate
-        completion_lengths=[100, 100, 100, 100],
-        num_turns=[1, 3, 2, 2],
+    result = efficiency_shaping_advantage(
+        _make_group(
+            rewards=[1.0, 1.0, 0.0, 1.0],
+            # token counts identical, but turns differ — turns penalty should still differentiate
+            completion_lengths=[100, 100, 100, 100],
+            num_turns=[1, 3, 2, 2],
+        ),
+        TurnsLengthPenaltyConfig(),
     )
-    result = default_advantage_fn(inputs, length_penalty=TurnsLengthPenaltyConfig())
 
     # mean_correct_turns = (1+3+2)/3 = 2
     # bonus = clamp(1 - [1,3,2,2]/2, 0, 1) = [0.5, 0, 0, 0]
-    assert result.advantages == pytest.approx([0.625, 0.125, -0.875, 0.125], abs=1e-6)
-
-
-def _train_rollouts(rewards: list[float]) -> list[TrainRollout]:
-    """Wrap a list of rewards into ``TrainRollout``\\ s sharing a single
-    ``group_id`` — ``assign_advantages`` works on one group at a time
-    (the sink groups by ``group_id`` upstream)."""
-    gid = uuid.uuid4()
-    return [
-        TrainRollout(
-            raw={"reward": r, "trajectory": []},
-            env_name="test",
-            example_id=0,
-            group_id=gid,
-            policy_version=0,
-            off_policy_steps=0,
+    assert result == pytest.approx([0.625, 0.125, -0.875, 0.125], abs=1e-6)
+
+
+def test_linear_penalty_is_grpo_plus_centered_penalty():
+    """The linear penalty is a separate additive advantage: grpo_advantage + length_penalty_advantage
+    is identical to folding the penalty into the reward before centering (centering is linear)."""
+    group = _make_group(rewards=[1.0, 1.0, 0.0, 0.0], completion_lengths=[100, 200, 100, 200])
+    cfg = LinearLengthPenaltyConfig(coef=0.25)
+    summed = [a + p for a, p in zip(grpo_advantage(group), length_penalty_advantage(group, cfg, 1000), strict=True)]
+
+    # folded reference: (reward - penalty) centered by the plain mean
+    rewards = [1.0, 1.0, 0.0, 0.0]
+    pass_rate = sum(rewards) / len(rewards)  # 0.5
+    penalty = [0.25 * pass_rate * (length / 1000) for length in (100, 200, 100, 200)]
+    folded_raw = [r - p for r, p in zip(rewards, penalty)]
+    mean = sum(folded_raw) / len(folded_raw)
+    folded = [x - mean for x in folded_raw]
+
+    assert summed == pytest.approx(folded, abs=1e-6)
+    assert summed == pytest.approx([0.50625, 0.49375, -0.49375, -0.50625], abs=1e-6)
+    assert sum(summed) == pytest.approx(0.0, abs=1e-6)
+    # shorter beats longer within each outcome
+    assert summed[0] > summed[1]
+    assert summed[2] > summed[3]
+
+
+def test_length_penalty_advantage_zero_pass_rate_is_zero():
+    """A never-solved group (mean reward 0) has zero penalty everywhere — it adds nothing to GRPO."""
+    group = _make_group(rewards=[0.0, 0.0, 0.0], completion_lengths=[10, 20, 30])
+    penalty = length_penalty_advantage(group, LinearLengthPenaltyConfig(coef=0.25), max_seq_len=100)
+    assert penalty == pytest.approx([0.0, 0.0, 0.0], abs=1e-6)
+
+
+def test_length_penalty_advantage_uniform_lengths_is_zero():
+    """Equal lengths → uniform penalty → the centered term is zero, so it is a no-op on GRPO."""
+    group = _make_group(rewards=[1.0, 1.0, 0.0, 0.0], completion_lengths=[100, 100, 100, 100])
+    penalty = length_penalty_advantage(group, LinearLengthPenaltyConfig(coef=0.25), max_seq_len=100)
+    assert penalty == pytest.approx([0.0, 0.0, 0.0, 0.0], abs=1e-6)
+
+
+def test_length_penalty_advantage_gate_by_correctness():
+    """gate_by_correctness penalizes only correct rollouts; with equal lengths the centered
+    penalty term is non-zero (correct rollouts pushed down, incorrect ones up)."""
+    group = _make_group(rewards=[1.0, 1.0, 0.0, 0.0], completion_lengths=[100, 100, 100, 100])
+    penalty = length_penalty_advantage(
+        group, LinearLengthPenaltyConfig(coef=0.25, gate_by_correctness=True), max_seq_len=100
+    )
+    # penalty_i = 0.25 * 0.5 * 1 * reward = [0.125, 0.125, 0, 0]; centered = mean - penalty
+    assert penalty == pytest.approx([-0.0625, -0.0625, 0.0625, 0.0625], abs=1e-6)
+
+
+def test_linear_penalty_with_length_weighted_baseline_matches_folded():
+    """With length_weighted_baseline, GRPO + penalty still equals folding the penalty
+    into the reward before length-weighted centering (the original #2702 behavior)."""
+    rewards = [1.0, 1.0, 0.0, 0.0]
+    lengths = [100, 200, 100, 300]
+    group = _make_group(rewards=rewards, completion_lengths=lengths)
+    cfg = LinearLengthPenaltyConfig(coef=0.25)
+    summed = [
+        a + p
+        for a, p in zip(
+            grpo_advantage(group, length_weighted_baseline=True),
+            length_penalty_advantage(group, cfg, 1000, length_weighted_baseline=True),
+            strict=True,
         )
-        for r in rewards
     ]
 
+    # folded reference: (reward - penalty) centered by the token-length-weighted mean
+    pass_rate = sum(rewards) / len(rewards)
+    penalty = [0.25 * pass_rate * (length / 1000) for length in lengths]
+    folded_raw = [r - p for r, p in zip(rewards, penalty)]
+    lw_mean = sum(length * x for length, x in zip(lengths, folded_raw)) / sum(lengths)
+    folded = [x - lw_mean for x in folded_raw]
 
-def test_assign_advantages_writes_field():
-    rollouts = _train_rollouts([1.0, 0.5, 0.8])
-    fn = setup_advantage_fn(DefaultAdvantageConfig())
-    assign_advantages(rollouts, fn)
-    advs = [r.advantage for r in rollouts]
-    assert sum(advs) == pytest.approx(0.0, abs=1e-6)
+    assert summed == pytest.approx(folded, abs=1e-6)
+    # advantages are length-weighted-zero, like the folded original (float32 accumulation)
+    assert sum(length * a for length, a in zip(lengths, summed)) == pytest.approx(0.0, abs=1e-3)
+
+
+def test_length_penalty_advantage_requires_max_seq_len():
+    """The linear penalty's denominator is orchestrator.seq_len — missing it is an error, not a guess."""
+    group = _make_group(rewards=[1.0, 0.0], completion_lengths=[100, 100])
+    with pytest.raises(ValueError, match="max_seq_len"):
+        length_penalty_advantage(group, LinearLengthPenaltyConfig(), max_seq_len=None)
 
 
-def test_assign_advantages_without_fn_is_reward():
-    """``advantage_fn=None`` falls back to ``advantage = reward``."""
-    rollouts = _train_rollouts([1.0, 0.5, 0.8])
-    assign_advantages(rollouts, None)
-    assert [r.advantage for r in rollouts] == [1.0, 0.5, 0.8]
+def test_grpo_length_weighted_baseline():
+    """The length-weighted baseline centers by per-token expected reward:
+    sum(len_i * reward_i) / sum(len_i) instead of the plain mean."""
+    group = _make_group(rewards=[1.0, 0.0], completion_lengths=[100, 300])
+    result = grpo_advantage(group, length_weighted_baseline=True)
 
+    # baseline = (100*1 + 300*0) / 400 = 0.25
+    assert result == pytest.approx([0.75, -0.25], abs=1e-6)
+    # advantages are length-weighted-zero (not mean-zero)
+    assert (100 * result[0] + 300 * result[1]) == pytest.approx(0.0, abs=1e-6)
 
-def test_assign_advantages_singleton_group_is_zero():
+
+def test_grpo_algorithm_sums_linear_penalty_end_to_end():
+    """build_algorithm injects max_seq_len; GRPOAlgorithm.score_group writes
+    grpo_advantage + length_penalty_advantage onto each rollout."""
+    config = AlgorithmConfig.model_validate(
+        {"advantage": {"type": "grpo", "length_penalty": {"type": "linear", "coef": 0.25}}}
+    )
+    algorithm = build_algorithm(config, policy_pool=None, renderer=None, max_seq_len=1000)
+
+    rollouts = [
+        _train_rollout(_make_rollout(reward, completion_len=length))
+        for reward, length in [(1.0, 100), (1.0, 200), (0.0, 100), (0.0, 200)]
+    ]
+    asyncio.run(algorithm.score_group([RolloutView(r) for r in rollouts]))
+
+    # single-token samples → each rollout's stream is its scalar advantage
+    advantages = [r.advantages[0] for r in rollouts]
+    assert advantages == pytest.approx([0.50625, 0.49375, -0.49375, -0.50625], abs=1e-6)
+
+
+def test_rollout_view_assign_advantages_broadcasts_scalar():
+    """A scalar broadcasts uniformly over the rollout's completion tokens."""
+    rollout = _train_rollout({"reward": 0.0, "trajectory": []}, completion_ids=(2, 3))
+    RolloutView(rollout).assign_advantages(0.7)
+    assert rollout.advantages == [0.7, 0.7]
+
+
+def test_rollout_view_assign_advantages_rejects_misaligned():
+    rollout = _train_rollout({"reward": 0.0, "trajectory": []}, completion_ids=(2, 3))
+    with pytest.raises(ValueError, match="align"):
+        RolloutView(rollout).assign_advantages([0.5])
+
+
+def test_apply_advantage_fn_broadcasts_group_norm():
+    rollouts = [_train_rollout({"reward": r, "trajectory": []}, completion_ids=(2, 3)) for r in (1.0, 0.5, 0.8)]
+    apply_advantage_fn([RolloutView(r) for r in rollouts], grpo_advantage)
+    streams = [r.advantages for r in rollouts]
+    # group credit broadcasts uniformly over each rollout's completion tokens
+    assert all(len(s) == 2 and s[0] == s[1] for s in streams)
+    assert sum(s[0] for s in streams) == pytest.approx(0.0, abs=1e-6)
+
+
+def test_apply_advantage_fn_singleton_group_is_zero():
     """A group of size 1 has reward == mean, so its advantage is 0."""
-    rollouts = _train_rollouts([0.7])
-    fn = setup_advantage_fn(DefaultAdvantageConfig())
-    assign_advantages(rollouts, fn)
-    assert rollouts[0].advantage == pytest.approx(0.0, abs=1e-6)
+    rollouts = [_train_rollout({"reward": 0.7, "trajectory": []}, completion_ids=(2, 3))]
+    apply_advantage_fn([RolloutView(r) for r in rollouts], grpo_advantage)
+    assert rollouts[0].advantages == pytest.approx([0.0, 0.0], abs=1e-6)
 
 
-def test_setup_advantage_fn_with_custom_config():
-    config = CustomAdvantageConfig(
-        import_path="tests.unit.orchestrator.test_advantage._dummy_custom_advantage",
-        kwargs={"scale": 2.0},
+def test_custom_advantage_algorithm():
+    config = AlgorithmConfig.model_validate(
+        {
+            "advantage": {
+                "type": "custom",
+                "import_path": "tests.unit.orchestrator.test_advantage._dummy_custom_advantage",
+                "kwargs": {"scale": 2.0},
+            }
+        }
     )
-    advantage_fn = setup_advantage_fn(config)
+    algorithm = CustomAlgorithm(config.advantage, policy_pool=None, renderer=None)
 
-    inputs = _make_group(rewards=[1.0, 0.5, 0.8], completion_lengths=[10, 12, 8])
+    group = _make_group(rewards=[1.0, 0.5, 0.8], completion_lengths=[10, 12, 8])
 
-    result = advantage_fn(inputs)
-    assert isinstance(result, AdvantageOutputs)
-    assert result.advantages == pytest.approx([2.0, 1.0, 1.6], abs=1e-6)
+    result = algorithm.advantage_fn(group)
+    assert result == pytest.approx([2.0, 1.0, 1.6], abs=1e-6)
 
 
-def _dummy_custom_advantage(inputs: AdvantageInputs, scale: float = 1.0) -> AdvantageOutputs:
-    """A simple custom advantage for testing."""
-    return AdvantageOutputs(advantages=[r["reward"] * scale for r in inputs.rollouts])
+def _dummy_custom_advantage(group: list[RolloutView], scale: float = 1.0) -> list[float]:
+    """A simple custom advantage for testing — one scalar per rollout."""
+    return [view.reward * scale for view in group]
diff --git a/tests/unit/orchestrator/test_algorithms.py b/tests/unit/orchestrator/test_algorithms.py
new file mode 100644
index 0000000000..b8b5409733
--- /dev/null
+++ b/tests/unit/orchestrator/test_algorithms.py
@@ -0,0 +1,316 @@
+import asyncio
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+import verifiers as vf
+
+from prime_rl.configs.algorithm import AlgorithmConfig, FrozenModelConfig
+from prime_rl.orchestrator.algo import EchoAlgorithm, stamp_advantages, stamp_loss_routing
+from prime_rl.orchestrator.trajectories import interleave_rollout
+from prime_rl.orchestrator.types import RolloutView, TrainRollout
+from prime_rl.transport.types import TrainingSample
+
+FROZEN = {"name": "org/ref-model", "base_url": ["http://ref:8001/v1"]}
+
+
+def _ref_kind(ref):
+    """Collapse a resolved reference to a comparable marker."""
+    return "frozen" if isinstance(ref, FrozenModelConfig) else ref
+
+
+@pytest.mark.parametrize(
+    ("advantage_type", "model", "source", "advantage_model", "action_loss_type"),
+    [
+        ("grpo", None, "policy", None, "rl"),
+        ("max_rl", None, "policy", None, "rl"),
+        ("opd", FROZEN, "policy", "frozen", "ref_kl"),
+        ("sft", FROZEN, "frozen", None, "ce"),
+        ("opsd", None, "policy", "policy", "ref_kl"),
+        ("echo", None, "policy", None, "rl"),
+    ],
+)
+def test_type_defaults_are_the_vetted_algorithms(advantage_type, model, source, advantage_model, action_loss_type):
+    algo = AlgorithmConfig(advantage={"type": advantage_type}, model=model)
+    assert _ref_kind(algo.sampling.source) == source
+    assert algo.advantage.type == advantage_type
+    assert _ref_kind(getattr(algo.advantage, "model", None)) == advantage_model
+    assert algo.advantage.action_loss_type == action_loss_type
+
+
+def test_echo_roles_replace_the_default_table():
+    algo = AlgorithmConfig(advantage={"type": "echo", "roles": {"user": {"alpha": 0.5}}})
+    assert algo.advantage.type == "echo"
+    assert algo.advantage.roles.user.alpha == 0.5
+    # Setting any role replaces the whole table: the tool default is gone
+    assert algo.advantage.roles.tool is None
+
+
+def test_echo_defaults_to_tool_bodies():
+    algo = AlgorithmConfig(advantage={"type": "echo"})
+    assert algo.advantage.roles.tool.alpha == 0.1
+    assert algo.advantage.roles.system is None
+    assert algo.advantage.roles.user is None
+    assert algo.advantage.roles.assistant is None
+
+
+def test_echo_roles_require_at_least_one():
+    with pytest.raises(ValueError, match="at least one role"):
+        AlgorithmConfig(advantage={"type": "echo", "roles": {}})
+
+
+def test_opd_requires_teacher():
+    with pytest.raises(ValueError, match="needs a teacher"):
+        AlgorithmConfig(advantage={"type": "opd"})
+
+
+def test_sft_requires_teacher():
+    with pytest.raises(ValueError, match="needs a teacher to sample rollouts from"):
+        AlgorithmConfig(advantage={"type": "sft"})
+
+
+def test_teacher_aliases_model_shorthand():
+    algo = AlgorithmConfig.model_validate({"advantage": {"type": "opd"}, "teacher": FROZEN})
+    assert isinstance(algo.advantage.model, FrozenModelConfig)
+    assert algo.advantage.model.name == "org/ref-model"
+
+
+def test_model_shorthand_without_target_errors():
+    with pytest.raises(ValueError, match="no component reference accepts it"):
+        AlgorithmConfig(model=FROZEN)
+
+
+def test_model_shorthand_redundant_but_consistent_is_accepted():
+    algo = AlgorithmConfig(model=FROZEN, advantage={"type": "opd", "model": FROZEN})
+    assert isinstance(algo.advantage.model, FrozenModelConfig)
+
+
+def test_opd_rejects_policy():
+    with pytest.raises(ValueError, match="degenerate"):
+        AlgorithmConfig(advantage={"type": "opd"}, model="policy")
+
+
+def test_rl_loss_type_incompatible_with_frozen_sampling():
+    with pytest.raises(ValueError, match="sampling.source is a frozen model"):
+        AlgorithmConfig(sampling={"source": FROZEN}, advantage={"type": "grpo"})
+
+
+def _make_sample(ce_weights: list[float] | None = None) -> TrainingSample:
+    return TrainingSample(
+        prompt_ids=[1, 2],
+        prompt_mask=[False, False],
+        completion_ids=[3, 4, 5, 6],
+        completion_mask=[True, True, False, True],
+        completion_logprobs=[-0.1, -0.2, 0.0, -0.3],
+        completion_temperatures=[],
+        env_name="test-env",
+        ce_weights=ce_weights,
+    )
+
+
+def test_stamp_loss_routing_uniform_rl():
+    sample = _make_sample()
+    stamp_loss_routing(sample, "rl")
+    # Hot path: absent streams mean rl weight 1.0 on the loss mask
+    assert sample.rl_weights is None
+    assert sample.ce_weights is None
+    assert sample.ref_kl_weights is None
+
+
+def test_stamp_loss_routing_ref_kl_action():
+    sample = _make_sample()
+    stamp_loss_routing(sample, "ref_kl")
+    # Action tokens (completion_mask True) feed the ref_kl component; rl is off
+    assert sample.rl_weights == [0.0] * 6
+    assert sample.ref_kl_weights == [0.0, 0.0] + [1.0, 1.0, 0.0, 1.0]
+    assert sample.ce_weights is None
+
+
+def test_stamp_loss_routing_ce_action():
+    sample = _make_sample()
+    stamp_loss_routing(sample, "ce")
+    assert sample.rl_weights == [0.0] * 6
+    assert sample.ce_weights == [0.0, 0.0] + [1.0, 1.0, 0.0, 1.0]
+    assert sample.ref_kl_weights is None
+
+
+def test_stamp_loss_routing_keeps_algorithm_written_ce_stream():
+    # Echo writes ce_weights directly at group time (observation at
+    # completion index 2, outside completion_mask); rl routing must not
+    # clobber it — the rl component still ships no streams (hot path).
+    sample = _make_sample(ce_weights=[0.0, 0.0] + [0.0, 0.0, 0.1, 0.0])
+    stamp_loss_routing(sample, "rl")
+    assert sample.rl_weights is None
+    assert sample.ce_weights == [0.0, 0.0] + [0.0, 0.0, 0.1, 0.0]
+    assert sample.ref_kl_weights is None
+
+
+def test_stamp_loss_routing_merges_action_weights_into_ce_stream():
+    # A ce-action algorithm that also weighted observation tokens: action
+    # tokens merge into the existing stream instead of replacing it.
+    sample = _make_sample(ce_weights=[0.0, 0.0] + [0.0, 0.0, 0.1, 0.0])
+    stamp_loss_routing(sample, "ce")
+    assert sample.rl_weights == [0.0] * 6
+    assert sample.ce_weights == [0.0, 0.0] + [1.0, 1.0, 0.1, 1.0]
+    assert sample.ref_kl_weights is None
+
+
+def _make_rollout(
+    samples: list[TrainingSample],
+    advantages: list[float] | None = None,
+    raw: vf.RolloutOutput | None = None,
+) -> TrainRollout:
+    return TrainRollout(
+        raw=raw if raw is not None else {},
+        env_name="test-env",
+        example_id=0,
+        group_id=uuid.uuid4(),
+        policy_version=0,
+        off_policy_steps=0,
+        samples=samples,
+        advantages=advantages,
+    )
+
+
+def test_stamp_advantages_pads_prompt():
+    rollout = _make_rollout([_make_sample()], advantages=[0.5, -0.5, 0.0, 1.0])
+    stamp_advantages(rollout)
+    # 2 prompt positions padded with 0.0 + 4 completion-aligned advantages
+    assert rollout.samples[0].advantages == [0.0, 0.0, 0.5, -0.5, 0.0, 1.0]
+
+
+def test_stamp_advantages_slices_across_samples():
+    samples = [_make_sample(), _make_sample()]
+    rollout = _make_rollout(samples, advantages=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])
+    stamp_advantages(rollout)
+    assert rollout.samples[0].advantages == [0.0, 0.0, 1.0, 2.0, 3.0, 4.0]
+    assert rollout.samples[1].advantages == [0.0, 0.0, 5.0, 6.0, 7.0, 8.0]
+
+
+def test_stamp_advantages_no_credit_ships_none():
+    rollout = _make_rollout([_make_sample()])
+    stamp_advantages(rollout)
+    assert rollout.samples[0].advantages is None
+
+
+def test_stamp_advantages_rejects_misaligned():
+    rollout = _make_rollout([_make_sample()], advantages=[0.5])
+    with pytest.raises(ValueError, match="align"):
+        stamp_advantages(rollout)
+
+
+def _echo_algorithm(roles: dict | None = None, filter_fn=None) -> EchoAlgorithm:
+    advantage: dict = {"type": "echo"}
+    if roles is not None:
+        advantage["roles"] = roles
+    algo = EchoAlgorithm(AlgorithmConfig(advantage=advantage).advantage, MagicMock(), MagicMock())
+    algo.filter_fn = filter_fn
+    return algo
+
+
+def _two_step_rollout(attribution: dict | None = None) -> vf.RolloutOutput:
+    def step(prompt_ids, completion_ids, logprobs, prompt_attribution=None):
+        tokens = vf.TrajectoryStepTokens(
+            prompt_ids=prompt_ids,
+            prompt_mask=[0] * len(prompt_ids),
+            completion_ids=completion_ids,
+            completion_mask=[1] * len(completion_ids),
+            completion_logprobs=logprobs,
+            overlong_prompt=False,
+            is_truncated=False,
+        )
+        if prompt_attribution is not None:
+            tokens["prompt_attribution"] = prompt_attribution
+        return vf.TrajectoryStep(
+            prompt=[{"role": "user", "content": "U"}],
+            completion=[{"role": "assistant", "content": "A"}],
+            response=MagicMock(),
+            tokens=tokens,
+            reward=None,
+            advantage=None,
+            is_truncated=False,
+            trajectory_id="1",
+            extras={},
+        )
+
+    # Renderer rollouts carry attribution on every step; the first step's
+    # prompt never lands as observation tokens, so a minimal one suffices.
+    first_attribution = {"message_indices": [0, 0], "message_roles": ["user"]} if attribution is not None else None
+    return vf.RolloutOutput(
+        example_id=0,
+        reward=1.0,
+        trajectory=[
+            step([1, 2], [3, 4], [-0.1, -0.2], prompt_attribution=first_attribution),
+            # Extension: prompt re-includes [1,2,3,4]; tokens [5,6] are the
+            # env's observation; [7,8] the next action.
+            step([1, 2, 3, 4, 5, 6], [7, 8], [-0.3, -0.4], prompt_attribution=attribution),
+        ],
+        error=None,
+    )
+
+
+def _echo_rollout(output: vf.RolloutOutput) -> TrainRollout:
+    samples = interleave_rollout(output, env_name="test-env")
+    assert samples is not None
+    return _make_rollout(samples, raw=output)
+
+
+def test_echo_weights_observations_by_role():
+    # Span tokens [5,6] (positions 4,5) belong to a tool message; is_content
+    # excludes the wrap token, so only the body token gets the tool weight.
+    attribution = {
+        "message_indices": [0, 0, 1, 1, 2, 2],
+        "message_roles": ["user", "assistant", "tool"],
+        "is_content": [True, True, True, True, False, True],
+    }
+    rollout = _echo_rollout(_two_step_rollout(attribution))
+    algo = _echo_algorithm()  # the default table: tool bodies at 0.1
+    asyncio.run(algo.score_rollout(RolloutView(rollout)))
+    sample = rollout.samples[0]
+    assert sample.completion_ids == [3, 4, 5, 6, 7, 8]
+    # [3,4] step-1 action, [5,6] observation, [7,8] step-2 action
+    assert sample.ce_weights == [0.0, 0.0] + [0.0, 0.0, 0.0, 0.1, 0.0, 0.0]
+    assert sample.completion_mask == [True, True, False, False, True, True]
+
+    # Without is_content, whole messages count; each role carries its own weight.
+    attribution = {"message_indices": [0, 0, 1, 1, 2, 3], "message_roles": ["user", "assistant", "tool", "user"]}
+    rollout = _echo_rollout(_two_step_rollout(attribution))
+    algo = _echo_algorithm(roles={"tool": {"alpha": 0.1}, "user": {"alpha": 0.05}})
+    asyncio.run(algo.score_rollout(RolloutView(rollout)))
+    assert rollout.samples[0].ce_weights == [0.0, 0.0] + [0.0, 0.0, 0.1, 0.05, 0.0, 0.0]
+
+    # MITO rollouts carry no attribution: loud error, not a silent no-op.
+    with pytest.raises(ValueError, match="attribution"):
+        asyncio.run(_echo_algorithm().score_rollout(RolloutView(_echo_rollout(_two_step_rollout()))))
+
+
+def test_echo_filter_narrows_selection():
+    attribution = {"message_indices": [0, 0, 1, 1, 2, 2], "message_roles": ["user", "assistant", "tool"]}
+
+    def keep_last_only(output):
+        # One keep-mask per step over prompt+completion; drops span position 4.
+        return [[True] * 4, [True, True, True, True, False, True, True, True]]
+
+    rollout = _echo_rollout(_two_step_rollout(attribution))
+    algo = _echo_algorithm(filter_fn=keep_last_only)
+    asyncio.run(algo.score_rollout(RolloutView(rollout)))
+    assert rollout.samples[0].ce_weights == [0.0, 0.0] + [0.0, 0.0, 0.0, 0.1, 0.0, 0.0]
+
+    # Shape violations fail loudly: wrong step count, wrong per-step length.
+    rollout = _echo_rollout(_two_step_rollout(attribution))
+    with pytest.raises(ValueError, match="per trajectory step"):
+        asyncio.run(_echo_algorithm(filter_fn=lambda output: [[True] * 4]).score_rollout(RolloutView(rollout)))
+    with pytest.raises(ValueError, match="prompt\\+completion"):
+        asyncio.run(
+            _echo_algorithm(filter_fn=lambda output: [[True] * 4, [True] * 6]).score_rollout(RolloutView(rollout))
+        )
+
+
+def test_interleave_records_obs_spans():
+    samples = interleave_rollout(_two_step_rollout(), env_name="test-env")
+    assert samples is not None
+    # The step-2 prompt extension [5,6] lands at completion positions 2-3,
+    # sourced from step 1's prompt at offset 4, length 2.
+    assert samples[0].obs_spans == [[2, 1, 4, 2]]
+    # Provenance only — no algorithm wrote a ce stream.
+    assert samples[0].ce_weights is None
diff --git a/tests/unit/orchestrator/test_batch.py b/tests/unit/orchestrator/test_batch.py
index bbe0e61724..720148c3ce 100644
--- a/tests/unit/orchestrator/test_batch.py
+++ b/tests/unit/orchestrator/test_batch.py
@@ -25,7 +25,8 @@ def _routed_experts(data, dtype=np.uint8):
 def make_training_example():
     def _make_training_example(
         temperature: float = 1.0,
-        training_mode: str = "rl",
+        ce_weights: list[float] | None = None,
+        rl_weights: list[float] | None = None,
         env_name: str = "test-env",
     ) -> TrainingSample:
         return TrainingSample(
@@ -35,10 +36,11 @@ def _make_training_example(
             completion_mask=[True, True],
             completion_logprobs=[-0.1, -0.2],
             completion_temperatures=[temperature, temperature],  # Per-token temperatures
-            teacher_logprobs=[0.0, 0.0, 0.0, 0.0],
-            advantage=1.0,
+            ref_logprobs=[0.0, 0.0, 0.0, 0.0],
+            advantages=[0.0, 0.0, 1.0, 1.0],
             env_name=env_name,
-            training_mode=training_mode,
+            ce_weights=ce_weights,
+            rl_weights=rl_weights,
         )
 
     return _make_training_example
@@ -54,7 +56,7 @@ def make_sized_training_example(length: int, env_name: str = "test-env") -> Trai
         completion_mask=[True],
         completion_logprobs=[-0.1],
         completion_temperatures=[1.0],
-        advantage=1.0,
+        advantages=[1.0] * length,
         env_name=env_name,
     )
 
@@ -83,6 +85,58 @@ def make_flops_config():
     )
 
 
+def test_training_sample_requires_env_name():
+    with pytest.raises(TypeError, match="env_name"):
+        TrainingSample(
+            prompt_ids=[1, 2],
+            prompt_mask=[False, False],
+            completion_ids=[3, 4],
+            completion_mask=[True, True],
+            completion_logprobs=[-0.1, -0.2],
+            completion_temperatures=[1.0, 1.0],
+        )
+
+
+@pytest.mark.parametrize(
+    ("rollout_count", "num_train_workers", "expected_batches_per_worker"), [(4, 2, 2), (5, 2, 3), (7, 1, 7), (11, 4, 3)]
+)
+def test_prepare_batch_balances_micro_batches_across_workers(
+    make_training_example, rollout_count, num_train_workers, expected_batches_per_worker
+):
+    examples = [make_training_example() for i in range(rollout_count)]
+
+    batches_per_gpu = prepare_batch(
+        rollouts=examples,
+        seq_len=4,
+        num_train_workers=num_train_workers,
+        idxs=[0] * rollout_count,
+        num_loras=1,
+        bin_cost=build_bin_cost(None),
+    )
+
+    assert all(len(worker_batches) == expected_batches_per_worker for worker_batches in batches_per_gpu)
+
+    flat_batches = _flatten_batches(batches_per_gpu)
+    assert len(examples) <= len(flat_batches) < len(examples) + num_train_workers
+
+    # Identify real vs padding batches by content, not position — the packer
+    # distributes by workload, so a dummy can land anywhere in the order.
+    real_batches = [batch for batch in flat_batches if _has_loss_tokens(batch)]
+    dummy_batches = [batch for batch in flat_batches if not _has_loss_tokens(batch)]
+    assert len(real_batches) == len(examples)
+
+    # Verify real rollouts have expected non-zero advantages and loss mask
+    # (the advantage stream is 0.0 on prompt positions, the scalar on completion)
+    for batch in real_batches:
+        assert sum(1 for advantage in batch.advantages if advantage != 0.0) == 2
+        assert sum(1 for loss_mask in batch.loss_mask if loss_mask) == 2
+
+    # Verify padded batches have zero advantages and loss mask
+    for batch in dummy_batches:
+        assert sum(1 for advantage in batch.advantages if advantage != 0.0) == 0
+        assert sum(1 for loss_mask in batch.loss_mask if loss_mask) == 0
+
+
 def test_randomized_packing_invariants():
     rng = np.random.default_rng(0)
 
@@ -158,25 +212,6 @@ def test_pad_micro_batch_preserves_explicit_sequence_lengths(make_training_examp
     assert padded.loss_mask[-2:] == [False, False]
 
 
-def test_prepare_batch_does_not_pack_mixed_training_mode(make_training_example):
-    rl_example = make_training_example(training_mode="rl")
-    sft_example = make_training_example(training_mode="sft")
-
-    batches_per_gpu = prepare_batch(
-        rollouts=[rl_example, sft_example],
-        seq_len=16,
-        num_train_workers=1,
-        idxs=[0, 0],
-        num_loras=1,
-        bin_cost=build_bin_cost(None),
-    )
-
-    flat_batches = _flatten_batches(batches_per_gpu)
-    assert len(flat_batches) == 2
-    assert {batch.training_mode for batch in flat_batches} == {"rl", "sft"}
-    assert [batch.sequence_lengths for batch in flat_batches] == [[4], [4]]
-
-
 def test_split_to_align_avoids_dummy_micro_batches():
     examples = [make_sized_training_example(length) for length in [6, 6, 5, 5, 4, 4]]
 
@@ -244,6 +279,131 @@ def test_flop_aware_split_to_align_splits_heaviest_flop_bin():
     assert sum(len(batch.sequence_lengths) > 1 for batch in real_batches) == 3
 
 
+def test_prepare_sample_propagates_weight_streams(make_training_example):
+    example = make_training_example(ce_weights=[0.0, 0.0, 1.0, 1.0], rl_weights=[0.0, 0.0, 0.0, 0.0])
+
+    micro_batch = prepare_sample(example, seq_len=16)
+
+    assert micro_batch.ce_weights == [0.0, 0.0, 1.0, 1.0]
+    assert micro_batch.rl_weights == [0.0, 0.0, 0.0, 0.0]
+
+
+def test_prepare_sample_uniform_rl_keeps_streams_none(make_training_example):
+    micro_batch = prepare_sample(make_training_example(), seq_len=16)
+
+    assert micro_batch.rl_weights is None
+    assert micro_batch.ce_weights is None
+    assert micro_batch.ref_kl_weights is None
+
+
+@pytest.mark.parametrize("streams_on_longer", [True, False])
+def test_prepare_batch_packs_mixed_components(make_training_example, streams_on_longer):
+    """Component membership is per token, so samples feeding different
+    components pack together. The stream-less sample's positions must backfill
+    with the stream defaults (rl 1.0, ce 0.0) on whichever side of the pack
+    boundary it lands — a wrong-side backfill silently reroutes tokens between
+    components while keeping every array length-aligned."""
+    longer = TrainingSample(
+        prompt_ids=[1, 2, 3],
+        prompt_mask=[False] * 3,
+        completion_ids=[4, 5, 6],
+        completion_mask=[True] * 3,
+        completion_logprobs=[-0.1] * 3,
+        completion_temperatures=[1.0] * 3,
+        advantages=[0.0] * 3 + [1.0] * 3,
+        env_name="test-env",
+        ce_weights=[0.0, 0.0, 0.0, 1.0, 1.0, 1.0] if streams_on_longer else None,
+        rl_weights=[0.0] * 6 if streams_on_longer else None,
+    )
+    shorter = make_training_example(
+        ce_weights=None if streams_on_longer else [0.0, 0.0, 1.0, 1.0],
+        rl_weights=None if streams_on_longer else [0.0, 0.0, 0.0, 0.0],
+    )
+
+    batches_per_gpu = prepare_batch(
+        rollouts=[longer, shorter],
+        seq_len=16,
+        num_train_workers=1,
+        idxs=[0, 0],
+        num_loras=1,
+        bin_cost=build_bin_cost(None),
+    )
+
+    flat_batches = _flatten_batches(batches_per_gpu)
+    assert len(flat_batches) == 1
+    batch = flat_batches[0]
+    # FFD places the longer sample first; every stream value must sit at its
+    # sample's offset, with the stream-less side backfilled.
+    if streams_on_longer:
+        assert batch.rl_weights == [0.0] * 6 + [1.0] * 4
+        assert batch.ce_weights == [0.0, 0.0, 0.0, 1.0, 1.0, 1.0] + [0.0] * 4
+    else:
+        assert batch.rl_weights == [1.0] * 6 + [0.0] * 4
+        assert batch.ce_weights == [0.0] * 6 + [0.0, 0.0, 1.0, 1.0]
+
+
+@pytest.mark.parametrize("refs_on_longer", [True, False])
+def test_prepare_batch_aligns_ref_logprobs_in_mixed_bins(make_training_example, refs_on_longer):
+    """Packing a ref-bearing sample (e.g. OPD) with a ref-less one (e.g. GRPO)
+    must keep ``ref_logprobs`` position-aligned with ``input_ids`` — placeholder
+    0.0s on the ref-less tokens, both when the bin gains refs after ref-less
+    content (backfill) and when ref-less content lands in a ref-bearing bin."""
+    longer = TrainingSample(
+        prompt_ids=[1, 2, 3],
+        prompt_mask=[False] * 3,
+        completion_ids=[4, 5, 6],
+        completion_mask=[True] * 3,
+        completion_logprobs=[-0.1] * 3,
+        completion_temperatures=[1.0] * 3,
+        ref_logprobs=[-1.5] * 6 if refs_on_longer else None,
+        advantages=[0.0] * 3 + [1.0] * 3,
+        env_name="test-env",
+    )
+    shorter = make_training_example()
+    shorter.ref_logprobs = None if refs_on_longer else [-1.5] * 4
+
+    batches_per_gpu = prepare_batch(
+        rollouts=[longer, shorter],
+        seq_len=16,
+        pad_to_multiple_of=1,
+        num_train_workers=1,
+        idxs=[0, 0],
+        num_loras=1,
+        bin_cost=build_bin_cost(None),
+    )
+    flat_batches = _flatten_batches(batches_per_gpu)
+    assert len(flat_batches) == 1  # both samples share one bin
+    bin_content = flat_batches[0]
+    assert len(bin_content.ref_logprobs) == len(bin_content.input_ids)
+    # FFD places the longer sample first; refs must sit at their sample's offset
+    if refs_on_longer:
+        assert bin_content.ref_logprobs == [-1.5] * 6 + [0.0] * 4
+    else:
+        assert bin_content.ref_logprobs == [0.0] * 6 + [-1.5] * 4
+
+
+def test_prepare_sample_with_routed_experts():
+    """Routed experts are passed through prepare_sample and match input_ids length."""
+    # 2 prompt + 2 completion = 4 tokens, 2 layers, topk=2
+    routed_experts = [[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[0, 2], [1, 3]], [[1, 0], [3, 2]]]
+    routed_payload = _routed_experts(routed_experts)
+    sample = TrainingSample(
+        prompt_ids=[1, 2],
+        prompt_mask=[False, False],
+        completion_ids=[3, 4],
+        completion_mask=[True, True],
+        completion_logprobs=[-0.1, -0.2],
+        completion_temperatures=[1.0, 1.0],
+        advantages=[0.0, 0.0, 1.0, 1.0],
+        env_name="test-env",
+        routed_experts=routed_payload,
+    )
+
+    micro_batch = prepare_sample(sample, seq_len=8)
+    assert micro_batch.routed_experts is not None
+    assert micro_batch.routed_experts == routed_payload
+
+
 def test_prepare_sample_truncates_routed_experts():
     """Routed experts are truncated to seq_len when input exceeds it."""
     routed_experts = [[[0, 1]], [[2, 3]], [[4, 5]], [[6, 7]]]
@@ -256,7 +416,7 @@ def test_prepare_sample_truncates_routed_experts():
         completion_mask=[True, True],
         completion_logprobs=[-0.1, -0.2],
         completion_temperatures=[1.0, 1.0],
-        advantage=1.0,
+        advantages=[0.0, 0.0, 1.0, 1.0],
         env_name="test-env",
         routed_experts=routed_payload,
     )
@@ -265,3 +425,20 @@ def test_prepare_sample_truncates_routed_experts():
     assert micro_batch.routed_experts is not None
     assert micro_batch.routed_experts == expected_payload
     assert micro_batch.env_names == ["test-env"] * 3
+
+
+def test_prepare_sample_none_routed_experts():
+    """When routed_experts is None, micro_batch.routed_experts is None."""
+    sample = TrainingSample(
+        prompt_ids=[1, 2],
+        prompt_mask=[False, False],
+        completion_ids=[3, 4],
+        completion_mask=[True, True],
+        completion_logprobs=[-0.1, -0.2],
+        completion_temperatures=[1.0, 1.0],
+        advantages=[0.0, 0.0, 1.0, 1.0],
+        env_name="test-env",
+    )
+
+    micro_batch = prepare_sample(sample, seq_len=8)
+    assert micro_batch.routed_experts is None
diff --git a/tests/unit/orchestrator/test_orchestrator_setup.py b/tests/unit/orchestrator/test_orchestrator_setup.py
index 2372b004fd..47147e6597 100644
--- a/tests/unit/orchestrator/test_orchestrator_setup.py
+++ b/tests/unit/orchestrator/test_orchestrator_setup.py
@@ -4,21 +4,21 @@
 
 from renderers import Qwen3VLRendererConfig
 
-from prime_rl.orchestrator.utils import setup_student_inference_pool
+from prime_rl.orchestrator.utils import setup_policy_inference_pool
 
 
-def test_setup_student_inference_pool_uses_renderer_when_enabled():
+def test_setup_policy_inference_pool_uses_renderer_when_enabled():
     async def run() -> None:
         tokenizer = object()
         renderer_settings = Qwen3VLRendererConfig()
         config = SimpleNamespace(
-            training_mode="rl",
-            student=SimpleNamespace(
+            model=SimpleNamespace(
                 client=SimpleNamespace(base_url=["http://localhost:8000/v1"]),
-                model=SimpleNamespace(name="student-model"),
+                name="policy-model",
             ),
             renderer=renderer_settings,
             pool_size=None,
+            any_policy_sourced=True,
         )
         renderer = object()
         inference_pool = object()
@@ -30,7 +30,7 @@ async def run() -> None:
                 new=AsyncMock(return_value=inference_pool),
             ) as setup_pool_mock,
         ):
-            returned_renderer, returned_pool = await setup_student_inference_pool(
+            returned_renderer, returned_pool = await setup_policy_inference_pool(
                 config=config,
                 tokenizer=tokenizer,
             )
@@ -39,8 +39,8 @@ async def run() -> None:
         assert returned_pool is inference_pool
         create_renderer_mock.assert_called_once_with(tokenizer, renderer_settings)
         setup_pool_mock.assert_awaited_once_with(
-            config.student.client,
-            model_name="student-model",
+            config.model.client,
+            model_name="policy-model",
             train_client_type="renderer",
             eval_client_type="openai_chat_completions",
             renderer_config=renderer_settings,
@@ -50,18 +50,17 @@ async def run() -> None:
     asyncio.run(run())
 
 
-def test_setup_student_inference_pool_defaults_to_mito():
+def test_setup_policy_inference_pool_defaults_to_mito():
     """No renderer -> plain MITO chat completions."""
 
     async def run() -> None:
         tokenizer = object()
         config = SimpleNamespace(
-            training_mode="rl",
             renderer=None,
             pool_size=None,
-            student=SimpleNamespace(
+            model=SimpleNamespace(
                 client=SimpleNamespace(base_url=["http://localhost:8000/v1"]),
-                model=SimpleNamespace(name="student-model"),
+                name="policy-model",
             ),
         )
         inference_pool = object()
@@ -73,7 +72,7 @@ async def run() -> None:
                 new=AsyncMock(return_value=inference_pool),
             ) as setup_pool_mock,
         ):
-            renderer, returned_pool = await setup_student_inference_pool(
+            renderer, returned_pool = await setup_policy_inference_pool(
                 config=config,
                 tokenizer=tokenizer,
             )
@@ -82,8 +81,53 @@ async def run() -> None:
         assert returned_pool is inference_pool
         create_renderer_mock.assert_not_called()
         setup_pool_mock.assert_awaited_once_with(
-            config.student.client,
-            model_name="student-model",
+            config.model.client,
+            model_name="policy-model",
+            train_client_type="openai_chat_completions",
+            eval_client_type="openai_chat_completions",
+        )
+
+    asyncio.run(run())
+
+
+def test_setup_policy_inference_pool_keeps_renderer_without_policy_sampling():
+    """Frozen-sourced runs (e.g. sft) keep the renderer object for client-side
+    tokenization, but the policy pool serves plain chat completions — the
+    renderer-client sampling path is never wired."""
+
+    async def run() -> None:
+        tokenizer = object()
+        renderer_settings = Qwen3VLRendererConfig()
+        config = SimpleNamespace(
+            model=SimpleNamespace(
+                client=SimpleNamespace(base_url=["http://localhost:8000/v1"]),
+                name="policy-model",
+            ),
+            renderer=renderer_settings,
+            pool_size=None,
+            any_policy_sourced=False,
+        )
+        renderer = object()
+        inference_pool = object()
+
+        with (
+            patch("renderers.base.create_renderer", return_value=renderer) as create_renderer_mock,
+            patch(
+                "prime_rl.orchestrator.utils.setup_inference_pool",
+                new=AsyncMock(return_value=inference_pool),
+            ) as setup_pool_mock,
+        ):
+            returned_renderer, returned_pool = await setup_policy_inference_pool(
+                config=config,
+                tokenizer=tokenizer,
+            )
+
+        assert returned_renderer is renderer
+        assert returned_pool is inference_pool
+        create_renderer_mock.assert_called_once_with(tokenizer, renderer_settings)
+        setup_pool_mock.assert_awaited_once_with(
+            config.model.client,
+            model_name="policy-model",
             train_client_type="openai_chat_completions",
             eval_client_type="openai_chat_completions",
         )
diff --git a/tests/unit/orchestrator/test_teacher_logprobs.py b/tests/unit/orchestrator/test_prefill_logprobs.py
similarity index 75%
rename from tests/unit/orchestrator/test_teacher_logprobs.py
rename to tests/unit/orchestrator/test_prefill_logprobs.py
index d63fdce792..0d9c9d0581 100644
--- a/tests/unit/orchestrator/test_teacher_logprobs.py
+++ b/tests/unit/orchestrator/test_prefill_logprobs.py
@@ -5,7 +5,6 @@
 import verifiers as vf
 
 from prime_rl.orchestrator import utils as orchestrator_utils
-from prime_rl.transport import TrainingSample
 
 
 class _FakeOpenAIClient:
@@ -30,7 +29,7 @@ async def post(self, url, *, cast_to, body):
         )
 
 
-def test_compute_teacher_logprobs_uses_inference_generate(monkeypatch):
+def test_compute_prefill_logprobs_uses_inference_generate(monkeypatch):
     async def _run():
         fake_client = _FakeOpenAIClient(
             {
@@ -43,29 +42,19 @@ async def _run():
         )
         monkeypatch.setattr(orchestrator_utils, "setup_openai_client", lambda _: fake_client)
 
-        sample = TrainingSample(
-            prompt_ids=[1],
-            prompt_mask=[True],
-            completion_ids=[2, 3],
-            completion_mask=[True, True],
-            completion_logprobs=[-0.1, -0.2],
-            completion_temperatures=[1.0, 1.0],
-            env_name="test-env",
+        result = await orchestrator_utils.compute_prefill_logprobs(
+            vf.ClientConfig(),
+            model_name="ref-model",
+            token_ids=[1, 2, 3],
         )
 
-        result = await orchestrator_utils.compute_teacher_logprobs(
-            clients=[vf.ClientConfig()],
-            model_name="teacher-model",
-            samples=[sample],
-        )
-
-        assert result == [[0.0, -0.7, -0.3]]
+        assert result == [0.0, -0.7, -0.3]
         assert fake_client.calls == [
             {
                 "url": "http://fake-host:8000/inference/v1/generate",
                 "cast_to": httpx.Response,
                 "body": {
-                    "model": "teacher-model",
+                    "model": "ref-model",
                     "token_ids": [1, 2, 3],
                     "sampling_params": {
                         "max_tokens": 1,
diff --git a/tests/unit/test_configs.py b/tests/unit/test_configs.py
index 21d09baa29..4999f933d7 100644
--- a/tests/unit/test_configs.py
+++ b/tests/unit/test_configs.py
@@ -167,6 +167,38 @@ def test_removed_fused_lm_head_chunk_size_field_is_rejected():
         TrainerModelConfig.model_validate({"fused_lm_head_chunk_size": "auto"})
 
 
+def test_env_advantage_shorthand_assembles_own_algorithm():
+    config = OrchestratorConfig.model_validate(
+        {
+            "renderer": {"name": "qwen3"},  # echo needs the renderer's role attribution
+            "algo": {"advantage": {"type": "echo"}},
+            "train": {"env": [{"id": "a", "advantage": {"type": "reward"}}, {"id": "b"}]},
+        }
+    )
+    env_a, env_b = config.train.env
+    # The shorthand makes env a assemble its own algorithm (default sampling +
+    # the given advantage); only env b inherits the top-level echo algorithm.
+    assert env_a.algo is not None and env_a.algo.advantage.type == "reward"
+    assert env_b.algo is not None and env_b.algo.advantage.type == "echo"
+
+    # The shorthand is write-only sugar: resolved configs dump without it and round-trip.
+    dumped = config.model_dump(exclude_none=True)
+    assert "advantage" not in dumped and "advantage" not in dumped["train"]["env"][0]
+    reloaded = OrchestratorConfig.model_validate(dumped)
+    assert reloaded.train.env[0].algo is not None and reloaded.train.env[0].algo.advantage.type == "reward"
+
+
+def test_advantage_shorthand_conflicts_with_explicit_algo_advantage():
+    with pytest.raises(ValidationError, match="Set one"):
+        OrchestratorConfig.model_validate(
+            {
+                "renderer": None,
+                "advantage": {"type": "reward"},
+                "algo": {"advantage": {"type": "grpo"}},
+            }
+        )
+
+
 def test_trainer_enable_token_export_cli_flag():
     assert not cli(TrainerConfig, args=[]).enable_token_export
     assert cli(TrainerConfig, args=["--enable-token-export"]).enable_token_export
@@ -176,14 +208,12 @@ def test_orchestrator_vlm_requires_renderer():
     with pytest.raises(ValidationError, match="orchestrator.renderer must be set when model.vlm is set"):
         OrchestratorConfig.model_validate(
             {
-                "student": {
-                    "model": {
-                        "name": "Qwen/Qwen3-VL-4B-Instruct",
-                        "vlm": {
-                            "vision_encoder_attr": "model.visual",
-                            "language_model_attr": "model.language_model",
-                        },
-                    }
+                "model": {
+                    "name": "Qwen/Qwen3-VL-4B-Instruct",
+                    "vlm": {
+                        "vision_encoder_attr": "model.visual",
+                        "language_model_attr": "model.language_model",
+                    },
                 },
                 "renderer": None,
             }
@@ -191,14 +221,12 @@ def test_orchestrator_vlm_requires_renderer():
 
     config = OrchestratorConfig.model_validate(
         {
-            "student": {
-                "model": {
-                    "name": "Qwen/Qwen3-VL-4B-Instruct",
-                    "vlm": {
-                        "vision_encoder_attr": "model.visual",
-                        "language_model_attr": "model.language_model",
-                    },
-                }
+            "model": {
+                "name": "Qwen/Qwen3-VL-4B-Instruct",
+                "vlm": {
+                    "vision_encoder_attr": "model.visual",
+                    "language_model_attr": "model.language_model",
+                },
             },
         }
     )
@@ -222,7 +250,7 @@ def test_shared_model_name_propagates_to_subconfigs():
         }
     )
     assert config.trainer.model.name == model_name
-    assert config.orchestrator.student.model.name == model_name
+    assert config.orchestrator.model.name == model_name
     assert config.inference is not None and config.inference.model.name == model_name
     assert config.trainer.tokenizer.name == model_name
     assert config.orchestrator.tokenizer.name == model_name
@@ -277,7 +305,7 @@ def test_explicit_subconfig_tokenizer_name_survives_shared_model_propagation():
 
     This is the case that the old RL-level ``auto_setup_tokenizer`` fix-up got
     wrong: it unconditionally re-derived ``orchestrator.tokenizer.name`` from
-    ``orchestrator.student.model.name`` after propagation, silently overriding
+    ``orchestrator.model.name`` after propagation, silently overriding
     the user's explicit value. The ``mode="before"`` ``auto_setup_shared_configs``
     propagator fixes this because it propagates the model name into the raw
     dict before sub-configs are built, so ``OrchestratorConfig``'s own
@@ -297,7 +325,7 @@ def test_explicit_subconfig_tokenizer_name_survives_shared_model_propagation():
     )
     # Shared model.name reached every sub-config that didn't override it.
     assert config.trainer.model.name == "M"
-    assert config.orchestrator.student.model.name == "M"
+    assert config.orchestrator.model.name == "M"
     # Trainer didn't specify a tokenizer, so it falls back to the propagated model name.
     assert config.trainer.tokenizer.name == "M"
     # Orchestrator's explicit tokenizer name survived.
diff --git a/tests/unit/train/models/test_nemotron_h_kl.py b/tests/unit/train/models/test_nemotron_h_kl.py
index 09c0403359..282134b3d0 100644
--- a/tests/unit/train/models/test_nemotron_h_kl.py
+++ b/tests/unit/train/models/test_nemotron_h_kl.py
@@ -99,7 +99,7 @@ def test_kl_zero_when_identical():
         inputs = LossInputs(
             trainer_logprobs=logprobs[i],
             inference_logprobs=logprobs[i],
-            teacher_logprobs=None,
+            ref_logprobs=None,
             advantages=advantages,
             loss_mask=loss_mask,
         )
@@ -133,7 +133,7 @@ def test_kl_positive_after_perturbation():
         inputs = LossInputs(
             trainer_logprobs=policy_logprobs[i],
             inference_logprobs=ref_logprobs[i],
-            teacher_logprobs=None,
+            ref_logprobs=None,
             advantages=advantages,
             loss_mask=loss_mask,
         )
diff --git a/tests/unit/train/rl/test_loss.py b/tests/unit/train/rl/test_loss.py
index 1585dac7bd..84d1c7d4ae 100644
--- a/tests/unit/train/rl/test_loss.py
+++ b/tests/unit/train/rl/test_loss.py
@@ -2,7 +2,7 @@
 import torch
 
 from prime_rl.configs.trainer import CustomLossConfig, DefaultLossConfig
-from prime_rl.trainer.rl.loss import LossInputs, LossOutputs, compute_entropy, compute_loss, setup_loss_fns
+from prime_rl.trainer.rl.loss import LossInputs, LossOutputs, compute_entropy, compute_loss, setup_rl_loss_fn
 
 pytestmark = [pytest.mark.gpu]
 
@@ -10,19 +10,24 @@
 def test_grpo_loss():
     trainer_logprobs = [torch.randn(50, dtype=torch.float32).cuda(), torch.randn(30, dtype=torch.float32).cuda()]
     inference_logprobs = [torch.randn(50, dtype=torch.float32).cuda(), torch.randn(30, dtype=torch.float32).cuda()]
-    teacher_logprobs = [torch.randn(50, dtype=torch.float32).cuda(), torch.randn(30, dtype=torch.float32).cuda()]
+    ref_logprobs = [torch.randn(50, dtype=torch.float32).cuda(), torch.randn(30, dtype=torch.float32).cuda()]
     advantages = [torch.randn(50).cuda(), torch.randn(30).cuda()]
     loss_mask = [torch.ones(50, dtype=torch.bool).cuda(), torch.ones(30, dtype=torch.bool).cuda()]
 
-    loss_fns = setup_loss_fns(DefaultLossConfig(dppo_mask_high=10.0))
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig(dppo_mask_high=10.0))
     loss, _ = compute_loss(
         trainer_logprobs,
         inference_logprobs,
-        teacher_logprobs,
+        ref_logprobs,
         advantages,
         loss_mask=loss_mask,
-        loss_fns=loss_fns,
-        loss_scale=1.0,
+        rl_weights=None,
+        ce_weights=None,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
     )
     assert loss.shape == ()
 
@@ -30,19 +35,24 @@ def test_grpo_loss():
 def test_gspo_loss():
     trainer_logprobs = [torch.randn(40, dtype=torch.float32).cuda(), torch.randn(60, dtype=torch.float32).cuda()]
     inference_logprobs = [torch.randn(40, dtype=torch.float32).cuda(), torch.randn(60, dtype=torch.float32).cuda()]
-    teacher_logprobs = [torch.randn(40, dtype=torch.float32).cuda(), torch.randn(60, dtype=torch.float32).cuda()]
+    ref_logprobs = [torch.randn(40, dtype=torch.float32).cuda(), torch.randn(60, dtype=torch.float32).cuda()]
     advantages = [torch.randn(40).cuda(), torch.randn(60).cuda()]
     loss_mask = [torch.ones(40, dtype=torch.bool).cuda(), torch.ones(60, dtype=torch.bool).cuda()]
 
-    loss_fns = setup_loss_fns(DefaultLossConfig(dppo_mask_high=10.0))
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig(dppo_mask_high=10.0))
     loss, _ = compute_loss(
         trainer_logprobs,
         inference_logprobs,
-        teacher_logprobs,
+        ref_logprobs,
         advantages,
         loss_mask=loss_mask,
-        loss_fns=loss_fns,
-        loss_scale=1.0,
+        rl_weights=None,
+        ce_weights=None,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
     )
     assert loss.shape == ()
 
@@ -53,72 +63,215 @@ def test_entropy_loss():
     assert entropy.shape == (10, 10)
 
 
-def test_setup_loss_fns_with_custom_config():
-    """Test setup_loss_fns with CustomLossConfig importing a custom loss."""
+def test_setup_rl_loss_fn_with_custom_config():
+    """Test setup_rl_loss_fn with CustomLossConfig importing a custom loss."""
     loss_config = CustomLossConfig(
         import_path="tests.unit.train.rl.test_loss._dummy_custom_loss",
         kwargs={"multiplier": 2.0},
     )
-    loss_fns = setup_loss_fns(loss_config)
+    rl_loss_fn = setup_rl_loss_fn(loss_config)
 
     inputs = LossInputs(
         trainer_logprobs=torch.randn(50, dtype=torch.float32).cuda(),
         inference_logprobs=torch.randn(50, dtype=torch.float32).cuda(),
-        teacher_logprobs=None,
+        ref_logprobs=None,
         advantages=torch.randn(50).cuda(),
         loss_mask=torch.ones(50, dtype=torch.bool).cuda(),
     )
 
-    result = loss_fns["rl"](inputs)
+    result = rl_loss_fn(inputs)
     assert isinstance(result, LossOutputs)
     assert result.loss.shape == ()
     assert "custom_metric" in result.metrics
 
 
-def test_sft_loss_matches_masked_nll():
+def test_ce_component_matches_masked_nll():
     trainer_logprobs = [torch.tensor([-0.1, -0.5, -0.2], dtype=torch.float32).cuda()]
     inference_logprobs = [torch.zeros(3, dtype=torch.float32).cuda()]
     advantages = [torch.zeros(3, dtype=torch.float32).cuda()]
     loss_mask = [torch.tensor([True, False, True], dtype=torch.bool).cuda()]
+    rl_weights = [torch.zeros(3, dtype=torch.float32).cuda()]
+    ce_weights = [torch.tensor([1.0, 0.0, 1.0], dtype=torch.float32).cuda()]
 
-    loss_fns = setup_loss_fns(DefaultLossConfig())
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig())
     loss, metrics = compute_loss(
         trainer_logprobs=trainer_logprobs,
         inference_logprobs=inference_logprobs,
-        teacher_logprobs=None,
+        ref_logprobs=None,
         advantages=advantages,
         loss_mask=loss_mask,
-        loss_fns=loss_fns,
-        loss_scale=2,
-        training_mode="sft",
+        rl_weights=rl_weights,
+        ce_weights=ce_weights,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=2,
+        ref_kl_scale=1,
     )
 
-    # loss = -sum(masked logprobs) / loss_scale = -(-0.1 - 0.2) / 2 = 0.15
+    # loss = -sum(member logprobs) / ce_scale = -(-0.1 - 0.2) / 2 = 0.15
     assert torch.isclose(loss, torch.tensor(0.15, device=loss.device), atol=1e-6)
     assert "nll" in metrics
+    assert "mismatch_kl" not in metrics
 
 
-def test_sft_loss_override_uses_masked_nll_with_default_loss_config():
+def test_ce_component_applies_weights():
+    """ECHO-style observation training: the ce weight stream scales the NLL per token."""
     trainer_logprobs = [torch.tensor([-0.1, -0.5, -0.2], dtype=torch.float32).cuda()]
     inference_logprobs = [torch.zeros(3, dtype=torch.float32).cuda()]
-    advantages = [torch.ones(3, dtype=torch.float32).cuda()]
+    advantages = [torch.zeros(3, dtype=torch.float32).cuda()]
     loss_mask = [torch.tensor([True, False, True], dtype=torch.bool).cuda()]
+    rl_weights = [torch.zeros(3, dtype=torch.float32).cuda()]
+    ce_weights = [torch.tensor([0.1, 0.0, 0.1], dtype=torch.float32).cuda()]
+
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig())
+    loss, _ = compute_loss(
+        trainer_logprobs=trainer_logprobs,
+        inference_logprobs=inference_logprobs,
+        ref_logprobs=None,
+        advantages=advantages,
+        loss_mask=loss_mask,
+        rl_weights=rl_weights,
+        ce_weights=ce_weights,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
+    )
+
+    # loss = 0.1 * (0.1 + 0.2) = 0.03
+    assert torch.isclose(loss, torch.tensor(0.03, device=loss.device), atol=1e-6)
+
 
-    loss_fns = setup_loss_fns(DefaultLossConfig())
+def test_explicit_rl_weights_match_absent_stream():
+    """An explicit all-ones rl stream must equal the rl_weights=None hot path."""
+    torch.manual_seed(0)
+    trainer_logprobs = [torch.randn(50, dtype=torch.float32).cuda()]
+    inference_logprobs = [torch.randn(50, dtype=torch.float32).cuda()]
+    advantages = [torch.randn(50).cuda()]
+    loss_mask = [torch.rand(50).cuda() > 0.3]
+
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig())
+    kwargs = dict(
+        trainer_logprobs=trainer_logprobs,
+        inference_logprobs=inference_logprobs,
+        ref_logprobs=None,
+        advantages=advantages,
+        loss_mask=loss_mask,
+        ce_weights=None,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
+    )
+    loss_absent, _ = compute_loss(rl_weights=None, **kwargs)
+    loss_explicit, _ = compute_loss(rl_weights=[torch.ones(50, dtype=torch.float32).cuda()], **kwargs)
+
+    assert torch.equal(loss_absent, loss_explicit)
+
+
+def test_disjoint_components_in_one_sequence():
+    """ECHO/OPD-shaped sequence: rl, ce, and ref_kl on disjoint token sets."""
+    n = 12
+    torch.manual_seed(1)
+    trainer_logprobs = [torch.randn(n, dtype=torch.float32).cuda()]
+    inference_logprobs = [torch.randn(n, dtype=torch.float32).cuda()]
+    ref_logprobs = [torch.randn(n, dtype=torch.float32).cuda()]
+    advantages = [torch.randn(n).cuda()]
+    loss_mask = [torch.ones(n, dtype=torch.bool).cuda()]
+    rl_weights = torch.zeros(n, dtype=torch.float32)
+    rl_weights[:4] = 1.0
+    ce_weights = torch.zeros(n, dtype=torch.float32)
+    ce_weights[4:8] = 1.0
+    ref_kl_weights = torch.zeros(n, dtype=torch.float32)
+    ref_kl_weights[8:] = 1.0
+
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig(dppo_mask_high=10.0))
     loss, metrics = compute_loss(
         trainer_logprobs=trainer_logprobs,
         inference_logprobs=inference_logprobs,
-        teacher_logprobs=None,
+        ref_logprobs=ref_logprobs,
         advantages=advantages,
         loss_mask=loss_mask,
-        loss_fns=loss_fns,
-        loss_scale=2,
-        training_mode="sft",
+        rl_weights=[rl_weights.cuda()],
+        ce_weights=[ce_weights.cuda()],
+        ref_kl_weights=[ref_kl_weights.cuda()],
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
     )
 
-    assert torch.isclose(loss, torch.tensor(0.15, device=loss.device), atol=1e-6)
+    assert loss.shape == ()
     assert "nll" in metrics
-    assert "mismatch_kl" not in metrics
+    assert "ref_kl" in metrics
+    assert "is_masked" in metrics
+
+
+def test_empty_components_keep_backward_valid():
+    """A fully truncated distillation sample (stamped streams survive truncation
+    as all-zero prefixes) must train as a zero-gradient no-op, not crash backward."""
+    trainer_logprobs = [torch.randn(6, dtype=torch.float32, device="cuda", requires_grad=True)]
+    inference_logprobs = [torch.zeros(6, dtype=torch.float32).cuda()]
+    advantages = [torch.zeros(6, dtype=torch.float32).cuda()]
+    loss_mask = [torch.zeros(6, dtype=torch.bool).cuda()]
+    rl_weights = [torch.zeros(6, dtype=torch.float32).cuda()]
+    ce_weights = [torch.zeros(6, dtype=torch.float32).cuda()]
+
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig())
+    loss, _ = compute_loss(
+        trainer_logprobs=trainer_logprobs,
+        inference_logprobs=inference_logprobs,
+        ref_logprobs=None,
+        advantages=advantages,
+        loss_mask=loss_mask,
+        rl_weights=rl_weights,
+        ce_weights=ce_weights,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=1,
+        ce_scale=1,
+        ref_kl_scale=1,
+    )
+
+    assert torch.equal(loss, torch.zeros_like(loss))
+    loss.backward()
+    assert trainer_logprobs[0].grad is not None
+    assert torch.equal(trainer_logprobs[0].grad, torch.zeros_like(trainer_logprobs[0].grad))
+
+
+def test_overlapping_components_sum():
+    """Components may overlap on the same token (e.g. RL + a CE behavior-cloning
+    regularizer): the total is the sum of each component computed alone, each
+    over its own normalization."""
+    n = 8
+    torch.manual_seed(2)
+    trainer_logprobs = [torch.randn(n, dtype=torch.float32).cuda()]
+    inference_logprobs = [torch.randn(n, dtype=torch.float32).cuda()]
+    advantages = [torch.randn(n).cuda()]
+    loss_mask = [torch.ones(n, dtype=torch.bool).cuda()]
+    ce_weights = [torch.full((n,), 0.5, dtype=torch.float32).cuda()]
+
+    rl_loss_fn = setup_rl_loss_fn(DefaultLossConfig(dppo_mask_high=10.0))
+    kwargs = dict(
+        trainer_logprobs=trainer_logprobs,
+        inference_logprobs=inference_logprobs,
+        ref_logprobs=None,
+        advantages=advantages,
+        loss_mask=loss_mask,
+        ref_kl_weights=None,
+        rl_loss_fn=rl_loss_fn,
+        rl_scale=4,
+        ce_scale=8,
+        ref_kl_scale=1,
+    )
+    rl_only, _ = compute_loss(rl_weights=None, ce_weights=None, **kwargs)
+    ce_only, _ = compute_loss(rl_weights=[torch.zeros(n, dtype=torch.float32).cuda()], ce_weights=ce_weights, **kwargs)
+    both, _ = compute_loss(rl_weights=None, ce_weights=ce_weights, **kwargs)
+
+    assert torch.isclose(both, rl_only + ce_only, atol=1e-6)
 
 
 def _dummy_custom_loss(inputs: LossInputs, multiplier: float = 1.0) -> LossOutputs:
diff --git a/tests/unit/train/rl/test_packer.py b/tests/unit/train/rl/test_packer.py
index 1cf6beac0d..0ec2371b24 100644
--- a/tests/unit/train/rl/test_packer.py
+++ b/tests/unit/train/rl/test_packer.py
@@ -49,6 +49,7 @@ def make_training_sample() -> TrainingSample:
         completion_mask=[True],
         completion_logprobs=[-0.1],
         completion_temperatures=[1.0],
+        advantages=[0.0, 1.0],
         env_name="test-env",
     )
 
diff --git a/tests/unit/train/test_runs.py b/tests/unit/train/test_runs.py
index 6bb6623c06..81c3c65ddc 100644
--- a/tests/unit/train/test_runs.py
+++ b/tests/unit/train/test_runs.py
@@ -217,7 +217,7 @@ def test_config_loading(tmp_path: Path) -> None:
 
     # Access config as OrchestratorConfig object
     config = multi_run_manager.config[run_idx]
-    assert config.student.model.name == "test-model"
+    assert config.model.name == "test-model"
     assert config.batch_size == 32
     assert config.max_steps == 1000
 
diff --git a/tests/utils.py b/tests/utils.py
index dbad6c55ad..eee659e22f 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -107,18 +107,21 @@ def check_loss_goes_down(lines: list[str]):
     return check_number_goes_up_or_down(lines, go_up=False, pattern=r"Loss:?\s+(\d+\.\d{4})")
 
 
-def check_eval_avg_goes_up(lines: list[str], env_name: str):
-    """Assert that the last `Evaluated {env_name} (Step N) | ... | Reward X.XXXX`
-    line reports a higher score than the first one. Use for smoke tests with
-    `interval = 1` evals."""
+def check_final_eval_reward_above(lines: list[str], env_name: str, min_threshold: float):
+    """Assert the LAST `Evaluated {env_name} (Step N) | ... | Reward X.XXXX`
+    line reports a reward above ``min_threshold``.
+
+    Robust for short distill smokes: until the policy converges, eval reward is
+    truncation-dominated noise, so an endpoint "did it go up?" compare is a coin
+    flip (e.g. distillation here doesn't surface until ~step 13). A threshold
+    the converged plateau clears — but the early noise floor doesn't —
+    validates that training actually worked. Pick one between the two."""
     pattern = rf"Evaluated {re.escape(env_name)} .*Reward:?\s+(\d+\.\d{{4}})"
     eval_lines = [line for line in lines if "SUCCESS" in line and re.search(pattern, line)]
     assert len(eval_lines) >= 2, f"Need at least 2 eval lines for {env_name!r}, found {len(eval_lines)}"
-    start = float(re.search(pattern, eval_lines[0]).group(1))
-    end = float(re.search(pattern, eval_lines[-1]).group(1))
-    assert end > start, (
-        f"Eval avg for {env_name!r} did not go up: first={start} last={end}\n"
-        f"first line: {eval_lines[0]}\nlast line: {eval_lines[-1]}"
+    final = float(re.search(pattern, eval_lines[-1]).group(1))
+    assert final >= min_threshold, (
+        f"Final eval reward for {env_name!r} below threshold: {final} < {min_threshold}\nlast line: {eval_lines[-1]}"
     )