diff --git a/infra/src/lib/stacks/observability-stack.ts b/infra/src/lib/stacks/observability-stack.ts index 635b7c32..5d2183ce 100644 --- a/infra/src/lib/stacks/observability-stack.ts +++ b/infra/src/lib/stacks/observability-stack.ts @@ -828,11 +828,33 @@ export class ObservabilityStack extends cdk.Stack { // path back to OK latches permanently on first breach and is then silent // for every later incident. // - // The probe scopes both counts to the USDT quote leg and bounds them to a - // rolling 7-day window. Both are load-bearing — exotic-quoted zeros are by + // The probe scopes both counts to the USDT quote leg and bounds each to a + // rolling window. Both are load-bearing — exotic-quoted zeros are by // design (~74M rows), and an unbounded scan every 15 min is task 0111's // outage wearing a health check's clothes. See // packages/rollup-freshness-probe/src/usd_sanity.rs. + // + // ⚠️ THE TWO DIRECTIONS READ DIFFERENT TIERS AND DIFFERENT WINDOWS (task + // 0213). `peg_applied` reads `price_ohlcv_1m` over 48 h — the tier + // enrichment WRITES; `stranded` reads `price_ohlcv_1h` over 7 days, where a + // zero rolls up faithfully and the 48 h grace matches BE's loss window. + // Reading one tier for both is what made the peg direction publish a + // confident 0 over 1,564,045 wrong rows. + // + // 🔴 A GREEN `usd-peg-applied` IS NOT EVIDENCE THE USDT LEG IS HEALTHY. + // Measured on prod 2026-08-20 inside the 48 h window: 684 rows scanned, ALL + // 684 at close_usd = 0, zero peg-valued and zero correctly priced. The leg + // has been unpriced since 2026-08-13 (task 0209), so this direction has + // nothing to judge and reads 0 for want of input. `usd-stranded` is what + // carries that condition, and it is latched. Never read one without the + // other. + // + // ⚠️ An earlier version of this comment said the peg ladder must not be + // deployed before 0212/0209 because it would ship permanently breached. + // The measurement above falsifies that — the 1.5 M peg population sits at + // timestamps <= 2026-08-13, entirely outside the window. That figure + // applies to an unbounded repoint, not to the query that ships; do not + // re-size these rungs against it. const usdSanityRungs = ( metricName: string, idPrefix: string, @@ -869,7 +891,7 @@ export class ObservabilityStack extends cdk.Stack { 'UsdPegAppliedAlarmCount', 'usd-peg-applied', (count) => - `${count} or more USDT-quoted candles written in the last 7 days carry a close_usd within 2% of their close — valued as if USDT were still pegged at $1. USDT depegged in June 2022 and trades at ~0.13-0.15 (task 0172), so these values are roughly 7.4x too high. Something is applying the peg path to the USDT leg again: check the enrichment tiers (USDT must be a PIVOT reference, never a peg member) and prices.oracle_prices for rows mis-attributed to the USDT identity — that is how tasks 0196 and 0168 reintroduced this WITHOUT touching the writer, so a green writer test proves nothing here. Task 0182 corrected 567,760 such candles; find what is writing them before re-running any repair. Rungs are operator-tunable via config.opsAlarms.usdSanityEscalationCounts.`, + `${count} or more USDT-quoted price_ohlcv_1m candles written in the last 48 h carry a close_usd within 2% of their close — valued as if USDT were still pegged at $1. USDT depegged in June 2022 and trades at ~0.13-0.15 (task 0172), so these values are roughly 7.4x too high. Something is applying the peg path to the USDT leg again: check the enrichment tiers (USDT must be a PIVOT reference, never a peg member) and prices.oracle_prices for rows mis-attributed to the USDT identity — that is how tasks 0196 and 0168 reintroduced this WITHOUT touching the writer. VERIFY ON _1m, NEVER a coarse tier: task 0182 repaired the coarse tables directly, so _1h reads clean over a broken _1m (tasks 0212, 0213). Find what is writing them before re-running any repair. NOTE this metric reads 0 whenever the leg is unpriced, which it has been since 2026-08-13 - a green ladder is NOT proof of correct USD valuation, check prices-{env}-usd-stranded too. Rungs tunable via config.opsAlarms.usdSanityEscalationCounts.`, ); this.usdStrandedAlarms = usdSanityRungs( diff --git a/lore/1-tasks/active/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md b/lore/1-tasks/active/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md new file mode 100644 index 00000000..0ef07a5b --- /dev/null +++ b/lore/1-tasks/active/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md @@ -0,0 +1,290 @@ +--- +id: "0213" +title: "The USD peg check reads _1h — the tier 0182 repaired — so it publishes 0 over 1.5M wrong _1m rows" +type: BUG +status: active +related_adr: [] +related_tasks: ["0204", "0212", "0209", "0182", "0172"] +tags: ["priority-medium", "effort-small", "observability", "clickhouse", "data-correctness", "milestone-M2"] +milestone: 2 +links: + - "../../../packages/rollup-freshness-probe/src/usd_sanity.rs" + - "../../../infra/src/lib/stacks/observability-stack.ts" +history: + - date: 2026-08-20 + status: backlog + who: okarcz + note: > + Spawned from 0204 gap 4 during the pre-deploy prod baseline. The + peg-applied ladder reads price_ohlcv_1h, which 0182's repair wrote, while + the peg values live in price_ohlcv_1m, which that repair never touched — + 1,564,045 of them. The check would publish a confident 0. The stranded + direction is unaffected and is what found 0209. + - date: 2026-08-20 + status: blocked + who: okarcz + note: > + Code complete and green; BLOCKED on 0212 (and 0209 behind it) for deploy + only. The peg direction now reads price_ohlcv_1m over a 48 h window with + its own guards and refusal, the stranded direction is untouched on + price_ohlcv_1h over 7 days, and the two publish independently. 51 unit + tests (+14) and 19 integration tests (+4) against CH 26.3.10.60. Verified + NON-VACUOUS by reverting PEG_TABLE to price_ohlcv_1h: 6 ITs and 2 unit + tests fail. Acceptance criterion 3 cannot be met until 0212 repairs the + 1.5 M rows — deploying before that ships a permanently-breached ladder. + - date: 2026-08-20 + status: blocked + who: okarcz + note: > + Prod measurements taken. 🔴 THE DEPLOY BLOCK WAS FALSIFIED — with 0212 + unlanded the ladder reads scanned 684 / peg_applied 0, because the 1.5 M + peg rows all sit at timestamps <= 2026-08-13 and the 48 h window does not + reach them. The "ships permanently breached" claim was inherited from the + unbounded-repoint argument and never re-checked against the query that + ships; corrected in usd_sanity.rs, observability-stack.ts, the PR body and + the alarm text. All 684 rows are close_usd = 0, so the direction reads 0 + for want of input, not want of defects — a green peg ladder is NOT + evidence of a healthy leg while 0209 stands. Empty-scan concern also + closed: longest silent gap in 30 days is 6.5 h against a 48 h window. + NOW BLOCKED ON ONE THING ONLY: the peg scan's read cost on prod, which + decides whether the probe's 1-minute timeout still holds. + - date: 2026-08-20 + status: active + who: okarcz + note: > + UNBLOCKED. The last open measurement came back clean: EXPLAIN ESTIMATE + puts the peg scan at 1,250,223 rows / 6 parts / 156 marks against + 997,376 / 5 / 125 for the _1h scan that runs in 41-50 ms — 1.25x, so the + probe's 1-minute timeout needs no change. Every acceptance criterion the + code can meet is now met and measured. Remaining work is the deploy, and + it needs BOTH stacks: the probe binary ships from eventbridge-stack and + the alarm text from observability-stack, so an Observability-only deploy + leaves the ladder blind (0204's own mistake). Rebuild and `strings` the + lambda assets first (0141). +--- + +# The peg check reads the repaired tier, not the source + +## Summary + +[[0204]] gap 4 alarms on two directions. The **stranded** one works: a zero in +`_1m` rolls up as a zero in `_1h`, so reading the coarse tier detects it — that +is how [[0209]] was found. The **peg-applied** one does not, because a *repaired* +value in `_1h` says nothing about the row it was rolled from. + +Measured on prod 2026-08-20: + +| table | USDT-quoted rows at `close_usd / close ≈ 1.0` | +|---|---| +| `price_ohlcv_1h` — what the check reads | **0** | +| `price_ohlcv_1m` — where enrichment writes | **1,564,045** | + +[[0182]]'s repair wrote the five coarse tiers directly and never touched `_1m` +([[0212]]). So the alarm reads clean over 1.5 M wrong values and would have gone +on doing so indefinitely. + +⚠️ **This is the task's own founding failure, reproduced inside the guard built +against it** — a check scoring healthy because it looked at the surface least +able to show the defect. + +## Why the obvious fix is wrong + +⛔ **Do not simply repoint `SANITY_TABLE` at `price_ohlcv_1m`.** + +1. It would read 1,564,045 immediately — above every rung of + `usdSanityEscalationCounts` (`[1, 100, 10000]`) — and sit permanently in + ALARM. A permanently-firing alarm gets muted, which is the exact end-state + [[0204]] exists to prevent. +2. `_1m` is **retention-managed at 7 days** while `_1h` is a forever-table. The + check's 7-day `LOOKBACK_SECONDS` sits exactly on that boundary, so the window + reasoning has to be redone rather than inherited. +3. The stranded direction is *correct* on `_1h` and would be made worse by moving + — the 48 h grace is calibrated to BE's loss window on the hourly tier. + +## Implementation + +- Split the two directions: keep `stranded` on `_1h` unchanged; give + `peg_applied` its own `_1m`-scoped query, window and ladder. +- ⚠️ Sequence it **after [[0212]]** has repaired the 1.5 M rows, or the new + ladder ships permanently breached — which is the muting failure above. +- Re-verify the scan cost by what it **reads**, not what it returns; `_1m` at + 7 days on one quote leg is a different shape from `_1h`. +- The IT already writes a par-valued candle into a real ClickHouse + (`usd_sanity_counts_both_induced_defects`); extend it to `_1m` so the tier + distinction is induced rather than reasoned about. + +## Acceptance Criteria + +- [x] The peg-applied metric is computed from **`price_ohlcv_1m`**, and an IT + proves it counts a par-valued `_1m` row that no coarse tier carries — + `a_peg_row_only_in_1m_is_counted_although_every_coarse_tier_reads_clean`. +- [x] The stranded metric still reads `_1h` and its 48 h grace still means + BE's loss window. `STRANDED_TABLE`, `STRANDED_LOOKBACK_SECONDS` and + `STRANDED_GRACE_SECONDS` are unchanged in value; only the names moved. +- [x] The ladder reads **0** on prod. ⚠️ **But the criterion's stated logic + ("i.e. [[0212]] landed first") was FALSIFIED.** Measured 2026-08-20 with + 0212 unlanded: `scanned` 684, `peg_applied` **0**. The 1.5 M peg rows all + sit at timestamps ≤ 2026-08-13, entirely outside the 48 h window — the + window choice moved this task out from under its own blocker, and nobody + noticed until the measurement. 🔴 **It reads 0 because all 684 rows are + `close_usd = 0`** (0 peg-valued, 0 correctly priced): the leg is dark, so + the direction has nothing to judge. A green ladder is NOT evidence of + correct USD valuation — recorded at `PEG_TABLE` and in the alarm text. +- [x] ✅ **MEASURED 2026-08-20 — the read cost is a non-issue.** `EXPLAIN + ESTIMATE` on prod: the peg scan reads **1,250,223 rows / 6 parts / 156 + marks**, against **997,376 / 5 / 125** for the `_1h` stranded scan that + measures 41-50 ms. **1.25x the known-good baseline**, so the probe's + 1-minute timeout holds with large margin and needs no change. ⚠️ The + window *does* prune despite `timestamp` not being a primary-key prefix — + `toYYYYMM` partition pruning plus per-part min/max does the work, and 156 + marks x 8192 ≈ 1.28 M corroborates the row estimate. ⛔ Re-take this if + the window is ever widened. Original concern: **the peg scan's read cost + on prod.** The + probe's 1-minute timeout was sized on the `_1h` scan alone (~1.37M rows / + ~70 MiB / 41-50 ms) and was not revisited for this second `FINAL` scan + against the **735M-row** `_1m`. ⚠️ **The 48 h window does not bound the + read**: both tables are `PARTITION BY toYYYYMM(timestamp)` with + `ORDER BY (asset_id, quote_asset_id, source, timestamp)`, so `timestamp` + is not a primary-key prefix and the scan prunes to a whole **monthly + partition** (two across a month boundary). Measure by `EXPLAIN` / + `read_rows`, never by rows returned. A timeout here is not a Rust `Err` — + it kills the invocation with nothing published, and step 4 (MV drift) + never runs. +- [x] ✅ **MEASURED 2026-08-20 — `scanned == 0` is not reachable.** The longest + stretch with no USDT-quoted `_1m` row in the preceding 30 days is + **6.5 hours** against the 48 h window: 7.4x margin, so the empty-scan + refusal will not page on a healthy leg. No code change needed. ⛔ Do not + shorten the window below ~24 h without re-taking this. Original concern: + **can `scanned` legitimately reach 0 on the peg tier?** `EmptyScan` was calibrated for a 7-day window on `_1h`; the peg + direction applies the same `scanned == 0` refusal to **48 h on a sparse + leg** — measured at ~16 USDT-quoted `_1m` rows for 2026-08-17, so ~32 in + a 48 h window. A quiet spell makes zero legitimate, and the refusal then + fails the invocation and pages ops via the probe's own `-errors` alarm + **every 15 minutes on a healthy system** — the muting failure again. + Measure the daily minimum over ≥30 days before deploying; if zero is + reachable, the guard needs a different discriminator (candidate: refuse + only when the *stranded* direction also scanned nothing, which + distinguishes a quiet window from a renumbered identity without inventing + a threshold). +- [x] A note records why `_1m`'s 7-day retention does not undermine the + lookback — `PEG_LOOKBACK_SECONDS`, plus the + `the_peg_window_stays_clear_of_the_1m_retention_frontier` unit test and + the `the_peg_window_excludes_rows_a_cleanup_run_could_delete` IT. + +## Implementation Notes + +Four files. The shape is a **split, not a repoint** — everything that made the +stranded direction correct on `_1h` is preserved verbatim. + +| file | change | +|---|---| +| `usd_sanity.rs` | `SANITY_TABLE` → `STRANDED_TABLE` + `PEG_TABLE`; `LOOKBACK_SECONDS` → `STRANDED_LOOKBACK_SECONDS` (7 d) + `PEG_LOOKBACK_SECONDS` (48 h); `sanity_query` → `stranded_query` + `peg_query`; `SanityCounts` → `StrandedCounts` + `PegCounts` behind a `ScanGuards` trait; `sanity_metrics` → `stranded_metric` + `peg_metric` | +| `main.rs` | two reads and two publishes instead of one, each failing independently | +| `rollup_freshness_it.rs` | fixtures parameterised by tier; 4 new ITs | +| `observability-stack.ts` | peg alarm description names `_1m` and 48 h; ladder comment carries the deploy ordering | + +**Verified:** `cargo fmt --all --check` · `cargo clippy -p rollup-freshness-probe +--all-targets --features lambda` (0 warnings) · `cargo test --workspace` (92 +suites, 0 failures) · 19 ITs against a local CH pinned to **26.3.10.60** · +`nx run-many -t lint typecheck` · `nx build infra` · `make -C infra +synth-production` · all six USD alarms render 856–937 chars against the 1024 cap. + +## Issues Encountered + +- 🔴 **The first version of the new tests was VACUOUS, and the fixtures were + why.** `insert_usdt_minute_candle` was written as + `insert_candle_into(c, PEG_TABLE, …)`. A fixture expressed in terms of the + constant under test follows it wherever it points, so reverting `PEG_TABLE` + to `price_ohlcv_1h` moved the *writes* too and the tier assertions kept + passing. Found only by doing the revert. Fixed by spelling both tables + literally in the fixtures. ⚠️ **This is the task's own founding failure a + third time** — after the check that read the repaired tier, and 0182 verified + against its own output: a test that cannot distinguish the thing it asserts. + The non-vacuity check is now the acceptance evidence, not a nicety. +- `main.rs` claimed `"{} of 4 probe checks failed"`. Check 3 can now contribute + two failures, so the denominator became false; it now reads + `"{} probe check(s) failed"`. + +## Design Decisions + +### From Plan + +1. **Split the directions rather than repoint one table.** The three costs of a + bare repoint are recorded verbatim on `PEG_TABLE`, so the next person to have + the same idea meets the reasoning before the change. +2. **Stranded stays on `_1h`, untouched.** A zero rolls up as a zero, and the + 48 h grace is calibrated to BE's loss window on the hourly tier. + +### Emerged + +3. **The peg window is 48 h, not 7 days.** The plan said the window "has to be + redone rather than inherited" but not to what. 48 h keeps five days of margin + below `_1m`'s 7-day retention, so a cleanup run can never truncate the scan. + It costs nothing because this is a re-introduction guard, not a historical + audit — a regressed writer writes continuously and shows up in the newest + rows. Frozen history is 0212's population. +4. **No grace on the peg direction.** A stranded row is one enrichment has not + reached *yet*; a peg-valued row is one enrichment has already written + *wrongly*. Waiting cannot improve the second, so a grace would only delay + saying so. +5. **The two directions publish and refuse INDEPENDENTLY.** Previously one + refusal suppressed both metrics — harmless while they came from one query, + and the muting failure from the other side once they read different tables. + A `_1m` scan that matched nothing says nothing about `_1h`. +6. **Guards duplicated per direction, via a `ScanGuards` trait.** Sharing one + `resolved_legs`/`scanned` pair across two tiers would score the tier that was + never examined as healthy — this task's own defect, one level up. +7. **`SanityRefusal` now carries the table and the lookback.** With two tiers, + "the scan matched nothing" is no longer a fact about one place, and a refusal + that did not say which would send the operator to a table that was fine. + +## Review Round + +A code review of PR #235 returned six findings; each was verified against the +code before acting. Four were defects in this change and are fixed in the PR: + +- **`both_queries_read_final` could not fail.** It asserted + `sql.contains(" FINAL ")`, already satisfied by `FROM assets FINAL` in the + shared CTE, so deleting `FINAL` from either candle table would have left it + green. Now asserts each candle table by name. A sibling assertion, + `!peg_query().contains("STRANDED_GRACE")`, was checking for Rust identifier + text in generated SQL and could never fail either. +- **One IT assertion tested the window, not the tier.** In + `each_direction_only_scans_its_own_tier`, the `_1h`-only half seeded at + `now() - INTERVAL 3 DAY`, which the 48 h peg window excludes whichever table + is read — so it passed with `PEG_TABLE` reverted. Seeded at `3 HOUR` now. + ⚠️ **Third instance of this task's founding failure in one change**, after + the fixture coupling above. +- **The "reads 1,564,045 — above every rung" claim was wrong for the query that + ships.** That is the all-history population ([[0212]]); bounded to 48 h the + reading is the recent arrival rate, tens of rows, which clears rung 1 and + nothing above it. Corrected in `usd_sanity.rs` and `observability-stack.ts`, + because it is exactly the number someone would use to re-size the ladder. +- **Broken intra-doc links** to the deleted `LOOKBACK_SECONDS`, `SanityCounts` + and `SANITY_TABLE`, plus a now-false "prunes by partition" claim. Fixed, and + the crate's unresolved-link count went **9 → 6** (the rest are in `mv_drift` + and `lib.rs`, out of scope here). + +The remaining two were promoted to acceptance criteria. **Both were then +measured on prod, and one of them overturned this task's central claim:** + +- ✅ **Empty scan is not reachable.** Longest silent stretch on the USDT `_1m` + leg over 30 days is **6.5 h** against the 48 h window — 7.4x margin. No code + change needed. +- 🔴 **The deploy block was wrong.** It read `peg_applied = 0` with 0212 + unlanded, because the peg population is entirely outside the window. ⚠️ The + claim was carried from the *unbounded repoint* argument and never re-checked + against the query that actually ships — the same class of error as the + original defect: reasoning about a surface other than the one in use. +- ✅ **Read cost measured and fine** — 1.25x the known-good `_1h` scan. The + concern was well-founded (a window on a non-prefix column need not prune) but + the measurement clears it. + +**All acceptance criteria that code can meet are met.** What remains is the +deploy itself, which is the operator's action, not a code gate. + +## Future Work + +None spawned. The remaining work is the deploy, which is gated by [[0212]] and +[[0209]] — both already exist and both already carry this ordering. ⛔ Do not +close this task on the code being green. diff --git a/lore/1-tasks/backlog/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md b/lore/1-tasks/backlog/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md deleted file mode 100644 index 15506a34..00000000 --- a/lore/1-tasks/backlog/0213_BUG_gap4-peg-check-reads-the-repaired-tier-not-the-source.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -id: "0213" -title: "The USD peg check reads _1h — the tier 0182 repaired — so it publishes 0 over 1.5M wrong _1m rows" -type: BUG -status: backlog -related_adr: [] -related_tasks: ["0204", "0212", "0209", "0182", "0172"] -tags: ["priority-medium", "effort-small", "observability", "clickhouse", "data-correctness", "milestone-M2"] -milestone: 2 -links: - - "../../../packages/rollup-freshness-probe/src/usd_sanity.rs" - - "../../../infra/src/lib/stacks/observability-stack.ts" -history: - - date: 2026-08-20 - status: backlog - who: okarcz - note: > - Spawned from 0204 gap 4 during the pre-deploy prod baseline. The - peg-applied ladder reads price_ohlcv_1h, which 0182's repair wrote, while - the peg values live in price_ohlcv_1m, which that repair never touched — - 1,564,045 of them. The check would publish a confident 0. The stranded - direction is unaffected and is what found 0209. ---- - -# The peg check reads the repaired tier, not the source - -## Summary - -[[0204]] gap 4 alarms on two directions. The **stranded** one works: a zero in -`_1m` rolls up as a zero in `_1h`, so reading the coarse tier detects it — that -is how [[0209]] was found. The **peg-applied** one does not, because a *repaired* -value in `_1h` says nothing about the row it was rolled from. - -Measured on prod 2026-08-20: - -| table | USDT-quoted rows at `close_usd / close ≈ 1.0` | -|---|---| -| `price_ohlcv_1h` — what the check reads | **0** | -| `price_ohlcv_1m` — where enrichment writes | **1,564,045** | - -[[0182]]'s repair wrote the five coarse tiers directly and never touched `_1m` -([[0212]]). So the alarm reads clean over 1.5 M wrong values and would have gone -on doing so indefinitely. - -⚠️ **This is the task's own founding failure, reproduced inside the guard built -against it** — a check scoring healthy because it looked at the surface least -able to show the defect. - -## Why the obvious fix is wrong - -⛔ **Do not simply repoint `SANITY_TABLE` at `price_ohlcv_1m`.** - -1. It would read 1,564,045 immediately — above every rung of - `usdSanityEscalationCounts` (`[1, 100, 10000]`) — and sit permanently in - ALARM. A permanently-firing alarm gets muted, which is the exact end-state - [[0204]] exists to prevent. -2. `_1m` is **retention-managed at 7 days** while `_1h` is a forever-table. The - check's 7-day `LOOKBACK_SECONDS` sits exactly on that boundary, so the window - reasoning has to be redone rather than inherited. -3. The stranded direction is *correct* on `_1h` and would be made worse by moving - — the 48 h grace is calibrated to BE's loss window on the hourly tier. - -## Implementation - -- Split the two directions: keep `stranded` on `_1h` unchanged; give - `peg_applied` its own `_1m`-scoped query, window and ladder. -- ⚠️ Sequence it **after [[0212]]** has repaired the 1.5 M rows, or the new - ladder ships permanently breached — which is the muting failure above. -- Re-verify the scan cost by what it **reads**, not what it returns; `_1m` at - 7 days on one quote leg is a different shape from `_1h`. -- The IT already writes a par-valued candle into a real ClickHouse - (`usd_sanity_counts_both_induced_defects`); extend it to `_1m` so the tier - distinction is induced rather than reasoned about. - -## Acceptance Criteria - -- [ ] The peg-applied metric is computed from **`price_ohlcv_1m`**, and an IT - proves it counts a par-valued `_1m` row that no coarse tier carries. -- [ ] The stranded metric still reads `_1h` and its 48 h grace still means - BE's loss window. -- [ ] The ladder reads **0** on prod at deploy time — i.e. [[0212]] landed first. -- [ ] A note records why `_1m`'s 7-day retention does not undermine the lookback. diff --git a/packages/rollup-freshness-probe/src/main.rs b/packages/rollup-freshness-probe/src/main.rs index a02d7c00..bdca8497 100644 --- a/packages/rollup-freshness-probe/src/main.rs +++ b/packages/rollup-freshness-probe/src/main.rs @@ -21,7 +21,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { visible_objects_query, }; use rollup_freshness_probe::usd_sanity::{ - SanityCounts, publish_sanity, sanity_metrics, sanity_query, + PegCounts, StrandedCounts, peg_metric, peg_query, publish_sanity, stranded_metric, + stranded_query, }; use rollup_freshness_probe::{TableLag, freshness_query, lag_metrics, publish}; use std::sync::Arc; @@ -41,7 +42,12 @@ async fn main() -> Result<(), lambda_runtime::Error> { // filesystem *functions* instead. No new grant is required for either. let ch = Arc::new(prices_clickhouse::mtls::client_from_lambda_env("prices").await?); let query = Arc::new(freshness_query()); - let sanity_query = Arc::new(sanity_query()); + // ⚠️ Two queries since task 0213, not one. The peg direction reads + // `price_ohlcv_1m` — the tier enrichment writes — while the stranded + // direction stays on `price_ohlcv_1h`. See `usd_sanity::PEG_TABLE` for why + // a single shared tier made the peg direction structurally blind. + let stranded_query = Arc::new(stranded_query()); + let peg_query = Arc::new(peg_query()); let aws_cfg = aws_config::defaults(aws_config::BehaviorVersion::latest()) .load() @@ -54,7 +60,8 @@ async fn main() -> Result<(), lambda_runtime::Error> { let ch = ch.clone(); let cw = cw.clone(); let query = query.clone(); - let sanity_query = sanity_query.clone(); + let stranded_query = stranded_query.clone(); + let peg_query = peg_query.clone(); let environment = environment.clone(); async move { // ⚠️ EVERY CHECK RUNS, AND A FAILURE IN ONE MUST NOT SUPPRESS @@ -128,25 +135,59 @@ async fn main() -> Result<(), lambda_runtime::Error> { } // ---- 3. USD-value correctness (task 0204, gap 4) -------------- - let mut sanity_counts: Option = None; - match ch.query(&sanity_query).fetch_one::().await { + // + // ⚠️ TWO INDEPENDENT READS, and the independence is the fix task + // 0213 shipped. They read different tiers, so a `_1m` scan that + // matched nothing says nothing whatever about `_1h`. Letting one + // refusal suppress the other's metric would re-create the + // suppressed-signal failure at the invocation level, having just + // removed it at the query level — and it is the same invariant the + // block comment at the top of this function is about. + let mut stranded_counts: Option = None; + match ch + .query(&stranded_query) + .fetch_one::() + .await + { Ok(counts) => { - sanity_counts = Some(counts); + stranded_counts = Some(counts); // An `Err` here is a check that did NOT RUN — an // unresolvable USDT identity, or one that resolved to an id - // the candles no longer carry. Either way its two zeros must - // not be published, because NOT_BREACHING would score them - // as a clean bill of health. - match sanity_metrics(&counts) { - Ok(sanity) => { - if let Err(e) = publish_sanity(&cw, &environment, &sanity).await { - failures.push(format!("usd-sanity publish: {e}")); + // the candles no longer carry. Either way its zero must not + // be published, because NOT_BREACHING would score it as a + // clean bill of health. + match stranded_metric(&counts) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)) + .await + { + failures.push(format!("usd-stranded publish: {e}")); + } + } + Err(refusal) => failures.push(format!("usd-stranded: {refusal}")), + } + } + Err(e) => failures.push(format!("usd-stranded read: {e}")), + } + + let mut peg_counts: Option = None; + match ch.query(&peg_query).fetch_one::().await { + Ok(counts) => { + peg_counts = Some(counts); + match peg_metric(&counts) { + Ok(metric) => { + if let Err(e) = + publish_sanity(&cw, &environment, std::slice::from_ref(&metric)) + .await + { + failures.push(format!("usd-peg-applied publish: {e}")); } } - Err(refusal) => failures.push(format!("usd-sanity: {refusal}")), + Err(refusal) => failures.push(format!("usd-peg-applied: {refusal}")), } } - Err(e) => failures.push(format!("usd-sanity read: {e}")), + Err(e) => failures.push(format!("usd-peg-applied read: {e}")), } // ---- 4. Materialized-view drift (task 0204, gap 3) ------------ @@ -199,9 +240,10 @@ async fn main() -> Result<(), lambda_runtime::Error> { checks_failed = failures.len(), disk_free_percent = free_percent.unwrap_or_default(), disk_available_bytes = disk_reading.map(|u| u.available_bytes).unwrap_or_default(), - usd_peg_applied = sanity_counts.map(|c| c.peg_applied).unwrap_or_default(), - usd_stranded = sanity_counts.map(|c| c.stranded).unwrap_or_default(), - usd_scanned = sanity_counts.map(|c| c.scanned).unwrap_or_default(), + usd_peg_applied = peg_counts.map(|c| c.peg_applied).unwrap_or_default(), + usd_peg_scanned = peg_counts.map(|c| c.scanned).unwrap_or_default(), + usd_stranded = stranded_counts.map(|c| c.stranded).unwrap_or_default(), + usd_stranded_scanned = stranded_counts.map(|c| c.scanned).unwrap_or_default(), mv_drift_critical = drift_critical, mv_drift = drift_count, mv_visible_objects = visible_objects.unwrap_or_default(), @@ -211,7 +253,7 @@ async fn main() -> Result<(), lambda_runtime::Error> { if !failures.is_empty() { return Err(lambda_runtime::Error::from(format!( - "{} of 4 probe checks failed (the rest published normally): {}", + "{} probe check(s) failed (the rest published normally): {}", failures.len(), failures.join("; ") ))); @@ -225,9 +267,10 @@ async fn main() -> Result<(), lambda_runtime::Error> { "capacity_bytes": disk_reading.map(|u| u.capacity_bytes), }, "usd_sanity": { - "peg_applied": sanity_counts.map(|c| c.peg_applied), - "stranded": sanity_counts.map(|c| c.stranded), - "scanned": sanity_counts.map(|c| c.scanned), + "peg_applied": peg_counts.map(|c| c.peg_applied), + "peg_scanned": peg_counts.map(|c| c.scanned), + "stranded": stranded_counts.map(|c| c.stranded), + "stranded_scanned": stranded_counts.map(|c| c.scanned), }, "mv_drift": { "critical": drift_critical, diff --git a/packages/rollup-freshness-probe/src/usd_sanity.rs b/packages/rollup-freshness-probe/src/usd_sanity.rs index 7e1e0514..9d5746cc 100644 --- a/packages/rollup-freshness-probe/src/usd_sanity.rs +++ b/packages/rollup-freshness-probe/src/usd_sanity.rs @@ -41,18 +41,34 @@ //! to end. //! 2. **Give the stranded direction a grace period.** Enrichment fills //! `close_usd` asynchronously, so a freshly written candle is *legitimately* -//! zero until the sweep reaches it. Without [`STRANDED_GRACE_SECONDS`] this +//! zero until the sweep reaches it. Without +//! [`crate::usd_sanity::STRANDED_GRACE_SECONDS`] this //! metric would never read zero. See that constant for why the grace is 48 h //! specifically and not an arbitrary round number. //! 3. **Bound the window.** An unbounded scan of the OHLCV tables every 15 -//! minutes is task 0111's outage, reintroduced as a health check. The window -//! is [`LOOKBACK_SECONDS`] and prunes by partition. +//! minutes is task 0111's outage, reintroduced as a health check. The +//! windows are [`crate::usd_sanity::STRANDED_LOOKBACK_SECONDS`] and +//! [`crate::usd_sanity::PEG_LOOKBACK_SECONDS`]. +//! +//! ⚠️ **A window bound is not a read bound.** Both tables are +//! `PARTITION BY toYYYYMM(timestamp)` with `ORDER BY (asset_id, +//! quote_asset_id, source, timestamp)`, so `timestamp` is not a primary-key +//! prefix and a window cannot be assumed to prune. ⛔ Size these on what the +//! query READS (`EXPLAIN ESTIMATE` / `read_rows`), never on the span it +//! names — the same rule the tier choice was decided by. +//! +//! Measured on prod 2026-08-20: the 48 h `_1m` peg scan estimates +//! **1,250,223 rows / 6 parts / 156 marks**, against **997,376 / 5 / 125** +//! for the 7-day `_1h` stranded scan that runs in 41-50 ms — **1.25x**, so +//! the probe's 1-minute timeout holds. `toYYYYMM` pruning plus per-part +//! min/max does the work the key order does not. //! //! ⚠️ **This is a re-introduction guard, not a historical audit.** It watches //! recent writes, because that is where a regression shows up. A frozen //! historical corruption inside the window would latch the alarm rather than -//! re-notify — see [`SanityCounts`] on why the counts still climb in the case -//! this actually guards against. +//! re-notify — see [`crate::usd_sanity::ScanGuards`] on why a reading that +//! examined nothing is +//! refused rather than published as a zero. //! //! ⚠️ **If USDT ever returns to par, the peg-applied direction stops being //! diagnostic** and this check must be revisited. It reads a ratio near 1.0 as @@ -72,7 +88,7 @@ pub const PEG_APPLIED_METRIC: &str = "UsdtPegAppliedCandles"; /// `prices-{env}-usd-stranded` alarm ladder. pub const STRANDED_METRIC: &str = "UsdtStrandedCandles"; -/// How far back the check looks, in seconds. +/// How far back the **stranded** direction looks, in seconds. /// /// Seven days. Long enough that a regression cannot slip through between runs /// or hide behind a weekend, short enough that the scan prunes to one or two @@ -80,7 +96,47 @@ pub const STRANDED_METRIC: &str = "UsdtStrandedCandles"; /// full-table scan of task 0111, which caused a four-day production outage, and /// re-introducing it inside a 15-minute probe would be worse than the defect /// this alarm watches for. -pub const LOOKBACK_SECONDS: i64 = 7 * 86_400; +/// +/// ⚠️ This bound is safe at seven days **because [`STRANDED_TABLE`] is a +/// forever-table**. The peg direction reads a retention-managed tier and cannot +/// inherit it — see [`PEG_LOOKBACK_SECONDS`]. +pub const STRANDED_LOOKBACK_SECONDS: i64 = 7 * 86_400; + +/// How far back the **peg-applied** direction looks, in seconds. +/// +/// **48 hours, and deliberately not the seven days the stranded direction +/// uses.** [`PEG_TABLE`] is `price_ohlcv_1m`, which is retention-managed at +/// seven days, so a seven-day window would sit exactly on the deletion +/// frontier: the oldest hours of the window would be removed out from under the +/// scan while it runs, and the count would move for a reason that is not a +/// defect. 48 h leaves five days of margin, so no cleanup run can ever truncate +/// this window. +/// +/// ⚠️ **Retention here is a JOB, not a TTL** — the same trap that produced task +/// 0174. `_1m` is pruned by the cleanup worker, which is currently DISABLED +/// (task 0200), so today the tier holds far more than seven days. This bound is +/// sized to be correct under **both** states rather than under whichever one +/// happens to be live, because 0200 may re-enable it at any time and nothing +/// would fail loudly if it did. +/// +/// ⚠️ Shortening the window costs nothing here because this is a +/// **re-introduction guard, not a historical audit** (see the module docs). A +/// writer that has started applying the peg again writes it continuously, so it +/// shows up in the newest rows; it does not need seven days of history to be +/// seen. A frozen historical population is task 0212's job, not this alarm's. +/// +/// ⚠️ **It must also stay wide enough that `scanned == 0` is anomalous**, or +/// [`crate::usd_sanity::ScanGuards`]'s empty-scan refusal fails the invocation +/// on a quiet leg and pages ops every 15 minutes on a healthy system. Measured +/// on prod 2026-08-20: the longest stretch with no USDT-quoted `_1m` row in the +/// preceding 30 days is **6.5 hours**, against this 48 h bound — 7.4x margin. +/// ⛔ Do not shorten this below ~24 h without re-taking that measurement. +/// +/// ⚠️ **Consequence to accept, not to fix by widening:** because the window is +/// short and enrichment drains oldest-first behind task 0111, this direction +/// reads 0 whenever the leg is unpriced — see [`PEG_TABLE`] on why a zero here +/// is not a clean bill of health. +pub const PEG_LOOKBACK_SECONDS: i64 = 2 * 86_400; /// How old a candle must be before a `close_usd` of zero counts as *stranded* /// rather than *not yet enriched*. @@ -93,7 +149,7 @@ pub const LOOKBACK_SECONDS: i64 = 7 * 86_400; /// waking someone for, and not before. /// /// ⚠️ It must also comfortably exceed real enrichment lag or the metric never -/// reads zero, and it holds **only on the hourly tier** — see [`SANITY_TABLE`] +/// reads zero, and it holds **only on the hourly tier** — see [`STRANDED_TABLE`] /// for why a coarser tier spends its own bucket width out of this grace before /// enrichment can even begin. /// @@ -133,11 +189,10 @@ pub const PEG_RATIO_TOLERANCE: f64 = 0.02; /// and is the value 0182's verification settled on. pub const REPRESENTABLE_CLOSE_FLOOR: &str = "0.00000000000005"; -/// Which OHLCV granularity the check reads. +/// Which OHLCV granularity the **stranded** direction reads. /// -/// `price_ohlcv_1h`. The defect appears on every tier — 0182 corrected all five -/// — so any one of them is diagnostic, and the choice is therefore about how the -/// tier interacts with [`STRANDED_GRACE_SECONDS`], not about coverage. +/// `price_ohlcv_1h`, and the choice is about how the tier interacts with +/// [`STRANDED_GRACE_SECONDS`], not about coverage. /// /// ⚠️ **This was `price_ohlcv_1d` and was changed on measurement (2026-08-19).** /// The original reasoning — `_1d` is cheapest, `_1h` would scan "~24× the rows" @@ -158,63 +213,177 @@ pub const REPRESENTABLE_CLOSE_FLOOR: &str = "0.00000000000005"; /// USDT-quoted `_1h` candle is priced by 30 hours of age" — was an artefact and /// is corrected in [`STRANDED_GRACE_SECONDS`]. /// -/// ⚠️ `_1h` is a forever-table (no retention job, unlike `_1m`/`_15m`), so the -/// 7-day [`LOOKBACK_SECONDS`] window can never outrun what is kept. +/// ⚠️ `_1h` is a forever-table (no retention job, unlike `_1m`/`_15m`), so +/// [`STRANDED_LOOKBACK_SECONDS`] can never outrun what is kept. /// -/// # 🔴 This check is one tier ABOVE the tier that carries the defect +/// # Why the stranded direction is correct on a derived tier +/// +/// A zero rolls up as a zero: `argMaxIf(close_usd, …, close_usd > 0)` has +/// nothing to select, so an unpriced `_1m` row surfaces as an unpriced `_1h` +/// row. Reading the coarse tier therefore detects the condition faithfully — +/// which is how task 0209 was found at all. ⛔ **Do not move this direction to +/// `_1m` alongside the peg direction.** The 48 h grace is calibrated to BE's +/// loss window *on the hourly tier*, and the tier swap would silently change +/// what the alarm asserts. +pub const STRANDED_TABLE: &str = "price_ohlcv_1h"; + +/// Which OHLCV granularity the **peg-applied** direction reads. /// -/// `close_usd` is written by the enrichment worker into **`price_ohlcv_1m`**; -/// every coarse tier rolls from it. Reading `_1h` therefore measures a -/// *derived* surface, and on 2026-08-20 that difference was not academic: +/// `price_ohlcv_1m` — **the tier enrichment actually writes**, not one rolled +/// from it. Task 0213; this was `price_ohlcv_1h` and was blind. /// -/// | | `_1h` | `_1m` | +/// # 🔴 Why a derived tier cannot carry this direction +/// +/// A *repaired* value in a coarse tier says nothing about the row it was rolled +/// from. Task 0182's repair wrote the five coarse tables **directly** and never +/// touched `_1m`, so on 2026-08-20 the two tiers disagreed completely: +/// +/// | | `_1h` — what this check used to read | `_1m` | /// |---|---|---| /// | USDT-quoted rows at the $1 peg | 0 | **1,564,045** | /// | 2026-08-17, USDT leg | 13 priced / 0 | 0 priced / 16 | /// -/// The coarse tiers read clean because task 0182's repair wrote them directly. -/// `_1m` was outside that repair and still carries the peg — task 0212 — so -/// **`peg_applied` would have published a confident 0 over 1.5 M wrong values.** +/// The check would have published **a confident 0 over 1.5 M wrong values**, and +/// gone on doing so indefinitely. +/// +/// ⚠️ **This is task 0204's own founding failure, reproduced inside the guard +/// built against it** — a check scoring healthy because it looked at the surface +/// least able to show the defect. It is the same shape as 0182 being verified +/// against the tiers its own repair had written. +/// +/// # Why this is a separate constant and not a repointed `SANITY_TABLE` +/// +/// Pointing one shared table name at `_1m` would have been the small change, and +/// it is wrong three times over: +/// +/// 1. An unbounded `_1m` read finds the whole 1,564,045-row peg population +/// (task 0212) — above every rung of `usdSanityEscalationCounts` +/// (`[1, 100, 10000]`) — and sits permanently in ALARM. A permanently-firing +/// alarm gets muted, the exact end-state task 0204 exists to prevent. +/// +/// ⚠️ **That figure applies to the repoint, NOT to the query that ships.** +/// Bounded to [`PEG_LOOKBACK_SECONDS`] the reading is the recent arrival +/// rate, and the whole peg population sits at timestamps at or before +/// 2026-08-13, outside the window. Measured on prod 2026-08-20: `scanned` +/// 684, `peg_applied` **0**. Do not quote 1.5 M as what this alarm reads. +/// 2. `_1m` is retention-managed while `_1h` is a forever-table, so the window +/// reasoning has to be redone rather than inherited — see +/// [`PEG_LOOKBACK_SECONDS`]. +/// 3. The stranded direction is *correct* on `_1h` and would be made worse by +/// moving, because its grace is calibrated to that tier. +/// +/// # 🔴 A zero here does NOT mean the USDT leg is healthy +/// +/// Measured on prod 2026-08-20, inside this window: **684 rows scanned, 684 of +/// them `close_usd = 0`, zero peg-valued and zero correctly priced.** The leg +/// has not been priced at all since 2026-08-13 (task 0209), so the peg +/// direction currently has *nothing to judge* and reads 0 for want of input +/// rather than for want of defects. +/// +/// ⚠️ That is correct behaviour for a re-introduction guard — it watches new +/// writes, and there are none — but it means **`usd-peg-applied` reading green +/// is not evidence of correct USD valuation while the leg is dark.** The +/// `usd-stranded` ladder is what carries that condition, and it is latched. +/// Never read this metric alone. /// -/// ⚠️ The *stranded* direction is unaffected: a zero rolls up as a zero, so -/// reading `_1h` detects it correctly, which is how 0209 was found at all. -/// Only the **peg-applied** direction is blinded by reading a repaired tier. +/// ⚠️ It stays vacuous longer than 0209 alone: enrichment drains oldest-first +/// behind task 0111's backlog, so even once the pivot writes again it will +/// price *history* first, and nothing enters this window until the backlog +/// reaches the present. Widening the window to compensate walks straight back +/// into point 1 above. /// -/// ⛔ **Do not "fix" this by simply pointing `SANITY_TABLE` at `_1m`.** That -/// trades a blind spot for a permanently-breaching alarm (1.5 M rows is above -/// every rung) and re-introduces a retention interaction the forever-table note -/// above exists to avoid. The peg-applied direction needs its own scoped `_1m` -/// query with its own ladder, decided deliberately — see task 0204 and task 0213. -pub const SANITY_TABLE: &str = "price_ohlcv_1h"; +/// # Deploy ordering +/// +/// ⚠️ **An earlier version of this comment said the alarm must not be deployed +/// before 0212 and 0209, because it would ship permanently breached.** The +/// prod measurement above falsifies that: bounded to 48 h it reads 0 today, +/// with 0212 unlanded, because the peg population is entirely outside the +/// window. The claim was inherited from the repoint argument in point 1 and +/// never re-checked against the query that actually ships. +/// +/// So this is deployable ahead of 0209/0212 — but deploy it for the right +/// reason: it arms a guard for when the leg is priced again, and it proves +/// nothing about the leg until then. +pub const PEG_TABLE: &str = "price_ohlcv_1m"; -/// One reading of the USDT quote leg's USD-value health. +/// Shared guards against the silent all-clear, carried by every reading. /// -/// `resolved_legs` and `scanned` are the two guards against the silent -/// all-clear. If the USDT identity cannot be resolved — the registry moved, an -/// issuer changed, task 0139 renumbered something — the quote-leg filter matches -/// nothing, both counts come back `0`, and the alarm scores a **healthy** result +/// If the USDT identity cannot be resolved — the registry moved, an issuer +/// changed, task 0139 renumbered something — the quote-leg filter matches +/// nothing, the counts come back `0`, and the alarm scores a **healthy** result /// for a check that did not run. `resolved_legs` catches the identity being /// missing; `scanned` catches it resolving to an id the candles no longer carry, /// which the first guard cannot see. Carrying both in the same row makes each -/// state detectable; [`sanity_metrics`] refuses them. +/// state detectable; [`stranded_metric`] and [`peg_metric`] refuse them. +/// +/// ⚠️ **Each direction carries its own pair, and they are not interchangeable.** +/// The two now read different tables, so `_1m` can match nothing while `_1h` +/// reads fine — a state a single shared guard would score as healthy on the +/// tier that was never examined. That is precisely the defect task 0213 exists +/// to close, so it must not be reintroduced one level up. +pub trait ScanGuards { + fn resolved_legs(&self) -> u64; + fn scanned(&self) -> u64; + /// Which table this reading examined, for the refusal message. + fn table(&self) -> &'static str; + /// The window this reading covered, in seconds, for the refusal message. + fn lookback_seconds(&self) -> i64; +} + +/// One reading of the **stranded** direction, from [`STRANDED_TABLE`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] -pub struct SanityCounts { +pub struct StrandedCounts { /// Number of `assets` rows matching the canonical USDT identity. Must be 1. pub resolved_legs: u64, - /// USDT-quoted candles in the window whose `close_usd / close` sits within - /// [`PEG_RATIO_TOLERANCE`] of 1.0. - pub peg_applied: u64, /// USDT-quoted candles older than [`STRANDED_GRACE_SECONDS`] still at /// `close_usd = 0` with a `close` above [`REPRESENTABLE_CLOSE_FLOOR`]. pub stranded: u64, - /// USDT-quoted candles examined. - /// - /// Not alarmed on, but **guarded on**: zero means the query matched nothing, - /// so the two counts above are zero for want of data rather than for want of - /// defects. See [`SanityRefusal::EmptyScan`]. + /// USDT-quoted candles examined. See [`ScanGuards`]. + pub scanned: u64, +} + +impl ScanGuards for StrandedCounts { + fn resolved_legs(&self) -> u64 { + self.resolved_legs + } + fn scanned(&self) -> u64 { + self.scanned + } + fn table(&self) -> &'static str { + STRANDED_TABLE + } + fn lookback_seconds(&self) -> i64 { + STRANDED_LOOKBACK_SECONDS + } +} + +/// One reading of the **peg-applied** direction, from [`PEG_TABLE`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clickhouse::Row, serde::Deserialize)] +pub struct PegCounts { + /// Number of `assets` rows matching the canonical USDT identity. Must be 1. + pub resolved_legs: u64, + /// USDT-quoted candles in the window whose `close_usd / close` sits within + /// [`PEG_RATIO_TOLERANCE`] of 1.0. + pub peg_applied: u64, + /// USDT-quoted candles examined. See [`ScanGuards`]. pub scanned: u64, } +impl ScanGuards for PegCounts { + fn resolved_legs(&self) -> u64 { + self.resolved_legs + } + fn scanned(&self) -> u64 { + self.scanned + } + fn table(&self) -> &'static str { + PEG_TABLE + } + fn lookback_seconds(&self) -> i64 { + PEG_LOOKBACK_SECONDS + } +} + /// One CloudWatch datum about USD-value correctness. Counts, so /// `StandardUnit::Count` — mirrored without dragging the AWS SDK into the /// default build, exactly as [`crate::disk::DiskMetric`] does. @@ -224,23 +393,26 @@ pub struct SanityMetric { pub value: f64, } -/// The one-row query behind [`SanityCounts`]. +/// The `WITH` clause both queries open with: the USDT identity, resolved +/// **inline by code and issuer** rather than hard-coded as an `asset_id`. /// -/// The USDT identity is resolved **inline by code and issuer** rather than -/// hard-coded as an `asset_id`. The numeric id is not a stable contract while -/// task 0139 is open, and a probe that silently watches the wrong leg is worse -/// than one that fails — see [`SanityCounts::resolved_legs`]. -/// -/// Tables are unqualified: the probe binds its client to the `prices` database, -/// the same convention as [`crate::freshness_query`] and [`crate::disk::disk_query`]. -/// -/// ⚠️ `FINAL` is required. These are `ReplacingMergeTree` tables and a repair -/// re-inserts corrected rows at a higher `version`; without `FINAL` this would -/// read superseded rows and report a defect that has already been fixed — -/// alarming on history rather than on state. +/// The numeric id is not a stable contract while task 0139 is open, and a probe +/// that silently watches the wrong leg is worse than one that fails — see +/// [`ScanGuards`]. +fn usdt_cte() -> String { + format!( + "WITH usdt AS ( \ + SELECT asset_id FROM assets FINAL \ + WHERE asset_code = 'USDT' AND issuer_address = '{usdt}' \ + ) ", + usdt = USDT_ISSUER, + ) +} + +/// The `resolved_legs` projection, and the reason it looks the way it does. /// -/// ⚠️ **`resolved_legs` is wrapped in `toUInt64(ifNull(…))` and must stay that -/// way.** A bare scalar subquery — `(SELECT count() FROM usdt)` — comes back as +/// ⚠️ **It is wrapped in `toUInt64(ifNull(…))` and must stay that way.** A bare +/// scalar subquery — `(SELECT count() FROM usdt)` — comes back as /// **`Nullable(UInt64)`**, and in RowBinary a nullable column is a one-byte null /// flag followed by the value. Deserializing that into a plain `u64` does not /// error: it consumes the flag as the low byte and silently returns **256** for @@ -249,15 +421,23 @@ pub struct SanityMetric { /// crate's IT file was written for (PR #97), which at least failed loudly — here /// the probe would refuse every healthy run as "ambiguous identity" and the /// gap-4 alarms would never publish. -pub fn sanity_query() -> String { +const RESOLVED_LEGS_PROJECTION: &str = + "toUInt64(ifNull((SELECT count() FROM usdt), 0)) AS resolved_legs"; + +/// The one-row query behind [`StrandedCounts`], over [`STRANDED_TABLE`]. +/// +/// Tables are unqualified: the probe binds its client to the `prices` database, +/// the same convention as [`crate::freshness_query`] and [`crate::disk::disk_query`]. +/// +/// ⚠️ `FINAL` is required. These are `ReplacingMergeTree` tables and a repair +/// re-inserts corrected rows at a higher `version`; without `FINAL` this would +/// read superseded rows and report a defect that has already been fixed — +/// alarming on history rather than on state. +pub fn stranded_query() -> String { format!( - "WITH usdt AS ( \ - SELECT asset_id FROM assets FINAL \ - WHERE asset_code = 'USDT' AND issuer_address = '{usdt}' \ - ) \ + "{cte}\ SELECT \ - toUInt64(ifNull((SELECT count() FROM usdt), 0)) AS resolved_legs, \ - countIf(close_usd > 0 AND abs(close_usd / close - 1) < {tol}) AS peg_applied, \ + {legs}, \ countIf( \ close_usd = 0 \ AND close > {floor} \ @@ -268,47 +448,98 @@ pub fn sanity_query() -> String { WHERE quote_asset_id IN (SELECT asset_id FROM usdt) \ AND close > 0 \ AND timestamp >= now() - INTERVAL {lookback} SECOND", - usdt = USDT_ISSUER, - tol = PEG_RATIO_TOLERANCE, + cte = usdt_cte(), + legs = RESOLVED_LEGS_PROJECTION, floor = REPRESENTABLE_CLOSE_FLOOR, grace = STRANDED_GRACE_SECONDS, - table = SANITY_TABLE, - lookback = LOOKBACK_SECONDS, + table = STRANDED_TABLE, + lookback = STRANDED_LOOKBACK_SECONDS, + ) +} + +/// The one-row query behind [`PegCounts`], over [`PEG_TABLE`]. +/// +/// Structurally the sibling of [`stranded_query`], and deliberately a **separate +/// query rather than a second column on one** — see [`PEG_TABLE`] for why the +/// two directions cannot share a tier, and [`ScanGuards`] for why they cannot +/// share a guard either. +/// +/// ⚠️ No grace period, and that asymmetry is the point. A stranded row is a row +/// enrichment has **not reached yet**, so it needs time before it means damage. +/// A peg-valued row is a row enrichment has already **written wrongly** — it is +/// wrong the instant it appears and waiting cannot improve it. +/// +/// ⚠️ `FINAL` for the same `ReplacingMergeTree` reason as its sibling. It +/// matters more here: task 0212's repair re-inserts corrected rows at a higher +/// `version`, so without `FINAL` this alarm would keep reading the pre-repair +/// rows and stay breached over data that had been fixed. +pub fn peg_query() -> String { + format!( + "{cte}\ + SELECT \ + {legs}, \ + countIf(close_usd > 0 AND abs(close_usd / close - 1) < {tol}) AS peg_applied, \ + count() AS scanned \ + FROM {table} FINAL \ + WHERE quote_asset_id IN (SELECT asset_id FROM usdt) \ + AND close > 0 \ + AND timestamp >= now() - INTERVAL {lookback} SECOND", + cte = usdt_cte(), + legs = RESOLVED_LEGS_PROJECTION, + tol = PEG_RATIO_TOLERANCE, + table = PEG_TABLE, + lookback = PEG_LOOKBACK_SECONDS, ) } /// Why a reading was refused instead of published. /// /// Both variants describe the **same hazard from different distances**: a query -/// that matched nothing still returns two perfectly publishable zeros, and the -/// gap-4 alarms are `treatMissingData: NOT_BREACHING`, so those zeros are scored +/// that matched nothing still returns a perfectly publishable zero, and the +/// gap-4 alarms are `treatMissingData: NOT_BREACHING`, so that zero is scored /// as a clean bill of health forever. The variants are separate because the /// operator's first move differs — one points at the asset registry, the other /// at the candles — and an alarm that names the wrong one costs an hour. +/// +/// ⚠️ **Both carry the table.** Since task 0213 the two directions read +/// different tiers, so "the scan matched nothing" is no longer a single fact +/// about one place. A refusal that did not say which tier would send the +/// operator to look at a table that was fine. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SanityRefusal { /// The canonical USDT identity did not resolve to exactly one `assets` row. - UnresolvableLeg { resolved_legs: u64 }, + UnresolvableLeg { + resolved_legs: u64, + table: &'static str, + }, /// The leg resolved, but matched **no candles at all** in the window. - EmptyScan, + EmptyScan { + table: &'static str, + lookback_seconds: i64, + }, } impl std::fmt::Display for SanityRefusal { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::UnresolvableLeg { resolved_legs } => write!( + Self::UnresolvableLeg { + resolved_legs, + table, + } => write!( f, "the canonical USDT identity resolved to {resolved_legs} assets, expected \ - exactly 1 — check prices.assets for the USDT code/issuer pair" + exactly 1 — check prices.assets for the USDT code/issuer pair (reading {table})" ), - Self::EmptyScan => write!( + Self::EmptyScan { + table, + lookback_seconds, + } => write!( f, "the USDT identity resolved, but no USDT-quoted candles were found in the \ last {days} days of {table} — the asset_id in the registry no longer matches \ the quote_asset_id stored in the candles (task 0139), or the leg has stopped \ trading entirely", - days = LOOKBACK_SECONDS / 86_400, - table = SANITY_TABLE, + days = lookback_seconds / 86_400, ), } } @@ -316,7 +547,7 @@ impl std::fmt::Display for SanityRefusal { impl std::error::Error for SanityRefusal {} -/// Shape one [`SanityCounts`] reading into the CloudWatch data to publish. +/// The two silent-all-clear guards, applied to either direction's reading. /// /// `Err` when the check **did not actually run**, which is not a clean bill of /// health and must never be published as one. The caller turns an `Err` into a @@ -328,27 +559,47 @@ impl std::error::Error for SanityRefusal {} /// resolves to an `asset_id` no longer present in the candles — precisely the /// renumber risk task 0139 is open for, and the reason the leg is resolved by /// issuer rather than by id — passes that guard and then matches zero rows. -/// Both counts publish `0` and every gap-4 alarm reads healthy forever. A scan +/// The count publishes `0` and every gap-4 alarm reads healthy forever. A scan /// that examined nothing has measured nothing. -pub fn sanity_metrics(counts: &SanityCounts) -> Result, SanityRefusal> { - if counts.resolved_legs != 1 { +fn guard(counts: &G) -> Result<(), SanityRefusal> { + if counts.resolved_legs() != 1 { return Err(SanityRefusal::UnresolvableLeg { - resolved_legs: counts.resolved_legs, + resolved_legs: counts.resolved_legs(), + table: counts.table(), + }); + } + if counts.scanned() == 0 { + return Err(SanityRefusal::EmptyScan { + table: counts.table(), + lookback_seconds: counts.lookback_seconds(), }); } - if counts.scanned == 0 { - return Err(SanityRefusal::EmptyScan); - } - Ok(vec![ - SanityMetric { - name: PEG_APPLIED_METRIC, - value: counts.peg_applied as f64, - }, - SanityMetric { - name: STRANDED_METRIC, - value: counts.stranded as f64, - }, - ]) + Ok(()) +} + +/// Shape one [`StrandedCounts`] reading into the CloudWatch datum to publish. +pub fn stranded_metric(counts: &StrandedCounts) -> Result { + guard(counts)?; + Ok(SanityMetric { + name: STRANDED_METRIC, + value: counts.stranded as f64, + }) +} + +/// Shape one [`PegCounts`] reading into the CloudWatch datum to publish. +/// +/// ⚠️ **Independent of [`stranded_metric`], deliberately.** Before task 0213 one +/// refusal suppressed both metrics, which was harmless while they came from one +/// query. Now they read different tables, and a `_1m` scan that matched nothing +/// says nothing whatever about `_1h` — suppressing a working signal because an +/// unrelated tier failed would be the muting failure again, arrived at from the +/// other side. Each direction publishes or refuses on its own evidence. +pub fn peg_metric(counts: &PegCounts) -> Result { + guard(counts)?; + Ok(SanityMetric { + name: PEG_APPLIED_METRIC, + value: counts.peg_applied as f64, + }) } /// Publish USD-sanity counts to CloudWatch under [`crate::METRIC_NAMESPACE`], @@ -401,42 +652,84 @@ pub async fn publish_sanity( mod tests { use super::*; - fn healthy() -> SanityCounts { - SanityCounts { + fn healthy_stranded() -> StrandedCounts { + StrandedCounts { resolved_legs: 1, - peg_applied: 0, stranded: 0, scanned: 4_503, } } + fn healthy_peg() -> PegCounts { + PegCounts { + resolved_legs: 1, + peg_applied: 0, + scanned: 4_503, + } + } + /// The trap this whole module is scoped around. Exotic-quoted candles sit at /// `close_usd = 0` by design (~74M on `_1h` alone, task 0182), so a check /// that did not filter by quote leg would breach permanently on healthy - /// data. + /// data. Both directions need it. #[test] - fn the_query_is_scoped_to_the_usdt_quote_leg() { - let sql = sanity_query(); - assert!(sql.contains("WHERE quote_asset_id IN (SELECT asset_id FROM usdt)")); + fn both_queries_are_scoped_to_the_usdt_quote_leg() { + for sql in [stranded_query(), peg_query()] { + assert!(sql.contains("WHERE quote_asset_id IN (SELECT asset_id FROM usdt)")); + } } /// The numeric `asset_id` is not a stable contract while task 0139 is open, /// so the leg is resolved by code and issuer at query time. #[test] fn the_usdt_leg_is_resolved_by_issuer_not_a_hardcoded_id() { - let sql = sanity_query(); - assert!(sql.contains(USDT_ISSUER)); - assert!(sql.contains("asset_code = 'USDT'")); + for sql in [stranded_query(), peg_query()] { + assert!(sql.contains(USDT_ISSUER)); + assert!(sql.contains("asset_code = 'USDT'")); + } } - /// Both directions must be counted. 0182's repair caused the second one, so - /// a check for only the peg would have passed while 157 candles sat + /// Both directions must still be counted. 0182's repair caused the stranded + /// one, so a check for only the peg would have passed while 157 candles sat /// destroyed. #[test] - fn the_query_counts_both_failure_directions() { - let sql = sanity_query(); - assert!(sql.contains("AS peg_applied")); - assert!(sql.contains("AS stranded")); + fn the_two_directions_are_still_both_counted() { + assert!(stranded_query().contains("AS stranded")); + assert!(peg_query().contains("AS peg_applied")); + } + + /// 🔴 **The defect task 0213 exists to close.** `close_usd` is written by + /// enrichment into `_1m`; every coarse tier rolls from it. Reading `_1h` + /// measured a *derived* surface that task 0182's repair had written + /// directly, so the peg direction published a confident 0 over 1,564,045 + /// wrong values (task 0212). + #[test] + fn the_peg_direction_reads_the_tier_enrichment_writes_not_a_rolled_one() { + assert_eq!(PEG_TABLE, "price_ohlcv_1m"); + assert!(peg_query().contains(&format!("FROM {PEG_TABLE} FINAL"))); + assert!( + !peg_query().contains("price_ohlcv_1h"), + "the peg direction must not read a tier 0182's repair wrote directly" + ); + } + + /// ⚠️ The stranded direction is **correct** on the coarse tier and must not + /// be moved along with the peg direction: a zero rolls up as a zero, and the + /// 48 h grace is calibrated to BE's loss window on the hourly tier. + #[test] + fn the_stranded_direction_stays_on_the_hourly_tier() { + assert_eq!(STRANDED_TABLE, "price_ohlcv_1h"); + assert!(stranded_query().contains(&format!("FROM {STRANDED_TABLE} FINAL"))); + } + + /// The two directions must not be collapsed back into one scan. Pinned + /// because "one query, two columns" is the obvious tidy-up and it is exactly + /// what re-introduces the blind spot. + #[test] + fn the_two_directions_read_different_tables() { + assert_ne!(STRANDED_TABLE, PEG_TABLE); + assert!(!stranded_query().contains("peg_applied")); + assert!(!peg_query().contains("AS stranded")); } /// Without the grace period the stranded count can never read zero: @@ -444,37 +737,78 @@ mod tests { /// legitimately at zero on every single run. #[test] fn the_stranded_count_excludes_candles_too_young_to_be_enriched() { - let sql = sanity_query(); - assert!(sql.contains(&format!( + assert!(stranded_query().contains(&format!( "timestamp < now() - INTERVAL {STRANDED_GRACE_SECONDS} SECOND" ))); } + /// ⚠️ The asymmetry is deliberate. A stranded row is one enrichment has not + /// reached yet, so it needs time before it means damage. A peg-valued row is + /// one enrichment has already written **wrongly** — it is wrong the instant + /// it appears, and a grace period would only delay saying so. + #[test] + fn the_peg_count_has_no_grace_because_a_wrong_write_is_wrong_immediately() { + // The generated SQL never contains Rust identifier text, so asserting + // the absence of "STRANDED_GRACE" could not fail and proved nothing. + // Assert the absence of the rendered clause instead. + assert!(!peg_query().contains(&format!( + "timestamp < now() - INTERVAL {STRANDED_GRACE_SECONDS} SECOND" + ))); + assert!(!peg_query().contains("close_usd = 0")); + } + /// An unbounded scan every 15 minutes is task 0111's outage wearing a health - /// check's clothes. + /// check's clothes. Both directions stay bounded. #[test] - fn the_scan_is_bounded_to_a_recent_window() { - let sql = sanity_query(); - assert!(sql.contains(&format!( - "timestamp >= now() - INTERVAL {LOOKBACK_SECONDS} SECOND" + fn both_scans_are_bounded_to_a_recent_window() { + assert!(stranded_query().contains(&format!( + "timestamp >= now() - INTERVAL {STRANDED_LOOKBACK_SECONDS} SECOND" + ))); + assert!(peg_query().contains(&format!( + "timestamp >= now() - INTERVAL {PEG_LOOKBACK_SECONDS} SECOND" ))); } + /// ⚠️ **The retention interaction, pinned.** `_1m` is pruned by the cleanup + /// worker at 7 days — a JOB, not a TTL (task 0174's lesson, currently + /// DISABLED under task 0200). A 7-day peg window would sit exactly on the + /// deletion frontier and move for reasons that are not defects, so it keeps + /// a wide margin instead of inheriting the stranded direction's bound. + #[test] + fn the_peg_window_stays_clear_of_the_1m_retention_frontier() { + let retention_seconds = 7 * 86_400_i64; + assert!( + PEG_LOOKBACK_SECONDS * 3 <= retention_seconds, + "the peg window must keep real margin below _1m retention, or a cleanup \ + run truncates the scan and the count moves without a defect" + ); + // The stranded direction reads a forever-table and needs no such margin. + assert_eq!(STRANDED_LOOKBACK_SECONDS, retention_seconds); + } + /// `ReplacingMergeTree` + a repair that re-inserts at a higher `version`. /// Without `FINAL` this reads superseded rows and alarms on a defect that - /// has already been corrected. + /// has already been corrected — which for the peg direction means staying + /// breached over exactly the rows task 0212 just fixed. #[test] - fn the_query_reads_final() { - let sql = sanity_query(); - assert!(sql.contains(&format!("FROM {SANITY_TABLE} FINAL"))); - assert!(sql.contains("FROM assets FINAL")); + fn both_queries_read_final() { + // ⚠️ Assert the CANDLE table's FINAL by name. A bare `contains(" FINAL ")` + // is already satisfied by `FROM assets FINAL` in the shared CTE, so + // deleting FINAL from either candle table would leave it green — a test + // unable to distinguish the thing it asserts, in the module whose whole + // subject is that failure. + assert!(stranded_query().contains(&format!("FROM {STRANDED_TABLE} FINAL"))); + assert!(peg_query().contains(&format!("FROM {PEG_TABLE} FINAL"))); + for sql in [stranded_query(), peg_query()] { + assert!(sql.contains("FROM assets FINAL")); + } } /// The dust bound is the arithmetic one. 0182's first attempt used `1e-11` /// and counted rows that had priced perfectly well. #[test] fn the_representable_floor_is_the_underflow_bound_not_a_round_number() { - assert!(sanity_query().contains(REPRESENTABLE_CLOSE_FLOOR)); + assert!(stranded_query().contains(REPRESENTABLE_CLOSE_FLOOR)); let floor: f64 = REPRESENTABLE_CLOSE_FLOOR.parse().expect("parses"); // Comfortably below 1e-11, and above where a ~0.15 rate underflows. assert!(floor < 1e-13); @@ -487,52 +821,78 @@ mod tests { /// `u64` yields **256** for a count of 1 without any error. Measured on /// 26.3.10.60. The probe would then reject every healthy run. #[test] - fn the_resolved_leg_count_is_forced_non_nullable() { - let sql = sanity_query(); - assert!(sql.contains("toUInt64(ifNull((SELECT count() FROM usdt), 0)) AS resolved_legs")); + fn the_resolved_leg_count_is_forced_non_nullable_in_both_queries() { + for sql in [stranded_query(), peg_query()] { + assert!( + sql.contains("toUInt64(ifNull((SELECT count() FROM usdt), 0)) AS resolved_legs") + ); + } } #[test] - fn a_healthy_reading_publishes_two_zeroes() { - let m = sanity_metrics(&healthy()).expect("leg resolved"); + fn healthy_readings_publish_zeroes() { assert_eq!( - m, - vec![ - SanityMetric { - name: PEG_APPLIED_METRIC, - value: 0.0 - }, - SanityMetric { - name: STRANDED_METRIC, - value: 0.0 - }, - ] + stranded_metric(&healthy_stranded()).expect("leg resolved"), + SanityMetric { + name: STRANDED_METRIC, + value: 0.0 + } + ); + assert_eq!( + peg_metric(&healthy_peg()).expect("leg resolved"), + SanityMetric { + name: PEG_APPLIED_METRIC, + value: 0.0 + } ); } #[test] fn counts_are_published_as_they_are_read() { - let m = sanity_metrics(&SanityCounts { - peg_applied: 44_657, - stranded: 157, - ..healthy() - }) - .expect("leg resolved"); - assert_eq!(m[0].value, 44_657.0); - assert_eq!(m[1].value, 157.0); + assert_eq!( + peg_metric(&PegCounts { + peg_applied: 1_564_045, + ..healthy_peg() + }) + .expect("leg resolved") + .value, + 1_564_045.0 + ); + assert_eq!( + stranded_metric(&StrandedCounts { + stranded: 157, + ..healthy_stranded() + }) + .expect("leg resolved") + .value, + 157.0 + ); } - /// The silent all-clear this module's `resolved_legs` column exists to - /// prevent: an unresolvable leg matches no candles, both counts read zero, - /// and a `NOT_BREACHING` alarm scores it healthy forever. + /// The silent all-clear the `resolved_legs` column exists to prevent: an + /// unresolvable leg matches no candles, the count reads zero, and a + /// `NOT_BREACHING` alarm scores it healthy forever. #[test] fn an_unresolvable_usdt_leg_is_refused_rather_than_reported_healthy() { assert_eq!( - sanity_metrics(&SanityCounts { + stranded_metric(&StrandedCounts { + resolved_legs: 0, + ..healthy_stranded() + }), + Err(SanityRefusal::UnresolvableLeg { + resolved_legs: 0, + table: STRANDED_TABLE + }) + ); + assert_eq!( + peg_metric(&PegCounts { resolved_legs: 0, - ..healthy() + ..healthy_peg() }), - Err(SanityRefusal::UnresolvableLeg { resolved_legs: 0 }) + Err(SanityRefusal::UnresolvableLeg { + resolved_legs: 0, + table: PEG_TABLE + }) ); } @@ -542,55 +902,104 @@ mod tests { #[test] fn an_ambiguous_usdt_identity_is_refused() { assert_eq!( - sanity_metrics(&SanityCounts { + peg_metric(&PegCounts { resolved_legs: 2, - ..healthy() + ..healthy_peg() }), - Err(SanityRefusal::UnresolvableLeg { resolved_legs: 2 }) + Err(SanityRefusal::UnresolvableLeg { + resolved_legs: 2, + table: PEG_TABLE + }) ); } /// The guard `resolved_legs` cannot provide. The identity resolves cleanly, /// so the first guard passes — but the `asset_id` it resolves to is not the /// `quote_asset_id` the candles carry (task 0139 renumbering, a registry - /// rewrite), so the scan matches nothing. Both counts are zero because - /// nothing was examined, and `NOT_BREACHING` would score that healthy for as - /// long as it lasted. + /// rewrite), so the scan matches nothing. The count is zero because nothing + /// was examined, and `NOT_BREACHING` would score that healthy. #[test] fn a_resolved_leg_that_matches_no_candles_is_refused() { assert_eq!( - sanity_metrics(&SanityCounts { + peg_metric(&PegCounts { scanned: 0, - ..healthy() + ..healthy_peg() }), - Err(SanityRefusal::EmptyScan) + Err(SanityRefusal::EmptyScan { + table: PEG_TABLE, + lookback_seconds: PEG_LOOKBACK_SECONDS + }) ); } + /// ⚠️ The whole point of splitting the guards. A `_1m` scan that matched + /// nothing says nothing about `_1h`; suppressing a working signal because an + /// unrelated tier failed would be the muting failure arrived at from the + /// other side. + #[test] + fn one_direction_refusing_does_not_suppress_the_other() { + let peg = peg_metric(&PegCounts { + scanned: 0, + ..healthy_peg() + }); + let stranded = stranded_metric(&StrandedCounts { + stranded: 42, + ..healthy_stranded() + }); + assert!(peg.is_err()); + assert_eq!(stranded.expect("published on its own evidence").value, 42.0); + } + /// The two refusals must not read alike: one sends the operator to the asset /// registry, the other to the candles. #[test] fn the_two_refusals_name_different_first_moves() { - let unresolvable = SanityRefusal::UnresolvableLeg { resolved_legs: 0 }.to_string(); - let empty = SanityRefusal::EmptyScan.to_string(); + let unresolvable = SanityRefusal::UnresolvableLeg { + resolved_legs: 0, + table: PEG_TABLE, + } + .to_string(); + let empty = SanityRefusal::EmptyScan { + table: PEG_TABLE, + lookback_seconds: PEG_LOOKBACK_SECONDS, + } + .to_string(); assert!(unresolvable.contains("prices.assets")); assert!(empty.contains("quote_asset_id")); assert_ne!(unresolvable, empty); } - /// ⚠️ Pins the tier against a "cheaper coarser table" optimisation, which is - /// exactly the change that was measured and REVERSED on 2026-08-19. + /// ⚠️ Since the directions read different tiers, a refusal that did not name + /// the table would send the operator to look at a table that was fine. + #[test] + fn a_refusal_names_the_tier_it_examined() { + assert!( + SanityRefusal::EmptyScan { + table: PEG_TABLE, + lookback_seconds: PEG_LOOKBACK_SECONDS, + } + .to_string() + .contains(PEG_TABLE) + ); + assert!( + SanityRefusal::UnresolvableLeg { + resolved_legs: 0, + table: STRANDED_TABLE, + } + .to_string() + .contains(STRANDED_TABLE) + ); + } + + /// ⚠️ Pins the stranded tier against a "cheaper coarser table" optimisation, + /// which is exactly the change that was measured and REVERSED on 2026-08-19. /// /// The grace is measured from a bucket's `timestamp`, which is its START, so /// a coarse tier spends its own width out of the grace before its data even - /// exists — 24 of a 48 h grace on `_1d`, one hour on `_1h`. Prod measurement: - /// every USDT-quoted `_1h` candle is priced by 30 h of age, while the `_1d` - /// bucket for the same day was still half unpriced at ~41 h. The cost saving - /// that motivated `_1d` was not real either (1.4×, same wall time), because - /// the scan is dominated by the `FINAL` merge and the `assets` lookup. + /// exists — 24 of a 48 h grace on `_1d`, one hour on `_1h`. #[test] - fn the_check_reads_the_hourly_tier_so_the_grace_is_not_eaten_by_bucket_width() { - assert_eq!(SANITY_TABLE, "price_ohlcv_1h"); + fn the_stranded_tier_does_not_let_bucket_width_eat_the_grace() { + assert_eq!(STRANDED_TABLE, "price_ohlcv_1h"); // The bucket width must stay small relative to the grace. At `_1d` this // ratio is 1/2 — half the grace gone before enrichment can start. diff --git a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs index ab88ee03..b0954542 100644 --- a/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs +++ b/packages/rollup-freshness-probe/tests/rollup_freshness_it.rs @@ -34,7 +34,8 @@ use rollup_freshness_probe::mv_drift::{ drift_metrics, visible_objects_query, }; use rollup_freshness_probe::usd_sanity::{ - SANITY_TABLE, SanityCounts, SanityRefusal, sanity_metrics, sanity_query, + PEG_TABLE, PegCounts, STRANDED_TABLE, SanityRefusal, StrandedCounts, peg_metric, peg_query, + stranded_metric, stranded_query, }; use rollup_freshness_probe::{ROLLUP_TIERS, TableLag, freshness_query, lag_metrics}; @@ -379,11 +380,19 @@ async fn seed_usdt_identity(c: &Client, asset_id: u32) { .await; } -/// Insert one USDT-quoted daily candle with an explicit `close` / `close_usd`. +/// Insert one USDT-quoted candle with an explicit `close` / `close_usd` into a +/// named tier. +/// +/// ⚠️ **The table is a parameter since task 0213.** The two directions read +/// different tiers, and the defect that task exists to close was invisible +/// precisely because a test could not tell them apart. `_1h` is created as +/// `AS price_ohlcv_1m`, so one statement shape serves both. +/// /// `ts_sql` is a server-side expression so `now()` arithmetic matches the /// probe's own window and grace bounds exactly. -async fn insert_usdt_candle( +async fn insert_candle_into( c: &Client, + table: &str, usdt_id: u32, asset_id: u32, ts_sql: &str, @@ -393,7 +402,7 @@ async fn insert_usdt_candle( exec( c, &format!( - "INSERT INTO prices.{SANITY_TABLE} \ + "INSERT INTO prices.{table} \ (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ SELECT {ts_sql}, {asset_id}, {usdt_id}, 'sdex', {close}, {close}, {close}, {close}, \ @@ -403,11 +412,75 @@ async fn insert_usdt_candle( .await; } -async fn read_counts(c: &Client) -> SanityCounts { - c.query(&sanity_query()) - .fetch_one::() +/// Insert into the **stranded** tier (`price_ohlcv_1h`). +async fn insert_usdt_candle( + c: &Client, + usdt_id: u32, + asset_id: u32, + ts_sql: &str, + close: &str, + close_usd: &str, +) { + insert_candle_into( + c, + STRANDED_TABLE, + usdt_id, + asset_id, + ts_sql, + close, + close_usd, + ) + .await; +} + +/// Insert into the **peg** tier — the tier enrichment writes. +/// +/// ⚠️ **The table is spelled literally, not as `PEG_TABLE`.** A fixture written +/// in terms of the constant under test follows it wherever it points, so the +/// tier assertions below would hold for `_1h` just as happily and would prove +/// nothing. Found by reverting `PEG_TABLE` to `price_ohlcv_1h` and watching +/// tests that should have failed keep passing. +async fn insert_usdt_minute_candle( + c: &Client, + usdt_id: u32, + asset_id: u32, + ts_sql: &str, + close: &str, + close_usd: &str, +) { + insert_candle_into( + c, + "price_ohlcv_1m", + usdt_id, + asset_id, + ts_sql, + close, + close_usd, + ) + .await; +} + +/// Clear both tiers and the registry. Both, always — a test that truncated only +/// the tier it was about would inherit the other's rows and read a count nobody +/// wrote. +async fn reset_sanity_tables(c: &Client) { + exec(c, "TRUNCATE TABLE prices.price_ohlcv_1h").await; + exec(c, "TRUNCATE TABLE prices.price_ohlcv_1m").await; + exec(c, "TRUNCATE TABLE prices.assets").await; +} + +async fn read_stranded(c: &Client) -> StrandedCounts { + c.query(&stranded_query()) + .fetch_one::() + .await + .expect("stranded query executes and deserializes") +} + +async fn read_peg(c: &Client) -> PegCounts { + c.query(&peg_query()) + .fetch_one::() .await - .expect("sanity query executes and deserializes") + .expect("peg query executes and deserializes") } /// The query must **execute and deserialize** against a real ClickHouse — the @@ -419,19 +492,23 @@ async fn read_counts(c: &Client) -> SanityCounts { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn usd_sanity_query_executes_and_reads_a_healthy_leg_as_zero() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; - // A correctly-priced USDT-quoted candle: USDT at its measured ~0.15, so - // close_usd is nowhere near close. + // A correctly-priced USDT-quoted candle on each tier: USDT at its measured + // ~0.15, so close_usd is nowhere near close. insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "15").await; + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "15").await; + + let stranded = read_stranded(&c).await; + assert_eq!(stranded.resolved_legs, 1, "the USDT identity must resolve"); + assert_eq!(stranded.stranded, 0, "a priced candle is not stranded"); + assert_eq!(stranded.scanned, 1); - let counts = read_counts(&c).await; - assert_eq!(counts.resolved_legs, 1, "the USDT identity must resolve"); - assert_eq!(counts.peg_applied, 0, "a 0.15 rate is not the peg"); - assert_eq!(counts.stranded, 0, "a priced candle is not stranded"); - assert_eq!(counts.scanned, 1); + let peg = read_peg(&c).await; + assert_eq!(peg.resolved_legs, 1, "the USDT identity must resolve"); + assert_eq!(peg.peg_applied, 0, "a 0.15 rate is not the peg"); + assert_eq!(peg.scanned, 1); } /// ⚠️ **Induce the condition, do not read the CDK.** This is task 0137's lesson @@ -441,19 +518,27 @@ async fn usd_sanity_query_executes_and_reads_a_healthy_leg_as_zero() { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn usd_sanity_counts_both_induced_defects() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; - // Defect 1 — the peg re-applied: close_usd == close (task 0172 / 0182). - insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "100").await; - // Defect 2 — stranded past the grace period: zero on a representable close - // (what task 0182's own reset produced on 2026-08-19). + // Defect 1 — the peg re-applied, on the tier enrichment WRITES: close_usd + // == close (task 0172 / 0212). + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; + // Defect 2 — stranded past the grace period, on the tier the consumer + // reads: zero on a representable close (what task 0182's own reset produced + // on 2026-08-19). insert_usdt_candle(&c, 111, 6, "now() - INTERVAL 3 DAY", "100", "0").await; - let counts = read_counts(&c).await; - assert_eq!(counts.peg_applied, 1, "close_usd == close must be counted"); - assert_eq!(counts.stranded, 1, "an aged zero must be counted"); + assert_eq!( + read_peg(&c).await.peg_applied, + 1, + "close_usd == close must be counted" + ); + assert_eq!( + read_stranded(&c).await.stranded, + 1, + "an aged zero must be counted" + ); } /// The grace period is what makes the stranded metric usable at all: enrichment @@ -464,18 +549,17 @@ async fn usd_sanity_counts_both_induced_defects() { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn a_freshly_written_zero_is_not_yet_stranded() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; // Inside the 48 h grace — awaiting enrichment, not damaged. insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 1 HOUR", "100", "0").await; - assert_eq!(read_counts(&c).await.stranded, 0, "still within grace"); + assert_eq!(read_stranded(&c).await.stranded, 0, "still within grace"); // The same row, aged past the grace, is the defect. - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; + exec(&c, "TRUNCATE TABLE prices.price_ohlcv_1h").await; insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "0").await; - assert_eq!(read_counts(&c).await.stranded, 1, "past grace = stranded"); + assert_eq!(read_stranded(&c).await.stranded, 1, "past grace = stranded"); } /// Dust is not damage. A `close` below the `Decimal(38, 14)` underflow bound @@ -487,8 +571,7 @@ async fn a_freshly_written_zero_is_not_yet_stranded() { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn dust_below_the_underflow_bound_is_not_counted_as_stranded() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; insert_usdt_candle( @@ -500,7 +583,7 @@ async fn dust_below_the_underflow_bound_is_not_counted_as_stranded() { "0", ) .await; - assert_eq!(read_counts(&c).await.stranded, 0, "1e-14 close is dust"); + assert_eq!(read_stranded(&c).await.stranded, 0, "1e-14 close is dust"); } /// ⚠️ The trap that makes this check scoped rather than global: exotic-quoted @@ -512,14 +595,13 @@ async fn dust_below_the_underflow_bound_is_not_counted_as_stranded() { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn an_exotic_quoted_zero_is_ignored_because_it_is_by_design() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; // quote_asset_id 999 is not the USDT leg — an unpriceable exotic pair. insert_usdt_candle(&c, 999, 5, "now() - INTERVAL 3 DAY", "100", "0").await; - let counts = read_counts(&c).await; + let counts = read_stranded(&c).await; assert_eq!(counts.scanned, 0, "the exotic leg is out of scope entirely"); assert_eq!(counts.stranded, 0); } @@ -532,34 +614,27 @@ async fn an_exotic_quoted_zero_is_ignored_because_it_is_by_design() { #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn a_repaired_candle_stops_counting_once_a_higher_version_supersedes_it() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; seed_usdt_identity(&c, 111).await; - insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "100").await; - assert_eq!( - read_counts(&c).await.peg_applied, - 1, - "the defect is present" - ); + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; + assert_eq!(read_peg(&c).await.peg_applied, 1, "the defect is present"); // The repair: same primary key, corrected value, version + 1. exec( &c, - &format!( - "INSERT INTO prices.{SANITY_TABLE} \ - (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ - volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ - SELECT timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ - volume_base, volume_quote, volume_quote, 15, vwap, trade_count, version + 1 \ - FROM prices.{SANITY_TABLE} FINAL \ - WHERE quote_asset_id = 111 AND close_usd = close" - ), + "INSERT INTO prices.price_ohlcv_1m \ + (timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote_usd, close_usd, vwap, trade_count, version) \ + SELECT timestamp, asset_id, quote_asset_id, source, open, high, low, close, \ + volume_base, volume_quote, volume_quote, 15, vwap, trade_count, version + 1 \ + FROM prices.price_ohlcv_1m FINAL \ + WHERE quote_asset_id = 111 AND close_usd = close", ) .await; assert_eq!( - read_counts(&c).await.peg_applied, + read_peg(&c).await.peg_applied, 0, "FINAL must collapse to the repaired row" ); @@ -568,29 +643,175 @@ async fn a_repaired_candle_stops_counting_once_a_higher_version_supersedes_it() /// The silent all-clear. With no USDT identity in the registry the quote-leg /// filter matches nothing, both counts read zero, and a `NOT_BREACHING` alarm /// would score a check that never ran as perfectly healthy. `resolved_legs` -/// exists so `sanity_metrics` can refuse it, and `main.rs` fails the invocation. +/// exists so `peg_metric` can refuse it, and `main.rs` fails the invocation. #[tokio::test] #[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] async fn an_unresolvable_usdt_leg_reads_as_zero_and_is_therefore_refused() { let c = client(); - exec(&c, &format!("TRUNCATE TABLE prices.{SANITY_TABLE}")).await; - exec(&c, "TRUNCATE TABLE prices.assets").await; + reset_sanity_tables(&c).await; // Deliberately no USDT identity seeded. - insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "100").await; + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; - let counts = read_counts(&c).await; + let counts = read_peg(&c).await; assert_eq!(counts.resolved_legs, 0); assert_eq!( counts.peg_applied, 0, "a real defect is invisible without the identity — hence the refusal" ); assert_eq!( - sanity_metrics(&counts), - Err(SanityRefusal::UnresolvableLeg { resolved_legs: 0 }), + peg_metric(&counts), + Err(SanityRefusal::UnresolvableLeg { + resolved_legs: 0, + table: PEG_TABLE + }), "this reading must never be published as healthy" ); } +/// 🔴 **The defect task 0213 exists to close, induced rather than reasoned +/// about.** +/// +/// Reproduces the exact production state measured on 2026-08-20: `price_ohlcv_1m` +/// carries peg-valued rows (1,564,045 of them) while every coarse tier reads +/// clean, because task 0182's repair wrote the coarse tables **directly** and +/// never touched the tier they roll from (task 0212). +/// +/// Before this task the peg direction read `_1h` and would have published a +/// confident **0** over that population. The assertion that matters is the +/// second one: the tier the check used to read shows nothing wrong. +#[tokio::test] +#[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] +async fn a_peg_row_only_in_1m_is_counted_although_every_coarse_tier_reads_clean() { + let c = client(); + reset_sanity_tables(&c).await; + seed_usdt_identity(&c, 111).await; + + // The tier enrichment writes: the peg re-applied. + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; + // The repaired coarse tier: the SAME candle, correctly valued at ~0.15 — + // which is precisely what 0182's repair left behind. + insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "15").await; + + assert_eq!( + read_peg(&c).await.peg_applied, + 1, + "the peg direction must see the tier enrichment writes" + ); + + // ⚠️ The regression this pins. A check reading the repaired tier sees a + // healthy leg and publishes zero — the silent all-clear, from a scan that + // really did run and really did examine rows. + let stranded = read_stranded(&c).await; + assert_eq!( + stranded.scanned, 1, + "the coarse tier was genuinely examined — this is not an empty scan" + ); + assert_eq!( + stranded.stranded, 0, + "and it reads perfectly healthy, which is exactly why the peg direction \ + cannot live here" + ); +} + +/// The two directions must not be able to see each other's rows. Pinned because +/// the tempting simplification — one query over one tier — is what made the peg +/// direction blind, and a future "let's just union them" would restore it. +#[tokio::test] +#[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] +async fn each_direction_only_scans_its_own_tier() { + let c = client(); + reset_sanity_tables(&c).await; + seed_usdt_identity(&c, 111).await; + + // Rows in `_1m` only. + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; + assert_eq!(read_peg(&c).await.scanned, 1); + assert_eq!( + read_stranded(&c).await.scanned, + 0, + "the stranded direction must not see _1m rows" + ); + + // Rows in `_1h` only. + // + // ⚠️ Seeded at 3 HOURS, not 3 days, and that is the whole point of the + // assertion. A 3-day-old row falls outside the 48 h peg window whichever + // table the peg direction reads, so it would pass with `PEG_TABLE` reverted + // to `price_ohlcv_1h` — testing the window instead of the tier. Inside the + // peg window, only the tier can explain a zero. + reset_sanity_tables(&c).await; + seed_usdt_identity(&c, 111).await; + insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "0").await; + assert_eq!(read_stranded(&c).await.scanned, 1); + assert_eq!( + read_peg(&c).await.scanned, + 0, + "the peg direction must not see _1h rows" + ); +} + +/// ⚠️ **The window bound the peg direction does not inherit.** `_1m` is +/// retention-managed at 7 days, so the peg scan keeps a wide margin below that +/// frontier (48 h) rather than reusing the stranded direction's 7 days. A row +/// older than the peg window is out of scope even though it is still in the +/// table — which is the property that makes a cleanup run unable to move the +/// count. +#[tokio::test] +#[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] +async fn the_peg_window_excludes_rows_a_cleanup_run_could_delete() { + let c = client(); + reset_sanity_tables(&c).await; + seed_usdt_identity(&c, 111).await; + + // Inside the 48 h peg window — counted. + insert_usdt_minute_candle(&c, 111, 5, "now() - INTERVAL 3 HOUR", "100", "100").await; + // Older than the peg window but still well inside `_1m`'s 7-day retention, + // i.e. exactly the band a widened window would have picked up and a cleanup + // run could then remove underneath it. + insert_usdt_minute_candle(&c, 111, 6, "now() - INTERVAL 5 DAY", "100", "100").await; + + let peg = read_peg(&c).await; + assert_eq!(peg.scanned, 1, "only the in-window row is examined"); + assert_eq!( + peg.peg_applied, 1, + "a defect outside the window is task 0212's population, not this alarm's" + ); +} + +/// ⚠️ A `_1m` scan that matched nothing must not suppress a working `_1h` +/// reading. Before task 0213 one refusal killed both metrics — harmless while +/// they came from one query, and the muting failure from the other side once +/// they read different tiers. +#[tokio::test] +#[ignore = "requires a local ClickHouse (docker compose up -d clickhouse)"] +async fn an_empty_peg_scan_does_not_suppress_the_stranded_metric() { + let c = client(); + reset_sanity_tables(&c).await; + seed_usdt_identity(&c, 111).await; + + // `_1m` is empty; `_1h` carries a real stranded candle. + insert_usdt_candle(&c, 111, 5, "now() - INTERVAL 3 DAY", "100", "0").await; + + let peg = read_peg(&c).await; + assert_eq!( + peg_metric(&peg), + Err(SanityRefusal::EmptyScan { + table: PEG_TABLE, + lookback_seconds: rollup_freshness_probe::usd_sanity::PEG_LOOKBACK_SECONDS, + }), + "an unexamined tier must be refused, not published as zero" + ); + + let stranded = read_stranded(&c).await; + assert_eq!( + stranded_metric(&stranded) + .expect("published on its own evidence") + .value, + 1.0, + "the working direction must still publish" + ); +} + // --------------------------------------------------------------------------- // Task 0204, gap 3 — materialized-view drift on a schedule. // ---------------------------------------------------------------------------