Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 44 additions & 11 deletions infra/src/lib/stacks/observability-stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ export interface ObservabilityStackProps extends cdk.StackProps {
* collide on `alarmName` and mask the real defect instead of fixing it.
*/
export interface WorkerHealthAlarms {
/** Duration approaching the configured timeout — warns BEFORE it becomes errors. */
readonly duration: cloudwatch.Alarm;
/**
* Duration approaching the configured timeout — warns BEFORE it becomes
* errors. Absent for a worker declared with `noDurationAlarm`.
*/
readonly duration?: cloudwatch.Alarm;
/** Zero invocations — the worker is not running at all. */
readonly noInvocations: cloudwatch.Alarm;
}
Expand All @@ -68,6 +71,17 @@ interface WorkerHealthAlarmProps {
readonly cadence: cdk.Duration;
/** Appended to each alarm description: what breaks when this worker stops. */
readonly impact: string;
/**
* Set — to the REASON — for a worker whose run length is a budget rather
* than a symptom, and skip the duration alarm for it. The 80%-of-timeout
* threshold assumes a run that grows only when something is wrong; a worker
* that deliberately walks until a wall-clock budget sits at that threshold
* every single run, and the alarm would fire permanently (task 0223 found
* this on supply: budget 240 s, timeout 300 s, threshold 240 000 ms). The
* liveness alarm is unaffected — it is the one that matters for such a
* worker anyway.
*/
readonly noDurationAlarm?: string;
}

/**
Expand Down Expand Up @@ -115,7 +129,15 @@ function addWorkerHealthAlarms(
snsAction: cw_actions.SnsAction,
props: WorkerHealthAlarmProps,
): WorkerHealthAlarms {
const { name, idPrefix, functionName, timeout, cadence, impact } = props;
const {
name,
idPrefix,
functionName,
timeout,
cadence,
impact,
noDurationAlarm,
} = props;

const metric = (
metricName: string,
Expand All @@ -133,11 +155,13 @@ function addWorkerHealthAlarms(
// 80% of the timeout. A worker creeping toward its limit is the leading
// indicator; once it crosses, every run fails and the errors alarm is
// reporting an outage that already started.
//
// Skipped, with the reason recorded at the call site, for a worker whose run
// length is a budget by design — see `noDurationAlarm`.
const durationThresholdMs = Math.floor(timeout.toMilliseconds() * 0.8);
const duration = new cloudwatch.Alarm(
scope,
`${idPrefix}WorkerDurationAlarm`,
{
let duration: cloudwatch.Alarm | undefined;
if (noDurationAlarm === undefined) {
duration = new cloudwatch.Alarm(scope, `${idPrefix}WorkerDurationAlarm`, {
alarmName: `prices-${envName}-${name}-duration-near-timeout`,
alarmDescription: `The ${name} worker is running at ≥80% of its ${timeout.toHumanString()} Lambda timeout (Duration.Maximum ≥ ${durationThresholdMs} ms for two consecutive periods). It has not failed yet, but it is trending at the wall and will start timing out. ${impact} Investigate before it becomes an outage — this is the warning enrichment did not have in 2026-07. OK also means "nothing ran" — liveness is prices-${envName}-${name}-no-invocations (task 0223).`,
metric: metric('Duration', 'Maximum', cadence),
Expand All @@ -147,10 +171,10 @@ function addWorkerHealthAlarms(
comparisonOperator:
cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
},
);
duration.addAlarmAction(snsAction);
duration.addOkAction(snsAction);
});
duration.addAlarmAction(snsAction);
duration.addOkAction(snsAction);
}

// Three cadences of total silence: the schedule rule was disabled, deleted,
// or is failing to invoke. Lambda publishes NO Invocations datapoint for a
Expand Down Expand Up @@ -1534,6 +1558,15 @@ export class ObservabilityStack extends cdk.Stack {
cadence: cdk.Duration.hours(1),
impact:
'market_cap_usd in current_prices is price × token_supply from prices.asset_supply, whose only writer is this worker: market caps go stale silently while prices keep moving.',
// Measured 2026-09-15 before the first deploy: Duration.Maximum is
// ~240.5 s EVERY run, because the Horizon walk stops at
// DEFAULT_TIME_BUDGET_SECS = 240 (supply-worker/src/lib.rs, task 0084)
// and the timeout is 300 s — exactly the 80% threshold. A duration
// alarm here would have latched on its second evaluation and been
// re-surfaced by the 0214 digest every day: the failure 0223 exists
// to remove, self-inflicted.
noDurationAlarm:
'run length is a wall-clock budget (240 s of a 300 s timeout), not a symptom',
},
{
name: 'enrichment',
Expand Down
Loading