diff --git a/.claude/skills/commit/SKILL.md b/.claude/skills/commit/SKILL.md deleted file mode 100644 index f3a574f8..00000000 --- a/.claude/skills/commit/SKILL.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -description: Create git commits with properly formatted messages using (type): message convention. Use when committing code changes, staging files, or finalizing work. Analyzes diffs to determine commit type (feat, fix, docs, refactor, test, chore, style), writes concise subject lines (max 80 chars), and adds descriptive body (max 300 chars). Not for pushing to remote or creating PRs. ---- - -# Commit - -Create a git commit with a properly formatted message. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## Steps - -1. Run `git status` to see staged and unstaged changes -2. Run `git diff --cached` to review staged changes -3. If no changes are staged, ask the user what to stage -4. Analyze the changes and determine the commit type: - - `feat`: New feature - - `fix`: Bug fix - - `docs`: Documentation only - - `refactor`: Code refactoring - - `test`: Adding or updating tests - - `chore`: Maintenance tasks - - `style`: Code style/formatting changes -5. Write a commit message following this format: - - First line: `(type): short description` (max 80 chars) - - Blank line - - Body: Longer description explaining what and why (max 300 chars) -6. Create the commit -7. **Do NOT include any AI model attribution in the commit message** - -## Format - -``` -(type): short description - -Longer explanation of what changed and why (max 300 characters). -``` - -## Rules - -- **Type**: Must be one of `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `style` -- **Short description**: Max 80 characters, imperative mood ("add" not "added") -- **Long description**: Max 300 characters, explain what and why -- **No AI attribution**: Do not include Co-Authored-By or any mention of AI - -## Examples - -``` -(feat): add user authentication endpoint - -Implements JWT-based authentication with refresh tokens. Includes login, logout, and token refresh routes. -``` - -``` -(fix): resolve null pointer in payment processing - -Handles edge case where customer payment method is undefined during checkout flow. -``` - -## Related Skills -- [[pr]] — commits feed into pull requests -- [[pr-review]] — reviewers check commit quality -- [[naming]] — file naming affects commit scope - -## Self-Improvement - -After completing this skill, if you discovered: -- A check that should be added -- A better review approach -- An edge case not covered - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync \ No newline at end of file diff --git a/.claude/skills/health/SKILL.md b/.claude/skills/health/SKILL.md deleted file mode 100644 index 7c645d29..00000000 --- a/.claude/skills/health/SKILL.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -description: "Audit skill system health and consistency. TRIGGER when asked to check skill health, validate skills, or when starting work on the minions repo itself. Checks for stale references, missing files, cross-skill consistency, hook validity, and repo-scope correctness. Not for improving individual skills (use /improve) or creating new skills (use /new-skill)." ---- - -# Health — Skill System Audit - -Diagnose issues across the entire skill system: stale references, missing files, broken cross-links, hook config problems, and sync inconsistencies. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## When to Run - -- Periodically (e.g., after major skill changes) -- When skills aren't behaving as expected -- After deleting or renaming skills -- When onboarding a new repo to the skill system -- When asked: "check health", "validate skills", "audit skills" - -## Checks - -Run all checks below. Report findings as a table with severity (ERROR, WARN, INFO). - -### 1. Skill File Integrity - -For every skill in `~/Desktop/Riverline/minions/skills/`: -- [ ] Has `SKILL.md` with valid frontmatter (`---\ndescription: ...\n---`) -- [ ] Has `LEARNINGS.md` -- [ ] `SKILL.md` has a `## Related Skills` section with [[wikilinks]] -- [ ] `SKILL.md` has a Self-Improvement section -- [ ] Description includes trigger conditions AND exclusions ("Not for...") - -```bash -# Find skills missing LEARNINGS.md or proper frontmatter -for skill_dir in ~/Desktop/Riverline/minions/skills/*/*/; do - skill_name=$(basename "$skill_dir") - scope=$(basename "$(dirname "$skill_dir")") - [ ! -f "$skill_dir/SKILL.md" ] && echo "ERROR: $scope/$skill_name missing SKILL.md" - [ ! -f "$skill_dir/LEARNINGS.md" ] && echo "ERROR: $scope/$skill_name missing LEARNINGS.md" - head -1 "$skill_dir/SKILL.md" 2>/dev/null | grep -q "^---" || echo "WARN: $scope/$skill_name SKILL.md missing frontmatter" -done -``` - -### 2. Cross-Skill References & Wikilinks - -Check that skills referencing other skills point to skills that exist: -- Search all SKILL.md files for `[[wikilinks]]` and verify each linked skill exists -- Search for patterns like `/skillname`, `skills/{scope}/{name}`, or "invoke the X skill" -- Flag any references to deleted/renamed skills -- Flag skills missing `## Related Skills` section entirely - -```bash -# Check wikilinks point to real skills -grep -rn '\[\[' ~/Desktop/Riverline/minions/skills/ --include="*.md" | while read line; do - linked=$(echo "$line" | grep -o '\[\[[^]]*\]\]' | tr -d '[]') - for skill in $linked; do - found=false - for scope_dir in ~/Desktop/Riverline/minions/skills/*/; do - [ -d "$scope_dir/$skill" ] && found=true && break - done - $found || echo "ERROR: Wikilink [[$skill]] points to non-existent skill — in $(echo "$line" | cut -d: -f1)" - done -done - -# Check for missing Related Skills sections -for skill_dir in ~/Desktop/Riverline/minions/skills/*/*/; do - skill_name=$(basename "$skill_dir") - scope=$(basename "$(dirname "$skill_dir")") - grep -q "## Related Skills" "$skill_dir/SKILL.md" 2>/dev/null || echo "WARN: $scope/$skill_name missing Related Skills section" -done -``` - -### 3. Repo-Scope Validity - -For each repo with a `.claude/repo-scope` file: -- [ ] Every scope listed has a corresponding directory in `minions/skills/` -- [ ] `shared` is NOT listed (it's always included automatically) - -```bash -# Check all repo-scope files -RIVERLINE_DIR=~/Desktop/Riverline -for repo_dir in "$RIVERLINE_DIR"/*/; do - scope_file="$repo_dir/.claude/repo-scope" - [ ! -f "$scope_file" ] && continue - repo_name=$(basename "$repo_dir") - while IFS= read -r scope || [ -n "$scope" ]; do - scope=$(echo "$scope" | tr -d '[:space:]') - [ -z "$scope" ] && continue - [ "$scope" = "shared" ] && echo "WARN: $repo_name repo-scope lists 'shared' (unnecessary, always included)" - [ ! -d ~/Desktop/Riverline/minions/skills/"$scope" ] && echo "ERROR: $repo_name repo-scope references non-existent scope '$scope'" - done < "$scope_file" -done -``` - -### 4. Sync Consistency - -Compare skills in each repo's `.claude/skills/` against what minions should be syncing: -- [ ] No extra skills that aren't in minions source -- [ ] No missing skills that should have been synced -- [ ] No stale copies (source newer than target) - -```bash -# Compare synced vs source for a repo -repo_dir="$PWD" # or specify -for skill_dir in "$repo_dir/.claude/skills"/*/; do - skill_name=$(basename "$skill_dir") - # Check if source exists in any expected scope - found=false - for scope_dir in ~/Desktop/Riverline/minions/skills/shared ~/Desktop/Riverline/minions/skills/*/; do - [ -d "$scope_dir/$skill_name" ] && found=true && break - done - $found || echo "WARN: $skill_name in .claude/skills/ but not found in minions source" -done -``` - -### 5. Hook Configuration - -Check `~/.claude/settings.json`: -- [ ] SessionStart hook exists and points to a valid script -- [ ] Hook script is executable -- [ ] Hook timeout is reasonable (10-30s) - -```bash -# Verify hook setup -settings=~/.claude/settings.json -[ ! -f "$settings" ] && echo "ERROR: ~/.claude/settings.json missing" -grep -q "SessionStart" "$settings" 2>/dev/null || echo "ERROR: No SessionStart hook configured" -[ -x ~/.claude/hooks/sync-minions.sh ] || echo "ERROR: sync-minions.sh not executable" -``` - -### 6. Stale Content Detection - -Search all skill files for known stale patterns: -- References to `plugins/` or `/plugin` commands -- References to `marketplace` -- Status codes that don't match sync script output (`content_updated`, `structural_change`) -- References to skills that no longer exist - -```bash -# Check for stale patterns -grep -rn "plugin" ~/Desktop/Riverline/minions/skills/ --include="*.md" -i | grep -v "claude-plugin" | grep -v "Plugin for" -grep -rn "marketplace" ~/Desktop/Riverline/minions/skills/ --include="*.md" -i -grep -rn "content_updated\|structural_change" ~/Desktop/Riverline/minions/skills/ --include="*.md" -``` - -## Output Format - -``` -# Skill System Health Report - -## Summary -- X errors, Y warnings, Z info -- Last sync: {timestamp from sync script} - -## Findings - -| Severity | Check | Issue | Location | -|----------|-------|-------|----------| -| ERROR | Cross-Ref | Skill 'review' referenced but doesn't exist | skills/shared/improve/SKILL.md:121 | -| WARN | Stale | References '/plugin' command | skills/shared/foo/SKILL.md:45 | -| INFO | Sync | 2 skills newer in source than target | torrent/.claude/skills/ | - -## Recommended Actions -1. ... -2. ... -``` - -## After Running - -If issues are found: -1. Fix ERRORs immediately (broken references, missing files) -2. Fix WARNs when convenient (stale content, unnecessary config) -3. Log INFOs in LEARNINGS.md if they represent known patterns -4. Invoke `/improve` for any skill that needs updating - -## Related Skills -- [[improve]] — health finds issues, improve fixes them - -## Self-Improvement - -After completing this skill, if you discovered: -- New stale patterns to check for -- False positives that should be excluded -- Additional consistency checks worth adding - -Then invoke `/improve` to update this skill. - - ---- - -# Accumulated Learnings - -> Auto-merged from LEARNINGS.md. Apply these edge cases, patterns, and preferences when executing this skill. - - - -## Known False Positives - -- `plugin` in text like "Claude Code Plugin" or "claude-plugin" directory references are not stale -- `/improve` and `/new-skill` references in Self-Improvement sections are intentional cross-refs -- `skills/shared/improve/SKILL.md` references scope examples that may not all exist yet - -## Edge Cases - -_None documented yet_ - -## Patterns - -- After deleting a skill, always grep for its name across all other skills -- `grep -qw` can cause false matches (e.g., "review" matching "pr-review") — use space-delimited exact matching -- repo-scope files without trailing newlines cause the last scope to be silently dropped by `read` diff --git a/.claude/skills/improve/SKILL.md b/.claude/skills/improve/SKILL.md deleted file mode 100644 index 3809d543..00000000 --- a/.claude/skills/improve/SKILL.md +++ /dev/null @@ -1,197 +0,0 @@ ---- -description: "Auto-improve skills when learnings are discovered. TRIGGER PROACTIVELY when: (1) user provides new rules, standards, preferences, or corrections, (2) you discover a workaround or better pattern than what a skill documents, (3) you correct your own mistake on something a skill handles, (4) you encounter unexpected behavior that a skill should account for, (5) a skill has wrong or missing instructions, (6) the post-skill reflection hook flags a learning. Edits SKILL.md for core changes or LEARNINGS.md for edge cases. Follows [[Related Skills]] wikilinks to propagate updates to connected skills. Commits and pushes to minions repo. Not for creating new skills (use /new-skill)." ---- - -# Improve Skill - -Update a skill based on new learnings discovered during usage. - -## Context - -Before executing, read `LEARNINGS.md` in this skill folder for additional context and patterns. - -**Note:** LEARNINGS.md content is auto-merged into the synced SKILL.md during session start. When Claude invokes a skill, it reads both the instructions AND accumulated learnings in one file. Source files in `minions/skills/` stay separate — always edit the source LEARNINGS.md, never the synced copy. - -## When to Use - -**IMPORTANT: Trigger this skill PROACTIVELY.** Do NOT wait for the user to explicitly say "update the skill." - -### Auto-trigger signals (invoke immediately when any occur): - -**From user feedback:** -- User provides a new convention, rule, or standard -- User corrects your behavior on something a skill handles -- User says "always do X" or "never do Y" about a skill-covered topic - -**From your own discovery:** -- You find a workaround for something that should work differently -- You correct your own approach mid-task (the wrong approach was what a skill taught you) -- You discover a skill has wrong, missing, or outdated instructions -- You find a better pattern than what's documented in a skill -- You hit an edge case not covered by an existing skill - -**From the reflection hook:** -- After a skill finishes, the post-skill hook asks: "did you discover anything?" -- If yes, invoke this skill immediately - -**Self-triggering examples:** -- User says "commits should use (type): message format" → Immediately update the commit skill -- You used `/debug` and found the common issues table was missing a pattern → Update debug skill -- You used `/naming` and realized scheduler files have a new convention → Update naming skill -- User says "PR reviews should check for X" → Update the pr-review skill -- You discovered that a deleted skill is still referenced by another → Fix the reference - -**Do NOT** just acknowledge the feedback and wait. Act on it immediately. - -## Skill File Structure - -Skills are organized in the minions repo by scope: -``` -~/Desktop/Riverline/minions/skills/ -├── shared/ # Skills for all repos -│ └── {skill-name}/ -│ ├── SKILL.md -│ └── LEARNINGS.md -├── torrent/ # Torrent-specific skills -│ └── {skill-name}/ -│ ├── SKILL.md -│ └── LEARNINGS.md -├── seabird/ # Seabird workflow skills -└── infosec/ # Security & compliance skills -``` - -## Critical Rules - -**ALWAYS edit the source files in the minions repo:** -- Source: `~/Desktop/Riverline/minions/skills/{scope}/{skill-name}/` -- NEVER edit `.claude/skills/` in project repos — those are copies synced by the hook -- Changes to project copies will be OVERWRITTEN on next session start - -## Steps - -1. **Identify the skill to update** - - Determine which skill the learning applies to - - Find it in: `~/Desktop/Riverline/minions/skills/{shared|torrent|seabird|infosec}/{skill}/` - -2. **Read current skill files** - - Read both `SKILL.md` and `LEARNINGS.md` - - Understand current instructions and existing learnings - -3. **Decide where to add the improvement** - - | Type of Learning | Where to Add | - |------------------|--------------| - | Edge case discovered | LEARNINGS.md → Edge Cases | - | User preference | LEARNINGS.md → User Preferences | - | Pattern noticed | LEARNINGS.md → Patterns | - | Core instruction change | SKILL.md (update relevant section) | - | New step required | SKILL.md (update Steps section) | - | Format/rule change | SKILL.md (update Rules section) | - -4. **Apply the improvement** - - Edit the appropriate file - - For LEARNINGS.md: Add under correct section with date - - For SKILL.md: Keep changes minimal and focused - - Update LEARNINGS.md changelog for any change - - **Verify `## Related Skills` exists** — if missing, add [[wikilinks]] to connected skills - -5. **Commit and push** - ```bash - cd ~/Desktop/Riverline/minions - - # Pull latest changes first - git pull origin main - - # Stage and commit - git add skills/ - git commit -m "(chore): update {skill-name} skill - {brief description} - - {What was added/changed and why - max 300 chars}" - git push - ``` - -6. **Propagate to related skills (reweave) — MANDATORY** - This step is NOT optional. Skipping it causes skill drift. - - [ ] Read the `## Related Skills` section of the skill you just updated - - [ ] For EACH [[linked skill]]: - - Read its SKILL.md - - Determine if the change affects it (shared concept, overlapping domain, upstream/downstream) - - If yes: apply the same learning to that skill - - If no: skip (but you MUST check, not assume) - - [ ] List the skills you checked and your decision for each in the commit message - -7. **Notify user** - - Tell user what was updated and in which file(s) - - List related skills checked and which ones were also updated - - Skills will auto-sync to all repos on next Claude session start (learnings are merged into SKILL.md during sync) - -## Commit Message Format - -``` -(chore): update {skill-name} skill - {brief description} - -{What was added/changed and why it improves the skill - max 300 chars} -``` - -**Rules:** -- Type: Use `chore` for skill improvements -- Subject: Max 80 characters -- Description: Max 300 characters -- No AI attribution - -## Examples - -### Example 1: Edge case (goes to LEARNINGS.md) - -User: "When reviewing PRs, also check for console.log statements left in code" - -Actions: -1. Read `skills/shared/pr-review/SKILL.md` and `LEARNINGS.md` -2. This is an edge case → Add to LEARNINGS.md under "Edge Cases" -3. Add entry: - ``` - ### 2026-01-23 - - Check for leftover console.log statements in PR reviews - ``` -4. Update changelog in LEARNINGS.md -5. Commit and push -6. Notify user - -### Example 2: Core instruction change (goes to SKILL.md) - -User: "Commits should follow this format: (type): message with max 80 chars" - -Actions: -1. Read `skills/shared/commit/SKILL.md` and `LEARNINGS.md` -2. This is a format/rule change → Update SKILL.md -3. Update the Format and Rules sections in SKILL.md -4. Add changelog entry in LEARNINGS.md -5. Commit and push -6. Notify user - -### Example 3: User preference (goes to LEARNINGS.md) - -User: "I prefer verbose commit messages with context" - -Actions: -1. Read `skills/shared/commit/SKILL.md` and `LEARNINGS.md` -2. This is a user preference → Add to LEARNINGS.md under "User Preferences" -3. Add entry: - ``` - ### 2026-01-23 - - User prefers verbose commit messages with additional context - ``` -4. Commit and push -5. Notify user - -## Related Skills -- [[new-skill]] — improve updates existing skills, new-skill creates them -- [[health]] — health audits find issues that improve fixes - -## Self-Improvement - -If you discover improvements to THIS skill while using it: -1. Add to `LEARNINGS.md` in this folder -2. Update `SKILL.md` if needed -3. Commit and push -4. Notify user diff --git a/.claude/skills/linear/SKILL.md b/.claude/skills/linear/SKILL.md deleted file mode 100644 index 0be9e69f..00000000 --- a/.claude/skills/linear/SKILL.md +++ /dev/null @@ -1,424 +0,0 @@ ---- -description: Standardize how the team interacts with Linear from Claude Code. Use when creating issues, updating issue status, querying issues, managing projects, or any Linear operation. Enforces consistent naming, labeling, parent/sub-issue structure, status transitions, and branch naming. Invoked automatically by /plan, /pr, and /pr-review when they interact with Linear. Also use directly when engineer says "create an issue", "update issue", "move issue to done", "what's in progress", or any Linear-related request. ---- - -# Linear - -Standardize all Linear interactions from Claude Code — issue creation, status updates, queries, and conventions. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## Core Principles - -``` -1. CONSISTENCY - Every engineer's agent creates issues the same way -2. TRACEABILITY - Issues link to PRs, plan docs, and branches -3. MINIMAL OVERHEAD - Linear tracks work, it doesn't create work -4. AGENT-FRIENDLY - Issue descriptions are detailed enough for agents to execute -``` - ---- - -## Workspace Structure - -### Teams - -| Team | Purpose | Issue Prefix | -|------|---------|-------------| -| **Engineering** | All development work | `ENG-` | -| **Operations** | Non-engineering operational work | `OPS-` | - -### Issue Statuses (Engineering) - -``` -Backlog → Todo → In Progress → In Review → Done - ↘ Canceled - ↘ Duplicate -``` - -| Status | When to Use | -|--------|-------------| -| **Backlog** | Idea captured, not yet prioritized | -| **Todo** | Prioritized, ready to be picked up | -| **In Progress** | Engineer is actively working on it | -| **In Review** | PR opened, awaiting review | -| **Done** | PR merged, feature shipped | -| **Canceled** | Decided not to do | -| **Duplicate** | Already exists as another issue | - -### Labels - -| Label | When to Use | -|-------|-------------| -| **Feature** | New functionality | -| **Bug** | Something broken | -| **Improvement** | Enhancement to existing feature | -| **Need More Clarity** | Requirements are unclear, needs discussion | - -### Priority - -| Priority | Meaning | -|----------|---------| -| **Urgent** (1) | Drop everything, fix now | -| **High** (2) | Do this week | -| **Normal** (3) | Do this sprint/cycle | -| **Low** (4) | Nice to have, when time permits | - ---- - -## Issue Conventions - -### Issue Title Format - -``` -{Action verb} {what} {context if needed} -``` - -**Action verbs:** Add, Create, Build, Implement, Fix, Update, Remove, Refactor, Migrate, Integrate - -**Examples:** -- `Add call analysis queue and worker` -- `Fix duplicate webhook processing in call handler` -- `Update customer schema with callAnalysis fields` -- `Integrate transcription API for call recordings` - -**Bad titles:** -- `Call analysis` — too vague -- `Bug` — no description -- `WIP: maybe add something` — not actionable -- `ENG-123 follow-up` — meaningless without context - -### Issue Description Format - -#### For Parent Issues (Features) - -```markdown -## Overview -{What this feature does and why, 2-3 sentences} - -## Plan -See: `docs/plans/{feature-name}.md` - -## Flow -1. {Step 1} -2. {Step 2} -3. {Step 3} - -## Decisions -| Decision | Choice | Reasoning | -|----------|--------|-----------| -| {what} | {choice} | {why} | - -## Sub-Tasks -- [ ] {Sub-task 1} -- [ ] {Sub-task 2} -- [ ] {Sub-task 3} -``` - -#### For Sub-Issues (Tasks) - -```markdown -## What -{Detailed description of what to implement} - -## Files -- CREATE: `path/to/new/file.ts` -- MODIFY: `path/to/existing/file.ts` - -## Dependencies -- Depends on: ENG-{number} (must be done first) - -## Pattern Reference -Follow pattern in: `path/to/reference/file.ts` - -## Acceptance Criteria -- [ ] {Specific verifiable outcome} -- [ ] {Specific verifiable outcome} -``` - -#### For Bug Issues - -```markdown -## Bug -{What's happening vs what should happen} - -## Reproduction -1. {Step to reproduce} -2. {Step to reproduce} -3. {Observe: bug} - -## Expected Behavior -{What should happen instead} - -## Root Cause -{If known, otherwise "To investigate"} - -## Impact -{Who/what is affected, severity} -``` - ---- - -## Parent/Sub-Issue Structure - -### When to Use Sub-Issues - -Use sub-issues when a feature has **3 or more distinct tasks** that: -- Can be worked on sequentially -- Each has clear completion criteria -- Track progress within a single feature - -**Important:** Sub-issues are for **tracking progress**, not for separate PRs. All sub-tasks ship together in **ONE PR** per feature. - -### Structure - -``` -ENG-100: Add call analysis system (parent — Feature label) -├── ENG-101: Add callAnalysis fields to Call schema -├── ENG-102: Create call-analysis BullMQ queue and worker -├── ENG-103: Add webhook handler for recording-ready events -├── ENG-104: Integrate transcription API service -├── ENG-105: Build LLM analysis service for call insights -├── ENG-106: Wire analysis pipeline end-to-end -└── ENG-107: Add API endpoint to fetch analysis results -``` - -### Rules - -- Parent issue has the **Feature** label and links to the plan doc -- Sub-issues inherit the parent's **project** and **priority** -- Sub-issues are ordered by dependency (earlier = do first) -- Sub-issues have detailed descriptions (agent-executable) -- **All sub-issues are worked on the SAME feature branch** -- **ONE PR covers all sub-issues** — sub-issues are moved to Done when their code is written -- When PR is opened → parent moves to In Review -- When PR is merged → parent moves to Done - ---- - -## Branch Naming - -Branches are derived from the **parent** issue identifier: - -``` -feature/{parent-issue-id}-{short-description} -fix/{issue-id}-{short-description} -``` - -**Examples:** -- `feature/ENG-100-call-analysis` (feature branch — all sub-tasks here) -- `fix/ENG-150-duplicate-webhook` - -**Rules:** -- Use the **parent issue ID** for feature branches (not sub-issue IDs) -- All sub-tasks are worked on the SAME branch -- Keep description to 3-4 words, kebab-case -- Always start with `feature/` or `fix/` - ---- - -## Status Transitions - -### When to Transition - -| Action | Parent Issue | Sub-Issues | -|--------|-------------|------------| -| Engineer starts working | Todo → In Progress | First sub-issue → In Progress | -| Sub-task code written | — | Sub-issue → Done | -| Feature PR opened | In Progress → In Review | All remaining → Done | -| PR has requested changes | In Review → In Progress | — | -| PR merged | In Review → Done | — | -| Work paused/blocked | In Progress → Todo | — | -| Feature abandoned | Any → Canceled | All → Canceled | - -### How to Transition - -Use Linear MCP tools: - -``` -# Move issue to In Progress -Use update_issue with state: "In Progress" - -# Move issue to In Review -Use update_issue with state: "In Review" - -# Move issue to Done -Use update_issue with state: "Done" -``` - -### Auto-Transitions by Other Skills - -| Skill | Transition | -|-------|-----------| -| `/plan` | Creates parent + sub-issues in **Backlog** or **Todo**, moves parent to **In Progress** when ready | -| `/pr` | Moves parent to **In Review**, all sub-issues to **Done** | -| `/pr-review` (approved + merged) | Moves parent to **Done** | - ---- - -## Common Operations - -### Create an Issue - -``` -1. Determine team (usually Engineering) -2. Determine label (Feature, Bug, Improvement) -3. Determine priority (ask engineer if unclear) -4. Determine project (ask engineer if unclear) -5. Write title following conventions -6. Write description following format -7. Use create_issue MCP tool -8. Report issue ID to engineer -``` - -### Query Issues - -``` -# My issues -Use list_issues with assignee: "me" - -# Issues in a project -Use list_issues with project: "{project name}" - -# Issues in progress -Use list_issues with state: "In Progress", team: "Engineering" - -# Search for issues -Use list_issues with query: "{search term}" -``` - -### Update an Issue - -``` -# Change status -Use update_issue with state: "{new status}" - -# Add comment (e.g., PR link, blocker, update) -Use create_comment with issueId and body -``` - -### Create Sub-Issues - -``` -1. Create parent issue first, note the ID -2. Create each sub-issue with parentId set to parent's ID -3. Sub-issues inherit team and project from parent -4. Set priority same as parent -``` - ---- - -## Projects - -### Active Projects - -| Project | Description | Lead | -|---------|-------------|------| -| **Torrent** | Core collection system | Jigyansu | -| **Shaastris** | New project | - | -| **Harbor** | Evals for Riverline | Vidhan | -| **Crew** | Team project | Jayanth | - -### When to Use Projects - -- Every feature issue should belong to a project -- Bug issues belong to the project they affect -- If unsure which project, ask the engineer - ---- - -## Linear MCP Tool Reference - -| Action | MCP Tool | -|--------|----------| -| Create issue | `create_issue` | -| Update issue | `update_issue` | -| List issues | `list_issues` | -| Get issue details | `get_issue` | -| Add comment | `create_comment` | -| List projects | `list_projects` | -| List teams | `list_teams` | -| List statuses | `list_issue_statuses` | -| List labels | `list_issue_labels` | - ---- - -## Anti-Patterns - -| Don't | Do Instead | -|-------|------------| -| Create issues without descriptions | Always include structured description | -| Use vague titles like "Fix bug" | Specific: "Fix duplicate webhook in call handler" | -| Skip linking to plan doc | Always link if plan doc exists | -| Leave parent issue open when all sub-issues done | Close parent when all children are done | -| Create issues in wrong team | Engineering for dev work, Operations for ops | -| Forget to set project | Always assign a project | -| Create duplicate issues | Search first using list_issues or query | - ---- - -## Related Skills -- [[pr]] — PRs link to Linear issues -- [[pr-review]] — review status syncs to Linear -- [[plan]] — planning references Linear issues - -## Self-Improvement - -After completing this skill, if you discovered: -- A new convention needed for issues -- A better description format -- A missing status transition rule -- A new project or label to document - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync - - ---- - -# Accumulated Learnings - -> Auto-merged from LEARNINGS.md. Apply these edge cases, patterns, and preferences when executing this skill. - - - -## Edge Cases - -- Linear MCP uses team names (e.g., "Engineering") not IDs for most operations. -- When creating sub-issues, the parentId must be the issue's UUID, not the display ID (ENG-123). Use get_issue to resolve the UUID first if needed. - -## Workspace Details - -### Teams -- Engineering (ID: c1a0e8f5-a0e0-479b-8b87-a018db6e4dcb) -- Operations (ID: fab0bf2f-56af-482f-8a54-9761cc7ae6a0) - -### Engineering Labels -- Feature (ID: 4bb389a9-f982-4c3b-8503-4198550914f7) -- Bug (ID: c826ee04-45c3-48c5-bf56-8c211cdc13de) -- Improvement (ID: c637cb9f-14f0-41dc-bcdb-06da967d8f8b) -- Need More Clarity (ID: 6534182b-fba5-4642-9461-237d97b1a3e9) - -### Engineering Statuses -- Backlog (ID: 7af28faf-177a-48c5-b762-31312cb241df) -- Todo (ID: 8db5fb2e-5837-4c37-8a10-282275d21c52) -- In Progress (ID: 67cd2410-d806-4734-b82f-48daabf5db4c) -- In Review (ID: abe8fafe-5f1a-4a88-97f0-6574c20061cf) -- Done (ID: ee592855-244f-4ef6-b3b5-31e5f112c515) -- Canceled (ID: 8a228608-467d-47a4-a869-3c92e0a3bbe1) -- Duplicate (ID: f860ebac-5315-4b8b-905a-f45077ac796c) - -## User Preferences - -_(None yet - will be populated as skill is used)_ - -## Patterns - -- Most work goes to Engineering team -- Active projects: Torrent (Jigyansu), Shaastris, Harbor (Vidhan), Crew (Jayanth) -- Default priority is Normal (3) unless specified diff --git a/.claude/skills/new-skill/SKILL.md b/.claude/skills/new-skill/SKILL.md deleted file mode 100644 index de16c68c..00000000 --- a/.claude/skills/new-skill/SKILL.md +++ /dev/null @@ -1,373 +0,0 @@ ---- -description: Create new Claude Code skills following established patterns. MANDATORY when asked to create, add, or build a new skill. Uses the new-feature workflow (Research → Plan → Execute → Finalize) to ensure skills are comprehensive, self-improving, and consistent with existing skills. Not for updating existing skills (use /improve). ---- - -# New Skill - -Create a new skill for Claude Code following established patterns. - -## Context - -Before executing, read `LEARNINGS.md` in this skill folder for edge cases and patterns. - -## Core Principles - -``` -1. SELF-IMPROVEMENT IS MANDATORY - Every skill must include self-improvement section -2. CONCISE BUT COMPLETE - Detailed enough to be useful, concise enough to be readable -3. LEARN FROM EXISTING - Study similar skills before creating new ones -4. FOLLOW THE WORKFLOW - Use Research → Plan → Execute → Finalize phases -``` - ---- - -## Workflow Overview - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ RESEARCH │ ──▶ │ PLAN │ ──▶ │ EXECUTE │ ──▶ │ FINALIZE │ -│ │ │ │ │ │ │ │ -│ • Explore │ │ • Structure │ │ • Write │ │ • Verify │ -│ • Similar │ │ • Sections │ │ • SKILL.md │ │ • Commit │ -│ • Questions │ │ • Approval │ │ • LEARNINGS │ │ • Notify │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ -``` - ---- - -## Phase 1: RESEARCH - -**Goal:** Understand what the skill should do and learn from similar skills. - -### 1.1 Clarify Requirements - -Ask the user: -- What problem does this skill solve? -- When should it trigger (automatic vs manual)? -- What are the expected inputs/outputs? -- Which scope? (shared for all repos, or repo-specific like torrent, waves, etc.) - -### 1.2 Study Similar Skills - -**MANDATORY: Read 2-3 existing skills before writing a new one.** - -```bash -# Find all skills in the minions repo -ls ~/Desktop/Riverline/minions/skills/*/SKILL.md -ls ~/Desktop/Riverline/minions/skills/*/*/SKILL.md -``` - -Study skills with similar characteristics: -| Skill Type | Study These | -|------------|-------------| -| Step-by-step process | `/commit`, `/queue` | -| Multi-phase workflow | `/new-feature`, `/debug`, `/plan` | -| Proactive triggering | `/improve`, `/naming` | -| Code generation | `/queue`, `/env` | -| External tool integration | `/linear`, `/pr` | - -### 1.3 Determine Scope - -| Skill Scope | Directory | Example | -|-------------|-----------|---------| -| Universal (all repos) | `skills/shared/` | `/commit`, `/pr-review`, `/plan`, `/linear` | -| Torrent-specific | `skills/torrent/` | `/queue`, `/env`, `/debug` | -| Waves-specific | `skills/waves/` | (future) | -| Spring-agent-specific | `skills/spring-agent/` | (future) | - ---- - -## Phase 2: PLAN - -**Goal:** Design the skill structure and get approval. - -### 2.1 Required Sections - -Every skill MUST have: - -```markdown ---- -description: [REQUIRED] Single sentence describing when to use. Critical for triggering. ---- - -# Skill Name - -[REQUIRED] One-line description. - -## Context - -[REQUIRED] Reference to LEARNINGS.md. - -## When to Use / Steps / Workflow - -[REQUIRED] Core instructions. - -## Related Skills - -[REQUIRED] Wikilinks to connected skills for propagation. - -## Self-Improvement - -[REQUIRED] Instructions to invoke /improve after use. -``` - -### 2.2 Optional Sections - -Add based on skill needs: - -| Section | When to Include | -|---------|-----------------| -| `## Core Principles` | Complex skills with key rules to remember | -| `## Directory Structure` | Skills that create/modify file structures | -| `## Examples` | When showing input/output is helpful | -| `## Checklist` | Multi-step processes that need verification | -| `## Patterns` | Reusable code patterns or approaches | -| `## Quick Reference` | Complex skills needing a cheat sheet | -| `## Rules` | Strict constraints that must be followed | - -### 2.3 Present Plan to User - -Show: -- Skill name and location -- Proposed sections -- Key behaviors (auto-trigger vs manual, etc.) -- Similar skills used as reference - -**Get approval before writing.** - ---- - -## Phase 3: EXECUTE - -**Goal:** Write the skill files. - -### 3.1 Write Frontmatter Description - -**This is the most critical part.** The description determines when Claude invokes the skill. - -```markdown ---- -description: [Action verb] + [what it does] + [when to use]. [What it handles]. [What it's NOT for]. ---- -``` - -**Formula:** -1. Start with action: "Create", "Review", "Debug", "Plan", "Update" -2. Describe what it does -3. List trigger conditions ("Use when...") -4. List what it handles -5. End with exclusions ("Not for...") - -**Good examples:** -``` -Create git commits with properly formatted messages using (type): message convention. Use when committing code changes, staging files, or finalizing work. Analyzes diffs to determine commit type. Not for pushing to remote or creating PRs. -``` - -``` -Update and improve skills based on new learnings. TRIGGER PROACTIVELY when user provides new rules, standards, preferences, or corrections. Edits SKILL.md for core instruction changes or LEARNINGS.md for edge cases. Not for creating new skills from scratch. -``` - -**Bad examples:** -``` -A skill for commits ← Too vague, won't trigger properly -``` - -### 3.2 Write SKILL.md Content - -**Style guidelines:** - -| Aspect | Guideline | -|--------|-----------| -| Tone | Direct, imperative ("Run this", not "You should run this") | -| Length | 100-300 lines typical, max 500 lines | -| Detail | Enough to execute without external context | -| Examples | Include 2-3 concrete examples | -| Code blocks | Use for commands, file paths, templates | -| Tables | Use for comparisons, mappings, checklists | - -**Structure tips:** -- Use headers liberally (easy to scan) -- Keep paragraphs short (2-3 sentences) -- Use bullet points for lists -- Use numbered lists for sequential steps -- Use ASCII diagrams for workflows - -### 3.3 Write Related Skills Section - -**MANDATORY for every skill.** Link to skills that share context or are affected by changes to this skill. - -```markdown -## Related Skills -- [[skill-name]] — brief reason for the relationship -- [[other-skill]] — how they connect -``` - -**How to determine related skills:** -- Which skills produce input for this skill? (upstream) -- Which skills consume output from this skill? (downstream) -- Which skills cover overlapping domain? (sibling) -- Which skills would need updating if this skill changes? (propagation) - -Review the full skill list at `~/Desktop/Riverline/minions/skills/` and identify at least 1-3 connections. - -### 3.4 Write Self-Improvement Section - -**MANDATORY for every skill.** Use this exact template: - -```markdown -## Self-Improvement - -After completing this skill, if you discovered: -- A missing step in the workflow -- A better approach -- An edge case not covered - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync -``` - -### 3.5 Create LEARNINGS.md - -Create the learnings file with this template: - -```markdown -# Learnings - -Edge cases, patterns, and preferences discovered while using this skill. - -## Edge Cases - -_(None yet - will be populated as skill is used)_ - -## User Preferences - -_(None yet - will be populated as skill is used)_ - -## Patterns - -_(None yet - will be populated as skill is used)_ - -## Changelog - -### YYYY-MM-DD -- Initial skill creation -``` - -### 3.6 File Locations - -``` -~/Desktop/Riverline/minions/skills/{scope}/{skill-name}/ -├── SKILL.md # Core instructions (source) -└── LEARNINGS.md # Edge cases, preferences, patterns (source) -``` - -**How sync works:** During session start, `sync-minions.sh` merges LEARNINGS.md content into the synced SKILL.md (in `.claude/skills/`). This means Claude automatically reads accumulated learnings when a skill loads — no extra tool call needed. Always edit the source files in `minions/skills/`, never the synced copies. - -**Naming:** -- Folder name: kebab-case (`new-skill`, not `newSkill`) -- Must match the command name (`/new-skill`) - ---- - -## Phase 4: FINALIZE - -**Goal:** Commit, push, and notify user. - -### 4.1 Verify Skill Structure - -Checklist: -- [ ] SKILL.md has frontmatter with description -- [ ] Description starts with action verb -- [ ] Description includes "Use when..." triggers -- [ ] Description includes "Not for..." exclusions -- [ ] Context section references LEARNINGS.md -- [ ] Related Skills section with [[wikilinks]] to connected skills -- [ ] Self-Improvement section is present and complete -- [ ] LEARNINGS.md created with template - -### 4.2 Commit and Push - -```bash -cd ~/Desktop/Riverline/minions - -# Pull latest changes first -git pull origin main - -# Stage and commit -git add skills/ -git commit -m "(feat): add {skill-name} skill - -Creates new skill for {brief description}. Includes self-improvement capability." -git push -``` - -### 4.3 Notify User - -Tell user: -1. Skill created at `skills/{scope}/{skill-name}/` -2. Skills will auto-sync to all repos on next Claude session start -3. To sync immediately, restart the Claude session - ---- - -## Quick Reference - -### Skill Anatomy - -``` -┌─────────────────────────────────────────────────────────┐ -│ --- │ -│ description: [ACTION] [WHAT] [WHEN]. [HANDLES]. [NOT]. │ ← CRITICAL -│ --- │ -│ │ -│ # Skill Name │ -│ │ -│ One-line summary. │ -│ │ -│ ## Context │ -│ Read LEARNINGS.md... │ ← REQUIRED -│ │ -│ ## When to Use / Steps / Workflow │ ← Core content -│ ... │ -│ │ -│ ## Related Skills │ ← REQUIRED -│ - [[skill]] — reason for connection │ -│ │ -│ ## Self-Improvement │ ← REQUIRED -│ After completing, invoke /improve... │ -└─────────────────────────────────────────────────────────┘ -``` - -### Common Mistakes - -| Mistake | Fix | -|---------|-----| -| Vague description | Be specific about triggers and exclusions | -| Missing self-improvement | Always include the template | -| No Related Skills | Add [[wikilinks]] to at least 1-3 connected skills | -| No LEARNINGS.md | Always create it, even if empty | -| Too verbose | Cut unnecessary words, use tables | -| No examples | Add 2-3 concrete examples | -| Wrong scope | shared for universal, {repo} for specific | - ---- - -## Related Skills -- [[improve]] — improve updates skills, new-skill creates them -- [[naming]] — new skills must follow naming conventions - -## Self-Improvement - -After completing this skill, if you discovered: -- A missing step in the workflow -- A better approach -- An edge case not covered - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync diff --git a/.claude/skills/plan/SKILL.md b/.claude/skills/plan/SKILL.md deleted file mode 100644 index bde07ebb..00000000 --- a/.claude/skills/plan/SKILL.md +++ /dev/null @@ -1,610 +0,0 @@ ---- -description: Ideate and plan features through structured conversation, then produce a plan doc and Linear issues. Use when engineer says "let's plan", "I want to build", "let's think through", or starts discussing a new feature. Drives the ideation conversation, writes docs/plans/feature.md in the repo, creates Linear parent issue with plan summary, and creates sub-issues for each task. Not for executing the plan (use /new-feature), not for simple bug fixes, not for tasks that don't need decomposition. ---- - -# Plan - -Ideate a feature through structured conversation, produce a plan document, and create Linear issues — all without leaving Claude Code. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## Core Principles - -``` -1. CONVERSATION FIRST - The plan emerges from dialogue, not from a template -2. NUANCES SURFACE - Ask about edge cases the engineer hasn't considered -3. PLAN IS THE ARTIFACT - The conversation produces a reusable document -4. LINEAR IS TRACKING - Issues are created from the plan, not vice versa -5. AGENT-READY - The plan doc must be detailed enough for an agent to execute each sub-task without further clarification -``` - ---- - -## Workflow Overview - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ IDEATE │ ──▶ │ DIAGRAM │ ──▶ │ DOCUMENT │ ──▶ │ LINEAR │ ──▶ │ READY │ -│ │ │ │ │ │ │ │ │ │ -│ • Converse │ │ • Data flow │ │ • Write │ │ • Parent │ │ • Branch │ -│ • Edge cases│ │ • System │ │ • plan doc │ │ • Sub-issues│ │ • Handoff │ -│ • Decisions │ │ • Approval │ │ • Diagrams │ │ • Links │ │ • Context │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ -``` - ---- - -## Phase 1: IDEATE - -**Goal:** Have a structured conversation that surfaces all requirements, edge cases, and decisions. - -### 1.1 Understand the Feature - -Start by understanding what the engineer wants. Ask: - -- What problem does this solve? -- What's the user/system flow end-to-end? -- What triggers this feature? (user action, webhook, cron, queue) -- What's the expected output/result? - -**Don't ask all at once.** One question at a time, build on answers. - -### 1.2 Explore the Codebase - -Before going deeper, understand what already exists: - -``` -Search for: -- Similar features already implemented -- Schemas/models that will be involved -- Services that can be reused or extended -- Existing patterns for this type of feature -- Related queues, schedulers, or webhooks -``` - -Share findings with the engineer: "I found X already exists, should we extend it or build separately?" - -### 1.3 Surface Edge Cases - -**This is the most important step.** Engineers often think about the happy path. Your job is to surface what they haven't considered. - -Ask about: - -| Category | Questions to Ask | -|----------|-----------------| -| **Failure modes** | What happens if X fails? Timeout? Partial failure? | -| **Concurrency** | Can this run in parallel? Race conditions? | -| **Idempotency** | What if the trigger fires twice? Duplicate handling? | -| **Data state** | What if the data is missing/null/stale? | -| **Scale** | How many records? Batch processing needed? | -| **Timing** | Immediate or async? Retry logic? Delays? | -| **Dependencies** | External APIs? What if they're down? | -| **Rollback** | If something goes wrong midway, can we recover? | - -**Don't dump all questions at once.** Ask the ones relevant to this specific feature. - -### 1.4 Make Decisions - -For each ambiguity, present options with trade-offs: - -``` -For fetching the recording after call ends: - -1. **Poll the API every 30s** (Simple) - - Pro: Easy to implement - - Con: Wastes API calls, delay up to 30s - -2. **Wait for recording-ready webhook** (Recommended) - - Pro: Immediate, no wasted calls - - Con: Need to handle webhook registration - -3. **Queue with exponential backoff** - - Pro: Resilient, handles delays - - Con: More complex, need retry limits - -I'd recommend option 2 because [reason]. What do you think? -``` - -**Record every decision** — these go into the plan doc. - -### 1.5 Determine App Location - -| Feature Type | App | Reason | -|---|---|---| -| Campaign triggers, webhooks, queues, schedulers, data sync | Execution | Critical path | -| Analytics, dashboards, CRUD, reporting | Operations | Non-critical | - -If unclear, ask: "Is this campaign execution, scheduling, webhooks, or data sync?" YES → Execution. NO → Operations. - -### 1.6 Identify Sub-Tasks - -As the conversation progresses, mentally note the natural task boundaries: - -- Schema changes -- New services (each with a clear responsibility) -- New routes/endpoints -- Queue/worker setup -- Webhook handlers -- Configuration/env changes -- Wiring everything together - ---- - -## Phase 2: DIAGRAM - -**Goal:** Visualize the system before writing the plan doc. Show to engineer for approval. - -After ideation is complete and before writing the plan doc, create diagrams that make the architecture concrete and reviewable. - -### 2.1 Choose Diagram Type - -Pick the diagram(s) most relevant to the feature: - -| Feature Type | Diagram Type | When to Use | -|-------------|-------------|-------------| -| Data pipeline (webhook → queue → service) | **Data Flow Diagram** | Data moves between systems/services | -| API endpoints | **Sequence Diagram** | Request/response between client, server, external APIs | -| State changes (campaign status, call lifecycle) | **State Diagram** | Entity transitions through defined states | -| Multi-service interaction | **System Flow Diagram** | Multiple services coordinate to complete a task | -| Schema relationships | **Data Model Diagram** | New collections or significant schema changes | - -For complex features, create **both** a data/system flow AND a data model diagram. - -### 2.2 Diagram Format - -Use **Mermaid syntax** — it renders in GitHub, Linear, and most markdown viewers. - -**Data Flow Diagram:** -```mermaid -flowchart LR - A[Webhook: recording-ready] --> B[Queue: call-analysis] - B --> C[Worker: transcribe] - C --> D[Worker: LLM analysis] - D --> E[(MongoDB: Call.callAnalysis)] - E --> F[API: GET /calls/:id/analysis] -``` - -**Sequence Diagram:** -```mermaid -sequenceDiagram - participant W as Webhook - participant Q as BullMQ Queue - participant T as Transcription API - participant L as LLM Service - participant DB as MongoDB - - W->>Q: recording-ready event - Q->>T: Send recording URL - T-->>Q: Transcript text - Q->>L: Analyze transcript - L-->>Q: Analysis result - Q->>DB: Save to Call.callAnalysis -``` - -**State Diagram:** -```mermaid -stateDiagram-v2 - [*] --> Pending: Call ends - Pending --> Processing: Recording ready - Processing --> Completed: Analysis done - Processing --> Failed: API error - Failed --> Processing: Retry (max 3) - Failed --> Abandoned: Max retries - Completed --> [*] -``` - -**Data Model Diagram:** -```mermaid -erDiagram - Call ||--o| CallAnalysis : has - Call { - ObjectId _id - string recordingUrl - string status - } - CallAnalysis { - string transcript - string sentiment - string summary - string[] actionItems - } -``` - -### 2.3 Present to Engineer - -Show the diagram(s) to the engineer and ask: - -``` -Here's how I see the system working: - -{diagram} - -Does this match your mental model? Anything missing or wrong? -``` - -**Wait for approval before proceeding to the plan doc.** - -If the engineer suggests changes, update the diagram and show again. The diagram must be accurate before it goes into the doc. - -### 2.4 What Good Diagrams Show - -- **Entry points** — what triggers the flow (webhook, cron, user action) -- **Data transformations** — what happens at each step -- **Storage** — where data is read from and written to -- **External dependencies** — third-party APIs, services -- **Failure points** — where things can go wrong (mark with ⚠️ if helpful) -- **Output** — what the end result looks like - -### 2.5 What to Avoid - -- Don't create diagrams for trivial features (simple CRUD, single endpoint) -- Don't over-detail — show the architecture, not every function call -- Don't mix concerns — separate data flow from state transitions into different diagrams - ---- - -## Phase 3: DOCUMENT - -**Goal:** Write the plan document in the repo, including the approved diagrams. - -### 3.1 Create the Plan File - -``` -Location: docs/plans/{feature-name}.md -``` - -Create the `docs/plans/` directory if it doesn't exist. - -### 3.2 Plan Document Structure - -```markdown -# {Feature Name} - -## Overview -One paragraph explaining what this feature does and why. - -## Flow -Step-by-step system/user flow. Use numbered steps. - -1. [Trigger] → ... -2. [Process] → ... -3. [Result] → ... - -## Diagrams - -{Include the approved Mermaid diagrams from Phase 2} - -### Data/System Flow -{flowchart or sequence diagram} - -### Data Model (if applicable) -{ER diagram showing schema relationships} - -### State Transitions (if applicable) -{state diagram} - -## Architecture - -### App: {Execution | Operations} -Reasoning: {why this app} - -### Files to Create/Modify -| File | Action | Purpose | -|------|--------|---------| -| `path/to/file.ts` | CREATE | Description | -| `path/to/existing.ts` | MODIFY | What changes | - -### Data Model Changes -{Schema changes, new fields, new collections — or "None"} - -### Queue/Worker Setup -{If applicable — queue name, job data, processing logic — or "N/A"} - -## Decisions - -| Decision | Choice | Alternatives Considered | Reasoning | -|----------|--------|------------------------|-----------| -| How to get recording | Webhook | Polling, queue backoff | Immediate, no waste | -| Where to store analysis | Call document | Separate collection | Co-located, simpler queries | - -## Edge Cases & Error Handling - -| Scenario | Handling | -|----------|----------| -| Recording not ready | Retry with backoff, max 5 attempts | -| Duplicate webhook | Idempotency check on callId | -| Analysis API timeout | Mark as failed, alert, manual retry | - -## Sub-Tasks - -1. **{Task title}** - - What: {detailed description} - - Files: {files involved} - - Dependencies: {what must be done first} - -2. **{Task title}** - - What: {detailed description} - - Files: {files involved} - - Dependencies: {what must be done first} - -{Repeat for each sub-task} - -## Open Questions -{Any unresolved questions — or "None, all clarified during planning"} -``` - -### 3.3 Write With Agent Execution in Mind - -Each sub-task description must be detailed enough that an agent can execute it without asking further questions. Include: - -- Exact file paths to create/modify -- Which existing patterns to follow (reference files) -- Schema field names, types, and validation -- API endpoint paths and request/response shapes -- Error handling expectations -- What to test - -**Bad sub-task:** -``` -Add the analysis service -``` - -**Good sub-task:** -``` -Create analysis service at apps/execution/src/services/call/call.analysis.ts - -- Input: { callId: string, recordingUrl: string } -- Output: { transcript: string, sentiment: string, summary: string, actionItems: string[] } -- Call transcription API (see @services/ai/transcription.ts for pattern) -- Pass transcript to LLM for structured analysis (see @ai-services/prompt.service.ts) -- Save result to Call.callAnalysis field -- Handle: API timeout (throw, let queue retry), invalid recording URL (mark failed) -- Follow pattern in: apps/execution/src/services/email/emailAutomation.service.ts -``` - -### 3.4 Review the Plan With Engineer - -After writing the plan doc, present a summary: - -``` -Plan written to docs/plans/{feature-name}.md - -Summary: -- {X} sub-tasks identified -- App: {Execution/Operations} -- New files: {count} -- Modified files: {count} -- Schema changes: {yes/no} -- Queue needed: {yes/no} - -Shall I create Linear issues from this plan? -``` - -**Wait for approval before creating Linear issues.** - ---- - -## Phase 4: LINEAR - -**Goal:** Create parent issue and sub-issues in Linear. - -### 4.1 Determine Linear Context - -Use Linear MCP tools to: -1. List teams → find the right team (usually "Engineering") -2. List projects → find or confirm the project -3. List labels → use appropriate labels - -### 4.2 Create Parent Issue - -Use `create_issue` MCP tool: - -``` -Title: {Feature name} — {one-line summary} -Team: Engineering -Project: {relevant project, e.g., "Torrent"} -Label: Feature -Priority: {ask engineer if not obvious} -Description: - ## Plan - See: `docs/plans/{feature-name}.md` in the repo - - ## Overview - {Copy the Overview section from plan doc} - - ## Flow - {Copy the Flow section from plan doc} - - ## Diagrams - {Copy the Mermaid diagrams from plan doc} - - ## Decisions - {Copy the Decisions table from plan doc} - - ## Edge Cases & Error Handling - {Copy the Edge Cases table from plan doc} - - ## Sub-Tasks - {List sub-task titles as checklist} - - [ ] Sub-task 1 - - [ ] Sub-task 2 - - [ ] Sub-task 3 -``` - -### 4.3 Create Sub-Issues (for tracking, not separate PRs) - -Sub-issues track progress within the feature. They all ship together in **one PR**. - -For each sub-task in the plan, create a sub-issue: - -``` -Title: {Sub-task title} -Team: Engineering -Parent: {parent issue ID} -Label: Feature -Priority: Same as parent -Description: - ## What - {Detailed description from plan doc sub-task} - - ## Files - {Files to create/modify} - - ## Dependencies - {Which sub-tasks must be done first} - - ## Pattern Reference - {Which existing files to follow} - - ## Plan Doc - See: `docs/plans/{feature-name}.md` for full context -``` - -**Note:** All sub-issues are worked on the same feature branch. When the feature is complete, one PR covers everything. Sub-issues are moved to "Done" when their code is written, and the parent issue moves to "In Review" when the PR is opened. - -### 4.4 Link Everything - -After creating all issues: -1. Note the parent issue identifier (e.g., `ENG-123`) -2. Add the issue identifier to the plan doc header: - -```markdown -# {Feature Name} - -**Linear:** ENG-123 | **Branch:** feature/ENG-123-{feature-name} -``` - -### 4.5 Report to Engineer - -``` -Linear issues created: - -Parent: ENG-123 — {Feature name} -├── ENG-124: {Sub-task 1} -├── ENG-125: {Sub-task 2} -├── ENG-126: {Sub-task 3} -├── ENG-127: {Sub-task 4} -└── ENG-128: {Sub-task 5} - -Plan doc: docs/plans/{feature-name}.md -Branch: feature/ENG-123-{feature-name} (not created yet) - -All sub-tasks will be worked on one branch: feature/ENG-123-{feature-name} -One PR will be opened when the feature is complete. - -Ready to start? -``` - ---- - -## Phase 5: READY - -**Goal:** Set up the engineer to start executing. - -### 5.1 Create Feature Branch - -```bash -git checkout main -git pull origin main -git checkout -b feature/{issue-id}-{feature-name} -``` - -### 5.2 Commit the Plan Doc - -```bash -git add docs/plans/{feature-name}.md -git commit -m "(docs): add plan for {feature-name} - -Planning doc for ENG-123. Covers flow, architecture, decisions, and sub-tasks." -``` - -### 5.3 Update Linear Status - -Move the parent issue to In Progress — implementation is starting: - -``` -Use update_issue MCP tool to move parent issue (ENG-123) to "In Progress" -``` - -### 5.4 Handoff - -The engineer can now: -- Work through sub-tasks sequentially on this single feature branch -- Each sub-task has enough context in the plan doc for an agent to execute -- Mark sub-issues as "Done" in Linear as each sub-task is completed -- When the full feature is ready, use `/pr` to open one PR for everything -- The reviewer uses `/pr-review` to review with plan context - ---- - -## Interplay With Other Skills - -``` -/plan → creates the plan doc + Linear parent + sub-issues - │ - ├── /new-feature → executes sub-tasks on ONE feature branch - ├── /pr → opens ONE PR for the entire feature, updates Linear with implementation doc - ├── /pr-review → reviews the feature PR with plan context - └── /linear → follows Linear conventions for all issue operations -``` - ---- - -## Anti-Patterns - -| Don't | Do Instead | -|-------|------------| -| Write the plan without conversation | Have the ideation dialogue first | -| Create Linear issues before plan doc | Plan doc is source of truth, issues derive from it | -| Make sub-tasks vague | Each sub-task must be agent-executable | -| Skip edge case discussion | Actively surface failure modes | -| Plan everything in one shot | Iterate — write draft, review with engineer, refine | -| Include implementation details in ideation | Stay at architecture level until documenting | - ---- - -## Related Skills -- [[new-feature]] — plans lead to feature implementation -- [[aryan-backend]] — backend features need planning -- [[linear]] — plans reference Linear issues - -## Self-Improvement - -After completing this skill, if you discovered: -- A question that should always be asked during ideation -- A plan doc section that was missing -- A better way to structure Linear issues -- An edge case category not covered - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync - - ---- - -# Accumulated Learnings - -> Auto-merged from LEARNINGS.md. Apply these edge cases, patterns, and preferences when executing this skill. - - - -## Edge Cases - -- `/new-feature` has overlapping Research + Plan phases. If `/plan` was used first, `/new-feature` should skip to Phase 3 (Execute) by reading the existing plan doc at `docs/plans/`. -- Plan docs should be deleted or archived after the feature is fully merged to avoid stale docs accumulating. - -## User Preferences - -_(None yet - will be populated as skill is used)_ - -## Patterns - -- Always check `docs/plans/` for existing plans before starting a new one — the engineer may have already planned this feature. -- Linear issue identifiers follow the pattern `ENG-{number}` for the Engineering team. -- The Engineering team has labels: Feature, Bug, Improvement, Need More Clarity. -- Issue statuses flow: Backlog → Todo → In Progress → In Review → Done. diff --git a/.claude/skills/pr-review/SKILL.md b/.claude/skills/pr-review/SKILL.md deleted file mode 100644 index 1841084c..00000000 --- a/.claude/skills/pr-review/SKILL.md +++ /dev/null @@ -1,555 +0,0 @@ ---- -description: Review another engineer's pull request for code quality, design decisions, and plan compliance. Use when reviewing a PR, when engineer says "review PR", "review this PR", "check PR #123", or provides a PR URL/number. Fetches PR diff via gh CLI, reads the linked plan doc for design context, evaluates both code quality AND architectural decisions. Uses Linear MCP for issue context. Not for reviewing your own staged changes (use /review), not for creating PRs (use /pr). ---- - -# PR Review - -Review a pull request for code quality, design decisions, and plan compliance. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## Core Principles - -``` -1. DESIGN BEFORE CODE - Check if the approach is right before checking if the code is clean -2. PLAN COMPLIANCE - Verify the PR implements what was planned -3. DECISIONS ARE REVIEWABLE - Evaluate the choices, not just the syntax -4. ACTIONABLE FEEDBACK - Every comment must be specific and fixable -5. TRUST THE TOOLS - Don't review formatting/linting — CI handles that -``` - ---- - -## Workflow - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ GATHER │ ──▶ │ DESIGN │ ──▶ │ CODE │ ──▶ │ VERDICT │ -│ │ │ REVIEW │ │ REVIEW │ │ │ -│ • PR diff │ │ • Plan doc │ │ • Bugs │ │ • Approve │ -│ • Plan doc │ │ • Decisions │ │ • Security │ │ • Request │ -│ • Linear │ │ • Approach │ │ • Patterns │ │ • Report │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ -``` - ---- - -## Step 1: GATHER Context - -### 1.1 Get PR Details - -```bash -# Get PR info (use PR number or URL provided by engineer) -gh pr view {pr-number} --json title,body,headRefName,baseRefName,files,additions,deletions,author - -# Get the full diff -gh pr diff {pr-number} -``` - -### 1.2 Read the PR Description - -From the PR body, extract: -- **Summary** — what does this PR claim to do? -- **Linear issue** — which issue is this for? -- **Plan doc** — is there a plan doc linked? -- **Decisions** — what choices were made? -- **Test plan** — how should this be verified? - -**If the PR has no description or a vague one:** Flag this immediately. - -``` -⚠️ PR has no description. Cannot review design decisions without context. -Ask the author to add a description using /pr skill, or provide context here. -``` - -### 1.3 Read the Plan Doc - -If a plan doc is linked (e.g., `docs/plans/{feature}.md`): - -```bash -# Checkout the branch to read the plan doc -git fetch origin {branch-name} -``` - -Read the plan doc and extract: -- The full feature scope (all sub-tasks should be in this one PR) -- Architecture decisions made during planning -- Edge cases that should be handled -- Expected files to create/modify - -### 1.4 Get Linear Issue Context - -If a Linear issue is referenced, use Linear MCP tools: -- Fetch the issue details -- Read the parent issue for full feature context -- Check if there are specific requirements in the description - ---- - -## Step 2: DESIGN REVIEW (Do This FIRST) - -**Before looking at code quality, evaluate the approach.** - -### 2.1 Plan Compliance - -| Check | Question | -|-------|----------| -| **Scope** | Does this PR cover the complete feature? All sub-tasks included? | -| **Architecture** | Does it follow the architecture described in the plan doc? | -| **App placement** | Is the code in the right app (Execution vs Operations)? | -| **Decisions** | Do the PR decisions align with plan decisions? Any contradictions? | -| **Files** | Are the files created/modified the ones expected by the plan? | - -If no plan doc exists, evaluate the approach on its own merits. - -### 2.2 Decision Evaluation - -For each decision listed in the PR description: - -``` -Decision: {what was decided} -Choice: {what they chose} - -Evaluation: -- Is this the right choice for the problem? -- Are there better alternatives they didn't consider? -- Does this align with existing patterns in the codebase? -- Will this cause problems at scale or over time? -``` - -**Flag decisions that seem wrong or unconsidered:** - -``` -⚠️ Decision: "Poll recording API every 30s" - Concern: There's already a recording-ready webhook in call.route.ts:142. - Suggestion: Use the existing webhook instead of polling. -``` - -### 2.3 Missing Considerations - -Check if the PR handles what the plan specified: - -- [ ] All edge cases from the plan doc addressed? -- [ ] Error handling matches the plan's error handling section? -- [ ] All files listed in the plan accounted for? -- [ ] Nothing extra that wasn't in the plan? - -### 2.4 Approach Assessment - -Even without a plan doc, evaluate: - -| Aspect | Question | -|--------|----------| -| **Simplicity** | Is this the simplest approach that works? | -| **Reuse** | Does similar code already exist that could be extended? | -| **Patterns** | Does it follow existing codebase patterns? | -| **Coupling** | Does it introduce unnecessary dependencies? | -| **Reversibility** | Can this be easily changed or rolled back? | - -### 2.5 Protected Files Check - -**MANDATORY: Check if the PR modifies any infrastructure or config files.** - -Scan the PR's changed files list for these patterns: - -``` -Protected file patterns: -- .github/workflows/** — CI/CD pipelines -- .github/** — Any GitHub config -- .eslintrc* — ESLint config -- eslint.config.* — ESLint flat config -- .prettierrc* — Prettier config -- prettier.config.* — Prettier config -- tsconfig*.json — TypeScript config -- .flake8 — Python linter config -- pyproject.toml — Python project config (may contain linter settings) -- Makefile — Build scripts -- Dockerfile* — Container config -- docker-compose*.yml — Container orchestration -- ecosystem.config.js — PM2 config -- package.json — Dependencies and scripts -- yarn.lock — Dependency lock file -``` - -**If ANY protected files are changed, flag them prominently:** - -``` -🚨 PROTECTED FILES MODIFIED — Requires careful review - -The following infrastructure/config files were changed in this PR: - -| File | What Changed | Risk | -|------|-------------|------| -| `.github/workflows/deploy.yml` | {describe what changed} | CI/CD pipeline — can break deployments | -| `eslint.config.mjs` | {describe what changed} | Lint rules — can silently weaken code quality | -| `package.json` | {describe what changed} | Dependencies — check for security, necessity | - -⚠️ These files affect ALL developers and the deployment pipeline. - Review each change carefully and confirm it is intentional and necessary. -``` - -**For each protected file, answer:** -1. **Is this change necessary?** Does the feature actually require this config change? -2. **Is it correct?** Will it break CI, deployments, or other developers? -3. **Is it scoped?** Does it change only what's needed, or does it have side effects? -4. **Was it discussed?** Is this change mentioned in the plan doc or PR description? - -**If a protected file was changed but NOT mentioned in the PR description:** Flag this as a critical issue. Config changes should never be silent. - ---- - -## Step 3: CODE REVIEW - -**Only after design review passes, review the code quality.** - -### 3.1 Bugs & Logic Errors - -- Null/undefined access without checks -- Missing `await` on async operations -- Unhandled promise rejections -- Off-by-one errors in loops/pagination -- Race conditions in concurrent operations -- Incorrect boolean logic -- Missing return statements - -### 3.2 Security (OWASP) - -- **Auth/Authz**: Missing authentication or authorization checks on endpoints -- **Injection**: Unsanitized input in DB queries, command execution -- **XSS**: User input reflected without sanitization -- **Secrets**: Hardcoded credentials, API keys in code -- **Validation**: Missing input validation at API boundaries -- **Logging**: Sensitive data (passwords, tokens, PII) in logs - -### 3.3 Torrent Conventions - -**Routes:** -- [ ] Thin routes — HTTP concerns only, business logic in services -- [ ] ObjectIds validated with `validateObjectId()` -- [ ] Specific routes before parameterized (`/all` before `/:id`) -- [ ] Response format: `{ success, message, data/error }` - -**Services:** -- [ ] Pure functions — no `req`/`res` objects -- [ ] All parameters and return values typed -- [ ] `.lean()` on read-only queries -- [ ] Descriptive error messages - -**TypeScript:** -- [ ] No `any` type -- [ ] No `var`, `.then()` chains, or `require()` -- [ ] Proper import order (external → @torrent/db → aliases → relative) - -### 3.4 Code Reuse - -Search the codebase for: -- Similar functions that already exist -- Utilities that could replace inline code -- Services that could be extended instead of duplicated - -``` -⚠️ Duplicate: formatPhoneNumber() at line 45 already exists in @utils/phone.ts - Suggestion: Import from @utils/phone instead of re-implementing -``` - -### 3.5 Error Handling - -For Execution app code: -- [ ] Sentry error tracking on critical paths -- [ ] Cronitor monitoring on schedulers -- [ ] Structured logging with tags `[Module]` - -For all code: -- [ ] try/catch around external API calls -- [ ] Meaningful error messages (not generic "Something went wrong") -- [ ] Errors thrown in services, caught in routes - -### 3.6 Database - -- [ ] Indexes exist for fields used in `.find()` filters -- [ ] `.select()` used to limit fields returned -- [ ] `.lean()` on read-only queries -- [ ] `findById()` instead of `findOne({ _id })` where applicable -- [ ] ObjectId validation before queries - ---- - -## Step 4: VERDICT - -### 4.1 Output Format - -```markdown -## PR Review: {PR title} - -**PR:** #{number} | **Author:** {author} | **Branch:** {branch} -**Linear:** {issue ID} | **Plan:** {plan doc path or "none"} - ---- - -### Design Review - -**Plan Compliance:** ✅ Matches plan / ⚠️ Deviates / ❌ Contradicts -{Details if not compliant} - -**Approach:** ✅ Sound / ⚠️ Concerns / ❌ Wrong approach -{Details if concerns} - -**Decision Evaluation:** -| Decision | Verdict | Note | -|----------|---------|------| -| {decision} | ✅/⚠️/❌ | {note} | - ---- - -### Code Review - -#### Critical (Must fix) -- `file:line` — {issue description} - -#### Warning (Should fix) -- `file:line` — {issue description} - -#### Suggestion (Nice to have) -- `file:line` — {suggestion} - -#### Positive (Good patterns) -- `file:line` — {what's good} - ---- - -### Verdict: {APPROVE / REQUEST CHANGES / NEEDS DISCUSSION} - -{Summary — 1-2 sentences on overall assessment} - -### Action Items -- [ ] {Specific thing to fix} -- [ ] {Specific thing to fix} -``` - -### 4.2 Verdict Criteria - -| Verdict | When | -|---------|------| -| **APPROVE** | Design is sound, no critical issues, warnings are minor | -| **REQUEST CHANGES** | Critical issues found, or design approach is wrong | -| **NEEDS DISCUSSION** | Design decision needs team input, not just author's fix | - -### 4.3 Update Linear - -**Always** add a review comment to the parent issue: - -``` -Use create_comment on parent issue: -"PR #{number} reviewed — {verdict}. {1-line summary of findings}" -``` - -**Then update statuses based on verdict:** - -| Verdict | Parent Issue | Sub-Issues | -|---------|-------------|------------| -| **APPROVE** | Stay In Review (author merges) | No change (already Done) | -| **REQUEST CHANGES** | In Review → In Progress | No change | -| **NEEDS DISCUSSION** | No change | No change | - -**After PR is merged** (author or reviewer merges): - -``` -Use update_issue to move parent issue to "Done" -Use update_issue to move any remaining sub-issues to "Done" -``` - -**Fetch all sub-issues** using the parent issue ID to ensure none are missed: - -``` -Use get_issue on parent to get sub-issue IDs -For each sub-issue not already Done → move to Done -``` - ---- - -## Step 5: POST-MERGE DEPLOYMENT MONITORING - -**After the PR is merged to main, monitor the GitHub Actions deployment.** - -This step happens after merge — the reviewer (or merge author) should invoke this or the agent should do it automatically. - -### 5.1 Watch the GitHub Action - -```bash -# Get the latest workflow run triggered by the merge commit -gh run list --branch main --limit 1 --json databaseId,status,conclusion,name,headSha - -# Watch it until completion (polls every 30 seconds) -gh run watch {run-id} -``` - -### 5.2 On Success — Report - -If the workflow succeeds: - -``` -✅ Deployment successful - -Workflow: {workflow name} -Run: {run URL} -Commit: {sha} -Duration: {duration} - -All apps are healthy and serving traffic. -``` - -Add a comment to the Linear parent issue: - -``` -Use create_comment: "✅ Deployed to production. Workflow run: {run URL}" -``` - -Move the parent issue to **Done** (if not already). - -### 5.3 On Failure — Diagnose and Notify - -If the workflow fails: - -```bash -# Get the failed run details -gh run view {run-id} --json jobs - -# Get logs from the failed job -gh run view {run-id} --log-failed -``` - -**Analyze the failure:** - -| Failure Type | Common Cause | Remediation | -|-------------|-------------|-------------| -| Type check failed | Code that passed locally but fails in CI (missing dependency, env diff) | Fix type errors, push to main | -| Lint failed | Unlinted code got merged | Run `yarn lint:fix`, push to main | -| Build failed | Missing dependency, import error | Check `yarn build` output, fix and push | -| Deploy failed (SSH) | EC2 connectivity issue | Check EC2 status, retry workflow | -| Deploy failed (PM2) | App crash on startup | Check PM2 logs, likely env var or runtime error | -| Health check failed | App started but not responding | Check app logs for startup errors | -| Docker build failed | Dockerfile or dependency issue | Check Docker build logs | - -**Report the failure:** - -``` -❌ Deployment FAILED - -Workflow: {workflow name} -Run: {run URL} -Failed job: {job name} -Failed step: {step name} - -Error: -{relevant error output from logs — keep concise, max 20 lines} - -Root cause: {your analysis} - -Remediation: -1. {specific step to fix} -2. {specific step to fix} - -⚠️ Main branch is currently broken. Fix urgently. -``` - -Add a comment to the Linear parent issue: - -``` -Use create_comment: "❌ Deployment failed after merge. {1-line cause}. See workflow: {run URL}" -``` - -**Do NOT move the parent issue to Done if deployment failed.** Keep it in In Review until the fix is deployed. - -### 5.4 If Fix is Needed - -If the failure is caused by the merged code: -1. Create a hotfix commit on main (or a hotfix branch if branch protection is on) -2. Push the fix -3. Monitor the new workflow run -4. Report success/failure again - ---- - -## Reviewing Without a Plan Doc - -If there's no plan doc: - -1. **Read the PR description carefully** — this is your only context -2. **Ask the author for context if needed** — don't guess at intent -3. **Focus more heavily on design review** — without a plan, bad approaches are more likely -4. **Suggest creating a plan doc** for complex features: "This feature is complex enough to benefit from a plan doc. Consider using `/plan` before implementing." - ---- - -## Reviewing Agent-Written Code - -Code written by AI agents has specific patterns to watch for: - -| Pattern | What to Check | -|---------|--------------| -| **Over-engineering** | Agent added abstractions, utilities, or error handling that isn't needed | -| **Hallucinated imports** | Agent imported a function/module that doesn't exist | -| **Pattern mismatch** | Agent followed a different pattern than what the codebase uses | -| **Missing edge cases** | Agent handled the happy path but missed failure modes | -| **Verbose code** | Agent wrote 50 lines where 10 would do | -| **Incorrect types** | Agent guessed at types instead of checking schemas | - ---- - -## Anti-Patterns - -| Don't | Do Instead | -|-------|------------| -| Review code before design | Always evaluate approach first | -| "LGTM" without reading the diff | Read every changed line | -| Flag linting/formatting issues | Trust CI to handle these | -| Approve because "the agent wrote it" | Agent code needs MORE scrutiny, not less | -| Block on style preferences | Only block on bugs, security, or wrong approach | -| Review without reading plan doc | Always read the plan doc if it exists | - ---- - -## Related Skills -- [[pr]] — reviews target pull requests -- [[commit]] — review checks commit quality -- [[linear]] — review status syncs to Linear -- [[naming]] — review checks naming conventions - -## Self-Improvement - -After completing this skill, if you discovered: -- A review check that was missing -- A common agent-written code pattern to watch for -- A better way to structure review feedback - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync - - ---- - -# Accumulated Learnings - -> Auto-merged from LEARNINGS.md. Apply these edge cases, patterns, and preferences when executing this skill. - - - -## Edge Cases - -- When reviewing agent-written code, pay extra attention to hallucinated imports and over-engineering. -- If no plan doc exists, the review should be more thorough on design decisions since there's no pre-approved architecture. - -## User Preferences - -_(None yet - will be populated as skill is used)_ - -## Patterns - -- Use `gh pr diff {number}` to get the full diff without checking out the branch -- Use `gh pr view {number} --json title,body,headRefName,files` for PR metadata -- Linear MCP tools can fetch issue details for context without leaving Claude Code diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md deleted file mode 100644 index f6a2cd15..00000000 --- a/.claude/skills/pr/SKILL.md +++ /dev/null @@ -1,416 +0,0 @@ ---- -description: Create pull requests with decision-exposing descriptions that link to plan docs and Linear issues. Use when opening a PR, pushing a branch for review, or when engineer says "open a PR", "create PR", "push this for review". Builds structured PR descriptions with summary, decisions made, alternatives considered, and links to plan doc and Linear issue. Updates the Linear parent issue with full implementation doc. Uses gh CLI. Not for reviewing PRs (use /pr-review), not for committing (use /commit), not for merging. ---- - -# Pull Request - -Create a pull request for a complete feature with a structured, decision-exposing description. - -## Context - -Learnings from previous usage (edge cases, patterns, preferences) are auto-merged into this file during sync. To add new learnings, edit the source `LEARNINGS.md` in this skill's folder in the minions repo. - -## Core Principles - -``` -1. ONE PR PER FEATURE - All sub-tasks ship together in a single PR -2. DECISIONS OVER DIFFS - The PR description explains WHY, the diff shows WHAT -3. LINK TO CONTEXT - Every PR connects to its plan doc and Linear issue -4. REVIEWER EFFICIENCY - A reviewer should understand the PR in 2 minutes without reading every line -``` - ---- - -## Workflow - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ GATHER │ ──▶ │ WRITE │ ──▶ │ CREATE │ ──▶ │ LINK │ -│ │ │ │ │ │ │ │ -│ • Full diff │ │ • Title │ │ • gh pr │ │ • Linear │ -│ • Plan doc │ │ • Body │ │ • Push │ │ • Impl doc │ -│ • Linear │ │ • Decisions │ │ • Labels │ │ • Status │ -└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ -``` - ---- - -## Step 1: GATHER Context - -### 1.1 Review All Changes - -```bash -# Full diff from the feature branch -git status -git diff --stat main...HEAD -git log --oneline main...HEAD -``` - -Understand the complete scope: -- Total commits on this branch -- All files changed across all sub-tasks -- Overall size of the feature - -### 1.2 Find the Plan Doc - -```bash -ls docs/plans/ -``` - -If a plan doc exists, read it. Extract: -- Feature overview and flow -- All sub-tasks and their status -- Architecture decisions made during planning -- Edge cases identified - -### 1.3 Find the Linear Parent Issue - -Look for the parent issue identifier in: -1. Branch name (e.g., `feature/ENG-123-call-analysis`) -2. Plan doc header -3. Ask the engineer if not found - -Fetch the parent issue and all sub-issues using Linear MCP tools. - ---- - -## Step 2: WRITE PR Description - -### 2.1 Title - -``` -{type}: {feature name} (max 70 chars) -``` - -Types: `feat`, `fix`, `refactor`, `chore`, `docs`, `test` - -Examples: -- `feat: add call analysis pipeline with transcription and LLM insights` -- `feat: whatsapp NPA nudge automation system` -- `fix: overhaul duplicate webhook handling across all channels` - -### 2.2 Body Structure - -The PR covers the entire feature. Structure the body to guide the reviewer through it: - -```markdown -## Summary -{2-4 bullet points explaining WHAT this feature does and WHY it's needed} - -## Linear -[ENG-123](https://linear.app/riverline/issue/ENG-123) {use actual issue ID and URL from Linear MCP} - -## Plan -`docs/plans/{feature}.md` - -## What's Included -{Group changes by sub-task so the reviewer can follow the logical structure} - -### {Sub-task 1 title} (ENG-124) -- {What was built} -- Files: `path/to/file.ts`, `path/to/other.ts` - -### {Sub-task 2 title} (ENG-125) -- {What was built} -- Files: `path/to/file.ts` - -### {Sub-task 3 title} (ENG-126) -- {What was built} -- Files: `path/to/file.ts` - -## Architecture -{Brief description of how the pieces fit together — data flow, service interactions} - -## Diagrams -{Copy the Mermaid diagrams from the plan doc — data flow, sequence, state, or ER diagrams as applicable} - -## Decisions Made -| Decision | Choice | Alternatives Considered | Reasoning | -|----------|--------|------------------------|-----------| -| {what} | {choice} | {alternatives} | {reasoning} | - -## Edge Cases Handled -| Scenario | Handling | -|----------|----------| -| {edge case} | {how it's handled} | - -## How to Review -{Guide the reviewer through the PR in order} -1. Start with {file/area} — this is the core logic -2. Then check {file/area} — this wires everything together -3. {file/area} is mechanical/boilerplate, skim it - -## Test Plan -- [ ] {How to verify the happy path} -- [ ] {Edge case to test} -- [ ] {Regression to check} -``` - -### 2.3 Decision Documentation - -**This is the most important section.** Document every non-obvious choice: - -| Type | Example | -|------|---------| -| Architecture | "Chose BullMQ over cron because processing time varies" | -| Data model | "Added field to Call schema instead of new collection for simpler queries" | -| Library choice | "Used zod over joi because existing validation uses zod" | -| Error handling | "Retry 3x with backoff instead of failing immediately because API is flaky" | -| Pattern | "Followed emailAutomation.service.ts pattern for consistency" | - -### 2.4 Adapt Based on Feature Type - -| Feature Type | Emphasis | -|-------------|----------| -| **New pipeline** (webhook → queue → service) | Data flow diagram, failure handling at each stage | -| **New API endpoints** | Request/response shapes, auth, rate limits | -| **Automation system** | Trigger conditions, scheduling, idempotency | -| **Integration** | External API behavior, retry logic, fallbacks | - ---- - -## Step 3: CREATE the PR - -### 3.1 Sync with Main & Resolve Conflicts - -**Before pushing, ensure the branch is up to date with main.** - -```bash -# Fetch latest main -git fetch origin main - -# Check if branch is behind main -BEHIND=$(git rev-list --count HEAD..origin/main) -echo "$BEHIND commits behind main" -``` - -**If behind (BEHIND > 0), rebase onto main:** - -```bash -git rebase origin/main -``` - -**If rebase has conflicts:** - -1. List conflicting files: -```bash -git diff --name-only --diff-filter=U -``` - -2. For each conflicting file: - - Read the file to understand both sides of the conflict - - Resolve by keeping the correct version (usually: keep main's structural changes + your feature's new code) - - **Never blindly accept one side** — understand what changed on main and why - -3. After resolving each file: -```bash -git add {resolved-file} -``` - -4. Continue the rebase: -```bash -git rebase --continue -``` - -5. If conflicts are too complex to resolve confidently: -```bash -git rebase --abort -``` -Then flag to the engineer: -``` -⚠️ Merge conflicts with main are too complex to auto-resolve. -Conflicting files: -- {file1} — {what conflicts} -- {file2} — {what conflicts} - -Please resolve manually or pair on this. -``` - -**If no conflicts, proceed.** - -### 3.2 Push Branch - -```bash -git push -u origin $(git branch --show-current) -# If rebased, may need force push: -git push -u origin $(git branch --show-current) --force-with-lease -``` - -**Always use `--force-with-lease`** (never `--force`) when force pushing after rebase — it protects against overwriting someone else's pushes. - -### 3.3 Get the Linear Issue URL - -Before creating the PR, fetch the Linear issue URL to include in the PR body: - -``` -Use get_issue MCP tool to get the parent issue details -Extract the issue URL (e.g., https://linear.app/riverline/issue/ENG-123) -Use this URL in the PR body's ## Linear section -``` - -### 3.4 Create PR with gh CLI - -```bash -gh pr create \ - --title "{type}: {feature description}" \ - --body "$(cat <<'EOF' -{full body from step 2, with Linear URL} -EOF -)" -``` - -After creation, capture the PR URL from the `gh pr create` output — you'll need it for the Linear update. - -### 3.5 Set PR Metadata - -```bash -# Add labels if applicable -gh pr edit --add-label "feature" - -# Add reviewers if engineer specifies -gh pr edit --add-reviewer {username} -``` - ---- - -## Step 4: LINK Back - -### 4.1 Update Linear Parent Issue with Implementation Doc - -Use Linear MCP `update_issue` to update the **parent issue** description with a full implementation section. This turns the Linear issue into the complete record — plan + implementation. - -**Append to the parent issue description:** - -```markdown ---- - -## Implementation - -**PR:** [#{pr_number} — {pr_title}]({github_pr_url}) -**Branch:** {branch} - -### What was built -{Concise summary of the complete feature — what the code does end-to-end} - -### Sub-tasks completed -| Sub-task | Linear | Status | -|----------|--------|--------| -| {title} | ENG-124 | Done | -| {title} | ENG-125 | Done | -| {title} | ENG-126 | Done | - -### Files changed -| File | Action | Purpose | -|------|--------|---------| -| `path/to/file.ts` | CREATED | {what it does} | -| `path/to/existing.ts` | MODIFIED | {what changed} | - -### Decisions Made -| Decision | Choice | Alternatives Considered | Reasoning | -|----------|--------|------------------------|-----------| -| {what} | {choice} | {alternatives} | {reasoning} | - -### Edge Cases Handled -| Scenario | Handling | -|----------|----------| -| {edge case 1} | {how it's handled} | -| {edge case 2} | {how it's handled} | - -### How to test -- [ ] {test step 1} -- [ ] {test step 2} -``` - -### 4.2 Update Linear Issue Statuses - -``` -Use update_issue to move parent issue to "In Review" -Use update_issue to move all sub-issues to "Done" (they're all in the PR) -``` - -### 4.3 Report to Engineer - -``` -PR created: {PR URL} - -Title: {title} -Branch: {branch} → main -Files changed: {count} -Commits: {count} -Linear: ENG-123 updated with implementation doc, moved to "In Review" -Sub-issues: ENG-124, ENG-125, ENG-126 moved to "Done" - -Reviewer can use /pr-review to review with full plan context. -``` - ---- - -## Multi-Commit PRs - -Feature PRs will typically have multiple commits. Include a commit summary: - -```markdown -## Commits -1. `abc1234` - Add callAnalysis schema fields -2. `def5678` - Create transcription service -3. `ghi9012` - Build LLM analysis service -4. `jkl3456` - Add queue worker and webhook trigger -5. `mno7890` - Wire pipeline end-to-end, add error handling -``` - ---- - -## Anti-Patterns - -| Don't | Do Instead | -|-------|------------| -| Generic "Updated code" title | Specific type + feature description | -| Empty PR description | Full structured description with all sections | -| Skip decisions section | Document every non-obvious choice | -| PR without Linear link | Always link to the parent issue | -| PR without plan reference | Link to plan doc if one exists | -| "LGTM" test plan | Specific, actionable test steps | -| List files without grouping | Group changes by sub-task for reviewability | - ---- - -## Related Skills -- [[commit]] — PRs are built from commits -- [[pr-review]] — PRs get reviewed -- [[linear]] — PRs link to Linear issues - -## Self-Improvement - -After completing this skill, if you discovered: -- A PR description section that was missing -- A better way to document decisions -- A pattern for specific PR types - -Then **automatically** invoke the `/improve` skill to: -1. Add the learning to `LEARNINGS.md` in this skill folder -2. Update `SKILL.md` if it's a core instruction change -3. Commit and push -4. Notify user to sync - - ---- - -# Accumulated Learnings - -> Auto-merged from LEARNINGS.md. Apply these edge cases, patterns, and preferences when executing this skill. - - - -## Edge Cases - -_(None yet - will be populated as skill is used)_ - -## User Preferences - -_(None yet - will be populated as skill is used)_ - -## Patterns - -- Plan docs live at `docs/plans/{feature-name}.md` in the repo -- Linear issue IDs are extracted from branch names: `feature/ENG-123-description` -- PR body uses heredoc with `gh pr create` to preserve formatting diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..118585b6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.next +.turbo +.vercel +.wrangler +**/.next +**/.turbo +**/.wrangler +**/dist +**/node_modules +.pnpm-store +.env +.env.* +!.env.example +*.har +*.log +/*.png +**/*.tsbuildinfo +coverage +node_modules +qa-output +supabase/.temp diff --git a/.env.example b/.env.example index 1a417c38..fe4756d3 100644 --- a/.env.example +++ b/.env.example @@ -1,51 +1,74 @@ -# Cloudflare -CLOUDFLARE_API_TOKEN= -CLOUDFLARE_ACCOUNT_ID= -OUTPUT_DOWNLOAD_SIGNING_SECRET= +# Root .env.local is the only local credential file. Cloudflare and Vercel +# receive production values directly in their dashboards; never copy this file there. -# Daytona +# Local Postgres. Generate four distinct URL-safe passwords and keep each Worker +# password identical to the password in only its own connection URL. +LOCAL_POSTGRES_PASSWORD=replace_with_64_hex_postgres_password +LOCAL_APP_GATEWAY_PASSWORD=replace_with_64_hex_app_gateway_password +LOCAL_APP_AGENT_PASSWORD=replace_with_64_hex_app_agent_password +LOCAL_APP_WEBHOOKS_PASSWORD=replace_with_64_hex_app_webhooks_password +LOCAL_DATABASE_PORT=54322 +CHEATCODE_LOCAL_DATABASE=true +SUPABASE_MIGRATION_URL=postgresql://postgres:replace_with_64_hex_postgres_password@database:5432/postgres +SUPABASE_MIGRATION_EXPECTED_HOST=database +SUPABASE_MIGRATION_EXPECTED_DATABASE=postgres +SUPABASE_MIGRATION_EXPECTED_ROLE=postgres +LOCAL_GATEWAY_DATABASE_URL=postgresql://app_gateway:replace_with_64_hex_app_gateway_password@database:5432/postgres +LOCAL_AGENT_DATABASE_URL=postgresql://app_agent:replace_with_64_hex_app_agent_password@database:5432/postgres +LOCAL_WEBHOOKS_DATABASE_URL=postgresql://app_webhooks:replace_with_64_hex_app_webhooks_password@database:5432/postgres + +# Clerk development instance. These test keys are for this laptop only; every +# Vercel environment uses the production Clerk instance. +NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_replace_me +CLERK_SECRET_KEY=sk_test_replace_me +# Optional only when receiving Clerk webhooks in local development. +CLERK_WEBHOOK_SIGNING_SECRET= + +# Local web routing. The real preview-proxy Worker is service-bound behind the +# gateway and serves each sandbox on *.localhost:8787. No second preview domain +# or cloud development deployment is required. +NEXT_PUBLIC_GATEWAY_URL=http://127.0.0.1:8787 +NEXT_PUBLIC_PREVIEW_HOSTNAME=localhost +NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA=development + +# Daytona development access. DAYTONA_API_KEY= DAYTONA_API_URL=https://app.daytona.io/api DAYTONA_PREVIEW_HOST_SUFFIXES=daytonaproxy01.net,proxy.daytona.work -DAYTONA_TARGET=us +# Required: set an explicit development snapshot so local startup can never +# inherit the production snapshot from the committed Worker configuration. DAYTONA_SANDBOX_SNAPSHOT= +DAYTONA_TARGET=us +DAYTONA_WORKSPACE_VOLUME=cheatcode-workspaces-development DAYTONA_WEBHOOK_SIGNING_SECRET= -PREVIEW_TOKEN_SECRET= - -# Supabase -DATABASE_URL= -# SUPABASE_MIGRATION_URL belongs in .env.migrate only, never here. +PREVIEW_TOKEN_SECRET=replace_with_a_distinct_32_byte_secret +# Optional when the Daytona account requires an explicit organization. +DAYTONA_ORG_ID= -# Clerk -# Local development uses pk_test_/sk_test_ keys. Configure pk_live_/sk_live_ only -# in Vercel Production and production Cloudflare Worker secrets. -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY= -CLERK_SECRET_KEY= -# Optional PEM public key for networkless Clerk token verification. -CLERK_JWT_KEY= -CLERK_AUTHORIZED_PARTIES=http://localhost:3000,http://127.0.0.1:3000 -CLERK_WEBHOOK_SIGNING_SECRET= +# Agent providers and integrations. +COMPOSIO_API_KEY= +COMPOSIO_AUTH_CONFIGS= +COMPOSIO_WEBHOOK_SECRET= +# Optional platform fallback; users can rely on BYOK instead. +DEEPSEEK_PLATFORM_API_KEY= -# Polar +# Polar local development always uses the sandbox account. POLAR_ACCESS_TOKEN= -POLAR_SERVER=production +POLAR_SERVER=sandbox POLAR_WEBHOOK_SECRET= POLAR_PRODUCT_ID_PRO= POLAR_PRODUCT_ID_PREMIUM= POLAR_PRODUCT_ID_ULTRA= POLAR_PRODUCT_ID_MAX= -# Composio -COMPOSIO_API_KEY= -COMPOSIO_AUTH_CONFIGS={"github":"ac_...","gmail":"ac_...","slack":"ac_...","notion":"ac_...","linear":"ac_..."} -COMPOSIO_WEBHOOK_SECRET= - -# Internal ops alerts -CLOUDFLARE_ANALYTICS_API_TOKEN= -INTERNAL_ALERT_WEBHOOK_SECRET= -# Reuse the same local value as INTERNAL_ALERT_WEBHOOK_SECRET unless rotating separately. -INTERNAL_MAINTENANCE_SECRET= - -# Gateway -NEXT_PUBLIC_GATEWAY_URL=http://localhost:8787 -NEXT_PUBLIC_PREVIEW_HOSTNAME=trycheatcode.com +# Internal local contracts. +DATABASE_CONTEXT_SIGNING_SECRET_AGENT=replace_with_a_distinct_32_byte_secret +DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY=replace_with_a_distinct_32_byte_secret +DATABASE_CONTEXT_SIGNING_SECRET_WEBHOOKS=replace_with_a_distinct_32_byte_secret +GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET=replace_with_a_distinct_32_byte_secret +WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET=replace_with_a_distinct_32_byte_secret +INTERNAL_WEBHOOK_REPLAY_SECRET=replace_with_a_distinct_32_byte_secret +RELEASE_DATABASE_READINESS_SECRET=replace_with_a_distinct_32_byte_secret +OUTPUT_DOWNLOAD_SIGNING_SECRET=replace_with_a_distinct_32_byte_secret +SKILL_RUNTIME_BASE_URL=https://gateway.trycheatcode.com/skill-runtime +SKILL_RUNTIME_TOKEN_SECRET=replace_with_a_distinct_32_byte_secret diff --git a/.env.migrate.example b/.env.migrate.example deleted file mode 100644 index 3e790a95..00000000 --- a/.env.migrate.example +++ /dev/null @@ -1,11 +0,0 @@ -# Local Supabase default. For production, replace this with the production -# Supabase admin/DDL connection for the same project used by Hyperdrive, or -# apply the migration through Supabase MCP and verify the Worker route. -SUPABASE_MIGRATION_URL=postgresql://postgres:postgres@localhost:54322/postgres -SUPABASE_MIGRATION_EXPECTED_HOST=localhost -SUPABASE_MIGRATION_EXPECTED_DATABASE=postgres -SUPABASE_MIGRATION_EXPECTED_ROLE=postgres -# Query once from the intended database: select system_identifier from pg_control_system(); -# SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER=replace_me -# Required for audit archive applies so Wrangler cannot select an unintended account. -# CLOUDFLARE_ACCOUNT_ID=00000000000000000000000000000000 diff --git a/.github/actions/setup-daytona/action.yml b/.github/actions/setup-daytona/action.yml new file mode 100644 index 00000000..45a53d55 --- /dev/null +++ b/.github/actions/setup-daytona/action.yml @@ -0,0 +1,49 @@ +name: Setup Daytona CLI +description: Install and authenticate the checksum-pinned Daytona CLI used by production operations. + +inputs: + api-key: + description: Daytona API key from the protected production environment. + required: true + +runs: + using: composite + steps: + - name: Install and authenticate checksum-pinned Daytona CLI + shell: bash + env: + DAYTONA_API_KEY: ${{ inputs.api-key }} + run: | + set +x + set -Eeuo pipefail + + readonly DAYTONA_CLI_VERSION=0.198.0 + readonly DAYTONA_CLI_SHA256=b5fbc05d195e016cb2db05ef009a19dc3a30b4e9f4729199d9b093acb30596a5 + readonly INSTALL_DIR="$RUNNER_TEMP/daytona-cli/bin" + readonly DOWNLOAD_PATH="$RUNNER_TEMP/daytona-cli/daytona-linux-amd64" + + if [ "$RUNNER_OS" != "Linux" ] || [ "$RUNNER_ARCH" != "X64" ]; then + echo "The pinned Daytona CLI action supports only Linux x64 runners." >&2 + exit 1 + fi + if [ -z "$DAYTONA_API_KEY" ]; then + echo "A protected Daytona API key is required." >&2 + exit 1 + fi + + rm -rf "$RUNNER_TEMP/daytona-cli" + mkdir -p "$INSTALL_DIR" + curl --fail --silent --show-error --location \ + --retry 3 --retry-all-errors --connect-timeout 10 --max-time 120 \ + "https://github.com/daytona/clients/releases/download/v${DAYTONA_CLI_VERSION}/daytona-linux-amd64" \ + --output "$DOWNLOAD_PATH" + echo "${DAYTONA_CLI_SHA256} $DOWNLOAD_PATH" | sha256sum --check --strict + install -m 0755 "$DOWNLOAD_PATH" "$INSTALL_DIR/daytona" + + version_output="$("$INSTALL_DIR/daytona" version)" + if [ "$version_output" != "Daytona CLI version v${DAYTONA_CLI_VERSION}" ]; then + echo "The installed Daytona CLI did not report the pinned version." >&2 + exit 1 + fi + "$INSTALL_DIR/daytona" login --api-key "$DAYTONA_API_KEY" + echo "$INSTALL_DIR" >> "$GITHUB_PATH" diff --git a/.github/actions/setup-repository/action.yml b/.github/actions/setup-repository/action.yml new file mode 100644 index 00000000..5ca487e0 --- /dev/null +++ b/.github/actions/setup-repository/action.yml @@ -0,0 +1,28 @@ +name: Setup repository +description: Verify an optional release commit and install the pinned Node and pnpm workspace. + +inputs: + release-sha: + description: Optional full commit SHA that the checked-out repository must match. + required: false + default: "" + +runs: + using: composite + steps: + - name: Verify release commit + if: inputs.release-sha != '' + shell: bash + env: + RELEASE_SHA: ${{ inputs.release-sha }} + run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: 11.8.0 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22.22.2 + cache: pnpm + - name: Install workspace dependencies + shell: bash + run: pnpm install --frozen-lockfile diff --git a/.github/workflows/audit-archive.yml b/.github/workflows/audit-archive.yml new file mode 100644 index 00000000..2b8c3116 --- /dev/null +++ b/.github/workflows/audit-archive.yml @@ -0,0 +1,177 @@ +name: Audit Archive + +run-name: Audit archive ${{ inputs.mode }} at ${{ github.sha }} + +on: + workflow_dispatch: + inputs: + mode: + description: Plan the operation or apply it to production. + required: true + default: dry-run + type: choice + options: + - dry-run + - apply + confirmation: + description: Type PLAN_AUDIT_ARCHIVE or APPLY_AUDIT_ARCHIVE to match the selected mode. + required: true + type: string + archive_before_days: + description: Archive complete partitions at least this many days old (minimum 30). + required: true + default: "90" + type: string + create_months_ahead: + description: Ensure this many future monthly partitions exist (1-60). + required: true + default: "24" + type: string + purge_verified_before_days: + description: Optional age for purging detached DB tables after verified R2 archival (minimum 30). + required: false + default: "" + type: string + purge_confirmation: + description: For apply with purge, type DROP_VERIFIED_AUDIT_PARTITIONS. + required: false + default: "" + type: string + +permissions: + actions: read + contents: read + +concurrency: + group: production-control-plane + cancel-in-progress: false + +jobs: + archive: + runs-on: ubuntu-24.04 + timeout-minutes: 240 + environment: production + steps: + - name: Authorize protected maintenance operation + env: + CONFIRMATION: ${{ inputs.confirmation }} + MODE: ${{ inputs.mode }} + PURGE_CONFIRMATION: ${{ inputs.purge_confirmation }} + PURGE_VERIFIED_BEFORE_DAYS: ${{ inputs.purge_verified_before_days }} + run: | + if [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo "Audit maintenance must be dispatched from main." >&2 + exit 1 + fi + case "$MODE" in + dry-run) + if [ "$CONFIRMATION" != "PLAN_AUDIT_ARCHIVE" ]; then + echo "Dry-run confirmation must exactly match PLAN_AUDIT_ARCHIVE." >&2 + exit 1 + fi + if [ -n "$PURGE_CONFIRMATION" ]; then + echo "Dry-run does not accept purge_confirmation." >&2 + exit 1 + fi + ;; + apply) + if [ "$CONFIRMATION" != "APPLY_AUDIT_ARCHIVE" ]; then + echo "Apply confirmation must exactly match APPLY_AUDIT_ARCHIVE." >&2 + exit 1 + fi + if [ -n "$PURGE_VERIFIED_BEFORE_DAYS" ] && + [ "$PURGE_CONFIRMATION" != "DROP_VERIFIED_AUDIT_PARTITIONS" ]; then + echo "Production purge requires DROP_VERIFIED_AUDIT_PARTITIONS." >&2 + exit 1 + fi + ;; + *) + echo "Unknown audit archive mode: $MODE" >&2 + exit 1 + ;; + esac + if [ -z "$PURGE_VERIFIED_BEFORE_DAYS" ] && [ -n "$PURGE_CONFIRMATION" ]; then + echo "purge_confirmation requires purge_verified_before_days." >&2 + exit 1 + fi + - name: Record absolute job deadline origin + run: echo "CHEATCODE_AUDIT_ARCHIVE_JOB_STARTED_AT_MS=$(date +%s%3N)" >> "$GITHUB_ENV" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + - uses: ./.github/actions/setup-repository + with: + release-sha: ${{ github.sha }} + - name: Require successful static checks for this commit + env: + GH_TOKEN: ${{ github.token }} + run: | + successful_runs="$(gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/static-checks.yml/runs" \ + --field branch=main \ + --field event=push \ + --field head_sha="$GITHUB_SHA" \ + --field per_page=1 \ + --field status=success \ + --jq '.total_count')" + if [ "$successful_runs" -lt 1 ]; then + echo "Static Checks has not succeeded for $GITHUB_SHA on main." >&2 + exit 1 + fi + - name: Validate protected production configuration + env: + AUDIT_ARCHIVE_CLOUDFLARE_API_TOKEN: ${{ inputs.mode == 'apply' && secrets.AUDIT_ARCHIVE_CLOUDFLARE_API_TOKEN || '' }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + MODE: ${{ inputs.mode }} + SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} + SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} + SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} + SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} + SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} + run: | + required=( + CLOUDFLARE_ACCOUNT_ID + SUPABASE_MIGRATION_EXPECTED_DATABASE + SUPABASE_MIGRATION_EXPECTED_HOST + SUPABASE_MIGRATION_EXPECTED_ROLE + SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER + SUPABASE_MIGRATION_URL + ) + if [ "$MODE" = "apply" ]; then + required+=(AUDIT_ARCHIVE_CLOUDFLARE_API_TOKEN) + fi + missing=() + for name in "${required[@]}"; do + if [ -z "${!name}" ]; then missing+=("$name"); fi + done + if [ "${#missing[@]}" -gt 0 ]; then + printf 'Missing production configuration: %s\n' "${missing[*]}" >&2 + exit 1 + fi + - name: Run bounded audit archive maintenance + env: + ARCHIVE_BEFORE_DAYS: ${{ inputs.archive_before_days }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ inputs.mode == 'apply' && secrets.AUDIT_ARCHIVE_CLOUDFLARE_API_TOKEN || '' }} + CREATE_MONTHS_AHEAD: ${{ inputs.create_months_ahead }} + MODE: ${{ inputs.mode }} + PURGE_CONFIRMATION: ${{ inputs.purge_confirmation }} + PURGE_VERIFIED_BEFORE_DAYS: ${{ inputs.purge_verified_before_days }} + SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} + SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} + SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} + SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} + SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} + run: | + args=( + "--$MODE" + --archive-before-days "$ARCHIVE_BEFORE_DAYS" + --create-months-ahead "$CREATE_MONTHS_AHEAD" + ) + if [ -n "$PURGE_VERIFIED_BEFORE_DAYS" ]; then + args+=(--purge-verified-before-days "$PURGE_VERIFIED_BEFORE_DAYS") + fi + if [ -n "$PURGE_CONFIRMATION" ]; then + args+=(--confirm-purge "$PURGE_CONFIRMATION") + fi + pnpm audit:archive -- "${args[@]}" diff --git a/.github/workflows/build-snapshot.yml b/.github/workflows/build-snapshot.yml index 6cb1b458..580543ae 100644 --- a/.github/workflows/build-snapshot.yml +++ b/.github/workflows/build-snapshot.yml @@ -25,9 +25,6 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 environment: production - env: - DAYTONA_CLI_VERSION: 0.197.0 - DAYTONA_CLI_SHA256: 75493c6a41e7f0a427e116f150310d1669fd01529d37e7b4abaf89034cb9193b steps: - name: Authorize snapshot publication env: @@ -67,30 +64,27 @@ jobs: - name: Derive immutable snapshot name id: snapshot run: | - snapshot_name="cheatcode-sandbox-viewer-bundle-$(git rev-parse --short=12 HEAD)-${GITHUB_RUN_ID}" + source_commit="$(git rev-parse HEAD)" + if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]] || + [[ ! "$GITHUB_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "Snapshot provenance did not resolve to canonical commit and workflow-run IDs." >&2 + exit 1 + fi + snapshot_name="cheatcode-sandbox-viewer-bundle-${source_commit:0:12}-${GITHUB_RUN_ID}" echo "name=$snapshot_name" >> "$GITHUB_OUTPUT" echo "SNAPSHOT_NAME=$snapshot_name" >> "$GITHUB_ENV" - - name: Install checksum-verified Daytona CLI - run: | - mkdir -p "$RUNNER_TEMP/bin" - curl --fail --silent --show-error --location \ - --retry 3 --retry-all-errors --connect-timeout 10 --max-time 120 \ - "https://github.com/daytona/clients/releases/download/v${DAYTONA_CLI_VERSION}/daytona-linux-amd64" \ - --output "$RUNNER_TEMP/daytona" - echo "${DAYTONA_CLI_SHA256} $RUNNER_TEMP/daytona" | sha256sum --check --strict - install -m 0755 "$RUNNER_TEMP/daytona" "$RUNNER_TEMP/bin/daytona" - echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" - - name: Authenticate Daytona CLI - env: - DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} - run: | - test -n "$DAYTONA_API_KEY" - "$RUNNER_TEMP/bin/daytona" version - "$RUNNER_TEMP/bin/daytona" login --api-key "$DAYTONA_API_KEY" + - name: Set up checksum-pinned Daytona CLI + uses: ./.github/actions/setup-daytona + with: + api-key: ${{ secrets.DAYTONA_API_KEY }} - name: Build immutable image candidate run: | image_tag="cheatcode-sandbox:$GITHUB_SHA" - docker build --platform=linux/amd64 --tag "$image_tag" infra/containers/sandbox + docker build \ + --platform=linux/amd64 \ + --build-context default_skills=skills \ + --tag "$image_tag" \ + infra/containers/sandbox echo "IMAGE_TAG=$image_tag" >> "$GITHUB_ENV" - name: Scan candidate configuration and image run: | @@ -128,8 +122,43 @@ jobs: libreoffice --version test -f /home/node/cheatcode-next-template/pnpm-lock.yaml test -f /home/node/cheatcode-expo-template/pnpm-lock.yaml + test -f /home/node/.cheatcode/default-skills/browser-use/SKILL.md + test -f /home/node/.cheatcode/default-skills/pptx/SKILL.md + test -f /home/node/.cheatcode/default-skills/pptx/scripts/thumbnail.py + test -f /home/node/.cheatcode/default-skills/docx/scripts/office/pack.py + test -f /home/node/.cheatcode/default-skills/docx/scripts/office/schemas/mce/mc.xsd + test -f /home/node/.cheatcode/default-skills/xlsx/scripts/recalc.py + test -f /home/node/.cheatcode/default-skills/webapp-testing/scripts/with_server.py + test -f /home/node/.cheatcode/default-skills/manage-skills/_shared.ts + test -f /home/node/.cheatcode/default-skills/skill-authoring/SKILL.md + test -f /home/node/.cheatcode/default-skills/skill-authoring/persist/save.ts + test -f /home/node/.cheatcode/default-skills/generate-media/SKILL.md + test -f /home/node/.cheatcode/default-skills/product-self-knowledge/SKILL.md + test "$(find /home/node/.cheatcode/default-skills -type f -name "*.py" | wc -l)" -eq 56 + test "$(find /home/node/.cheatcode/default-skills -type f -name "*.ts" | wc -l)" -eq 12 + test "$(find /home/node/.cheatcode/default-skills -type f -name "*.xsd" | wc -l)" -eq 117 + test ! -e /home/node/.cheatcode/default-skills/deploy + test ! -e /home/node/.cheatcode/default-skills/landing-page + test ! -e /home/node/.cheatcode/default-skills/slide-from-prd + test ! -e /home/node/.cheatcode/default-skills/social-post-pack + test ! -e /home/node/.cheatcode/default-skills/competitor-brief + test -z "$(find /home/node/.cheatcode/default-skills -mindepth 1 -maxdepth 1 -type d -iname "*deploy*" -print -quit)" test -f /opt/cheatcode-browser-driver/server.js node --check /opt/cheatcode-browser-driver/server.js + cheatcode-skills --help + cheatcode-browser --help + cheatcode-skills manage-skills/manage/list --help + cheatcode-skills skill-authoring/persist/save --help + python3 /home/node/.cheatcode/default-skills/pptx/scripts/thumbnail.py --help + python3 /home/node/.cheatcode/default-skills/docx/scripts/office/validate.py --help + python3 /home/node/.cheatcode/default-skills/xlsx/scripts/recalc.py --help + python3 /home/node/.cheatcode/default-skills/webapp-testing/scripts/with_server.py --help + python3 -c "import defusedxml, lxml, markitdown, pdf2image, pdfplumber, playwright, pypdf, pypdfium2, pytesseract, reportlab" + pandoc --version + pdfinfo -v + qpdf --version + tesseract --version + test -z "$(grep -R -E "(LEGACY_SKILL_API_KEY|ORCHIDS_API_KEY)" /home/node/.cheatcode/default-skills /opt/cheatcode-skill-runtime || true)" ' - name: Publish and verify immutable snapshot candidate run: | diff --git a/.github/workflows/db-migrate.yml b/.github/workflows/db-migrate.yml deleted file mode 100644 index 6c28802d..00000000 --- a/.github/workflows/db-migrate.yml +++ /dev/null @@ -1,434 +0,0 @@ -name: Production Release - -on: - pull_request: - paths: - - infra/supabase/migrations/** - - packages/db/src/schema/** - - packages/db/drizzle/** - - scripts/migrate.ts - - scripts/migration-drizzle.ts - - scripts/database-operation-safety.ts - - scripts/archive-audit-log.ts - - scripts/audit-archive-options.ts - - scripts/audit-archive-storage.ts - - packages/db/src/drizzle-migrations.ts - - packages/db/src/supabase-target.ts - - packages/env/src/migrate.ts - - .github/workflows/db-migrate.yml - - .github/workflows/deploy-workers.yml - - apps/web/vercel.json - workflow_dispatch: - inputs: - confirmation: - description: Type RELEASE_PRODUCTION to release the exact main-branch commit. - required: true - type: string - -permissions: - actions: read - contents: read - -# One lock covers the schema apply and every dependent deployment, so production -# releases cannot overlap. -concurrency: - group: ${{ github.event_name == 'workflow_dispatch' && 'production-release' || format('db-migrate-pr-{0}', github.event.pull_request.number) }} - cancel-in-progress: false - -jobs: - diff: - if: github.event_name == 'pull_request' - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - # drizzle.config.ts imports @cheatcode/env/migrate from its built output. - - run: pnpm turbo build --filter=@cheatcode/db^... - # Generation never connects. The placeholder only satisfies eager env parsing. - - run: pnpm --filter @cheatcode/db db:generate - env: - SUPABASE_MIGRATION_URL: postgres://placeholder@localhost:5432/placeholder - - name: Fail if Drizzle migrations drifted - run: git diff --exit-code packages/db/drizzle - - preflight: - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-24.04 - timeout-minutes: 10 - environment: production - steps: - - name: Authorize production release - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [ "$GITHUB_REF" != "refs/heads/main" ]; then - echo "Production releases must be dispatched from main." >&2 - exit 1 - fi - if [ "$CONFIRMATION" != "RELEASE_PRODUCTION" ]; then - echo "Confirmation must exactly match RELEASE_PRODUCTION." >&2 - exit 1 - fi - - name: Preflight production configuration - env: - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} - SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} - SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} - SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - required=( - CLOUDFLARE_ACCOUNT_ID - CLOUDFLARE_API_TOKEN - SUPABASE_MIGRATION_EXPECTED_DATABASE - SUPABASE_MIGRATION_EXPECTED_HOST - SUPABASE_MIGRATION_EXPECTED_ROLE - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER - SUPABASE_MIGRATION_URL - VERCEL_ORG_ID - VERCEL_PROJECT_ID - VERCEL_TOKEN - ) - missing=() - for name in "${required[@]}"; do - if [ -z "${!name}" ]; then - missing+=("$name") - fi - done - if [ "${#missing[@]}" -gt 0 ]; then - printf 'Missing production configuration: %s\n' "${missing[*]}" >&2 - exit 1 - fi - - name: Require successful static checks for this commit - env: - GH_TOKEN: ${{ github.token }} - run: | - successful_runs="$(gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/static-checks.yml/runs" \ - --field branch=main \ - --field event=push \ - --field head_sha="$GITHUB_SHA" \ - --field per_page=1 \ - --field status=success \ - --jq '.total_count')" - if [ "$successful_runs" -lt 1 ]; then - echo "Static Checks has not succeeded for $GITHUB_SHA on main." >&2 - exit 1 - fi - - prepare-frontend: - needs: preflight - if: needs.preflight.result == 'success' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - environment: production - outputs: - deployment_url: ${{ steps.deploy.outputs.url }} - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.sha }} - - name: Verify release commit - env: - RELEASE_SHA: ${{ github.sha }} - run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - - name: Pull Vercel production configuration - working-directory: apps/web - env: - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - test -n "$VERCEL_ORG_ID" - test -n "$VERCEL_PROJECT_ID" - test -n "$VERCEL_TOKEN" - pnpm exec vercel pull --yes --environment=production --token="$VERCEL_TOKEN" - - name: Build immutable Vercel artifact - working-directory: apps/web - env: - NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA: ${{ github.sha }} - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - run: pnpm exec vercel build --prod - # Create a production-target deployment without assigning production - # domains. This proves the exact artifact before any database or backend - # mutation; the later promote step is the only frontend traffic cutover. - - name: Stage exact frontend release - id: deploy - working-directory: apps/web - env: - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - deployment_json="$(pnpm exec vercel deploy \ - --archive=tgz \ - --format=json \ - --prebuilt \ - --prod \ - --skip-domain \ - --yes \ - --token="$VERCEL_TOKEN")" - deployment_url="$(jq --exit-status --raw-output \ - 'select(.target == "production" and .readyState == "READY") | - .url | select(type == "string")' <<<"$deployment_json")" - deployment_url="${deployment_url%/}" - if [[ ! "$deployment_url" =~ ^https://[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.vercel\.app$ ]]; then - echo "Vercel did not return a trusted deployment URL." >&2 - exit 1 - fi - echo "url=$deployment_url" >> "$GITHUB_OUTPUT" - - name: Verify staged frontend release - working-directory: apps/web - env: - DEPLOYMENT_URL: ${{ steps.deploy.outputs.url }} - RELEASE_SHA: ${{ github.sha }} - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - readonly MAX_ATTEMPTS=24 - readonly POLL_INTERVAL_SECONDS=5 - - if [[ ! "$DEPLOYMENT_URL" =~ ^https://[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.vercel\.app$ ]]; then - echo "Refusing to query an untrusted Vercel deployment origin." >&2 - exit 1 - fi - - for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1)); do - if response="$(pnpm exec vercel curl /api/health \ - --deployment "$DEPLOYMENT_URL" \ - --yes \ - --token="$VERCEL_TOKEN" \ - -- \ - --fail-with-body \ - --silent \ - --show-error \ - --connect-timeout 5 \ - --max-time 15 \ - --header "Accept: application/json" \ - --header "Cache-Control: no-cache")" && - jq --exit-status --arg sha "$RELEASE_SHA" ' - .ok == true and .service == "web" and .releaseSha == $sha - ' <<<"$response" > /dev/null 2>&1; then - echo "Staged Vercel deployment reports release $RELEASE_SHA." - exit 0 - fi - - echo "Staged Vercel deployment has not converged to $RELEASE_SHA (attempt $attempt/$MAX_ATTEMPTS)." - if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then - sleep "$POLL_INTERVAL_SECONDS" - fi - done - - echo "Staged Vercel deployment did not converge to $RELEASE_SHA within the release window." >&2 - exit 1 - - apply-pre-deploy: - needs: prepare-frontend - if: needs.prepare-frontend.result == 'success' - runs-on: ubuntu-24.04 - timeout-minutes: 30 - environment: production - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.sha }} - - name: Verify release commit - env: - RELEASE_SHA: ${{ github.sha }} - run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - - run: pnpm turbo build --filter=@cheatcode/db^... - - name: Show pre-deploy migration plan - run: pnpm tsx scripts/migrate.ts --dry-run --phase=pre-deploy - env: - SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} - SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} - SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} - SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} - - name: Apply pre-deploy migrations - env: - SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} - SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} - SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} - SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} - run: pnpm tsx scripts/migrate.ts --apply --phase=pre-deploy - - deploy-backend: - needs: [apply-pre-deploy, prepare-frontend] - if: needs['apply-pre-deploy'].result == 'success' && needs.prepare-frontend.result == 'success' - uses: ./.github/workflows/deploy-workers.yml - with: - confirmation: ${{ inputs.confirmation }} - - deploy-frontend: - needs: [prepare-frontend, deploy-backend] - if: needs.prepare-frontend.result == 'success' && needs.deploy-backend.result == 'success' - runs-on: ubuntu-24.04 - timeout-minutes: 30 - environment: production - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.sha }} - - name: Verify release commit - env: - RELEASE_SHA: ${{ github.sha }} - run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - - name: Pull Vercel project configuration - working-directory: apps/web - env: - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - test -n "$VERCEL_ORG_ID" - test -n "$VERCEL_PROJECT_ID" - test -n "$VERCEL_TOKEN" - pnpm exec vercel pull --yes --environment=production --token="$VERCEL_TOKEN" - - name: Promote verified frontend release - working-directory: apps/web - env: - DEPLOYMENT_URL: ${{ needs.prepare-frontend.outputs.deployment_url }} - VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: | - if [[ ! "$DEPLOYMENT_URL" =~ ^https://[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.vercel\.app$ ]]; then - echo "Refusing to promote an untrusted Vercel deployment URL." >&2 - exit 1 - fi - pnpm exec vercel promote "$DEPLOYMENT_URL" \ - --yes \ - --token="$VERCEL_TOKEN" - - name: Verify production frontend release - working-directory: apps/web - env: - PRODUCTION_URL: ${{ vars.VERCEL_PRODUCTION_URL || 'https://trycheatcode.com' }} - RELEASE_SHA: ${{ github.sha }} - run: | - readonly MAX_ATTEMPTS=24 - readonly POLL_INTERVAL_SECONDS=5 - - production_origin="${PRODUCTION_URL%/}" - if [[ ! "$production_origin" =~ ^https://([A-Za-z0-9-]+\.)*trycheatcode\.com$ ]]; then - echo "Refusing to query an untrusted production origin." >&2 - exit 1 - fi - - curl_options=( - --fail-with-body - --silent - --show-error - --connect-timeout 5 - --max-time 15 - --header "Accept: application/json" - --header "Cache-Control: no-cache" - ) - - for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1)); do - if response="$(curl "${curl_options[@]}" "$production_origin/api/health")" && - jq --exit-status --arg sha "$RELEASE_SHA" ' - .ok == true and .service == "web" and .releaseSha == $sha - ' <<<"$response" > /dev/null 2>&1; then - echo "Vercel production reports release $RELEASE_SHA." - exit 0 - fi - - echo "Vercel production has not converged to $RELEASE_SHA (attempt $attempt/$MAX_ATTEMPTS)." - if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then - sleep "$POLL_INTERVAL_SECONDS" - fi - done - - echo "Vercel production did not converge to $RELEASE_SHA within the release window." >&2 - exit 1 - - name: Record frontend release - env: - DEPLOYMENT_URL: ${{ needs.prepare-frontend.outputs.deployment_url }} - RELEASE_SHA: ${{ github.sha }} - run: | - { - echo "### Vercel frontend release" - echo "- Commit: \`$RELEASE_SHA\`" - echo "- Deployment: $DEPLOYMENT_URL" - } >> "$GITHUB_STEP_SUMMARY" - - post-deploy-migrations: - needs: deploy-frontend - if: needs.deploy-frontend.result == 'success' - runs-on: ubuntu-24.04 - timeout-minutes: 30 - environment: production - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.sha }} - - name: Verify release commit - env: - RELEASE_SHA: ${{ github.sha }} - run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - - run: pnpm turbo build --filter=@cheatcode/db^... - - name: Show post-deploy migration plan - run: pnpm tsx scripts/migrate.ts --dry-run --phase=post-deploy - env: - SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} - SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} - SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} - SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} - - name: Apply post-deploy migrations - run: pnpm tsx scripts/migrate.ts --apply --phase=post-deploy - env: - SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} - SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} - SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} - SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} - SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} diff --git a/.github/workflows/deploy-workers.yml b/.github/workflows/deploy-workers.yml index 3c9c469d..3b61814c 100644 --- a/.github/workflows/deploy-workers.yml +++ b/.github/workflows/deploy-workers.yml @@ -1,14 +1,37 @@ name: Deploy Workers -# This reusable workflow releases the Cloudflare backend for one immutable commit. -# Its caller prepares the matching Vercel artifact before invoking it and applies -# schema contractions only after both backend and frontend releases are verified. +# Reusable half-release for one immutable commit. `stage-closed` closes the +# gateway, lets already-admitted agent/webhook work drain, then closes every writer. +# `open` verifies that exact barrier, proves the dedicated signed database roles, +# performs the isolated V1 retirement, then opens agent and webhooks with the +# matching gateway always deployed last. on: workflow_call: inputs: confirmation: required: true type: string + frontend_deployment_id: + required: false + default: "" + type: string + phase: + required: true + type: string + production_web_origin: + required: false + default: "" + type: string + production_resource_fingerprint: + required: true + type: string + release_sha: + required: true + type: string + superseded_release_sha: + required: false + default: "" + type: string permissions: actions: read @@ -17,13 +40,27 @@ permissions: jobs: release-backend: runs-on: ubuntu-24.04 - timeout-minutes: 45 + # The script admits mutations only when its 230-minute stage budget and + # separate 50-minute fail-closed recovery reserve fit inside this job. + timeout-minutes: 360 environment: production steps: - - name: Authorize coordinated backend release + - name: Authorize coordinated backend release phase env: + CALLER_WORKFLOW_REF: ${{ github.workflow_ref }} CONFIRMATION: ${{ inputs.confirmation }} + FRONTEND_DEPLOYMENT_ID: ${{ inputs.frontend_deployment_id }} + PRODUCTION_WEB_ORIGIN: ${{ inputs.production_web_origin }} + PRODUCTION_RESOURCE_FINGERPRINT: ${{ inputs.production_resource_fingerprint }} + RELEASE_PHASE: ${{ inputs.phase }} + RELEASE_SHA: ${{ inputs.release_sha }} + SUPERSEDED_RELEASE_SHA: ${{ inputs.superseded_release_sha }} run: | + expected_caller_workflow_ref="${GITHUB_REPOSITORY}/.github/workflows/production-release.yml@refs/heads/main" + if [ "$CALLER_WORKFLOW_REF" != "$expected_caller_workflow_ref" ]; then + echo "Backend releases must be called by the exact main production-release workflow." >&2 + exit 1 + fi if [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ]; then echo "Backend releases must originate from a manual Production Release run." >&2 exit 1 @@ -32,124 +69,161 @@ jobs: echo "Backend releases must target main." >&2 exit 1 fi - if [ "$CONFIRMATION" != "RELEASE_PRODUCTION" ]; then - echo "Backend release confirmation must exactly match RELEASE_PRODUCTION." >&2 + if [[ ! "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "The backend release SHA must be a full lowercase 40-character Git commit SHA." >&2 + exit 1 + fi + if [[ ! "$PRODUCTION_RESOURCE_FINGERPRINT" =~ ^[0-9a-f]{64}$ ]]; then + echo "The backend release requires the exact production resource fingerprint." >&2 + exit 1 + fi + if [ -n "$SUPERSEDED_RELEASE_SHA" ] && { + [ "$RELEASE_PHASE" != "stage-closed" ] || + [[ ! "$SUPERSEDED_RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] || + [ "$SUPERSEDED_RELEASE_SHA" = "$RELEASE_SHA" ]; + }; then + echo "A superseded release must be a different closed SHA on stage-closed." >&2 exit 1 fi + case "$RELEASE_PHASE:$CONFIRMATION" in + stage-closed:STAGE_PRODUCTION_CLOSED) + if [ -n "$PRODUCTION_WEB_ORIGIN" ] || [ -n "$FRONTEND_DEPLOYMENT_ID" ]; then + echo "stage-closed must not receive production frontend identity." >&2 + exit 1 + fi + ;; + open:OPEN_PRODUCTION) + if [ "$PRODUCTION_WEB_ORIGIN" != "https://trycheatcode.com" ] || + [[ ! "$FRONTEND_DEPLOYMENT_ID" =~ ^dpl_[A-Za-z0-9]+$ ]]; then + echo "open requires the canonical origin and verified Vercel deployment ID." >&2 + exit 1 + fi + ;; + *) + echo "Confirmation does not authorize release phase $RELEASE_PHASE." >&2 + exit 1 + ;; + esac + started_at_seconds="$(date +%s)" + if [[ ! "$started_at_seconds" =~ ^[0-9]{10}$ ]]; then + echo "Could not establish the backend release job deadline." >&2 + exit 1 + fi + echo "CHEATCODE_RELEASE_JOB_STARTED_AT_MS=${started_at_seconds}000" >> "$GITHUB_ENV" - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + fetch-depth: 0 + ref: ${{ inputs.release_sha }} + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: .release-control ref: ${{ github.sha }} - - name: Verify release commit - env: - RELEASE_SHA: ${{ github.sha }} - run: test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + sparse-checkout: .github/actions/verify-release-control + - uses: ./.release-control/.github/actions/verify-release-control + with: + control-sha: ${{ github.sha }} + release-sha: ${{ inputs.release_sha }} + - run: rm -rf .release-control - name: Require successful static checks for this commit env: GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ inputs.release_sha }} run: | successful_runs="$(gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/workflows/static-checks.yml/runs" \ --field branch=main \ --field event=push \ - --field head_sha="$GITHUB_SHA" \ + --field head_sha="$RELEASE_SHA" \ --field per_page=1 \ --field status=success \ --jq '.total_count')" if [ "$successful_runs" -lt 1 ]; then - echo "Static Checks has not succeeded for $GITHUB_SHA on main." >&2 + echo "Static Checks has not succeeded for $RELEASE_SHA on main." >&2 exit 1 fi - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: ./.github/actions/setup-repository with: - version: 10.34.5 - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - with: - node-version: 22 - cache: pnpm - - run: pnpm install --frozen-lockfile - # Rebuild every backend deployable before the first Worker mutation. Static - # Checks already proved this exact commit before the expand migration gate. - - name: Build backend release - run: >- - pnpm turbo build - --filter=@cheatcode/gateway-worker - --filter=@cheatcode/agent-worker - --filter=@cheatcode/webhooks-worker - --filter=@cheatcode/preview-proxy - # The shared script deploys the new gateway with its public release gate - # closed, verifies the exact-SHA 503 barrier, deploys and verifies agent - # through that closed health endpoint, then reopens the same gateway build. - # Webhooks and preview follow only after gateway+agent are current. Partial - # production Worker deploys are rejected by the script. - - name: Deploy backend behind fail-closed release barrier - run: pnpm deploy:workers -- --apply --skip-build + release-sha: ${{ inputs.release_sha }} + - name: Build backend release phase + env: + RELEASE_PHASE: ${{ inputs.phase }} + run: | + if [ "$RELEASE_PHASE" = "open" ]; then + pnpm turbo build \ + --filter=@cheatcode/gateway-worker \ + --filter=@cheatcode/agent-worker \ + --filter=@cheatcode/webhooks-worker + else + pnpm turbo build \ + --filter=@cheatcode/gateway-worker \ + --filter=@cheatcode/agent-worker \ + --filter=@cheatcode/webhooks-worker \ + --filter=@cheatcode/preview-proxy + fi + - name: Pull Vercel production project identity + if: ${{ inputs.phase == 'open' }} + env: + VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + run: pnpm exec vercel pull --yes --environment=production --token="$VERCEL_TOKEN" + - name: Reverify canonical alias immediately before backend OPEN + if: ${{ inputs.phase == 'open' }} + env: + DEPLOYMENT_ID: ${{ inputs.frontend_deployment_id }} + PRODUCTION_WEB_ORIGIN: ${{ inputs.production_web_origin }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + run: | + project_name="$(jq --exit-status --raw-output '.projectName' .vercel/project.json)" + inspection="$(timeout --signal=TERM 15s pnpm exec vercel inspect \ + "$PRODUCTION_WEB_ORIGIN" --format=json --token="$VERCEL_TOKEN")" + jq --exit-status \ + --arg id "$DEPLOYMENT_ID" \ + --arg project "$project_name" ' + .id == $id and .name == $project and + .target == "production" and .readyState == "READY" + ' <<<"$inspection" > /dev/null + - name: Apply fail-closed backend release phase env: + CHEATCODE_MIGRATION_ATTESTATIONS: ${{ secrets.CHEATCODE_MIGRATION_ATTESTATIONS }} + CHEATCODE_FRONTEND_DEPLOYMENT_ID: ${{ inputs.frontend_deployment_id }} + CHEATCODE_RELEASE_SHA: ${{ inputs.release_sha }} + CHEATCODE_PRODUCTION_WEB_ORIGIN: ${{ inputs.production_web_origin }} + CHEATCODE_PRODUCTION_RESOURCE_FINGERPRINT: ${{ inputs.production_resource_fingerprint }} + CHEATCODE_SUPERSEDE_CLOSED_SHA: ${{ inputs.superseded_release_sha }} CHEATCODE_PROD_DEPLOY_APPROVED: "true" CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - - name: Verify exact backend release - env: - GATEWAY_HEALTH_URL: ${{ vars.GATEWAY_HEALTH_URL || 'https://gateway.trycheatcode.com/health' }} - PREVIEW_PROXY_HEALTH_URL: ${{ vars.PREVIEW_PROXY_HEALTH_URL || 'https://preview.trycheatcode.com/health' }} - RELEASE_SHA: ${{ github.sha }} - WEBHOOKS_HEALTH_URL: ${{ vars.WEBHOOKS_HEALTH_URL || 'https://webhooks.trycheatcode.com/health' }} - run: | - readonly MAX_ATTEMPTS=24 - readonly POLL_INTERVAL_SECONDS=5 - - wait_for_release() { - local health_url="$1" - local label="$2" - local jq_filter="$3" - local attempt - local response - - for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1)); do - if response="$(curl \ - --fail-with-body \ - --silent \ - --show-error \ - --connect-timeout 5 \ - --max-time 15 \ - --header "Accept: application/json" \ - --header "Cache-Control: no-cache" \ - "$health_url")" && - jq --exit-status --arg sha "$RELEASE_SHA" "$jq_filter" \ - <<<"$response" > /dev/null 2>&1; then - echo "$label reports release $RELEASE_SHA." - return 0 - fi - - echo "$label has not converged to $RELEASE_SHA (attempt $attempt/$MAX_ATTEMPTS)." - if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then - sleep "$POLL_INTERVAL_SECONDS" - fi - done - - echo "$label did not converge to $RELEASE_SHA within the release window." >&2 - return 1 - } - - wait_for_release "$GATEWAY_HEALTH_URL" "Gateway and agent" ' - .ok == true and - .releaseSha == $sha and - .agent.ok == true and - .agent.releaseSha == $sha - ' - wait_for_release "$WEBHOOKS_HEALTH_URL" "Webhooks worker" ' - .ok == true and .worker == "webhooks" and .releaseSha == $sha - ' - wait_for_release "$PREVIEW_PROXY_HEALTH_URL" "Preview proxy" ' - .ok == true and .worker == "preview-proxy" and .releaseSha == $sha - ' - - name: Record release + CLOUDFLARE_RESOURCE_READ_API_TOKEN: ${{ secrets.CLOUDFLARE_RESOURCE_READ_API_TOKEN }} + RELEASE_PHASE: ${{ inputs.phase }} + RELEASE_DATABASE_READINESS_SECRET: ${{ secrets.RELEASE_DATABASE_READINESS_SECRET }} + RELEASE_SHA: ${{ inputs.release_sha }} + SUPABASE_MIGRATION_EXPECTED_DATABASE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_DATABASE }} + SUPABASE_MIGRATION_EXPECTED_HOST: ${{ vars.SUPABASE_MIGRATION_EXPECTED_HOST }} + SUPABASE_MIGRATION_EXPECTED_ROLE: ${{ vars.SUPABASE_MIGRATION_EXPECTED_ROLE }} + SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER: ${{ vars.SUPABASE_MIGRATION_EXPECTED_SYSTEM_IDENTIFIER }} + SUPABASE_MIGRATION_URL: ${{ secrets.SUPABASE_MIGRATION_URL }} + VERCEL_ORG_ID: ${{ vars.VERCEL_ORG_ID }} + VERCEL_PROJECT_ID: ${{ vars.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ inputs.phase == 'open' && secrets.VERCEL_TOKEN || '' }} + run: pnpm deploy:workers -- --phase "$RELEASE_PHASE" --apply --skip-build + - name: Record release phase + continue-on-error: true env: - RELEASE_SHA: ${{ github.sha }} + RELEASE_PHASE: ${{ inputs.phase }} + RELEASE_SHA: ${{ inputs.release_sha }} + SUPERSEDED_RELEASE_SHA: ${{ inputs.superseded_release_sha }} run: | { - echo "### Cloudflare backend release" + echo "### Cloudflare backend: $RELEASE_PHASE" echo "- Commit: \`$RELEASE_SHA\`" - echo "- Release barrier: gateway closed and agent convergence verified before reopening" + if [ -n "$SUPERSEDED_RELEASE_SHA" ]; then + echo "- Superseded closed SHA: \`$SUPERSEDED_RELEASE_SHA\`" + fi echo "- Migration gate: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - echo "- Health: gateway, agent, webhooks, and preview proxy verified at this commit" + if [ "$RELEASE_PHASE" = "stage-closed" ]; then + echo "- Barrier: gateway, agent, and webhooks are verified CLOSED; every Workflow writer is drained; preview proxy matches this commit" + else + echo "- Barrier: dedicated database roles were proved before and after isolated V1 retirement; agent and webhooks reopened first; matching gateway was verified OPEN last after exact production web reverification" + fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 0094d091..40d9aa65 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -22,10 +22,10 @@ jobs: fetch-depth: 0 - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 with: - version: 10.34.5 + version: 11.8.0 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: - node-version: 22 + node-version: 22.22.2 cache: pnpm - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: @@ -36,6 +36,12 @@ jobs: cache: true version: v0.72.0 - run: pnpm install --frozen-lockfile + - name: Lint GitHub Actions and embedded shell + uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0 + with: + pyflakes: false + shellcheck: true + version: 1.7.12 - name: Audit pnpm lockfiles run: | trivy fs --scanners vuln --include-dev-deps --severity MEDIUM,HIGH,CRITICAL --exit-code 1 pnpm-lock.yaml @@ -67,6 +73,7 @@ jobs: CLERK_SECRET_KEY: sk_test_static_checks_do_not_authenticate NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: pk_test_c3RhdGljLWNoZWNrcy0wMC5jbGVyay5hY2NvdW50cy5kZXYk NEXT_PUBLIC_GATEWAY_URL: ${{ vars.NEXT_PUBLIC_GATEWAY_URL }} + NEXT_PUBLIC_PREVIEW_HOSTNAME: trycheatcode.com NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA: ${{ github.sha }} # Never expose a future remote-cache credential to pull-request code. TURBO_TOKEN: ${{ github.event_name == 'push' && secrets.TURBO_TOKEN || '' }} diff --git a/.gitignore b/.gitignore index 14ec158a..f13992a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store node_modules/ +.pnpm-store/ .turbo/ .wrangler/ .next/ @@ -14,7 +15,6 @@ qa-*.png .env .env.* !.env.example -!.env.migrate.example apps/*/.dev.vars apps/*/wrangler.local-dev.generated.jsonc apps/*/wrangler.production.*.generated.json @@ -22,7 +22,6 @@ supabase/.temp/ packages/skills/src/generated.ts # Local planning notes that are not part of the V2 workspace. -/docs/ /tasks/ /plans/ diff --git a/.nvmrc b/.nvmrc index 2bd5a0a9..db49bb14 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 +22.22.2 diff --git a/.superset/config.json b/.superset/config.json deleted file mode 100644 index 9d9aba5a..00000000 --- a/.superset/config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "setup": [], - "teardown": [] -} diff --git a/.vercelignore b/.vercelignore index 4afd9c46..bc48339e 100644 --- a/.vercelignore +++ b/.vercelignore @@ -4,7 +4,17 @@ /plans/ # Build artifacts (match anywhere; Vercel installs + builds fresh) node_modules +.pnpm-store .next .wrangler .turbo *.log +# Local-only credentials and container orchestration never enter Vercel uploads. +.env +.env.* +**/.dev.vars +**/.dev.vars.* +compose.yaml +.dockerignore +infra/containers/dev +supabase diff --git a/AGENTS.md b/AGENTS.md index 5059e94c..d4b9832c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,19 +14,19 @@ source tree on July 13, 2026; the repository now contains V2 code only. | Layer | Choice | |---|---| | Backend | Cloudflare Workers + Durable Objects + Workflows | -| Frontend | Next.js 16.2.9 + React 19.2.7 + Tailwind 4.3.1 + shadcn 4.6.0 + AI Elements + Streamdown on Vercel | -| Agent framework | Mastra 1.42.0 on Vercel AI SDK v6.0.205 | +| Frontend | Next.js 16.2.10 + React 19.2.7 + Tailwind 4.3.2 + shadcn 4.6.0 + AI Elements + Streamdown on Vercel | +| Agent framework | Mastra 1.51.0 on Vercel AI SDK v6.0.205 | | Sandbox | Daytona per-user sandboxes via REST-over-fetch | | Browser | Stagehand v3.7.0 LOCAL inside the Daytona sandbox snapshot | | Database | Supabase Postgres via Hyperdrive + Drizzle 0.45.2 | -| Auth | Clerk 7.5.2 | +| Auth | Clerk 7.5.19 | | Billing | Polar 0.48.1 | | OAuth tools | Composio v3.1 REST via bounded `@cheatcode/composio` client | | Storage | R2 (no Supabase Storage) | | Observability | Workers Logs + Workers Tracing + Workers Analytics Engine (no third-party APM in the initial release) | -| Lint/format | Biome 2.5.0 (single config, no ESLint+Prettier except next plugin) | +| Lint/format | Biome 2.5.2 (single config, no ESLint+Prettier except next plugin) | | QA | Direct `agent-browser --auto-connect --session cheatcode-debug` UI operation + console/network/log review; no scripted test harnesses | -| Monorepo | pnpm 10 + Turborepo 2.9.18 | +| Monorepo | pnpm 11.8.0 + Turborepo 2.10.5 | ## Repo layout @@ -51,13 +51,13 @@ packages/ Shared libraries skills/ 8 curated Anthropic SKILL.md skills infra/ Wrangler configs, Supabase migrations, Daytona sandbox Dockerfile -scripts/ Operational helpers only: build skills, secrets, deploy orchestration, migrations, audit archive +scripts/ Operational helpers only: build skills, local startup, deploy orchestration, migrations, audit archive ``` ## Build ```bash -pnpm install # Install workspace deps (use pnpm@10, not npm/yarn) +pnpm install # Install workspace deps (use pnpm@11.8.0, not npm/yarn) pnpm turbo skills:build # Bundle skills/ into packages/skills/src/generated.ts (REQUIRED before build) pnpm turbo db:generate # Generate Drizzle types from schema pnpm turbo build # Production build @@ -117,32 +117,37 @@ not write, run, or keep scripts to submit prompts, click UI, drive auth, wrap ## Run locally ```bash -pnpm dev # apps/web (Next dev) + all Workers (wrangler dev) + Miniflare +pnpm dev # Compose: Postgres + migrations + Next + chained Workers +pnpm dev:down # Stop the local Compose stack ``` Required local env vars in `.env.local` (template in `.env.example`): ``` -# Cloudflare -CLOUDFLARE_API_TOKEN= -CLOUDFLARE_ACCOUNT_ID= -CLOUDFLARE_ANALYTICS_API_TOKEN= -OUTPUT_DOWNLOAD_SIGNING_SECRET= +# Local Postgres + per-Worker roles (distinct URL-safe passwords) +LOCAL_POSTGRES_PASSWORD= +LOCAL_APP_GATEWAY_PASSWORD= +LOCAL_APP_AGENT_PASSWORD= +LOCAL_APP_WEBHOOKS_PASSWORD= +SUPABASE_MIGRATION_URL=postgresql://postgres:@database:5432/postgres +LOCAL_GATEWAY_DATABASE_URL=postgresql://app_gateway:@database:5432/postgres +LOCAL_AGENT_DATABASE_URL=postgresql://app_agent:@database:5432/postgres +LOCAL_WEBHOOKS_DATABASE_URL=postgresql://app_webhooks:@database:5432/postgres + +# Per-Worker signed tenant context (three distinct secrets, each at least 32 bytes) +DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY= +DATABASE_CONTEXT_SIGNING_SECRET_AGENT= +DATABASE_CONTEXT_SIGNING_SECRET_WEBHOOKS= # Daytona DAYTONA_API_KEY= DAYTONA_API_URL=https://app.daytona.io/api DAYTONA_TARGET=us DAYTONA_SANDBOX_SNAPSHOT= +DAYTONA_WORKSPACE_VOLUME=cheatcode-workspaces-development PREVIEW_TOKEN_SECRET= -# Supabase — Workers connect as app_worker only (never service_role). -DATABASE_URL= -# SUPABASE_MIGRATION_URL (admin/DDL role) is NOT here — it lives in a git-ignored -# .env.migrate, used only by scripts/migrate.ts and scripts/archive-audit-log.ts. -# Never bind it to a Worker. - -# Clerk +# Clerk development instance only NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY= CLERK_SECRET_KEY= @@ -157,14 +162,20 @@ COMPOSIO_WEBHOOK_SECRET= # Internal ops alerts INTERNAL_ALERT_WEBHOOK_SECRET= -INTERNAL_MAINTENANCE_SECRET= + +# Capability-scoped ccm2 contracts (four distinct secrets, each at least 32 bytes) +GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET= +WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET= +INTERNAL_WEBHOOK_REPLAY_SECRET= +RELEASE_DATABASE_READINESS_SECRET= # Gateway -NEXT_PUBLIC_GATEWAY_URL=https://gateway.trycheatcode.com +NEXT_PUBLIC_GATEWAY_URL=http://127.0.0.1:8787 ``` -Never commit `.env.local`. Use `pnpm sync:secrets -- --store-id --apply` to create or -rotate production Secrets Store entries by name without printing their values. +Never commit `.env.local`. It is the sole laptop credential file and accepts only +Clerk test keys plus sandbox/local credentials. Vercel and Cloudflare receive +production credentials directly through their protected production environments. ## Code conventions (CI-enforced) @@ -177,7 +188,9 @@ rotate production Secrets Store entries by name without printing their values. 7. **Zod-validate at trust boundaries** — HTTP input, LLM output, env, webhooks. 8. **Files ≤800 lines, functions ≤50 lines, cognitive complexity ≤15.** 9. **BYOK keys never logged.** Decrypt only inside the active `withUserContext()` transaction and pass request-scoped values downward. Do not cache plaintext in module scope, KV, DO storage, logs, or R2. -10. **Workers connect as `app_worker` Postgres role**, never `service_role`. +10. **Workers use separate least-privilege Postgres roles**: `app_gateway`, + `app_agent`, and `app_webhooks`. They never use `service_role`; the historical + `app_worker` transition role must not exist in the production-ready target. The enforced rules live in the root `biome.jsonc` and TypeScript configs. diff --git a/CLAUDE.md b/CLAUDE.md index 8ad0bc5d..8fec565e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,12 +14,12 @@ Direct competitors: Manus (generalist async agent), HappyCapy (GUI workstation + |---|---| | Language | **TypeScript** everywhere. No Python in backend. Python lives only inside the Daytona sandbox. | | Backend runtime | **Cloudflare Workers + Durable Objects + Workflows** | -| Frontend | **Next.js 16.2.9 + React 19.2.7 + Tailwind 4.3.1 + shadcn CLI 4.6.0 + AI Elements + Streamdown** on Vercel | -| Agent framework | **Mastra 1.42.0** on top of **Vercel AI SDK v6.0.205** | +| Frontend | **Next.js 16.2.10 + React 19.2.7 + Tailwind 4.3.2 + shadcn CLI 4.6.0 + AI Elements + Streamdown** on Vercel | +| Agent framework | **Mastra 1.51.0** on top of **Vercel AI SDK v6.0.205** | | Sandbox | **Daytona Sandboxes** via REST-over-fetch (no SDK in Workers; `packages/tools-code/src/daytona-client.ts`) — one persistent sandbox per user with isolated project folders | | Browser automation | **Stagehand v3.7.0 LOCAL mode** inside the Daytona sandbox image | -| Database | **Supabase Postgres via Cloudflare Hyperdrive** + **Drizzle 0.45.2** (no `service_role` from Workers — uses `app_worker` role) | -| Auth | **Clerk 7.5.2** (Workers JWT verify) | +| Database | **Supabase Postgres via Cloudflare Hyperdrive** + **Drizzle 0.45.2** (separate `app_gateway`, `app_agent`, and `app_webhooks` roles; no `service_role`) | +| Auth | **Clerk 7.5.19** (Workers JWT verify) | | Billing | **Polar 0.48.1** (no fixed cost, rev-share only) | | OAuth tool integrations | **Composio v3.1 REST via bounded `@cheatcode/composio` client** | | Storage | **R2** (no Supabase Storage; zero egress) | @@ -56,7 +56,7 @@ packages/ skills/ 8 curated Anthropic SKILL.md skills infra/ Wrangler configs, Supabase migrations, Daytona sandbox Dockerfile/snapshot -scripts/ Operational helpers only: build skills, secrets, deploy orchestration, migrations, audit archive +scripts/ Operational helpers only: build skills, local startup, deploy orchestration, migrations, audit archive ``` ## Critical conventions (non-negotiable) @@ -72,13 +72,14 @@ These are CI-enforced. Violating them blocks merge. 7. **Zod-validate all trust boundaries** — HTTP input, LLM output, env, webhooks, DB rows from external systems. 8. **Files ≤800 lines, functions ≤50 lines, cognitive complexity ≤15.** 9. **BYOK keys** are decrypted on demand via `packages/byok` Vault RPC inside `withUserContext()` and passed only as request-scoped values. **Never log them, never cache in module scope, never persist to KV/DOs/R2.** -10. **Workers connect to Postgres as `app_worker` role**, never `service_role`. RLS is enabled only on `provider_keys` and `audit_log`. +10. **Workers connect through separate `app_gateway`, `app_agent`, and `app_webhooks` roles**, never `service_role`. Every tenant-owned V2 table uses forced RLS and a role-specific signed transaction context. ## Common commands ```bash pnpm install # Install all workspace deps -pnpm dev # Run Next dev plus the backend Workers through Wrangler +pnpm dev # Compose: Postgres + migrations + Next + chained Workers +pnpm dev:down # Stop the local Compose stack pnpm turbo skills:build # Bundle skills/* into packages/skills/src/generated.ts pnpm turbo db:generate # Generate Drizzle types from schema pnpm turbo lint # Biome check (fails on warnings in CI) @@ -134,9 +135,9 @@ not write, run, or keep scripts to submit prompts, click UI, drive auth, wrap ## Skills system -8 curated skills bundled at build time into `packages/skills/src/generated.ts` (Workers have no filesystem at runtime). Anthropic SKILL.md format. V2 has no bundled skill scripts, no `evals/evals.json`, no local skill-eval runner, and no `skill_run_script` tool. +Curated skills are bundled at build time into `packages/skills/src/generated.ts` (Workers have no filesystem at runtime). Anthropic SKILL.md format. V2 has no bundled skill scripts, no `evals/evals.json`, no local skill-eval runner, and no `skill_run_script` tool. -The 8 skills: `pitch-deck`, `deep-research` (covers parallel fan-out research), `competitor-brief`, `slide-from-prd`, `csv-analyst`, `social-post-pack`, `landing-page`, `mobile-app`. External skill registry exports, skills.sh links, public publishing scripts, and launch-prep copy are outside V2 unless the user explicitly re-expands the plan. +The source-of-truth catalog is the set of skill folders under `skills/`; do not duplicate a manually maintained name list here. External skill registry exports, skills.sh links, public publishing scripts, and launch-prep copy are outside V2 unless the user explicitly re-expands the plan. The bundler contract lives in `scripts/build-skills.ts` and `packages/skills`. @@ -149,7 +150,7 @@ The bundler contract lives in `scripts/build-skills.ts` and `packages/skills`. - ❌ Don't add Sentry, Langfuse, or Axiom — use Workers-native observability only. - ❌ Don't expose Cheatcode as an MCP server or add shadcn registry MCP tooling. - ❌ Don't bypass `packages/byok` to access provider keys directly. -- ❌ Don't use `service_role` from Workers — `app_worker` only. +- ❌ Don't use `service_role`; the historical `app_worker` transition role must not exist in the production-ready target. - ❌ Don't use `postgres.js` — use `pg` (node-postgres) per Cloudflare's Hyperdrive + Drizzle guide. - ❌ Don't `drizzle-kit push` in production — always `generate` + review + `migrate`. - ❌ Don't add hard step, token, or cost ceilings to agent loops — semantic completion decides when work is done; cancellation and timeouts remain operational guards. diff --git a/README.md b/README.md index b421ff0a..7bfde0a3 100644 --- a/README.md +++ b/README.md @@ -7,29 +7,34 @@ The live source, package READMEs, migrations, and deployment configuration defin ## Local Setup ```bash -pnpm install -pnpm exec supabase start -cp .env.migrate.example .env.migrate -cp apps/gateway-worker/.dev.vars.example apps/gateway-worker/.dev.vars -pnpm turbo skills:build -pnpm typecheck:scripts -pnpm turbo db:generate -pnpm tsx scripts/migrate.ts --apply --phase=pre-deploy -pnpm tsx scripts/migrate.ts --apply --phase=post-deploy -pnpm turbo build +nvm use +cp .env.example .env.local +# Fill the Clerk development, Daytona, Polar sandbox, and integration values. +# Run `openssl rand -hex 32` thirteen times: four distinct local database passwords, +# three distinct per-Worker tenant-context HMAC secrets, and four distinct ccm2 +# capability secrets, plus the preview-token and output-download signing secrets. +# Repeat each Worker password only in that Worker's matching connection URL. +pnpm dev ``` -Run locally: +`pnpm dev` is the complete local entrypoint. Docker Compose builds the pinned +Node 22.22.2/pnpm 11.8.0 development image, starts the Vault-capable Postgres +database, provisions isolated `app_gateway`, `app_agent`, and `app_webhooks` +roles, applies all three local migration phases, and then starts Next.js plus the +chained Workers. Source changes sync into +the app container; migration changes restart the app behind the migration gate so +the stack cannot stay healthy against a stale schema, and dependency-file changes +rebuild it. Stop the stack with: ```bash -pnpm dev +pnpm dev:down ``` After Weeks 1-8 are implemented in code, product QA is manual browser operation only: ```bash -agent-browser --auto-connect --session cheatcode-debug open http://localhost:3000 +agent-browser --auto-connect --session cheatcode-debug open http://127.0.0.1:3000 agent-browser --auto-connect --session cheatcode-debug snapshot -i ``` @@ -43,7 +48,7 @@ scripts and source-level `*.test.ts` files are intentionally absent; real acceptance testing is the UI plus logs. Do not generate temporary validation scripts either; operate the UI directly and remove any throwaway product QA script that appears in the V2 tree. Operational scripts that remain -in the repo are for build, migration, secret sync, Docker cleanup, and deploy +in the repo are for build, migration, sandbox maintenance, and deploy guardrails only; they are never acceptance evidence. The `scripts/` directory is not a testing surface; delete any script that submits prompts, drives auth or browser flows, checks accessibility/load, or gathers final product evidence. @@ -66,135 +71,240 @@ console/network/app-log inspection. Delete any V2 script that submits prompts, drives auth/browser flows, checks accessibility/load, wraps `agent-browser`, or gathers final product evidence. -Docker local behavior: - -```bash -pnpm docker:clean -``` - V2 production uses Vercel for `apps/web`, Cloudflare for the backend Workers, Supabase, and hosted Daytona sandboxes. Build the sandbox image directly with the command in `infra/containers/sandbox/README.md`; its AMD64 platform matches -Daytona's runner. The obsolete standalone Docker Compose runtime was removed -because Daytona supplies the sandbox daemon and lifecycle in production. +Daytona's runner. Compose is local-only; it is not a production deployment path. `pnpm dev` writes ignored `wrangler.local-dev.generated.jsonc` files next to -each Worker config with production-only Secrets Store bindings removed. Local -Workers read secrets from `.dev.vars`; production deploys still use the -committed `wrangler.jsonc` Secrets Store bindings. Before local Workers start, -`pnpm dev` validates that `apps/agent-worker/.dev.vars` has the required -Daytona, preview, maintenance, and output-signing secrets. +each Worker config with production-only Secrets Store bindings removed. Root +`.env.local` is the sole laptop credential source; Wrangler receives only each +Worker's allowlisted bindings, and Next retains only its own allowlisted values +after loading that root file. Production deploys still use the committed +`wrangler.jsonc` Secrets Store bindings. Each generated Worker config receives +only its own role-specific local Hyperdrive connection. Local startup rejects +Clerk live keys, non-sandbox Polar configuration, production Daytona workspace +inheritance, missing explicit snapshot/runtime settings, and cloud-control +credentials. Expected local endpoints: -- `apps/web`: `http://localhost:3000` -- Gateway Worker: `http://localhost:8787` from one chained `wrangler dev` - process that includes gateway, agent, and webhooks Workers +- `apps/web`: `http://127.0.0.1:3000` +- Gateway Worker: `http://127.0.0.1:8787` from one chained `wrangler dev` + process that includes gateway, agent, webhooks, and the real preview-proxy + Worker. Sandbox previews use `*.localhost:8787`; no extra domain is required. - Wrangler inspector: `http://localhost:9239` (kept off 9229 so `agent-browser --auto-connect` attaches to Chrome on 9222, not workerd) -- Supabase Studio: `http://localhost:54323` +- Postgres: `postgres://localhost:54322/postgres` (loopback only) -Database migrations use an expand/deploy/contract sequence enforced by +Database migrations use an expand/contract/prove/finalize sequence enforced by `scripts/migrate.ts`: raw pre-SQL and Drizzle migrations run before the backend -release; destructive raw post-SQL runs only after the new Workers are live. -Every apply requires an explicit `--phase=pre-deploy` or -`--phase=post-deploy`; `--phase=all` is read-only planning only. Use -`SUPABASE_MIGRATION_URL` from a git-ignored `.env.migrate`; never bind it to a -Worker. - -`.env.migrate.example` points at local Supabase only. Before applying a -production migration, confirm `SUPABASE_MIGRATION_URL` targets the same -Supabase project/ref as the production Hyperdrive config, or apply the DDL via -Supabase MCP and verify the deployed Worker route that depends on it. +release; destructive runtime contractions run only behind the closed-writer +barrier. The one-time V1 retirement is a separate `release-finalization` stream: +production can apply it only after the exact dedicated-role CLOSED Workers pass +the signed database-readiness probe, and the release repeats that probe before +opening any writer. Every apply requires an explicit `--phase=pre-deploy`, +`--phase=post-deploy`, or `--phase=release-finalization`; `--phase=all` is +read-only planning only. Use +`SUPABASE_MIGRATION_URL` from root `.env.local` locally or protected CI +environment variables in production; it is never bound to a Worker. Production +DDL is applied only by the protected release workflow after it proves the pinned +database identity matches the three reviewed Hyperdrive targets. The Supabase +dashboard and MCP are read-only verification surfaces during a release, never an +alternative mutation path. `scripts/migrate.ts` validates the target before it prints or applies a migration plan. There is no standalone database validation script in V2; the guardrail runs inside the migration operation that needs it. -When rotating or replacing the production Hyperdrive configuration, update every -database-backed Worker `wrangler.jsonc` binding together with the reviewed -configuration ID: +Each database-backed Worker has its own Hyperdrive and Postgres login. Update all +three reviewed configuration IDs in one validated operation when provisioning or rotating them: ```bash -pnpm prod:set-hyperdrive -- --id -pnpm prod:set-hyperdrive -- --id --apply +pnpm prod:set-hyperdrive -- \ + --gateway-id \ + --agent-id \ + --webhooks-id +pnpm prod:set-hyperdrive -- \ + --gateway-id \ + --agent-id \ + --webhooks-id \ + --apply ``` The GitHub static-check workflow runs repository-wide lint, typecheck, and build gates. The `Production Release` workflow is a manual-only `workflow_dispatch` operation -gated by the exact `RELEASE_PRODUCTION` confirmation and the `production` -environment; a push to `main` must never deploy Cloudflare resources, promote -Vercel production, or mutate the production database by itself. Local production deploy -commands also refuse to run unless `CHEATCODE_PROD_DEPLOY_APPROVED=true` is set -after explicit approval. The release workflow builds, stages, and health-checks an -exact-SHA Vercel production candidate without assigning domains before it mutates -the database or backend. It then serializes pre-deploy migrations, the Cloudflare -backend, promotion of that already-verified Vercel deployment, production-domain -health checks, and post-deploy contractions. Product -correctness is verified through direct `agent-browser` UI operation and logs, -not standalone product-flow validation scripts. +gated by the phase-specific `STAGE_PRODUCTION_CLOSED`, +`RECONCILE_PRODUCTION_CLOSED`, or `OPEN_PRODUCTION` +confirmation and the `production` environment; a push to `main` must never deploy +Cloudflare resources, promote Vercel production, or mutate the production database +by itself. Local commands stay dry-run; the protected workflow alone supplies the +apply authorization and release-job deadline. Production release is deliberately split across three +manual dispatches for the same immutable commit. `stage-closed` first builds, +stages, and health-checks an exact-SHA Vercel production candidate without +assigning domains, then applies expand-only migrations while the current release +is still open. It next closes the gateway, deploys agent/webhooks in `draining` +so already admitted work can finish, proves stable quiescence, and deploys those +writers fully `closed`. The successful stage stores the exact Vercel deployment +ID, immutable URL, release SHA, control-workflow ref, and stage run identity in a +GitHub artifact; operators hand only that stage run ID to OPEN. +`reconcile-closed` re-proves ordinary Workflows drained, then runs the exact +release-scoped workspace and Daytona-snapshot Workflow across every active user while all writers +remain closed. +The `open` dispatch downloads and validates that immutable handoff and revalidates +the frontend artifact, writer barrier, and reconciliation evidence. Before any +contraction, it creates an encrypted custom-format dump of the `drizzle`, `public`, +and `vault` schemas plus an encrypted export of the decrypted Vault records. The +job decrypts the archive, verifies every file checksum, and makes `pg_restore` +parse the complete dump before the protected artifact is accepted. Only then does +OPEN apply contractions, promote that exact deployment ID, and prove the canonical +Vercel alias resolves to it. Backend OPEN then re-proves the contracted database target, +redeploys every writer CLOSED on its dedicated role, exercises all three roles +through a signed readiness aggregate, and reopens agent and webhooks before +gateway last. The critical section rechecks the exact canonical Vercel deployment +and production web SHA immediately before gateway opens. The +Workflow derives all project paths from live V2 data and the canonical generator; +it has no operator-supplied workspace map. Product correctness is +verified through direct `agent-browser` UI operation and logs, not standalone +product-flow validation scripts. The protected `Production` GitHub environment must provide the -`CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN`, `DAYTONA_API_KEY`, -`SUPABASE_MIGRATION_URL`, and `VERCEL_TOKEN` secrets. Repository variables provide +`CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN`, +`AUDIT_ARCHIVE_CLOUDFLARE_API_TOKEN`, `DAYTONA_API_KEY`, +`SUPABASE_MIGRATION_URL`, `VERCEL_TOKEN`, +`DATABASE_BACKUP_ENCRYPTION_KEY`, and +`RELEASE_DATABASE_READINESS_SECRET` secrets. The readiness secret must contain at +least 32 UTF-8 bytes and exists only in the protected release environment and the +three production writer Workers. The backup encryption key must also contain at +least 32 UTF-8 bytes. It exists only in the protected release environment and is +needed to recover the encrypted 90-day pre-contraction artifact; it is never bound +to an application Worker. Before any release phase, +the checksum-pinned Daytona CLI verifies that the configured immutable snapshot +exists exactly once, is active with the reviewed resources and region, and was +built from unchanged sandbox image source on the release's ancestry. It also requires the shared +production workspace volume exactly once in the same organization, ready and error-free, and +rejects duplicate canonical sandbox labels so Durable Object recovery is independent of physical +sandbox names. Daytona volumes use provider-managed object-store capacity and expose no fixed +region or size field; the target is instead checked on the snapshot and every canonical sandbox. +The one-time +`CHEATCODE_MIGRATION_ATTESTATIONS` protected secret is also required while the +attested V1 external-resource contraction remains pending and should be removed +after that raw migration is recorded. Repository variables provide the four `SUPABASE_MIGRATION_EXPECTED_*` identity values, `VERCEL_ORG_ID`, `VERCEL_PROJECT_ID`, and `NEXT_PUBLIC_GATEWAY_URL`. Static checks use an inert, test-shaped Clerk publishable value rather than a deployable credential. Protected deployment URLs are verified with authenticated `vercel curl` requests using the scoped CI token, so no separate -protection-bypass secret is distributed. `VERCEL_PRODUCTION_URL` is optional and -defaults to the canonical `https://trycheatcode.com` frontend origin. - -Publish a new immutable Daytona snapshot after changing `infra/containers/sandbox/`: +protection-bypass secret is distributed. `VERCEL_PRODUCTION_URL` is required and +must exactly equal the canonical `https://trycheatcode.com` frontend origin. +Use one least-privilege Cloudflare release token scoped to the production account +and the exact `trycheatcode.com` zone. It needs account `Workers Scripts Write` +(the Workflows list/detail/version/instance APIs accept this permission), exact-zone +`Workers Routes Write`, exact-zone `Zone Read`, and account `Workers R2 Storage Read` +for the read-only output lifecycle verification; add only a named read permission +that the pinned Wrangler version demonstrably requests. Do not grant broad Account +or Zone Write. The backend verifies the checked-in route contract and creates only +missing exact no-script exclusions for Clerk, documentation, and `www` before the +preview wildcard is deployed. The same fail-closed release gate verifies the exact +`cheatcode-outputs` lifecycle contract: abandoned multipart uploads expire after seven +days and unindexed output objects expire after 60 days. See Cloudflare's [permission catalog](https://developers.cloudflare.com/fundamentals/api/reference/permissions/) +and [Workflows API authorization](https://developers.cloudflare.com/api/resources/workflows/methods/list/). + +Publish a new immutable Daytona snapshot after changing `infra/containers/sandbox/` +by dispatching the protected workflow from `main`: ```bash -docker build --platform=linux/amd64 -t cheatcode-sandbox: infra/containers/sandbox -daytona snapshot push cheatcode-sandbox: --name --cpu 2 --memory 4 --disk 10 --region us +gh workflow run build-snapshot.yml --ref main -f confirmation=BUILD_SNAPSHOT ``` -Cloudflare Secrets Store sync is dry-run by default, never prints secret values, creates missing -entries, and rotates existing entries by name: +Review the emitted immutable snapshot name, then commit it in the agent Worker +configuration. A local AMD64 Docker build is an optional non-publishing check; +production Daytona credentials and snapshot publication stay in GitHub. + +Worker commands are dry-run locally: ```bash -pnpm sync:secrets -- --env-file apps/web/.env.local --env-file apps/webhooks-worker/.dev.vars -pnpm sync:secrets -- --env-file apps/web/.env.local --env-file apps/webhooks-worker/.dev.vars --store-id --apply +pnpm deploy:workers -- --phase stage-closed +pnpm deploy:workers -- --phase open +pnpm reconcile:production-workspaces -- --release-sha <40-character-sha> ``` -Worker deploys are also dry-run by default: +Production release mutation is authorized only through the protected three-dispatch +`Production Release` workflow. Do not run a partial local `--apply`, promote a +Vercel URL manually, or flip a Worker gate in the dashboard; those paths cannot +produce the immutable cross-provider handoff and recovery evidence. -```bash -pnpm deploy:workers -CHEATCODE_PROD_DEPLOY_APPROVED=true pnpm deploy:workers -- --apply -``` +Audit retention has a separate protected `Audit Archive` workflow because it is +maintenance rather than a release. It accepts an exact plan/apply confirmation, +uses the pinned production database identity, and receives a dedicated +bucket-scoped Cloudflare token. Production audit DDL and R2 operations cannot run +from a laptop. The committed Wrangler files are authoritative: an apply replaces Worker vars with a generated copy of the declared config plus the exact release SHA and does -not retain undeclared dashboard vars. It deploys the final gateway bundle closed, -waits for its exact-SHA `503`, deploys agent, verifies the agent SHA through the -still-closed gateway health response, and only then redeploys the same gateway -bundle open. Webhooks and preview proxy follow after that pair has converged. -If any barrier step fails, the operation re-deploys and verifies the closed gate -before stopping. If even that recovery cannot be verified, it reports the gate -state as unconfirmed and requires immediate inspection rather than claiming -production is closed. -The local deploy script releases only the Cloudflare backend. +not retain undeclared dashboard vars. The `stage-closed` phase closes the gateway +first, then deploys agent and webhooks at the exact SHA in `draining`. Fresh HTTP, +scheduled, and ordinary Workflow admissions are rejected while already-admitted +Workflow, Durable Object, sandbox, and persistence continuations finish. Two stable +drain proofs must pass before agent and webhooks are redeployed `closed`; the proofs +are then repeated and the current preview proxy is deployed. Stage-only generated +configs use the transition `app_worker` Hyperdrive until role grants expand; source +and OPEN configs always retain the dedicated least-privilege Hyperdrives. The +separate reconciliation phase runs the one exact release-scoped workspace Workflow +while every writer remains closed. The outer OPEN workflow applies contractions and +promotes the selected Vercel artifact before invoking the backend OPEN half; the +backend script independently verifies both facts before changing a writer. Its +360-minute job admits mutation only when the full 230-minute stage budget (or +160-minute OPEN budget) plus a separate 50-minute fail-closed recovery reserve +still fits. Every child process, health poll, Cloudflare request, Vercel proof, and +database query is clipped to that absolute budget. If a barrier-owned step +fails, the operation re-deploys and verifies gateway, agent, and webhooks closed +before stopping; an unverified recovery is reported as an urgent unconfirmed +writer state. The local deploy script releases only the Cloudflare backend. Production disables every Vercel Git auto-deploy and promotes the staged prebuilt frontend only from the coordinated release workflow. -After the script has verified the recovered closed gate, further release-barrier -recovery is deliberately manual. First fix the transient cause and rerun the -complete deployment from the same immutable commit; -the sequence is idempotent and re-verifies every release identity. If the release -must be abandoned, leave gateway closed, perform a reviewed rollback of agent if -it changed, then roll gateway back to the matching known-good open Worker version -and verify the public `/health` response before resuming frontend promotion or -post-deploy migrations. Never flip the gate open in the dashboard or reopen it -against an unverified agent version. - -The release barrier drains new public requests; it does not cancel requests or -Durable Object executions already in flight. A change to active `AgentRun` -behavior or gateway-owned Durable Object state must be handled as an explicit -drain/state migration rather than assuming the HTTP gate makes that state -transition atomic. +The `app_worker` binding override is deliberately one-shot. Before mutating any +Worker, stage verifies the pinned raw-migration ledger and refuses to run once +`0059_finalize_worker_database_roles.sql` has dropped that role. The immediate +steady-state follow-up release must remove the transition Hyperdrive ID, transition +binding mode, ledger guard, and all transition-only documentation/config branches; +its draining and closed deployments use the dedicated source Hyperdrives directly. +This hard stop prevents one-time cutover compatibility from becoming a permanent +release path. + +Release-barrier recovery is deliberately operator-dispatched. If automated +close-gate recovery is unconfirmed, inspect all three writer health responses, +fix the transient cause, and rerun the complete deployment from the same +immutable commit. The forward-only preflight permits that exact gateway-closed +SHA to resume even when a partial failure left downstream gates incomplete; +no different or older release receives that repair exception. Once recovery is +verified, the sequence is idempotent and re-verifies every release identity. If the +release +cannot continue, leave the gateway closed and dispatch a reviewed, +forward-compatible `stage-closed` release that explicitly names the superseded +closed SHA. Both the currently deployed open SHA and an explicitly superseded +closed SHA must be strict Git ancestors of the candidate; only resuming the exact +same closed SHA is exempt. Never deploy older code after a schema contraction, flip the gate +open in the dashboard, or reopen it against an unverified agent version. + +The release barrier rejects new admissions without canceling a pinned Workflow +or Durable Object continuation already in flight. Before any DDL, the draining +phase explicitly waits for relationally active +AgentRuns and every retained AgentRun/webhook/ops/resource-deletion Workflow to +become quiescent across stable Cloudflare API passes. +Errored and terminated instances fail immediately because Cloudflare can restart +them on that retiring version; `complete` is the only retained status ignored by +the drain gate. `infra/cloudflare/production-workflow-inventory.json` is the +authoritative account inventory: undeclared or duplicate Cheatcode resources fail +the release, and a `retiring` entry remains scanned until its exact resource is +purged. Newly created resources must have their Cloudflare IDs pinned in the first +steady-state follow-up commit. +Persisted Durable Object changes use in-place schema reconciliation rather than +assuming the HTTP gate makes that state transition atomic. Canonical workspace +reconciliation therefore runs inside the user's durable sandbox transition fence, +which rejects new workspace operations +and waits for operations already admitted by that object before touching a +folder. The repository contains only the active V2 implementation. The legacy V1 source tree was permanently removed on July 13, 2026 after explicit user authorization. diff --git a/apps/agent-worker/.dev.vars.example b/apps/agent-worker/.dev.vars.example deleted file mode 100644 index 4e55f1f8..00000000 --- a/apps/agent-worker/.dev.vars.example +++ /dev/null @@ -1,16 +0,0 @@ -# Daytona hosted sandbox credentials for local Worker dev. -DAYTONA_API_KEY= -PREVIEW_TOKEN_SECRET= - -# App-level Composio key used to execute connected-account actions. -COMPOSIO_API_KEY= - -# Used to sign generated-output download URLs served through agent-worker. -OUTPUT_DOWNLOAD_SIGNING_SECRET= - -# Used to verify internal DSR maintenance calls from webhooks-worker. -INTERNAL_MAINTENANCE_SECRET= - -# Optional platform DeepSeek API key used by the built-in fallback model. -# Leave blank to require a user BYOK/OpenRouter key locally. -DEEPSEEK_PLATFORM_API_KEY= diff --git a/apps/agent-worker/README.md b/apps/agent-worker/README.md index af597022..fc5cf0d2 100644 --- a/apps/agent-worker/README.md +++ b/apps/agent-worker/README.md @@ -1,7 +1,7 @@ # @cheatcode/agent-worker -Agent loop Worker with `AgentRun`, user-scoped `ProjectSandbox`, and the Daytona -sandbox adapter. +Agent loop Worker with `AgentRun`, its durable `AgentRunWorkflow` owner, +user-scoped `ProjectSandbox`, and the Daytona sandbox adapter. Each run Durable Object is keyed by run UUID. Each sandbox Durable Object is keyed by a one-way digest of the internal user UUID, so every project for that user shares one isolated @@ -12,32 +12,69 @@ The Worker implements the provider-neutral sandbox and artifact ports from `@cheatcode/sandbox-contracts`. Daytona control-plane and toolbox details remain behind `@cheatcode/tools-code` and do not leak into peer tool packages. +Generated artifacts use a crash-consistent Postgres/R2 protocol. Content determines the output +UUID, object key, and SHA-256 metadata. The Worker durably reserves that identity, +revalidates the live run/project ownership graph, and writes R2 with a create-only precondition; +an existing object is accepted only when its size, checksum, and complete custom identity match. +Reservation and pre-write guard each move a two-hour `cleanup_not_before` fence forward; it is a +remote-side-effect grace deadline, not an ownership token. The final database transaction inserts +the public output and removes the intent. Any post-write result other than committed deletes that +exact object before failing. A committed replay under a still-active run atomically renews output +retention before the output can be exposed through a fresh download capability. A terminal replay +can acknowledge only the same unexpired output and never renews it; every committed replay verifies +the exact R2 object again before returning. Terminal run +persistence records upload quiescence only after its execution promise has settled, while deletion +RPCs abort and join the same promise before returning. + +Artifact messages persist only the output UUID and presentation metadata. The authenticated +`POST /v1/outputs/:outputId/download-url` path rechecks tenant ownership, retention, and R2 +existence before minting a one-hour HMAC capability; the public signed download route is only the +streaming second hop. Expiring capabilities and internal R2 keys are never stored in transcripts or +returned by artifact tools. + Run creation validates the gateway payload with the shared `CreateRunSchema` from -`packages/types` before selecting the user-scoped `AgentRun` Durable Object. The +`packages/types` before selecting the run-scoped `AgentRun` Durable Object. The database binds a gateway-hashed idempotency key to the exact body and thread. After the pending run and thread pointer commit, start delivery is retried and then reconciled through an ordered run-key presence probe. A present object reconnects its stream (and finalizes a -detached run first); only an authoritative empty response fails the nonterminal database run +durable Workflow admission first); only an authoritative empty response fails the nonterminal database run and clears the matching thread pointer in one transaction. Transport or reconnect ambiguity leaves the pointer intact for the next idempotent replay. Active-run conflicts use the same reconciliation path instead of blindly returning a conflict. -There is no periodic stale-pending scheduler. A Worker termination after the database commit -but before the first Durable Object dispatch is repaired by the next idempotent replay or -active-run attempt; adding time-based repair requires a separately owned durable scheduler. +Each admitted semantic run has a deterministic chain of Cloudflare Workflow generations. A +generation keeps the Durable Object execution request attached in four-minute ownership epochs, +then checkpoints and renews the same in-memory coroutine. At 20,000 epochs (about 55 days), it +atomically reserves `generation + 1` in the run-keyed Durable Object and creates the deterministic +successor, leaving almost 5,000 steps below Cloudflare's configured 25,000-step platform ceiling. +The successor's first execution atomically promotes its exact generation, input hash, and instance +ID; any late callback from the predecessor becomes a no-op. A pending successor is recovered by +the existing admission alarm if creation was ambiguous. `draining` permits that continuation, +while `closed` fences it. Generations are an operational rollover mechanism, not a run-duration, +step, token, or cost limit. + +A persisted execution-start fence makes retries at-most-once: a warm retry joins the exact +promise, while a restart that lost that promise terminalizes the run instead of replaying model +calls, tools, or other non-idempotent external side effects. An alarm-backed lease also +terminalizes an admission or execution whose current Workflow owner stops renewing it. +Before a Worker release, the closed gateway plus draining agent gate requires every retained +`cheatcode-agent-runs` instance to be complete. Errored and terminated instances +remain restartable on their pinned Worker version, so they block deployment until +the exact retained instance has expired or been purged. Normal chat runs resolve provider credentials from Supabase Vault through `packages/byok`, pass only the request-scoped transport credential to Mastra, and execute tools against the project folder inside the user's Daytona sandbox. The product-level logical model ID remains separate from the provider-local transport provider/model pair: direct and OpenRouter-routed -requests retain the requested logical ID, while included DeepSeek and approved OpenAI fallback +requests retain the requested logical ID, while included DeepSeek and automatic OpenAI fallback attempts use their own canonical IDs. AgentRun writes that resolved logical ID to Postgres and its Durable Object immediately before every stream attempt. Before model execution, AgentRun loads the newest complete user/assistant transcript suffix -under the caller's Postgres context. PostgreSQL bounds the result to 33 persisted messages and -256 KiB of serialized records before they cross Hyperdrive; the Worker then validates every -record with the canonical UI-message schema and converts it with AI SDK +under the caller's Postgres context. PostgreSQL skips an individually oversized logical turn, +then bounds the result to 33 complete turns and 256 KiB of serialized segment records before +they cross Hyperdrive; the Worker coalesces segments only inside that bound, validates every +record with the canonical UI-message schema, and converts it with AI SDK `convertToModelMessages`. The current run's user message must be last and carry that run ID. Ephemeral app-builder context is appended only to that current model turn and is never stored. @@ -45,8 +82,18 @@ Ephemeral app-builder context is appended only to that current model turn and is every concern. Its HTTP adapter owns bounded request parsing and route dispatch; the run lifecycle module owns progress, terminal persistence, and sandbox-lease cleanup; the run-path module selects general or app-builder execution; and the output component owns replay and -answer segmentation. The shell retains only Durable Object identity, cancellation, approvals, -status, and dependency wiring. +answer segmentation. The Workflow controller owns admission, execution identity, the +at-most-once fence, and ownership leases. The shell retains only Durable Object identity, +cancellation, status, and dependency wiring. + +AgentRun keeps one compact exact SQLite shape for run identity, replay parts, and +coordination state. Dormant objects are reconciled transactionally on activation; +target detection checks column order, affinity, nullability, primary keys, and +defaults before accepting a table as current. Every persisted/replayed UI event is +losslessly normalized to at most 64 KiB, and SQLite reads return at most 32 rows and +256 KiB. Each run accepts at most eight concurrent replay/live streams, with a +256 KiB byte-based queue per stream; a slow client is disconnected and resumes from +its persisted sequence cursor instead of growing isolate memory. Composio actions use the app-level `COMPOSIO_API_KEY`, active rows in `v2_user_integrations`, and the gateway-owned `QuotaTracker` Durable Object before @@ -56,28 +103,45 @@ ProjectSandbox records elapsed sandbox-hours to the same `QuotaTracker` as a sof meter so Settings can show real monthly sandbox consumption without blocking sandbox file/process work. +Postgres is authoritative for user-authored skills. ProjectSandbox mirrors each +selected skill to `/workspace/.cheatcode/skills//SKILL.md` with a registry +revision so users can edit it in Files. An unchanged registry revision permits a +mirror edit to be promoted on the next invocation; concurrent database and file +edits fail closed to the database version while preserving the file for manual +resolution. Curated default skills are immutable snapshot files under +`/home/node/.cheatcode/default-skills/`. + +ProjectSandbox also writes `/workspace/.cheatcode/runtime.json` as an atomic, +generated projection of managed app-preview processes. The Durable Object process +records remain authoritative; the file is only an inspectable runtime manifest. + +Managed processes use required stable IDs and a maximum of 32 live metadata slots per user +sandbox. Reusing an ID atomically replaces that slot. At capacity, ProjectSandbox reconciles the +bounded record set against Daytona, removes missing or completed sessions and their port state, +and rejects a new distinct slot only when all 32 remain live. + Each user has one durable Daytona sandbox. Projects are lexically confined to their folders under `/workspace`, and run leases keep the sandbox active while the agent is working. Project folders share the sandbox's Unix identity, so this prevents accidental cross-project access but is not an operating-system security boundary within one user. -Sandbox lookup validates the Daytona name and ownership labels before trusting a cached -ID. A duplicate live label match fails closed for operator reconciliation instead of -deleting an arbitrary sandbox, and snapshot drift is emitted as structured operational -telemetry while an existing data-bearing sandbox remains usable. Newly created sandboxes -also carry their immutable snapshot name as a label. Destruction resolves that same -validated identity and deletes only the exact Daytona ID; name-based multi-delete cleanup -is deliberately forbidden. +Sandbox lookup validates canonical ownership labels before trusting a cached ID; the +physical Daytona name is deliberately not identity because a promoted replacement has a +release-scoped name. A missing/stale Durable Object cache therefore recovers the one +canonical sandbox by labels, while duplicate live canonical matches fail closed. New +sandboxes pin the configured immutable snapshot and mount the environment's shared Daytona +volume at `/workspace` with the user sandbox name as its isolated subpath. Canonical and +candidate checks require the provider's actual mount tuple as well as the matching labels; labels +alone cannot attest durable storage. A noncurrent +sandbox is maintenance-only and cannot serve product work. Preview URLs carry a 60-second `handoff` capability minted by `@cheatcode/auth`. -Both the production preview Worker and local preview path exchange it for a -distinct host-only, HttpOnly `session` capability capped at 10 minutes. Local -preview does not accept legacy tokens, referrer-carried credentials, or implicit -child credentials. - -The local code-server preview path shares its parent-frame bridge with the -production preview proxy through `@cheatcode/preview-bridge`. It injects only -bounded code-server workbench HTML and pins parent messaging to -`http://localhost:3000`; generated-app preview HTML remains streamed. +The preview-proxy Worker exchanges it for a distinct host-only, HttpOnly +`session` capability capped at 10 minutes in both production and local +development. Local Compose service-binds that same Worker behind +`*.localhost:8787`; the agent has no second proxy implementation. The shared +proxy injects only bounded code-server workbench HTML and pins parent messaging +to the environment's exact app origin; generated-app preview HTML remains +streamed. AgentRun does not count, persist, bill, or emit model-token or model-cost data, and it does not apply per-run or daily dollar caps. Provider usage remains an @@ -90,12 +154,20 @@ the resolved logical model. A failure before any stream attempt keeps planned at provider-local transport IDs remain structured-log context. R2-backed artifact persistence also atomically claims the user's durable first-artifact timestamp before emitting the `first_generated_artifact` activation signal, so output retention cannot make it fire twice. -Terminal database status updates are first persisted or durably queued in AgentRun's +Terminal database status updates are persisted or durably queued in AgentRun's SQLite storage; alarms retry transient database failures with bounded exponential delay -until the database accepts the update. -Final assistant transcript persistence is also run-idempotent: PostgreSQL permits only -one assistant message per run and accepts a replay only when its JSONB content and tenant -identity are semantically identical. +until the database accepts the update. A terminal Postgres status is deliberately held +behind the final transcript outbox: the alarm flushes the transcript first and only then +publishes the terminal status. Production drain can therefore treat every nonterminal +Postgres run as the complete set of unfinished transcript/database work. A closed release +gate performs no write but keeps an outstanding active-run/outbox alarm rearmed for a +same-SHA draining recovery. +Final assistant transcript persistence is run-idempotent. The Durable Object pages its +SQLite log into ordered JSONB segments of at most 128 KiB, using its terminal `completed_at` +as every segment's logical timestamp. PostgreSQL publishes a run only when its unique final +segment exists; retries compare each segment's JSONB, final marker, timestamp, and tenant +identity. Oversized structured parts use lossless bounded fragment envelopes rather than +truncation, and there is no transcript-length, step, token, or cost ceiling. Mastra tool-call chunks also emit `step_started`, `step_completed`, `tool_invoked`, and `skill_invoked` events when those chunks are present in the live stream. If the last stream subscriber disconnects while a run is still @@ -107,15 +179,67 @@ render task progress without polling a separate status endpoint. Project deletion first fences project/thread mutations, refuses an active run, records a durable cleanup request, then removes that project's workspace folder. The database marks cleanup complete only after the Agent service succeeds, so a repeated DELETE retries a -failed cleanup instead of silently leaking storage. Account deletion destroys shared +failed cleanup instead of silently leaking storage. Filesystem operations with an exact canonical +path remain concurrent across unrelated projects, but arbitrary code, shell execution, and process +launch always take a non-exclusive global lease because path parsing cannot prove their runtime +filesystem scope. Project cleanup fences and drains that lease, terminates every managed and +same-user untracked sandbox process, and only then removes the folder. Account deletion destroys shared sandbox state once and removes run Durable Objects in bounded pages. The account cleanup RPC synchronously fences new sandbox work, drains operations that already started, records -final sandbox usage, deletes the validated Daytona sandbox, and preserves only a durable -deletion tombstone. A restarted `ProjectSandbox` therefore cannot recreate resources for a -deleted account. Late lease renewal, lease release, and alarm delivery are safe no-ops after -the fence; every other sandbox RPC fails closed. Every destructive maintenance request is -verified through the shared `ccm1` method/path/millisecond-timestamp/body-hash contract -before the shared deletion payload schema is parsed. +final sandbox usage, clears the user's Daytona volume subpath, and deletes every validated +sandbox. A temporary durable tombstone makes an interrupted cleanup resume behind the same +fence. Once external cleanup succeeds, the configured 2026-07-15 Workers compatibility contract +lets one atomic `deleteAll()` remove that tombstone, owner keys, workspace SQLite schema, and +alarm so the object ceases to occupy storage. + +Constructors inspect existing identity and SQLite metadata without materializing an empty store. +An object with no registered owner absorbs late lease/alarm cleanup and rejects every other +operational RPC. Its only creation path first checks the exact user through the signed +`app_agent` database context, then persists the owner and workspace schema. Clerk deletion makes +gateway identity resolution fail immediately; the account Workflow later aborts and joins every +run before sandbox deletion, while the sandbox fence drains RPCs admitted by the current isolate. +After eviction, a deleted or missing Postgres user cannot register the empty object again, so a +late request cannot resurrect Daytona or durable state. Per-project workspace tombstones remain +durable for active accounts in one `STRICT` table whose checks bind each canonical slug to its +project UUID and enforce ordered millisecond timestamps. Every destructive maintenance request uses +the isolated `ccm2` agent-lifecycle capability and the exact `agent.internal` host. Before any +Durable Object or Daytona mutation, the Worker validates the account deletion fence or exact +project/thread soft-delete generation and verifies that every requested run belongs to that +scope. The 30-second signature window is therefore safe to retry and cannot authorize stale or +cross-tenant destruction; no shared key or legacy signature fallback exists. + +Workspace and sandbox releases use a separate signed internal RPC. For one exact +release SHA, the closed release gate and an in-memory mutation lease reject concurrent +workspace operations. Preparation stops affected processes, collision-checks and renames +Daytona folders, reconciles process and port state, and records only temporary KV evidence for +the canonical folders that existed. Finalization reloads the already-canonical Postgres +inventory and requires the same physical evidence before snapshot work begins. The release +workflow drains all AgentRuns before this phase, so no stale run can recreate a replaced path. +Generic Durable Object reconciliation deliberately runs first: it contracts the permanent +SQLite schema to the project tombstone table and removes the one-time transition and retired-slug +tables; prepare and finalize do not depend on either table. An owner with no materialized sandbox +state uses only the in-memory maintenance lease plus the temporary evidence key, so successful +reconciliation does not leave an empty SQLite store behind. + +Finalization also reconciles the user's existing Daytona sandbox to the exact configured +snapshot. Volume-backed replacements mount the same isolated subpath and compare complete tree +digests. The one-time adoption of a local-disk sandbox creates a deterministic archive and copies +it through durable 8 MiB chunks; there is no total workspace-size cap. A candidate never carries +the canonical label while the source does. After digest verification the source is retired, the +candidate receives the full canonical label set, the Durable Object atomically adopts its exact +ID, and only then is the old sandbox deleted. Every boundary is retryable by the temporary +upgrade phase and deterministic candidate identity. Once final verification succeeds, both the +workspace-transition evidence and snapshot-upgrade state are deleted; an ambiguous response can +therefore retry against the canonical physical state without leaving cutover residue. Account +deletion clears the user's shared-volume subpath before deleting all exact owned sandboxes, so +persistent volume data does not outlive the account. + +Production binds `CHEATCODE_RELEASE_GATE` explicitly. `draining` rejects public +run, sandbox, preview, download, and deletion admission while allowing already +admitted AgentRun Workflow/DO callbacks, sandbox operations, and persistence to +finish. `closed` additionally fences those continuation paths and serves only +`/health` plus the exact signed canonical-workspace reconciliation RPC. Stable +drain proofs run at both gates before DDL. Project ZIP generation and streaming share the exact `PROJECT_ARCHIVE_MAX_OUTPUT_BYTES` contract from `@cheatcode/types` (640 MiB). The @@ -126,6 +250,7 @@ the same bound while streaming. - `agentApp` - `AgentRun` +- `AgentRunWorkflow` - `ProjectSandbox` ## Code Checks @@ -138,24 +263,30 @@ pnpm --filter @cheatcode/agent-worker typecheck - `CHEATCODE_ENVIRONMENT` (`production` in committed Wrangler config; local generated config overrides it) - `CHEATCODE_RELEASE_SHA` (required for production deployments) +- `CHEATCODE_RELEASE_GATE` (`open` in source; coordinated releases inject `draining` and then `closed` until migration/reconciliation complete) - `CF_VERSION_METADATA` - `AGENT_RUN` +- `AGENT_RUN_WORKFLOW` - `PROJECT_SANDBOX` -- `HYPERDRIVE` +- `HYPERDRIVE` (dedicated config whose database login is exactly `app_agent`) +- `DATABASE_CONTEXT_SIGNING_SECRET_AGENT` (role-specific Secrets Store binding; + must match the `app_agent` Supabase Vault HMAC secret) - `DAYTONA_API_KEY` - `DAYTONA_API_URL` - `DAYTONA_TARGET` - `DAYTONA_SANDBOX_SNAPSHOT` +- `DAYTONA_WORKSPACE_VOLUME` (one shared environment volume; each user mounts only its sandbox-name subpath) - `PREVIEW_TOKEN_SECRET` - `COMPOSIO_API_KEY` - `DEEPSEEK_PLATFORM_API_KEY` - `OUTPUT_DOWNLOAD_SIGNING_SECRET` (Secrets Store binding) - `OUTPUT_DOWNLOAD_BASE_URL` -- `INTERNAL_MAINTENANCE_SECRET` +- `WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET` (ccm2 `agent-lifecycle` capability shared + only with the webhooks caller) +- `RELEASE_DATABASE_READINESS_SECRET` (ccm2 `database-readiness` verifier only) - `PREVIEW_HOSTNAME` - `QUOTA_TRACKER` - `R2_AUDIT` - `R2_OUTPUTS` -- `R2_OUTPUTS_BUCKET_NAME` - `SANDBOX_STATE` - `USER_EVENTS`, `AGENT_METRICS`, `ERROR_EVENTS`, `PERFORMANCE_METRICS` diff --git a/apps/agent-worker/package.json b/apps/agent-worker/package.json index e40f1735..6bb1cc7a 100644 --- a/apps/agent-worker/package.json +++ b/apps/agent-worker/package.json @@ -7,7 +7,7 @@ "types": "./dist/index.d.ts", "scripts": { "build": "wrangler deploy --dry-run", - "dev": "wrangler dev --port 8788 --var CHEATCODE_ENVIRONMENT:development", + "deploy": "wrangler deploy", "lint": "biome check .", "typecheck": "tsc -p tsconfig.json --noEmit" }, @@ -16,11 +16,14 @@ "@cheatcode/auth": "workspace:*", "@cheatcode/billing": "workspace:*", "@cheatcode/byok": "workspace:*", + "@cheatcode/composio": "workspace:*", "@cheatcode/db": "workspace:*", + "@cheatcode/durable-storage": "workspace:*", "@cheatcode/env": "workspace:*", "@cheatcode/observability": "workspace:*", "@cheatcode/preview-bridge": "workspace:*", "@cheatcode/sandbox-contracts": "workspace:*", + "@cheatcode/skills": "workspace:*", "@cheatcode/tools-code": "workspace:*", "@cheatcode/types": "workspace:*", "ai": "catalog:", diff --git a/apps/agent-worker/src/agent-api-run-routes.ts b/apps/agent-worker/src/agent-api-run-routes.ts index 9d438dad..7b10ac96 100644 --- a/apps/agent-worker/src/agent-api-run-routes.ts +++ b/apps/agent-worker/src/agent-api-run-routes.ts @@ -11,10 +11,13 @@ import { import { APIError, readJsonRequest } from "@cheatcode/observability"; import { type AgentRunId, - ApprovalDecisionRequestSchema, + BrowserTakeoverResumeResultSchema, + BrowserTakeoverResumeSchema, + BrowserTakeoverStatusSchema, type CreateRun, ThreadId, UserId as toUserId, + type UIMessagePart, type UserId, } from "@cheatcode/types"; import type { Context, Hono } from "hono"; @@ -42,11 +45,9 @@ import { userSandboxName, } from "./tenancy"; -const MAX_APPROVAL_BODY_BYTES = 4 * 1024; const MAX_CREATE_RUN_BODY_BYTES = 64 * 1024; const RUN_IDEMPOTENCY_KEY_HASH_HEADER = "X-Cheatcode-Idempotency-Key-Hash"; const RUN_REQUEST_BODY_HASH_HEADER = "X-Cheatcode-Request-Body-Hash"; -const ApprovalIdParamSchema = z.string().uuid(); const Sha256HexSchema = z.string().regex(/^[a-f0-9]{64}$/); type AgentContext = Context<{ Bindings: AgentEnv }>; type CreateRunResult = Awaited>; @@ -64,7 +65,9 @@ export function registerAgentRunHttpRoutes(app: Hono<{ Bindings: AgentEnv }>): v app.get("/v1/threads/:threadId/runs/stream", streamActiveRun); app.get("/v1/threads/:threadId/runs/status", activeRunStatus); app.post("/v1/runs/:runId/cancel", cancelRun); - app.post("/v1/runs/:runId/approvals/:approvalId", decideApproval); + app.get("/v1/threads/:threadId/browser-takeover", browserTakeoverStatus); + app.post("/v1/threads/:threadId/browser-takeover/start", startBrowserTakeover); + app.post("/v1/threads/:threadId/browser-takeover/resume", resumeBrowserTakeover); } async function createRun(c: AgentContext): Promise { @@ -111,7 +114,10 @@ async function loadRequestPersonalization( threadId: string, body: CreateRun, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, userId, async (tx) => { const thread = await getThread(tx, { threadId: ThreadId(threadId), userId }); @@ -135,7 +141,10 @@ async function persistRunRequest( userId: UserId; }, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, input.userId, async (tx) => { const created = await createAgentRunForThread(tx, { @@ -149,7 +158,7 @@ async function persistRunRequest( if (created.type === "created") { await createThreadMessage(tx, { agentRunId: created.run.runId, - parts: input.body.message.parts, + parts: persistedUserMessageParts(input.body), role: "user", threadId: created.run.threadId, userId: input.userId, @@ -162,6 +171,19 @@ async function persistRunRequest( } } +function persistedUserMessageParts(body: CreateRun): UIMessagePart[] { + const textPart = body.message.parts[0]; + if (!textPart) { + return []; + } + return body.intent === "skill-creator" + ? [ + { data: { intent: "skill-creator", v: 1 }, type: "data-run-intent" }, + { state: "done", text: textPart.text, type: "text" }, + ] + : [{ state: "done", text: textPart.text, type: "text" }]; +} + function rejectedRunError(result: RejectedRunResult): APIError { if (result.type === "thread-not-found") { return new APIError(404, "not_found_thread", "Thread not found", { retriable: false }); @@ -210,7 +232,10 @@ async function reconcileAbsentRunRow( userId: UserId, runId: AgentRunId, ): Promise<"failed" | "not-found" | "terminal"> { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, userId, (tx) => reconcileAbsentAgentRunStart(tx, { runId, userId }), @@ -273,19 +298,61 @@ async function cancelRun(c: AgentContext): Promise { }); } -async function decideApproval(c: AgentContext): Promise { +async function browserTakeoverStatus(c: AgentContext): Promise { const userId = readGatewayUserId(c.req.raw.headers); - const runId = parseRunRouteParam(c.req.param("runId") ?? ""); - const approvalId = parseApprovalRouteParam(c.req.param("approvalId") ?? ""); - const body = ApprovalDecisionRequestSchema.parse( - await readJsonRequest(c.req.raw, MAX_APPROVAL_BODY_BYTES, "Approval decision request"), + const run = await activeRunForThread(c, userId); + if (!run) { + return Response.json(BrowserTakeoverStatusSchema.parse({ status: "inactive" })); + } + return fetchAgentRun( + agentRunForRunId(c.env, run.runId), + "https://agent-run.internal/browser-takeover", + { headers: { "X-Cheatcode-User-Id": userId } }, ); - const run = await runForRoute(c.env, userId, runId); - return fetchAgentRun(agentRunForRunId(c.env, run.runId), "https://agent-run.internal/approval", { - body: JSON.stringify({ ...body, approvalId, userId }), - headers: { "X-Cheatcode-User-Id": userId }, - method: "POST", - }); +} + +async function startBrowserTakeover(c: AgentContext): Promise { + const userId = readGatewayUserId(c.req.raw.headers); + const run = await activeRunForThread(c, userId); + if (!run) { + throw new APIError(409, "conflict_state_invalid", "No active run can be taken over", { + hint: "Start a browser task first.", + retriable: false, + }); + } + return fetchAgentRun( + agentRunForRunId(c.env, run.runId), + "https://agent-run.internal/browser-takeover/start", + { headers: { "X-Cheatcode-User-Id": userId }, method: "POST" }, + ); +} + +async function resumeBrowserTakeover(c: AgentContext): Promise { + const userId = readGatewayUserId(c.req.raw.headers); + const body = BrowserTakeoverResumeSchema.parse( + await readJsonRequest(c.req.raw, 4 * 1024, "Browser takeover resume request"), + ); + const run = await activeRunForThread(c, userId); + if (!run) { + return Response.json(BrowserTakeoverResumeResultSchema.parse({ ok: true, status: "inactive" })); + } + return fetchAgentRun( + agentRunForRunId(c.env, run.runId), + "https://agent-run.internal/browser-takeover/resume", + { + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + "X-Cheatcode-User-Id": userId, + }, + method: "POST", + }, + ); +} + +async function activeRunForThread(c: AgentContext, userId: string): Promise { + const threadId = parseThreadRouteParam(c.req.param("threadId") ?? ""); + return activeRunForThreadRoute(c.env, userId, threadId); } function readRunRequestIdentity(headers: Headers): { bodyHash: string; keyHash: string } { @@ -300,11 +367,3 @@ function readRunRequestIdentity(headers: Headers): { bodyHash: string; keyHash: } return parsed.data; } - -function parseApprovalRouteParam(value: string): string { - const parsed = ApprovalIdParamSchema.safeParse(value); - if (!parsed.success) { - throw new APIError(400, "invalid_path_param", "Invalid approval id", { retriable: false }); - } - return parsed.data; -} diff --git a/apps/agent-worker/src/agent-api-system-routes.ts b/apps/agent-worker/src/agent-api-system-routes.ts index b1d0ff94..9bc75e42 100644 --- a/apps/agent-worker/src/agent-api-system-routes.ts +++ b/apps/agent-worker/src/agent-api-system-routes.ts @@ -1,9 +1,21 @@ -import { createDb, findGeneratedOutputOwner, getProject, withUserContext } from "@cheatcode/db"; +import { + createDb, + findGeneratedOutput, + getProject, + isAgentStateDeletionAuthorized, + loadWorkspaceTransitionOwner, + withUserContext, +} from "@cheatcode/db"; import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; import { APIError, readBoundedRequestText } from "@cheatcode/observability"; import { InternalAgentStateDeleteBodySchema, InternalStateDeleteResponseSchema, + InternalWorkspaceReconciliationBodySchema, + InternalWorkspaceReconciliationResponseSchema, + internalUserStateDeletePath, + internalUserWorkspaceReconciliationPath, + OutputIdSchema, ProjectId, UserId, } from "@cheatcode/types"; @@ -17,13 +29,14 @@ import { sandboxStubForUser, } from "./agent-routing"; import { + assertAgentInternalHostname, + assertAgentLifecycleCapability, parseInternalMaintenanceJson, - verifyAgentMaintenanceRequest, + verifyAgentLifecycleRequest, } from "./internal-maintenance"; -import { resolveLocalPreviewOrigin } from "./local-preview"; import { + createOutputDownloadCapability, OutputDownloadQuerySchema, - OutputIdSchema, verifySignedOutputDownload, } from "./output-download"; import { GatewayUserIdSchema, readGatewayUserId } from "./tenancy"; @@ -33,81 +46,126 @@ const RUN_STATE_DELETE_CONCURRENCY = 16; type AgentContext = Context<{ Bindings: AgentEnv }>; export function registerAgentSystemHttpRoutes(app: Hono<{ Bindings: AgentEnv }>): void { - app.get("/__internal/local-preview-origin", resolveInternalLocalPreviewOrigin); app.post("/internal/users/:userId/delete-state", deleteInternalUserState); + app.post("/internal/users/:userId/reconcile-workspaces", reconcileInternalUserWorkspaces); + app.post("/v1/outputs/:outputId/download-url", mintOutputDownloadUrl); app.get("/v1/outputs/:outputId/download", downloadOutput); app.post("/v1/projects/:projectId/download", downloadProjectArchive); } -async function resolveInternalLocalPreviewOrigin(c: AgentContext): Promise { - const previewUrl = c.req.header("X-Cheatcode-Local-Preview-Url"); - const previewHost = c.req.header("X-Cheatcode-Local-Preview-Host"); - if (!previewUrl || !previewHost) { - throw new APIError(400, "invalid_request_body", "Missing local preview origin headers", { - retriable: false, - }); - } - const headers = localPreviewHeaders(c, previewHost); - const resolved = await resolveLocalPreviewOrigin(new Request(previewUrl, { headers }), c.env); - if (!resolved) { - throw new APIError(404, "invalid_request_body", "Local preview origin not found", { - retriable: false, - }); - } - if (resolved.authorization.fromQuery) { +async function reconcileInternalUserWorkspaces(c: AgentContext): Promise { + if (c.env.CHEATCODE_RELEASE_GATE !== "closed") { throw new APIError( - 400, - "invalid_request_body", - "WebSocket preview requires an established session", + 409, + "conflict_state_invalid", + "Workspace reconciliation requires the closed release gate", { retriable: false }, ); } - return c.json({ - originalHost: resolved.originalHost, - signed: resolved.origin.signed, - token: resolved.origin.token, - url: resolved.origin.url, + assertAgentInternalHostname(c.req.raw); + assertAgentLifecycleCapability(c.req.raw); + const userId = UserId(GatewayUserIdSchema.parse(c.req.param("userId"))); + const rawBody = await readBoundedRequestText( + c.req.raw, + MAX_INTERNAL_MAINTENANCE_BODY_BYTES, + "Internal workspace reconciliation", + ); + await verifyAgentLifecycleRequest({ + expectedPathname: internalUserWorkspaceReconciliationPath(userId), + rawBody, + request: c.req.raw, + secrets: c.env, }); + const body = InternalWorkspaceReconciliationBodySchema.parse( + parseInternalMaintenanceJson(rawBody), + ); + if (c.env.CHEATCODE_RELEASE_SHA !== body.releaseSha) { + throw new APIError(409, "conflict_state_invalid", "Agent release does not match transition", { + details: { actualReleaseSha: c.env.CHEATCODE_RELEASE_SHA ?? null }, + retriable: false, + }); + } + await assertWorkspaceTransitionInventory(c.env, userId, body); + const sandbox = await sandboxStubForUser(c.env, userId); + const result = + body.phase === "prepare" + ? await sandbox.prepareWorkspaceTransition(body) + : await sandbox.finalizeWorkspaceTransition(body); + return c.json(InternalWorkspaceReconciliationResponseSchema.parse(result)); } -function localPreviewHeaders(c: AgentContext, previewHost: string): Headers { - const headers = new Headers({ Host: previewHost }); - copyHeader(c, headers, "X-Cheatcode-Local-Preview-Cookie", "Cookie"); - copyHeader(c, headers, "Origin", "Origin"); - copyHeader( - c, - headers, - "X-Cheatcode-Local-Preview-Client-Host", - "X-Cheatcode-Local-Preview-Client-Host", - ); - return headers; +async function assertWorkspaceTransitionInventory( + env: AgentEnv, + userId: UserId, + body: z.infer, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const owner = await withUserContext(db, userId, (transaction) => + loadWorkspaceTransitionOwner(transaction, userId), + ); + if (!owner || !workspaceInventoryMatches(owner.projects, body.projects, body.phase)) { + throw new APIError( + 409, + "conflict_state_invalid", + "Postgres workspace inventory does not match transition", + { retriable: false }, + ); + } + } finally { + await close(); + } } -function copyHeader( - c: AgentContext, - target: Headers, - sourceName: string, - targetName: string, -): void { - const value = c.req.header(sourceName); - if (value) { - target.set(targetName, value); +function workspaceInventoryMatches( + actual: Array<{ + canonicalWorkspaceSlug: string; + currentWorkspaceSlug: string; + projectId: string; + }>, + requested: Array<{ + canonicalWorkspaceSlug: string; + currentWorkspaceSlug: string; + projectId: string; + }>, + phase: "finalize" | "prepare", +): boolean { + if (actual.length !== requested.length) { + return false; } + const requestedById = new Map(requested.map((project) => [project.projectId, project])); + return actual.every((project) => { + const request = requestedById.get(project.projectId); + return ( + request?.canonicalWorkspaceSlug === project.canonicalWorkspaceSlug && + (phase === "finalize" + ? project.currentWorkspaceSlug === request.canonicalWorkspaceSlug + : project.currentWorkspaceSlug === request.currentWorkspaceSlug || + project.currentWorkspaceSlug === request.canonicalWorkspaceSlug) + ); + }); } async function deleteInternalUserState(c: AgentContext): Promise { + assertAgentInternalHostname(c.req.raw); + assertAgentLifecycleCapability(c.req.raw); + const userId = UserId(GatewayUserIdSchema.parse(c.req.param("userId"))); const rawBody = await readBoundedRequestText( c.req.raw, MAX_INTERNAL_MAINTENANCE_BODY_BYTES, "Internal maintenance request", ); - await verifyAgentMaintenanceRequest({ + await verifyAgentLifecycleRequest({ + expectedPathname: internalUserStateDeletePath(userId), rawBody, request: c.req.raw, - secret: c.env.INTERNAL_MAINTENANCE_SECRET, + secrets: c.env, }); - const userId = UserId(GatewayUserIdSchema.parse(c.req.param("userId"))); const body = InternalAgentStateDeleteBodySchema.parse(parseInternalMaintenanceJson(rawBody)); + await assertAgentStateDeletionAuthority(c.env, userId, body); if (body.scope === "runs") { await deleteRunStates(c.env, userId, body.runIds); return deletedStateResponse(c); @@ -118,10 +176,39 @@ async function deleteInternalUserState(c: AgentContext): Promise { return deletedStateResponse(c); } const sandbox = await sandboxForUser(c.env, userId); - await sandbox.cleanupProjectWorkspace({ workspaceSlug: body.workspaceSlug }); + await sandbox.cleanupProjectWorkspace({ + projectId: body.projectId, + workspaceSlug: body.workspaceSlug, + }); return deletedStateResponse(c); } +async function assertAgentStateDeletionAuthority( + env: AgentEnv, + userId: UserId, + body: z.infer, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const isAuthorized = await withUserContext(db, userId, (transaction) => + isAgentStateDeletionAuthorized(transaction, userId, body), + ); + if (!isAuthorized) { + throw new APIError( + 409, + "conflict_state_invalid", + "Agent state deletion no longer matches an authoritative database generation", + { retriable: false }, + ); + } + } finally { + await close(); + } +} + async function deleteRunStates(env: AgentEnv, userId: string, runIds: string[]): Promise { let nextIndex = 0; const worker = async (): Promise => { @@ -156,6 +243,25 @@ function deletedStateResponse(c: AgentContext): Response { return c.json(InternalStateDeleteResponseSchema.parse({ ok: true })); } +async function mintOutputDownloadUrl(c: AgentContext): Promise { + const outputId = parseOutputId(c.req.param("outputId")); + const userId = UserId(readGatewayUserId(c.req.raw.headers)); + const output = await findDownloadableOutput(c.env, outputId, userId); + if (!(await c.env.R2_OUTPUTS.head(output.r2Key))) { + throw new APIError(404, "not_found_output", "Output object not found", { retriable: false }); + } + const capability = await createOutputDownloadCapability({ + baseUrl: outputDownloadBaseUrl(c.env), + outputId, + secret: await resolveOutputSigningSecret(c.env.OUTPUT_DOWNLOAD_SIGNING_SECRET), + userId, + }); + const response = c.json(capability); + response.headers.set("Cache-Control", "private, max-age=0, no-store"); + response.headers.set("Referrer-Policy", "no-referrer"); + return response; +} + async function downloadOutput(c: AgentContext): Promise { const outputId = parseOutputId(c.req.param("outputId")); const query = parseOutputDownloadQuery(c); @@ -164,13 +270,14 @@ async function downloadOutput(c: AgentContext): Promise { outputId, secret: await resolveOutputSigningSecret(c.env.OUTPUT_DOWNLOAD_SIGNING_SECRET), signature: query.sig, + userId: query.userId, }); if (!isValid) { throw new APIError(403, "permission_denied", "Invalid or expired output download URL", { retriable: false, }); } - const output = await findDownloadableOutput(c.env, outputId); + const output = await findDownloadableOutput(c.env, outputId, query.userId); const object = await c.env.R2_OUTPUTS.get(output.r2Key); if (!object?.body) { throw new APIError(404, "not_found_output", "Output object not found", { retriable: false }); @@ -201,6 +308,7 @@ function parseOutputDownloadQuery(c: AgentContext): z.infer + findGeneratedOutput(tx, { outputId, userId }), + ); if (!output) { throw new APIError(404, "not_found_output", "Output not found", { retriable: false }); } - if (output.expiresAt.getTime() < Date.now()) { - throw new APIError(410, "gone_output_expired", "Output download has expired", { - retriable: false, - }); - } return output; } finally { await close(); @@ -239,6 +347,14 @@ async function resolveOutputSigningSecret(secret: WorkerSecret): Promise { const parsedProjectId = z.string().uuid().safeParse(c.req.param("projectId")); if (!parsedProjectId.success) { @@ -267,7 +383,10 @@ async function downloadProjectArchive(c: AgentContext): Promise { } async function loadProject(env: AgentEnv, userId: UserId, projectId: ProjectId) { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, userId, (tx) => getProject(tx, { projectId, userId })); } finally { @@ -276,13 +395,21 @@ async function loadProject(env: AgentEnv, userId: UserId, projectId: ProjectId) } function downloadContentDisposition(filename: string): string { - const safeName = Array.from(filename, (character) => { + const sanitized = Array.from(filename, (character) => { const codePoint = character.codePointAt(0) ?? 0; - return codePoint <= 31 || codePoint === 127 || character === "\\" || character === '"' + return codePoint <= 31 || + codePoint === 127 || + character === "/" || + character === "\\" || + character === '"' ? "_" : character; - }).join(""); - return `attachment; filename="${safeName}"; filename*=UTF-8''${encodeURIComponent(filename)}`; + }) + .slice(0, 200) + .join(""); + const safeName = sanitized || "cheatcode-output"; + const asciiFallback = safeName.replaceAll(/[^\x20-\x7e]/gu, "_"); + return `attachment; filename="${asciiFallback}"; filename*=UTF-8''${encodeURIComponent(safeName)}`; } function projectArchiveFilename(projectName: string): string { diff --git a/apps/agent-worker/src/agent-env.ts b/apps/agent-worker/src/agent-env.ts index 5ae8bb9b..b2ca00a7 100644 --- a/apps/agent-worker/src/agent-env.ts +++ b/apps/agent-worker/src/agent-env.ts @@ -1,21 +1,25 @@ import type { CloudflareVersionMetadata, WorkerSecret } from "@cheatcode/env"; import type { AnalyticsBindings } from "@cheatcode/observability"; import type { AgentRun } from "./durable-objects/agent-run"; +import type { AgentRunWorkflowPayload } from "./durable-objects/agent-run-workflow-protocol"; import type { ProjectSandbox } from "./durable-objects/project-sandbox"; export interface AgentEnv extends AnalyticsBindings { AGENT_RUN: DurableObjectNamespace; + AGENT_RUN_WORKFLOW: Workflow; CF_VERSION_METADATA?: CloudflareVersionMetadata; CHEATCODE_ENVIRONMENT: "development" | "production"; + CHEATCODE_RELEASE_GATE: "closed" | "draining" | "open"; CHEATCODE_RELEASE_SHA?: string; COMPOSIO_API_KEY?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; DAYTONA_API_KEY: WorkerSecret; DAYTONA_API_URL: string; DAYTONA_ORG_ID?: string; DAYTONA_PREVIEW_HOST_SUFFIXES?: string; DAYTONA_TARGET: string; + DAYTONA_WORKSPACE_VOLUME: string; HYPERDRIVE: Hyperdrive; - INTERNAL_MAINTENANCE_SECRET?: WorkerSecret; OUTPUT_DOWNLOAD_BASE_URL?: string; OUTPUT_DOWNLOAD_SIGNING_SECRET: WorkerSecret; PREVIEW_TOKEN_SECRET: WorkerSecret; @@ -24,6 +28,9 @@ export interface AgentEnv extends AnalyticsBindings { QUOTA_TRACKER: DurableObjectNamespace; R2_AUDIT: R2Bucket; R2_OUTPUTS: R2Bucket; - R2_OUTPUTS_BUCKET_NAME?: string; + RELEASE_DATABASE_READINESS_SECRET: WorkerSecret; SANDBOX_STATE?: KVNamespace; + SKILL_RUNTIME_BASE_URL: string; + SKILL_RUNTIME_TOKEN_SECRET: WorkerSecret; + WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET: WorkerSecret; } diff --git a/apps/agent-worker/src/agent-routing.ts b/apps/agent-worker/src/agent-routing.ts index 47b987e5..63a8586e 100644 --- a/apps/agent-worker/src/agent-routing.ts +++ b/apps/agent-worker/src/agent-routing.ts @@ -7,8 +7,8 @@ import { type AgentRunHandle, createDb, findActiveAgentRunForThread, + findAgentEntitlementByUserId, findAgentRunForUser, - findEntitlementByUserId, getProjectWriteState, getThread, type RunPersonalization, @@ -78,7 +78,10 @@ export async function requireWritableThreadProject( threadId: string, ): Promise { const parsedUserId = UserId(userId); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { await withUserContext(db, parsedUserId, async (tx) => { const thread = await getThread(tx, { threadId: ThreadId(threadId), userId: parsedUserId }); @@ -86,7 +89,7 @@ export async function requireWritableThreadProject( throw new APIError(404, "not_found_thread", "Thread not found", { retriable: false }); } if (!thread.projectId) { - // Project-less chat (no first run yet): nothing to gate — the run creates it. + // Project-less chats stay writable until a workspace-backed tool materializes a project. return; } const state = await getProjectWriteState(tx, { @@ -139,7 +142,6 @@ export async function startAgentRun( const stub = agentRunForRunId(env, run.runId); const startBody = JSON.stringify({ isFirstRun: Boolean(run.isFirstRun), - ...(run.masterInstructions ? { masterInstructions: run.masterInstructions } : {}), ...(personalization.agentDisplayName ? { agentDisplayName: personalization.agentDisplayName } : {}), @@ -149,8 +151,9 @@ export async function startAgentRun( messageText, model: run.modelId, modelExplicit, - projectId: run.projectId, - workspaceSlug: run.workspaceSlug, + ...(body.intent ? { runIntent: body.intent } : {}), + ...(run.projectId ? { projectId: run.projectId } : {}), + ...(run.workspaceSlug ? { workspaceSlug: run.workspaceSlug } : {}), ...(run.projectMode ? { projectMode: run.projectMode } : {}), runId: run.runId, sandboxName, @@ -253,13 +256,16 @@ export async function runEntitlementPolicy( env: AgentEnv, userId: string, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); let entitlement: EntitlementCache; let periodEnd: Date; try { ({ entitlement, periodEnd } = await withUserContext(db, UserId(userId), async (tx) => { const loadedEntitlement = entitlementCacheFromValues( - (await findEntitlementByUserId(tx, UserId(userId))) ?? { tier: "free" }, + (await findAgentEntitlementByUserId(tx, UserId(userId))) ?? { tier: "free" }, ); return { entitlement: loadedEntitlement, @@ -399,7 +405,10 @@ export async function activeRunForThreadRoute( userId: string, threadId: string, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, UserId(userId), (tx) => findActiveAgentRunForThread(tx, { @@ -417,7 +426,10 @@ export async function runForRoute( userId: string, runId: string, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { const run = await withUserContext(db, UserId(userId), (tx) => findAgentRunForUser(tx, { diff --git a/apps/agent-worker/src/database-readiness.ts b/apps/agent-worker/src/database-readiness.ts new file mode 100644 index 00000000..1f42c027 --- /dev/null +++ b/apps/agent-worker/src/database-readiness.ts @@ -0,0 +1,128 @@ +import { assertDatabaseRuntimeReadiness, createDb } from "@cheatcode/db"; +import { resolveWorkerSecret } from "@cheatcode/env"; +import { APIError, readBoundedRequestText } from "@cheatcode/observability"; +import { DaytonaClient } from "@cheatcode/tools-code"; +import { + AgentDatabaseReadinessResponseSchema, + DaytonaVolumeIdentitySchema, + INTERNAL_DATABASE_READINESS_PATH, + InternalDatabaseReadinessRequestSchema, +} from "@cheatcode/types"; +import type { Context, Hono } from "hono"; +import type { AgentEnv } from "./agent-env"; +import { + assertAgentDatabaseReadinessCapability, + assertAgentInternalHostname, + parseInternalMaintenanceJson, + verifyAgentDatabaseReadinessRequest, +} from "./internal-maintenance"; + +const MAX_READINESS_BODY_BYTES = 4 * 1024; +type AgentContext = Context<{ Bindings: AgentEnv }>; + +export function registerAgentDatabaseReadinessRoute(app: Hono<{ Bindings: AgentEnv }>): void { + app.post(INTERNAL_DATABASE_READINESS_PATH, handleDatabaseReadiness); +} + +async function handleDatabaseReadiness(c: AgentContext): Promise { + if (c.env.CHEATCODE_RELEASE_GATE !== "closed") { + throw releaseMismatch("Database readiness requires the closed release gate"); + } + assertAgentInternalHostname(c.req.raw); + assertAgentDatabaseReadinessCapability(c.req.raw); + const rawBody = await readBoundedRequestText( + c.req.raw, + MAX_READINESS_BODY_BYTES, + "Database readiness request", + ); + await verifyAgentDatabaseReadinessRequest({ + expectedPathname: INTERNAL_DATABASE_READINESS_PATH, + rawBody, + request: c.req.raw, + secrets: c.env, + }); + const request = InternalDatabaseReadinessRequestSchema.parse( + parseInternalMaintenanceJson(rawBody), + ); + if (c.env.CHEATCODE_RELEASE_SHA !== request.releaseSha) { + throw releaseMismatch("Database readiness release does not match the agent Worker"); + } + const [, daytona] = await Promise.all([ + assertAgentDatabaseReady(c.env), + readDaytonaVolumeIdentity(c.env), + ]); + return c.json( + AgentDatabaseReadinessResponseSchema.parse({ + databaseRole: "app_agent", + daytona, + ok: true, + releaseSha: request.releaseSha, + versionId: c.env.CF_VERSION_METADATA?.id ?? null, + worker: "agent", + }), + ); +} + +async function assertAgentDatabaseReady(env: AgentEnv): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + await assertDatabaseRuntimeReadiness(db, "app_agent"); + } catch (error) { + throw new APIError(503, "unavailable_maintenance", "Agent database readiness failed", { + cause: error, + retriable: true, + }); + } finally { + await close(); + } +} + +async function readDaytonaVolumeIdentity(env: AgentEnv) { + try { + const organizationId = requiredDaytonaOrganizationId(env); + const apiKey = await resolveWorkerSecret(env.DAYTONA_API_KEY); + if (!apiKey?.trim()) throw new Error("Daytona API key is unavailable"); + const client = new DaytonaClient({ + apiKey, + apiUrl: env.DAYTONA_API_URL, + organizationId, + requestTimeoutMs: 4_000, + target: env.DAYTONA_TARGET, + }); + const volume = await client.getVolumeByName(env.DAYTONA_WORKSPACE_VOLUME); + if ( + !volume || + volume.organizationId !== organizationId || + volume.state !== "ready" || + volume.errorReason + ) { + throw new Error("Daytona workspace volume is absent or not ready"); + } + return DaytonaVolumeIdentitySchema.parse({ + organizationId: volume.organizationId, + state: volume.state, + volumeId: volume.id, + volumeName: volume.name, + }); + } catch (error) { + throw new APIError(503, "unavailable_maintenance", "Daytona volume readiness failed", { + cause: error, + retriable: true, + }); + } +} + +function requiredDaytonaOrganizationId(env: AgentEnv): string { + const organizationId = env.DAYTONA_ORG_ID?.trim(); + if (!organizationId) { + throw new Error("DAYTONA_ORG_ID is required for release readiness"); + } + return organizationId; +} + +function releaseMismatch(message: string): APIError { + return new APIError(409, "conflict_state_invalid", message, { retriable: false }); +} diff --git a/apps/agent-worker/src/durable-object-storage.ts b/apps/agent-worker/src/durable-object-storage.ts new file mode 100644 index 00000000..cd3f0a2c --- /dev/null +++ b/apps/agent-worker/src/durable-object-storage.ts @@ -0,0 +1,71 @@ +import { APIError, readBoundedRequestText } from "@cheatcode/observability"; +import { + INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + InternalDurableObjectStorageRequestSchema, + InternalDurableObjectStorageResponseSchema, +} from "@cheatcode/types"; +import type { Context, Hono } from "hono"; +import type { AgentEnv } from "./agent-env"; +import { + assertAgentDurableObjectStorageCapability, + assertAgentInternalHostname, + parseInternalMaintenanceJson, + verifyAgentDurableObjectStorageRequest, +} from "./internal-maintenance"; + +const MAX_STORAGE_BODY_BYTES = 4 * 1024; +type AgentContext = Context<{ Bindings: AgentEnv }>; + +export function registerAgentDurableObjectStorageRoute(app: Hono<{ Bindings: AgentEnv }>): void { + app.post(INTERNAL_DURABLE_OBJECT_STORAGE_PATH, handleDurableObjectStorage); +} + +async function handleDurableObjectStorage(c: AgentContext): Promise { + assertClosedRelease(c.env); + assertAgentInternalHostname(c.req.raw); + assertAgentDurableObjectStorageCapability(c.req.raw); + const rawBody = await readBoundedRequestText( + c.req.raw, + MAX_STORAGE_BODY_BYTES, + "Durable Object storage request", + ); + await verifyAgentDurableObjectStorageRequest({ + expectedPathname: INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + rawBody, + request: c.req.raw, + secrets: c.env, + }); + const input = InternalDurableObjectStorageRequestSchema.parse( + parseInternalMaintenanceJson(rawBody), + ); + if (input.releaseSha !== c.env.CHEATCODE_RELEASE_SHA) { + throw releaseMismatch("Durable Object request does not match the agent release"); + } + if (input.className === "AgentRun") { + const id = c.env.AGENT_RUN.idFromString(input.objectId); + return c.json( + InternalDurableObjectStorageResponseSchema.parse( + await c.env.AGENT_RUN.get(id).reconcileStorageSchema(input), + ), + ); + } + if (input.className === "ProjectSandbox") { + const id = c.env.PROJECT_SANDBOX.idFromString(input.objectId); + return c.json( + InternalDurableObjectStorageResponseSchema.parse( + await c.env.PROJECT_SANDBOX.get(id).reconcileStorageSchema(input), + ), + ); + } + throw releaseMismatch("Durable Object class is not owned by the agent Worker"); +} + +function assertClosedRelease(env: AgentEnv): void { + if (env.CHEATCODE_RELEASE_GATE !== "closed") { + throw releaseMismatch("Durable Object reconciliation requires the closed release gate"); + } +} + +function releaseMismatch(message: string): APIError { + return new APIError(409, "conflict_state_invalid", message, { retriable: false }); +} diff --git a/apps/agent-worker/src/durable-objects/abort-timeout.ts b/apps/agent-worker/src/durable-objects/abort-timeout.ts index 222b39d4..ff3c98c6 100644 --- a/apps/agent-worker/src/durable-objects/abort-timeout.ts +++ b/apps/agent-worker/src/durable-objects/abort-timeout.ts @@ -2,18 +2,12 @@ export type AbortTimeoutResult = T | "timeout"; interface AbortTimeoutInput { abortController: AbortController; - /** - * Pending-decision interlock. While approval is pending, the same operation - * remains in flight and the timer polls without issuing a second read. - */ - extendWhile?: () => boolean; operation: Promise; timeoutMs: number; } export async function resolveWithAbortTimeout({ abortController, - extendWhile, operation, timeoutMs, }: AbortTimeoutInput): Promise> { @@ -26,15 +20,10 @@ export async function resolveWithAbortTimeout({ throw error; }); const timeout = new Promise<"timeout">((resolve) => { - const fire = () => { - if (extendWhile?.()) { - timeoutId = setTimeout(fire, 1_000); - return; - } + timeoutId = setTimeout(() => { abortController.abort(new Error("operation timed out")); resolve("timeout"); - }; - timeoutId = setTimeout(fire, timeoutMs); + }, timeoutMs); }); const aborted = new Promise<"timeout">((resolve) => { resolveAbort = resolve; diff --git a/apps/agent-worker/src/durable-objects/agent-run-alarm.ts b/apps/agent-worker/src/durable-objects/agent-run-alarm.ts new file mode 100644 index 00000000..231d25b3 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-alarm.ts @@ -0,0 +1,56 @@ +import { pendingAssistantMessageRetryAt } from "./agent-run-message-persistence"; +import { nextAgentRunAlarm } from "./agent-run-retention"; +import { pendingStatusRetryAt } from "./agent-run-status-persistence"; +import { getRunStateTimestamp, getRunStateValue } from "./agent-run-storage"; +import { + AGENT_RUN_WORKFLOW_ADMITTED_KEY, + AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY, + AGENT_RUN_WORKFLOW_RETRY_AT_KEY, +} from "./agent-run-workflow-protocol"; +import { hasActiveRun } from "./run-state"; + +const CLOSED_GATE_ALARM_RECHECK_MS = 60_000; + +/** Re-arms the Durable Object alarm to the earliest outstanding run obligation. */ +export async function armAgentRunAlarm(ctx: DurableObjectState): Promise { + if (!getRunStateValue(ctx, "run_id")) { + await ctx.storage.deleteAlarm(); + return; + } + const isRunActive = hasActiveRun(getRunStateValue(ctx, "status")); + const executionLeaseAlarm = isRunActive + ? (getRunStateTimestamp(ctx, AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY) ?? + Number.POSITIVE_INFINITY) + : Number.POSITIVE_INFINITY; + const admissionRetryAlarm = + isRunActive && getRunStateValue(ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) !== "true" + ? (getRunStateTimestamp(ctx, AGENT_RUN_WORKFLOW_RETRY_AT_KEY) ?? Number.POSITIVE_INFINITY) + : Number.POSITIVE_INFINITY; + const assistantMessageRetryAlarm = pendingAssistantMessageRetryAt(ctx); + const statusRetryAlarm = + assistantMessageRetryAlarm === Number.POSITIVE_INFINITY + ? pendingStatusRetryAt(ctx) + : Number.POSITIVE_INFINITY; + await ctx.storage.setAlarm( + Math.min( + admissionRetryAlarm, + executionLeaseAlarm, + assistantMessageRetryAlarm, + statusRetryAlarm, + nextAgentRunAlarm(Date.now()), + ), + ); +} + +/** Preserve admitted recovery work without executing it while a release is closed. */ +export async function armClosedAgentRunAlarm( + ctx: DurableObjectState, + status: string | undefined, +): Promise { + const hasDeferredDatabaseWrite = + pendingAssistantMessageRetryAt(ctx) !== Number.POSITIVE_INFINITY || + pendingStatusRetryAt(ctx) !== Number.POSITIVE_INFINITY; + if (hasActiveRun(status) || hasDeferredDatabaseWrite) { + await ctx.storage.setAlarm(Date.now() + CLOSED_GATE_ALARM_RECHECK_MS); + } +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-app-builder.ts b/apps/agent-worker/src/durable-objects/agent-run-app-builder.ts index 3422670c..3f333bc8 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-app-builder.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-app-builder.ts @@ -471,9 +471,9 @@ export async function warmSandbox( code: "print('ready')", language: "python", }); - const stdout = result.stdout ?? result.output ?? ""; + const stdout = result.stdout; logger.info("sandbox_warmed", { - success: result.success === true, + success: result.success, stdoutBytes: stdout.length, }); } diff --git a/apps/agent-worker/src/durable-objects/agent-run-approvals.ts b/apps/agent-worker/src/durable-objects/agent-run-approvals.ts deleted file mode 100644 index 31aebb4c..00000000 --- a/apps/agent-worker/src/durable-objects/agent-run-approvals.ts +++ /dev/null @@ -1,619 +0,0 @@ -import type { ApprovalBroker, ApprovalRequestInput, RunDecision } from "@cheatcode/agent-core"; -import { - type AnalyticsBindings, - APIError, - createLogger, - emitUserEvent, -} from "@cheatcode/observability"; -import { - ApprovalDecisionDataSchema, - type ApprovalDecisionResponse, - ApprovalRequestDataSchema, - type LogicalModelId, - type ModelFallbackData, - ModelFallbackDataSchema, -} from "@cheatcode/types"; -import type { UIMessageChunk } from "ai"; -import { z } from "zod"; -import { pendingAssistantMessageRetryAt } from "./agent-run-message-persistence"; -import { nextAgentRunAlarm } from "./agent-run-retention"; -import { pendingStatusRetryAt } from "./agent-run-status-persistence"; -import { deleteRunStateValues, getRunStateValue, setRunStateValue } from "./agent-run-storage"; - -/** Model-fallback auto-allow window. */ -const MODEL_FALLBACK_DECISION_TIMEOUT_MS = 120_000; -const APPROVAL_SUMMARY_MAX = 400; -const PENDING_APPROVAL_KEY = "pending_approval"; -const APPROVAL_DECISION_PREFIX = "approval_decision:"; - -const PendingApprovalSchema = z - .object({ - approvalId: z.string().uuid(), - expiresAt: z.number().int(), - kind: z.enum(["tool-approval", "model-fallback"]), - requestedAt: z.number().int(), - summary: z.string().min(1).max(APPROVAL_SUMMARY_MAX), - timeoutDecision: z.enum(["allow", "deny"]), - toolName: z.string().min(1).optional(), - }) - .strict(); -export type PendingApproval = z.infer; - -const ApprovalDecisionRecordSchema = z - .object({ - approvalId: z.string().uuid(), - decidedAt: z.number().int(), - decidedBy: z.enum(["user", "timeout", "cancel"]), - decision: z.enum(["allow", "deny"]), - reason: z.string().max(500).optional(), - }) - .strict(); -type ApprovalDecisionRecord = z.infer; - -/** Body of the DO `/approval` endpoint. */ -export const ApprovalDecisionInputSchema = z - .object({ - approvalId: z.string().uuid(), - decision: z.enum(["allow", "deny"]), - reason: z.string().trim().min(1).max(500).optional(), - userId: z.string().uuid(), - }) - .strict(); -export type ApprovalDecisionInput = z.infer; - -export interface RunIdentity { - runId: string; - threadId: string; - userId: string; -} - -export interface RunApprovalControllerDeps { - append: (chunk: UIMessageChunk, options?: { allowAfterCancelRequest?: boolean }) => Promise; - armAlarm: () => Promise; - ctx: DurableObjectState; - currentStatus: () => string | undefined; - env: AnalyticsBindings; - finalizeUnrecoverable: () => Promise; - identity: () => RunIdentity | null; - isCanceled: () => boolean; - setRunStatus: (status: "paused" | "running") => Promise; -} - -interface ResolveParams { - approvalId: string; - decidedBy: ApprovalDecisionRecord["decidedBy"]; - decision: ApprovalDecisionRecord["decision"]; - kind: PendingApproval["kind"]; - reason?: string; -} - -/** - * Per-run approval controller. Owns the in-memory resolver map plus the - * pause/resolve/alarm/cancel/orphan state machine. The DO wires thin closures - * into {@link RunApprovalControllerDeps} so the heavy logic lives here. - */ -export class RunApprovalController { - private requestChain: Promise = Promise.resolve(); - private readonly resolvers = new Map void>(); - private settlementChain: Promise = Promise.resolve(); - - public constructor(private readonly deps: RunApprovalControllerDeps) {} - - public hasPendingDecision(): boolean { - return this.resolvers.size > 0; - } - - public createBroker(): ApprovalBroker { - return { requestDecision: (input) => this.requestDecision(input) }; - } - - /** Serializes concurrent gated calls onto a single pending slot. */ - private requestDecision(input: ApprovalRequestInput): Promise { - const result = this.requestChain.then(() => this.beginRequest(input)); - this.requestChain = result.catch(() => undefined); - return result; - } - - private async beginRequest(input: ApprovalRequestInput): Promise { - const pending = buildPending(input); - let resolveDecision!: (decision: RunDecision) => void; - const decision = new Promise((resolve) => { - resolveDecision = resolve; - }); - const opened = await this.serializeSettlement(() => this.openRequest(pending, resolveDecision)); - return opened ? decision : { decidedBy: "cancel", decision: "deny" }; - } - - private async openRequest( - pending: PendingApproval, - resolveDecision: (decision: RunDecision) => void, - ): Promise { - const identity = this.deps.identity(); - if (this.deps.isCanceled() || !identity) { - return false; - } - this.resolvers.set(pending.approvalId, resolveDecision); - try { - await this.openApproval(pending, identity); - } catch (error) { - await this.rollbackOpenFailure(pending, identity); - throw error instanceof Error ? error : new Error("Approval setup failed."); - } - return true; - } - - private async rollbackOpenFailure( - pending: PendingApproval, - identity: RunIdentity, - ): Promise { - this.resolvers.delete(pending.approvalId); - try { - clearPendingApproval(this.deps.ctx); - } catch (error) { - this.logger(identity).error("approval_setup_cleanup_failed", { - approvalId: pending.approvalId, - error, - }); - } - if (!this.deps.isCanceled()) { - await this.deps.setRunStatus("running").catch(() => undefined); - } - await this.deps.armAlarm().catch(() => undefined); - } - - private async openApproval(pending: PendingApproval, identity: RunIdentity): Promise { - savePendingApproval(this.deps.ctx, pending); - await this.deps.setRunStatus("paused"); - await this.deps.append(approvalRequestChunk(pending, identity.runId)); - await this.deps.armAlarm(); - this.logRequested(pending, identity); - } - - /** POST `/approval` path (decidedBy: "user"); idempotent on replays. */ - public applyDecision( - input: Pick & { reason?: string }, - ): Promise { - return this.serializeSettlement(() => this.applyDecisionInternal(input)); - } - - private async applyDecisionInternal( - input: Pick & { reason?: string }, - ): Promise { - const recorded = readApprovalDecision(this.deps.ctx, input.approvalId); - if (recorded) { - return this.decisionResponseForInput(recorded, input.decision); - } - const pending = readPendingApproval(this.deps.ctx); - if (!pending || pending.approvalId !== input.approvalId) { - throw unknownApprovalError(); - } - if (this.deps.isCanceled()) { - const record = await this.resolveCanceled(pending); - return this.decisionResponseForInput(record, input.decision); - } - if (!this.resolvers.has(input.approvalId)) { - await this.finalizeOrphaned(pending); - throw orphanedApprovalError(); - } - const isExpired = Date.now() >= pending.expiresAt; - const record = await this.resolveInternal( - isExpired - ? { - approvalId: input.approvalId, - decidedBy: "timeout", - decision: pending.timeoutDecision, - kind: pending.kind, - } - : { - approvalId: input.approvalId, - decidedBy: "user", - decision: input.decision, - kind: pending.kind, - ...(input.reason ? { reason: input.reason } : {}), - }, - ); - return this.decisionResponseForInput(record, input.decision); - } - - /** Alarm path: apply the timeout decision, or finalize if orphaned. */ - public handleAlarmIfDue(): Promise { - return this.serializeSettlement(() => this.handleAlarmIfDueInternal()); - } - - private async handleAlarmIfDueInternal(): Promise { - const pending = readPendingApproval(this.deps.ctx); - if (!pending) { - return false; - } - if (this.deps.isCanceled()) { - await this.resolveCanceled(pending); - return true; - } - if (Date.now() < pending.expiresAt) { - await this.deps.armAlarm(); - return true; - } - if (!this.resolvers.has(pending.approvalId)) { - await this.finalizeOrphaned(pending); - return true; - } - await this.resolveInternal({ - approvalId: pending.approvalId, - decidedBy: "timeout", - decision: pending.timeoutDecision, - kind: pending.kind, - }); - return true; - } - - /** Cancel path: resolve any pending decision as deny+cancel. */ - public cancelPending(): Promise { - return this.serializeSettlement(() => this.cancelPendingInternal()); - } - - private async cancelPendingInternal(): Promise { - const pending = readPendingApproval(this.deps.ctx); - if (!pending) { - return false; - } - await this.resolveCanceled(pending); - return true; - } - - private resolveCanceled(pending: PendingApproval): Promise { - return this.resolveInternal({ - approvalId: pending.approvalId, - decidedBy: "cancel", - decision: "deny", - kind: pending.kind, - }); - } - - private serializeSettlement(operation: () => Promise): Promise { - const result = this.settlementChain.then(operation); - this.settlementChain = result.catch(() => undefined); - return result; - } - - private async resolveInternal(params: ResolveParams): Promise { - const identity = this.deps.identity(); - const runId = identity?.runId ?? getRunStateValue(this.deps.ctx, "run_id") ?? ""; - const record: ApprovalDecisionRecord = { - approvalId: params.approvalId, - decidedAt: Date.now(), - decidedBy: params.decidedBy, - decision: params.decision, - ...(params.reason ? { reason: params.reason } : {}), - }; - recordApprovalDecision(this.deps.ctx, record); - await this.settleDecisionSideEffect(record, identity, "append", () => - this.deps.append(approvalDecisionChunk(record, runId), { - allowAfterCancelRequest: params.decidedBy === "cancel", - }), - ); - await this.settleDecisionSideEffect(record, identity, "clear_pending", async () => { - clearPendingApproval(this.deps.ctx); - }); - if (params.decidedBy !== "cancel") { - await this.settleDecisionSideEffect(record, identity, "resume_run", () => - this.deps.setRunStatus("running"), - ); - } - await this.settleDecisionSideEffect(record, identity, "arm_alarm", () => this.deps.armAlarm()); - this.releaseResolver(record); - this.logDecided(record, params.kind, identity); - return record; - } - - private async settleDecisionSideEffect( - record: ApprovalDecisionRecord, - identity: RunIdentity | null, - operation: string, - effect: () => Promise, - ): Promise { - try { - await effect(); - } catch (error) { - const logger = identity ? this.logger(identity) : createLogger(); - logger.error("approval_decision_settle_failed", { - approvalId: record.approvalId, - error, - operation, - }); - } - } - - private releaseResolver(record: ApprovalDecisionRecord): void { - const resolver = this.resolvers.get(record.approvalId); - if (!resolver) { - return; - } - this.resolvers.delete(record.approvalId); - resolver({ - decidedBy: record.decidedBy, - decision: record.decision, - ...(record.reason ? { reason: record.reason } : {}), - }); - } - - private async finalizeOrphaned(pending: PendingApproval): Promise { - const identity = this.deps.identity(); - if (identity) { - this.logger(identity).error("tool_approval_unrecoverable", { - approvalId: pending.approvalId, - }); - } - clearPendingApproval(this.deps.ctx); - await this.deps.finalizeUnrecoverable(); - } - - private decisionResponse(record: ApprovalDecisionRecord): ApprovalDecisionResponse { - return { - approvalId: record.approvalId, - decidedBy: record.decidedBy, - decision: record.decision, - ok: true, - runStatus: this.currentRunStatus(), - }; - } - - private decisionResponseForInput( - record: ApprovalDecisionRecord, - submittedDecision: ApprovalDecisionRecord["decision"], - ): ApprovalDecisionResponse { - if (record.decision !== submittedDecision) { - throw conflictDecisionError(); - } - return this.decisionResponse(record); - } - - private currentRunStatus(): ApprovalDecisionResponse["runStatus"] { - const status = this.deps.currentStatus(); - if ( - status === "running" || - status === "paused" || - status === "completed" || - status === "failed" || - status === "canceled" - ) { - return status; - } - return "running"; - } - - private logRequested(pending: PendingApproval, identity: RunIdentity): void { - const logger = this.logger(identity); - if (pending.kind === "tool-approval") { - logger.info("tool_approval_requested", { - approvalId: pending.approvalId, - expiresAt: pending.expiresAt, - ...(pending.toolName ? { toolName: pending.toolName } : {}), - }); - this.emit("tool_approval_requested", identity, pending.toolName); - return; - } - logger.warn("llm_provider_fallback_offered", { - approvalId: pending.approvalId, - expiresAt: pending.expiresAt, - }); - this.emit("model_fallback_offered", identity, undefined); - } - - private logDecided( - record: ApprovalDecisionRecord, - kind: PendingApproval["kind"], - identity: RunIdentity | null, - ): void { - if (!identity) { - return; - } - const logger = this.logger(identity); - if (kind === "tool-approval") { - logger.info("tool_approval_decided", { - approvalId: record.approvalId, - decidedBy: record.decidedBy, - decision: record.decision, - }); - this.emit("tool_approval_decided", identity, undefined); - return; - } - logger.warn("llm_provider_fallback_decided", { - approvalId: record.approvalId, - decidedBy: record.decidedBy, - decision: record.decision, - }); - this.emit("model_fallback_decided", identity, undefined); - } - - private emit(eventName: string, identity: RunIdentity, toolName: string | undefined): void { - emitUserEvent(this.deps.env, { - eventName, - runId: identity.runId, - userId: identity.userId, - ...(toolName ? { toolName } : {}), - }); - } - - private logger(identity: RunIdentity): ReturnType { - return createLogger({ - runId: identity.runId, - threadId: identity.threadId, - userId: identity.userId, - }); - } -} - -/** - * Opens the interactive fallback pause. The caller appends the informational - * transition only after both approval and durable model attribution succeed. - */ -export async function offerModelFallback(params: { - broker: ApprovalBroker; - fromModel: LogicalModelId; - reason: ModelFallbackData["reason"]; - toModel: LogicalModelId; -}): Promise { - return params.broker.requestDecision({ - kind: "model-fallback", - summary: `Fall back from ${params.fromModel} to ${params.toModel} (${params.reason}).`.slice( - 0, - APPROVAL_SUMMARY_MAX, - ), - timeoutDecision: "allow", - timeoutMs: MODEL_FALLBACK_DECISION_TIMEOUT_MS, - }); -} - -/** Appends the transition only after fallback attribution is durably committed. */ -export async function appendModelFallbackTransition(params: { - append: (chunk: UIMessageChunk) => Promise; - fromModel: LogicalModelId; - reason: ModelFallbackData["reason"]; - toModel: LogicalModelId; -}): Promise { - await params.append(modelFallbackChunk(params.fromModel, params.toModel, params.reason)); -} - -/** Re-arms the DO alarm to the earliest of retention vs. the approval deadline. */ -export async function armAgentRunAlarm(ctx: DurableObjectState): Promise { - if (!getRunStateValue(ctx, "run_id")) { - await ctx.storage.deleteAlarm(); - return; - } - const pending = readPendingApproval(ctx); - const retentionAlarm = nextAgentRunAlarm(Date.now()); - const approvalAlarm = pending ? Math.min(pending.expiresAt, retentionAlarm) : retentionAlarm; - const target = Math.min( - approvalAlarm, - pendingAssistantMessageRetryAt(ctx), - pendingStatusRetryAt(ctx), - ); - await ctx.storage.setAlarm(target); -} - -/** Pending-approval shape for the `GET /runs/status` snapshot. */ -export function pendingApprovalSnapshot(ctx: DurableObjectState): PendingApproval | undefined { - return readPendingApproval(ctx) ?? undefined; -} - -function savePendingApproval(ctx: DurableObjectState, pending: PendingApproval): void { - setRunStateValue(ctx, PENDING_APPROVAL_KEY, JSON.stringify(pending)); -} - -function readPendingApproval(ctx: DurableObjectState): PendingApproval | null { - const raw = getRunStateValue(ctx, PENDING_APPROVAL_KEY); - if (!raw) { - return null; - } - const parsed = PendingApprovalSchema.safeParse(safeJsonParse(raw)); - return parsed.success ? parsed.data : null; -} - -function clearPendingApproval(ctx: DurableObjectState): void { - deleteRunStateValues(ctx, [PENDING_APPROVAL_KEY]); -} - -function recordApprovalDecision(ctx: DurableObjectState, record: ApprovalDecisionRecord): void { - setRunStateValue(ctx, `${APPROVAL_DECISION_PREFIX}${record.approvalId}`, JSON.stringify(record)); -} - -function readApprovalDecision( - ctx: DurableObjectState, - approvalId: string, -): ApprovalDecisionRecord | null { - const raw = getRunStateValue(ctx, `${APPROVAL_DECISION_PREFIX}${approvalId}`); - if (!raw) { - return null; - } - const parsed = ApprovalDecisionRecordSchema.safeParse(safeJsonParse(raw)); - return parsed.success ? parsed.data : null; -} - -function buildPending(input: ApprovalRequestInput): PendingApproval { - const requestedAt = Date.now(); - return { - approvalId: crypto.randomUUID(), - expiresAt: requestedAt + input.timeoutMs, - kind: input.kind, - requestedAt, - summary: input.summary.slice(0, APPROVAL_SUMMARY_MAX), - timeoutDecision: input.timeoutDecision, - ...(input.toolName ? { toolName: input.toolName } : {}), - }; -} - -function approvalRequestChunk(pending: PendingApproval, runId: string): UIMessageChunk { - return { - data: ApprovalRequestDataSchema.parse({ - approvalId: pending.approvalId, - expiresAt: pending.expiresAt, - kind: pending.kind, - requestedAt: pending.requestedAt, - runId, - summary: pending.summary, - timeoutDecision: pending.timeoutDecision, - v: 1, - ...(pending.toolName ? { toolName: pending.toolName } : {}), - }), - type: "data-approval-request", - }; -} - -function approvalDecisionChunk(record: ApprovalDecisionRecord, runId: string): UIMessageChunk { - return { - data: ApprovalDecisionDataSchema.parse({ - approvalId: record.approvalId, - decidedBy: record.decidedBy, - decision: record.decision, - runId, - v: 1, - ...(record.reason ? { reason: record.reason } : {}), - }), - type: "data-approval-decision", - }; -} - -function modelFallbackChunk( - fromModel: LogicalModelId, - toModel: LogicalModelId, - reason: ModelFallbackData["reason"], -): UIMessageChunk { - return { - data: ModelFallbackDataSchema.parse({ fromModel, reason, toModel, v: 1 }), - type: "data-model-fallback", - }; -} - -function safeJsonParse(raw: string): unknown { - try { - return JSON.parse(raw) as unknown; - } catch { - return null; - } -} - -function conflictDecisionError(): APIError { - return new APIError( - 409, - "conflict_state_invalid", - "A different decision was already recorded for this approval.", - { - hint: "The approval was already resolved with the opposite decision.", - retriable: false, - }, - ); -} - -function unknownApprovalError(): APIError { - return new APIError(409, "conflict_state_invalid", "Run is not awaiting this approval.", { - hint: "The approval id is unknown or already resolved.", - retriable: false, - }); -} - -function orphanedApprovalError(): APIError { - return new APIError(409, "conflict_state_invalid", "Run is no longer live — start a new run.", { - hint: "The run could not recover the pending approval after a restart.", - retriable: false, - }); -} diff --git a/apps/agent-worker/src/durable-objects/agent-run-artifacts.ts b/apps/agent-worker/src/durable-objects/agent-run-artifacts.ts index 53359a69..16f8d612 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-artifacts.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-artifacts.ts @@ -1,11 +1,13 @@ import { + type ArtifactUploadIdentity, createDb, type DatabaseHandle, - getProject, - saveGeneratedOutput, + finalizeArtifactUpload, + guardArtifactUpload, + reserveArtifactUpload, withUserContext, } from "@cheatcode/db"; -import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; +import type { WorkerSecret } from "@cheatcode/env"; import { type AnalyticsBindings, APIError, @@ -15,25 +17,19 @@ import { } from "@cheatcode/observability"; import type { ArtifactUploadInput, ArtifactUploadResult } from "@cheatcode/sandbox-contracts"; import type { AgentRunId, ProjectId, ThreadId, UserId } from "@cheatcode/types"; -import { createSignedOutputDownloadUrl } from "../output-download"; +import { ArtifactKindSchema } from "@cheatcode/types/artifacts"; +const ARTIFACT_DIGEST_DOMAIN = "cheatcode:artifact-upload:v2"; const MAX_ARTIFACT_BYTES = 32 * 1024 * 1024; -const MAX_ARTIFACT_METADATA_BYTES = 64 * 1024; const MAX_CONTENT_TYPE_LENGTH = 255; -const OUTPUTS_BUCKET_NAME = "cheatcode-outputs"; -const OUTPUT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; -const VALID_ARTIFACT_KINDS = new Set(["audio", "docx", "image", "pdf", "slide", "video", "xlsx"]); const VALID_CONTENT_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/iu; type AgentRunLogger = ReturnType; interface ArtifactEnv extends AnalyticsBindings { + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; HYPERDRIVE: Hyperdrive; - OUTPUT_DOWNLOAD_BASE_URL?: string; - OUTPUT_DOWNLOAD_SIGNING_SECRET: WorkerSecret; - PREVIEW_HOSTNAME: string; R2_OUTPUTS: R2Bucket; - R2_OUTPUTS_BUCKET_NAME?: string; } interface ArtifactRunInput { @@ -49,63 +45,130 @@ interface StoreAgentArtifactOptions { input: ArtifactRunInput; } +interface PreparedArtifact { + contentSha256: string; + filename: string; + identity: ArtifactUploadIdentity; + outputId: string; + r2Key: string; +} + export async function storeAgentArtifact({ artifact, env, input, }: StoreAgentArtifactOptions): Promise { assertArtifactUpload(artifact); - const outputId = crypto.randomUUID(); - const filename = sanitizeFilename(artifact.filename); - // Resolve the signed URL before writing either durable store. A missing signing secret must not - // leave an inaccessible R2 object or database row behind for a retry to duplicate. - const downloadUrl = await signedOutputUrl(outputId, env); - const bucketName = env.R2_OUTPUTS_BUCKET_NAME?.trim() || OUTPUTS_BUCKET_NAME; + const prepared = await prepareArtifact(artifact, input); const logger = createRunLogger({ threadId: input.threadId, userId: input.userId }); - const projectId = await requireArtifactProject(env, input, logger); + const dbHandle = artifactDatabase(env); + try { + await persistPreparedArtifact(dbHandle, env, artifact, input, prepared); + return artifactUploadResult(artifact, prepared); + } finally { + await closeDatabase(dbHandle, logger); + } +} + +async function prepareArtifact( + artifact: ArtifactUploadInput, + input: ArtifactRunInput, +): Promise { + const filename = sanitizeFilename(artifact.filename); + const contentSha256 = await sha256Hex(artifact.data); + const outputId = await deterministicOutputId(artifact, input.runId, filename, contentSha256); const r2Key = outputObjectKey({ agentRunId: input.runId, filename, outputId, - projectId, + projectId: input.projectId, userId: input.userId, }); - const sha256 = await sha256Hex(artifact.data); - await writeArtifactObject(env, artifact, { filename, outputId, r2Key }); - let alreadyHadGeneratedOutput: boolean; - try { - alreadyHadGeneratedOutput = await persistGeneratedArtifact({ - artifact, - bucketName, - env, - filename, - input, - outputId, - projectId, + return { + contentSha256, + filename, + identity: { + agentRunId: input.runId, + id: outputId, + projectId: input.projectId, r2Key, - sha256, userId: input.userId, - logger, - }); - } catch (error) { - await env.R2_OUTPUTS.delete(r2Key).catch(() => undefined); - throw error; + }, + outputId, + r2Key, + }; +} + +async function persistPreparedArtifact( + dbHandle: DatabaseHandle, + env: ArtifactEnv, + artifact: ArtifactUploadInput, + input: ArtifactRunInput, + prepared: PreparedArtifact, +): Promise { + const reservation = await withUserContext(dbHandle.db, input.userId, (db) => + reserveArtifactUpload(db, prepared.identity), + ); + if (reservation.state === "committed") { + await verifyCommittedArtifactObject(env, artifact, prepared); + return; + } + if (reservation.state === "fenced") { + throw unavailableArtifactOwnership(); + } + const guard = await withUserContext(dbHandle.db, input.userId, (db) => + guardArtifactUpload(db, prepared.identity), + ); + if (guard.state === "committed") { + await verifyCommittedArtifactObject(env, artifact, prepared); + return; + } + if (guard.state === "fenced") { + throw unavailableArtifactOwnership(); + } + if (guard.state === "reservation-lost") { + throw lostArtifactReservation(); + } + await putAndFinalize(dbHandle, env, artifact, input, prepared); +} + +async function putAndFinalize( + dbHandle: DatabaseHandle, + env: ArtifactEnv, + artifact: ArtifactUploadInput, + input: ArtifactRunInput, + prepared: PreparedArtifact, +): Promise { + await writeArtifactObject(env, artifact, prepared); + const createdAt = new Date(); + const finalized = await withUserContext(dbHandle.db, input.userId, (db) => + finalizeArtifactUpload(db, { + ...prepared.identity, + createdAt, + filename: prepared.filename, + mimeType: artifact.contentType, + }), + ); + if (finalized.state === "committed") { + emitFirstArtifactEvent(env, input, prepared.outputId, finalized.isFirstForUser); + return; } - emitFirstArtifactEvent(env, input, alreadyHadGeneratedOutput); - return artifactUploadResult(artifact, { downloadUrl, filename, outputId, r2Key }); + await env.R2_OUTPUTS.delete(prepared.r2Key); + if (finalized.state === "fenced") { + throw unavailableArtifactOwnership(); + } + throw lostArtifactReservation(); } function artifactUploadResult( artifact: ArtifactUploadInput, - identity: { downloadUrl: string; filename: string; outputId: string; r2Key: string }, + prepared: PreparedArtifact, ): ArtifactUploadResult { return { - downloadUrl: identity.downloadUrl, - filename: identity.filename, + filename: prepared.filename, kind: artifact.kind, mimeType: artifact.contentType, - outputId: identity.outputId, - r2Key: identity.r2Key, + outputId: prepared.outputId, sizeBytes: artifact.data.byteLength, }; } @@ -113,77 +176,110 @@ function artifactUploadResult( async function writeArtifactObject( env: ArtifactEnv, artifact: ArtifactUploadInput, - identity: { filename: string; outputId: string; r2Key: string }, + identity: PreparedArtifact, ): Promise { - await env.R2_OUTPUTS.put(identity.r2Key, artifact.data, { + const stored = await env.R2_OUTPUTS.put(identity.r2Key, artifact.data, { customMetadata: { + contentSha256: identity.contentSha256, filename: identity.filename, kind: artifact.kind, outputId: identity.outputId, }, httpMetadata: { contentType: artifact.contentType }, + onlyIf: { etagDoesNotMatch: "*" }, + sha256: identity.contentSha256, }); + const object = stored ?? (await env.R2_OUTPUTS.head(identity.r2Key)); + assertStoredArtifactObject(object, artifact, identity); } -async function persistGeneratedArtifact(options: { - artifact: ArtifactUploadInput; - bucketName: string; - env: ArtifactEnv; - filename: string; - input: ArtifactRunInput; - logger: AgentRunLogger; - outputId: string; - projectId: ProjectId; - r2Key: string; - sha256: string; - userId: UserId; -}): Promise { - const dbHandle = createDb(options.env.HYPERDRIVE); - try { - return await withUserContext(dbHandle.db, options.userId, async (db) => { - const saved = await saveGeneratedOutput(db, generatedOutputRecord(options)); - return !saved.isFirstForUser; - }); - } finally { - await closeDatabase(dbHandle, options.logger); +async function verifyCommittedArtifactObject( + env: ArtifactEnv, + artifact: ArtifactUploadInput, + identity: PreparedArtifact, +): Promise { + assertStoredArtifactObject(await env.R2_OUTPUTS.head(identity.r2Key), artifact, identity); +} + +function assertStoredArtifactObject( + object: R2Object | null, + artifact: ArtifactUploadInput, + identity: PreparedArtifact, +): void { + const metadata = object?.customMetadata; + const checksum = object?.checksums.sha256; + if ( + !object || + object.key !== identity.r2Key || + object.size !== artifact.data.byteLength || + object.httpMetadata?.contentType !== artifact.contentType || + !metadata || + metadata["contentSha256"] !== identity.contentSha256 || + metadata["filename"] !== identity.filename || + metadata["kind"] !== artifact.kind || + metadata["outputId"] !== identity.outputId || + !checksum || + bytesToHex(new Uint8Array(checksum)) !== identity.contentSha256 + ) { + throw invalidStoredArtifact(); } } -function generatedOutputRecord(options: { - artifact: ArtifactUploadInput; - bucketName: string; - filename: string; - input: ArtifactRunInput; - outputId: string; - projectId: ProjectId; - r2Key: string; - sha256: string; - userId: UserId; -}) { - return { - expiresAt: new Date(Date.now() + OUTPUT_RETENTION_MS), - filename: options.filename, - id: options.outputId, - kind: options.artifact.kind, - metadata: options.artifact.metadata ?? {}, - mimeType: options.artifact.contentType, - agentRunId: options.input.runId, - projectId: options.projectId, - r2Bucket: options.bucketName, - r2Key: options.r2Key, - sha256: options.sha256, - sizeBytes: options.artifact.data.byteLength, - userId: options.userId, - }; +async function deterministicOutputId( + artifact: ArtifactUploadInput, + runId: AgentRunId, + filename: string, + contentSha256: string, +): Promise { + const digestInput = new TextEncoder().encode( + [ + ARTIFACT_DIGEST_DOMAIN, + runId, + artifact.kind, + artifact.contentType, + filename, + contentSha256, + ].join("\0"), + ); + const bytes = new Uint8Array(await crypto.subtle.digest("SHA-256", digestInput)).slice(0, 16); + return uuidV8(bytes); +} + +async function sha256Hex(data: Uint8Array): Promise { + return bytesToHex(new Uint8Array(await crypto.subtle.digest("SHA-256", arrayBufferView(data)))); +} + +function arrayBufferView(data: Uint8Array): Uint8Array { + return data.buffer instanceof ArrayBuffer + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); +} + +function bytesToHex(bytes: Uint8Array): string { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function uuidV8(bytes: Uint8Array): string { + const versionByte = bytes[6]; + const variantByte = bytes[8]; + if (versionByte === undefined || variantByte === undefined) { + throw new Error("Artifact identity digest was incomplete"); + } + bytes[6] = (versionByte & 0x0f) | 0x80; + bytes[8] = (variantByte & 0x3f) | 0x80; + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; } function emitFirstArtifactEvent( env: ArtifactEnv, input: ArtifactRunInput, - alreadyHadGeneratedOutput: boolean, + outputId: string, + isFirstForUser: boolean, ): void { - if (!alreadyHadGeneratedOutput) { + if (isFirstForUser) { emitUserEvent(env, { + eventId: `artifact:${outputId}`, eventName: "first_generated_artifact", runId: input.runId, userId: input.userId, @@ -207,75 +303,48 @@ function assertArtifactUpload(artifact: ArtifactUploadInput): void { ) { throw invalidArtifact("Artifact content type is invalid"); } - if (!VALID_ARTIFACT_KINDS.has(artifact.kind)) { + if (!ArtifactKindSchema.safeParse(artifact.kind).success) { throw invalidArtifact("Artifact kind is invalid"); } - if (artifact.metadata && serializedByteLength(artifact.metadata) > MAX_ARTIFACT_METADATA_BYTES) { - throw invalidArtifact(`Artifact metadata exceeds ${MAX_ARTIFACT_METADATA_BYTES} bytes`); - } -} - -function serializedByteLength(value: unknown): number { - try { - const serialized = JSON.stringify(value); - if (serialized === undefined) { - throw new TypeError("Value is not JSON serializable"); - } - return new TextEncoder().encode(serialized).byteLength; - } catch { - throw invalidArtifact("Artifact metadata must be JSON serializable"); - } } function invalidArtifact(message: string): APIError { return new APIError(400, "tool_validation_failed", message, { retriable: false }); } -async function requireArtifactProject( - env: ArtifactEnv, - input: ArtifactRunInput, - logger: AgentRunLogger, -): Promise { - const dbHandle = createDb(env.HYPERDRIVE); - try { - const project = await withUserContext(dbHandle.db, input.userId, (db) => - getProject(db, { projectId: input.projectId, userId: input.userId }), - ); - if (!project) { - throw new APIError(404, "not_found_project", "Artifact project not found", { - retriable: false, - }); - } - return project.id; - } finally { - await closeDatabase(dbHandle, logger); - } +function unavailableArtifactOwnership(): APIError { + return new APIError(409, "conflict_state_invalid", "Artifact run is no longer active", { + retriable: false, + }); } -async function signedOutputUrl(outputId: string, env: ArtifactEnv): Promise { - const secret = await resolveWorkerSecret(env.OUTPUT_DOWNLOAD_SIGNING_SECRET); - return createSignedOutputDownloadUrl({ - baseUrl: outputDownloadBaseUrl(env), - outputId, - secret, +function lostArtifactReservation(): APIError { + return new APIError(409, "conflict_state_invalid", "Artifact upload reservation was removed", { + retriable: true, }); } -function outputDownloadBaseUrl(env: ArtifactEnv): string | undefined { - const previewHostname = env.PREVIEW_HOSTNAME.trim(); - if (previewHostname === "localhost:8787" || previewHostname === "127.0.0.1:8787") { - return `http://${previewHostname}`; - } - return env.OUTPUT_DOWNLOAD_BASE_URL; +function invalidStoredArtifact(): APIError { + return new APIError( + 409, + "conflict_state_invalid", + "Artifact object identity does not match its durable upload intent", + { retriable: false }, + ); +} + +function artifactDatabase(env: ArtifactEnv): DatabaseHandle { + return createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); } async function closeDatabase(dbHandle: DatabaseHandle, logger: AgentRunLogger): Promise { try { await dbHandle.close(); } catch (error) { - logger.warn("db_close_failed", { - error, - }); + logger.warn("db_close_failed", { error }); } } @@ -314,12 +383,3 @@ function outputObjectKey(input: { `${input.outputId}-${input.filename}`, ].join("/"); } - -async function sha256Hex(data: Uint8Array): Promise { - const buffer = new ArrayBuffer(data.byteLength); - new Uint8Array(buffer).set(data); - const digest = await crypto.subtle.digest("SHA-256", buffer); - return Array.from(new Uint8Array(digest)) - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); -} diff --git a/apps/agent-worker/src/durable-objects/agent-run-browser-takeover.ts b/apps/agent-worker/src/durable-objects/agent-run-browser-takeover.ts new file mode 100644 index 00000000..75caf6bc --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-browser-takeover.ts @@ -0,0 +1,210 @@ +import { APIError, createLogger } from "@cheatcode/observability"; +import { + BrowserTakeoverResumeResultSchema, + type BrowserTakeoverSession, + BrowserTakeoverSessionSchema, + BrowserTakeoverStatusSchema, +} from "@cheatcode/types"; +import type { AgentRunEnv } from "./agent-run-env"; +import { + deleteRunStateValues, + getRunStateTimestamp, + getRunStateValue, + setRunStateValue, +} from "./agent-run-storage"; +import { hasActiveRun } from "./run-state"; + +const TAKEOVER_ID_KEY = "browser_takeover_id"; +const TAKEOVER_EXPIRES_AT_KEY = "browser_takeover_expires_at"; +const TAKEOVER_TTL_SECONDS = 10 * 60; + +interface BrowserTakeoverGateDeps { + ctx: DurableObjectState; + env: AgentRunEnv; + getOwnerUserId: () => string | undefined; + getStatus: () => string | undefined; +} + +/** Durable human-control pause for one run's headed browser. */ +export class AgentRunBrowserTakeover { + private readonly waiters = new Set<() => void>(); + + public constructor(private readonly deps: BrowserTakeoverGateDeps) {} + + public async start(userId: string): Promise { + const denied = this.ownerError(userId); + if (denied) return denied; + if (!hasActiveRun(this.deps.getStatus())) { + return errorResponse(409, "The agent run is no longer active", "Start a new browser task."); + } + const runId = getRunStateValue(this.deps.ctx, "run_id"); + const sandboxName = getRunStateValue(this.deps.ctx, "sandbox_name"); + if (!runId || !sandboxName) { + return errorResponse(503, "Browser takeover state is incomplete", "Retry in a moment."); + } + const current = this.currentState(); + const takeoverId = current?.takeoverId ?? crypto.randomUUID(); + const expiresAtMs = Date.now() + TAKEOVER_TTL_SECONDS * 1_000; + this.storeActive(takeoverId, expiresAtMs); + try { + const sandbox = this.sandbox(sandboxName); + const session = await sandbox.exposeBrowserTakeover({ + expiresInSeconds: TAKEOVER_TTL_SECONDS, + runId, + takeoverId, + }); + const parsed = BrowserTakeoverSessionSchema.parse({ + ...session, + status: "active", + } satisfies BrowserTakeoverSession); + setRunStateValue( + this.deps.ctx, + TAKEOVER_EXPIRES_AT_KEY, + String(Date.parse(parsed.expiresAt)), + ); + return Response.json(parsed); + } catch (error) { + this.clear(); + if (error instanceof APIError) { + return error.toResponse(requestId()); + } + throw error; + } + } + + public async status(userId: string): Promise { + const denied = this.ownerError(userId); + if (denied) return denied; + const state = this.currentState(); + if (!state) { + return Response.json(BrowserTakeoverStatusSchema.parse({ status: "inactive" })); + } + if (state.expiresAtMs <= Date.now()) { + this.expire(); + return Response.json(BrowserTakeoverStatusSchema.parse({ status: "inactive" })); + } + return Response.json( + BrowserTakeoverStatusSchema.parse({ + expiresAt: new Date(state.expiresAtMs).toISOString(), + status: "active", + takeoverId: state.takeoverId, + }), + ); + } + + public async resume(userId: string, takeoverId: string): Promise { + const denied = this.ownerError(userId); + if (denied) return denied; + const state = this.currentState(); + if (!state) { + return Response.json( + BrowserTakeoverResumeResultSchema.parse({ ok: true, status: "inactive" }), + ); + } + if (state.takeoverId !== takeoverId) { + return errorResponse(409, "Browser takeover session changed", "Reconnect and try again."); + } + await this.stopProcess(); + this.clear(); + return Response.json(BrowserTakeoverResumeResultSchema.parse({ ok: true, status: "inactive" })); + } + + public async cleanup(): Promise { + await this.stopProcess().catch((error: unknown) => { + createLogger().warn("browser_takeover_cleanup_failed", { error }); + }); + this.clear(); + } + + /** Waits without consuming model chunks and returns the human-control duration. */ + public async wait(signal: AbortSignal): Promise { + const state = this.currentState(); + if (!state) return 0; + if (state.expiresAtMs <= Date.now()) { + this.expire(); + return 0; + } + const startedAt = Date.now(); + await waitForRelease(this.waiters, state.expiresAtMs - startedAt, signal); + const current = this.currentState(); + if (current && current.expiresAtMs <= Date.now()) { + this.expire(); + } + return Date.now() - startedAt; + } + + private currentState(): { expiresAtMs: number; takeoverId: string } | null { + const takeoverId = getRunStateValue(this.deps.ctx, TAKEOVER_ID_KEY); + const expiresAtMs = getRunStateTimestamp(this.deps.ctx, TAKEOVER_EXPIRES_AT_KEY); + return takeoverId && expiresAtMs !== null ? { expiresAtMs, takeoverId } : null; + } + + private storeActive(takeoverId: string, expiresAtMs: number): void { + setRunStateValue(this.deps.ctx, TAKEOVER_ID_KEY, takeoverId); + setRunStateValue(this.deps.ctx, TAKEOVER_EXPIRES_AT_KEY, String(expiresAtMs)); + } + + private clear(): void { + deleteRunStateValues(this.deps.ctx, [TAKEOVER_ID_KEY, TAKEOVER_EXPIRES_AT_KEY]); + for (const release of this.waiters) release(); + this.waiters.clear(); + } + + private expire(): void { + this.clear(); + this.deps.ctx.waitUntil( + this.stopProcess().catch((error: unknown) => { + createLogger().warn("browser_takeover_expiry_cleanup_failed", { error }); + }), + ); + } + + private async stopProcess(): Promise { + const sandboxName = getRunStateValue(this.deps.ctx, "sandbox_name"); + const runId = getRunStateValue(this.deps.ctx, "run_id"); + if (!sandboxName || !runId) return; + await this.sandbox(sandboxName).stopBrowserTakeover({ runId }); + } + + private sandbox(sandboxName: string) { + return this.deps.env.PROJECT_SANDBOX.get(this.deps.env.PROJECT_SANDBOX.idFromName(sandboxName)); + } + + private ownerError(userId: string): Response | null { + if (this.deps.getOwnerUserId() === userId) return null; + return new APIError(403, "permission_denied", "Run ownership mismatch", { + retriable: false, + }).toResponse(requestId()); + } +} + +function waitForRelease( + waiters: Set<() => void>, + timeoutMs: number, + signal: AbortSignal, +): Promise { + return new Promise((resolve) => { + let timeout: ReturnType | undefined; + const finish = () => { + if (timeout !== undefined) clearTimeout(timeout); + signal.removeEventListener("abort", finish); + waiters.delete(finish); + resolve(); + }; + waiters.add(finish); + signal.addEventListener("abort", finish, { once: true }); + timeout = setTimeout(finish, Math.max(0, timeoutMs)); + if (signal.aborted) finish(); + }); +} + +function errorResponse(status: number, message: string, hint: string): Response { + return new APIError(status, "conflict_state_invalid", message, { + hint, + retriable: status >= 500, + }).toResponse(requestId()); +} + +function requestId(): string { + return `req_${crypto.randomUUID().replaceAll("-", "")}`; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-chunk-telemetry.ts b/apps/agent-worker/src/durable-objects/agent-run-chunk-telemetry.ts index f690dbd8..4ba435e6 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-chunk-telemetry.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-chunk-telemetry.ts @@ -1,30 +1,45 @@ +import type { AgentChunkType } from "@cheatcode/agent-core"; import { emitUserEvent } from "@cheatcode/observability"; import type { AgentRunEnv } from "./agent-run-env"; import type { StartRunInput } from "./agent-run-schemas"; -import { getRunStateValue, setRunStateValue } from "./agent-run-storage"; +import { deleteRunStateValues, getRunStateValue, setRunStateValue } from "./agent-run-storage"; const STEP_IDX_KEY = "telemetry_step_idx"; interface ToolStartState { stepIdx: number; startedAt: number; - toolName: string; } +type ToolCallPayload = Extract["payload"]; +type ToolResultPayload = Extract["payload"]; +type ToolErrorPayload = Extract["payload"]; + export function emitMastraChunkTelemetry( ctx: DurableObjectState, env: AgentRunEnv, input: StartRunInput, - chunk: unknown, + chunk: AgentChunkType, ): void { - const record = asRecord(chunk); - const type = stringField(record, "type"); - if (type === "tool-call") { - emitToolStarted(ctx, env, input, record); + if (chunk.type === "tool-call") { + emitToolStarted(ctx, env, input, chunk.payload); return; } - if (type === "tool-result") { - emitToolCompleted(ctx, env, input, record); + if (chunk.type === "tool-result") { + const durationMs = emitToolCompleted( + ctx, + env, + input, + chunk.payload, + serializedResultBytes(chunk.payload.result), + ); + if (chunk.payload.toolName === "skill_invoke") { + emitSkillInvoked(env, input, chunk.payload, durationMs); + } + return; + } + if (chunk.type === "tool-error") { + emitToolCompleted(ctx, env, input, chunk.payload, serializedResultBytes(chunk.payload.error)); } } @@ -32,25 +47,20 @@ function emitToolStarted( ctx: DurableObjectState, env: AgentRunEnv, input: StartRunInput, - record: Record, + payload: ToolCallPayload, ): void { - const payload = chunkPayload(record); - const toolName = toolNameFromPayload(payload); - if (!toolName) { - return; - } const stepIdx = nextStepIdx(ctx); setRunStateValue( ctx, - toolStateKey(payload, toolName), - JSON.stringify({ stepIdx, startedAt: Date.now(), toolName }), + toolStateKey(payload.toolCallId), + JSON.stringify({ stepIdx, startedAt: Date.now() }), ); emitUserEvent(env, { eventName: "step_started", runId: input.runId, stepIdx, stepType: "tool", - toolName, + toolName: payload.toolName, userId: input.userId, }); } @@ -59,27 +69,23 @@ function emitToolCompleted( ctx: DurableObjectState, env: AgentRunEnv, input: StartRunInput, - record: Record, -): void { - const payload = chunkPayload(record); - const toolName = toolNameFromPayload(payload); - if (!toolName) { - return; - } - const state = readToolStartState(ctx, payload, toolName) ?? { + payload: ToolResultPayload | ToolErrorPayload, + resultBytes: number, +): number { + const key = toolStateKey(payload.toolCallId); + const state = readToolStartState(ctx, payload.toolCallId) ?? { startedAt: Date.now(), stepIdx: nextStepIdx(ctx), - toolName, }; + deleteRunStateValues(ctx, [key]); const durationMs = Math.max(0, Date.now() - state.startedAt); - const resultBytes = serializedResultBytes(payload); emitUserEvent(env, { durationMs, eventName: "tool_invoked", resultBytes, runId: input.runId, stepIdx: state.stepIdx, - toolName, + toolName: payload.toolName, userId: input.userId, }); emitUserEvent(env, { @@ -89,25 +95,21 @@ function emitToolCompleted( runId: input.runId, stepIdx: state.stepIdx, stepType: "tool", - toolName, + toolName: payload.toolName, userId: input.userId, }); - if (toolName === "skill_invoke") { - emitSkillInvoked(env, input, payload, durationMs); - } + return durationMs; } function emitSkillInvoked( env: AgentRunEnv, input: StartRunInput, - payload: Record, + payload: ToolResultPayload, durationMs: number, ): void { const skillName = - stringField(asRecord(payload["input"]), "skillName") || - stringField(asRecord(payload["args"]), "skillName") || - stringField(asRecord(payload["output"]), "name") || - stringField(asRecord(payload["result"]), "name"); + stringField(asRecord(payload.args), "skillName") || + stringField(asRecord(payload.result), "name"); emitUserEvent(env, { durationMs, eventName: "skill_invoked", @@ -124,12 +126,8 @@ function nextStepIdx(ctx: DurableObjectState): number { return next; } -function readToolStartState( - ctx: DurableObjectState, - payload: Record, - toolName: string, -): ToolStartState | null { - const raw = getRunStateValue(ctx, toolStateKey(payload, toolName)); +function readToolStartState(ctx: DurableObjectState, toolCallId: string): ToolStartState | null { + const raw = getRunStateValue(ctx, toolStateKey(toolCallId)); if (!raw) { return null; } @@ -144,33 +142,26 @@ function readToolStartState( } } -function toolStateKey(payload: Record, toolName: string): string { - const id = stringField(payload, "toolCallId") || stringField(payload, "id") || toolName; - return `telemetry_tool:${id}`; +function toolStateKey(toolCallId: string): string { + return `telemetry_tool:${toolCallId}`; } function isToolStartState(value: unknown): value is ToolStartState { return ( typeof value === "object" && value !== null && - typeof (value as Record)["toolName"] === "string" && typeof (value as Record)["stepIdx"] === "number" && typeof (value as Record)["startedAt"] === "number" ); } -function chunkPayload(record: Record): Record { - const payload = asRecord(record["payload"]); - return Object.keys(payload).length > 0 ? payload : record; -} - -function toolNameFromPayload(payload: Record): string { - return stringField(payload, "toolName") || stringField(payload, "tool"); -} - -function serializedResultBytes(payload: Record): number { - const result = payload["output"] ?? payload["result"] ?? payload; - return new TextEncoder().encode(JSON.stringify(result)).byteLength; +function serializedResultBytes(value: unknown): number { + try { + const serialized = JSON.stringify(value); + return serialized === undefined ? 0 : new TextEncoder().encode(serialized).byteLength; + } catch { + return 0; + } } function asRecord(value: unknown): Record { diff --git a/apps/agent-worker/src/durable-objects/agent-run-conversation.ts b/apps/agent-worker/src/durable-objects/agent-run-conversation.ts index 55ebe642..03847853 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-conversation.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-conversation.ts @@ -55,7 +55,10 @@ async function readContextRows( env: AgentRunEnv, input: StartRunInput, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, UserId(input.userId), (tx) => listRecentThreadContextMessages(tx, { @@ -73,6 +76,8 @@ async function readContextRows( function parseConversationMessage(row: ThreadContextMessageRecord): ConversationMessage { return ConversationMessageSchema.parse({ agentRunId: row.agentRunId, + agentRunSegment: row.agentRunSegment, + agentRunSegmentFinal: row.agentRunSegmentFinal, createdAt: row.createdAt.toISOString(), id: row.id, parts: row.parts, @@ -117,13 +122,6 @@ function modelRelevantParts(message: ConversationMessage): ConversationUIMessage text: part.text, type: "text", }); - } else if (part.type === "file") { - parts.push({ - ...(part.filename === undefined ? {} : { filename: part.filename }), - mediaType: part.mediaType, - type: "file", - url: part.url, - }); } } return parts; diff --git a/apps/agent-worker/src/durable-objects/agent-run-env.ts b/apps/agent-worker/src/durable-objects/agent-run-env.ts index 3262ede8..cef12c63 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-env.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-env.ts @@ -1,9 +1,14 @@ import type { WorkerSecret } from "@cheatcode/env"; import type { AnalyticsBindings } from "@cheatcode/observability"; +import type { AgentRunWorkflowPayload } from "./agent-run-workflow-protocol"; import type { ProjectSandbox } from "./project-sandbox"; export interface AgentRunEnv extends AnalyticsBindings { + AGENT_RUN_WORKFLOW: Workflow; + CHEATCODE_RELEASE_GATE: "closed" | "draining" | "open"; + CHEATCODE_RELEASE_SHA?: string; COMPOSIO_API_KEY?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; DEEPSEEK_PLATFORM_API_KEY?: WorkerSecret; HYPERDRIVE: Hyperdrive; OUTPUT_DOWNLOAD_BASE_URL?: string; @@ -12,5 +17,6 @@ export interface AgentRunEnv extends AnalyticsBindings { PROJECT_SANDBOX: DurableObjectNamespace; QUOTA_TRACKER: DurableObjectNamespace; R2_OUTPUTS: R2Bucket; - R2_OUTPUTS_BUCKET_NAME?: string; + SKILL_RUNTIME_BASE_URL: string; + SKILL_RUNTIME_TOKEN_SECRET: WorkerSecret; } diff --git a/apps/agent-worker/src/durable-objects/agent-run-http.ts b/apps/agent-worker/src/durable-objects/agent-run-http.ts index e76744f6..fb7f7da3 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-http.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-http.ts @@ -1,20 +1,32 @@ import { APIError, readJsonRequest } from "@cheatcode/observability"; -import { type ApprovalDecisionInput, ApprovalDecisionInputSchema } from "./agent-run-approvals"; +import { BrowserTakeoverResumeSchema } from "@cheatcode/types"; import { type StartRunInput, StartRunInputSchema } from "./agent-run-schemas"; import { missingInternalUserResponse } from "./agent-run-utils"; +import { + type AgentRunWorkflowCallbackInput, + AgentRunWorkflowCallbackInputSchema, + type AgentRunWorkflowFailureInput, + AgentRunWorkflowFailureInputSchema, +} from "./agent-run-workflow-protocol"; import { parseLastSeqParam } from "./run-state"; const INTERNAL_USER_HEADER = "X-Cheatcode-User-Id"; -const MAX_APPROVAL_REQUEST_BYTES = 4 * 1024; const MAX_START_RUN_REQUEST_BYTES = 128 * 1024; +const MAX_WORKFLOW_FAILURE_REQUEST_BYTES = 4 * 1024; +const MAX_WORKFLOW_EXECUTE_REQUEST_BYTES = 256 * 1024; +const MAX_BROWSER_TAKEOVER_REQUEST_BYTES = 4 * 1024; type ResponseResult = Promise | Response; export interface AgentRunHttpHandlers { - approval: (userId: string, body: ApprovalDecisionInput) => ResponseResult; cancel: (userId: string) => ResponseResult; + browserTakeoverResume: (userId: string, takeoverId: string) => ResponseResult; + browserTakeoverStart: (userId: string) => ResponseResult; + browserTakeoverStatus: (userId: string) => ResponseResult; deleteAll: (userId: string) => ResponseResult; - finalizeDetachedRun: () => Promise; + executeWorkflow: (input: AgentRunWorkflowCallbackInput) => ResponseResult; + failWorkflow: (input: AgentRunWorkflowFailureInput) => ResponseResult; + rolloverWorkflow: (input: AgentRunWorkflowCallbackInput) => ResponseResult; resume: (userId: string, lastSeq: number) => ResponseResult; start: (input: StartRunInput) => ResponseResult; status: (userId: string) => ResponseResult; @@ -45,9 +57,13 @@ async function handleGet( if (!userId) { return missingInternalUserResponse("status"); } - await handlers.finalizeDetachedRun(); return handlers.status(userId); } + if (url.pathname === "/browser-takeover") { + const userId = internalUser(request); + if (!userId) return missingInternalUserResponse("browser takeover"); + return handlers.browserTakeoverStatus(userId); + } if (url.pathname === "/stream") { return handleStream(request, url, handlers); } @@ -67,7 +83,6 @@ async function handleStream( if (!userId) { return missingInternalUserResponse("streams"); } - await handlers.finalizeDetachedRun(); return handlers.resume(userId, lastSeq); } @@ -80,10 +95,35 @@ async function handlePost( const input = StartRunInputSchema.parse( await readJsonRequest(request, MAX_START_RUN_REQUEST_BYTES, "Agent run start request"), ); - await handlers.finalizeDetachedRun(); return handlers.start(input); } - if (pathname !== "/cancel" && pathname !== "/approval" && pathname !== "/delete-all") { + if (pathname === "/workflow/execute") { + return handleWorkflowExecute(request, handlers); + } + if (pathname === "/workflow/failed") { + return handleWorkflowFailure(request, handlers); + } + if (pathname === "/workflow/rollover") { + return handleWorkflowRollover(request, handlers); + } + if (pathname === "/browser-takeover/start") { + const userId = internalUser(request); + if (!userId) return missingInternalUserResponse("browser takeover"); + return handlers.browserTakeoverStart(userId); + } + if (pathname === "/browser-takeover/resume") { + const userId = internalUser(request); + if (!userId) return missingInternalUserResponse("browser takeover"); + const body = BrowserTakeoverResumeSchema.parse( + await readJsonRequest( + request, + MAX_BROWSER_TAKEOVER_REQUEST_BYTES, + "Browser takeover resume request", + ), + ); + return handlers.browserTakeoverResume(userId, body.takeoverId); + } + if (pathname !== "/cancel" && pathname !== "/delete-all") { return notFound(); } const userId = internalUser(request); @@ -96,26 +136,53 @@ async function handlePost( if (pathname === "/delete-all") { return handlers.deleteAll(userId); } - if (pathname === "/approval") { - return handleApproval(request, userId, handlers); - } return notFound(); } -async function handleApproval( +async function handleWorkflowExecute( + request: Request, + handlers: AgentRunHttpHandlers, +): Promise { + const input = AgentRunWorkflowCallbackInputSchema.parse( + await readJsonRequest( + request, + MAX_WORKFLOW_EXECUTE_REQUEST_BYTES, + "AgentRun Workflow execution request", + ), + ); + return handlers.executeWorkflow(input); +} + +async function handleWorkflowFailure( + request: Request, + handlers: AgentRunHttpHandlers, +): Promise { + const input = AgentRunWorkflowFailureInputSchema.parse( + await readJsonRequest( + request, + MAX_WORKFLOW_FAILURE_REQUEST_BYTES, + "AgentRun Workflow failure request", + ), + ); + return handlers.failWorkflow(input); +} + +async function handleWorkflowRollover( request: Request, - userId: string, handlers: AgentRunHttpHandlers, ): Promise { - const body = ApprovalDecisionInputSchema.parse( - await readJsonRequest(request, MAX_APPROVAL_REQUEST_BYTES, "Approval decision request"), + const input = AgentRunWorkflowCallbackInputSchema.parse( + await readJsonRequest( + request, + MAX_WORKFLOW_EXECUTE_REQUEST_BYTES, + "AgentRun Workflow rollover request", + ), ); - return handlers.approval(userId, body); + return handlers.rolloverWorkflow(input); } -function postOperationName(pathname: string): "approval" | "cancel" | "delete-all" { +function postOperationName(pathname: string): "cancel" | "delete-all" { if (pathname === "/cancel") return "cancel"; - if (pathname === "/approval") return "approval"; return "delete-all"; } diff --git a/apps/agent-worker/src/durable-objects/agent-run-lifecycle.ts b/apps/agent-worker/src/durable-objects/agent-run-lifecycle.ts index 7de3f63d..873fa636 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-lifecycle.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-lifecycle.ts @@ -19,6 +19,7 @@ type TerminalRunStatus = "canceled" | "completed" | "failed"; export interface AgentRunLifecycleDeps { append: (chunk: UIMessageChunk) => Promise; + cleanupBrowserTakeover: () => Promise; ctx: DurableObjectState; env: AgentRunEnv; executeRunPath: ( @@ -84,7 +85,7 @@ function createRunExecution( async function executeActiveRun(execution: RunExecution): Promise { const { deps, input } = execution; await deps.persistRunStatus(input, "running"); - await deps.append({ type: "start" }); + await deps.append({ messageId: input.runId, type: "start" }); await deps.append(runPlanChunk()); await deps.append({ type: "data-sandbox-status", data: { v: 1, status: "starting" } }); deps.setRunStage("Preparing project sandbox."); @@ -187,6 +188,7 @@ async function cleanupRun(execution: RunExecution): Promise { if (execution.runLeaseHeartbeat !== undefined) { clearInterval(execution.runLeaseHeartbeat); } + await execution.deps.cleanupBrowserTakeover(); if (execution.sandbox.killProcess) { await execution.sandbox .killProcess({ processId: browserDriverProcessId(execution.input.runId) }) diff --git a/apps/agent-worker/src/durable-objects/agent-run-mastra-stream.ts b/apps/agent-worker/src/durable-objects/agent-run-mastra-stream.ts index 3ade698e..c9ecfe75 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-mastra-stream.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-mastra-stream.ts @@ -1,21 +1,28 @@ -import { type ApprovalBroker, createCodeRequestContext, mastra } from "@cheatcode/agent-core"; +import { type AgentChunkType, createCodeRequestContext, mastra } from "@cheatcode/agent-core"; import { workspacePathForSlug } from "@cheatcode/db"; import { APIError, type createLogger } from "@cheatcode/observability"; -import type { ArtifactRuntime, CodeRuntimeContext } from "@cheatcode/sandbox-contracts"; -import { isLoopFinished, type ModelMessage } from "ai"; +import type { + ArtifactRuntime, + CodeRuntimeContext, + WorkspaceResolver, +} from "@cheatcode/sandbox-contracts"; +import { hasToolCall, type ModelMessage, stepCountIs } from "ai"; import { resolveWithAbortTimeout } from "./abort-timeout"; import type { AgentRunEnv } from "./agent-run-env"; import type { StartRunInput } from "./agent-run-schemas"; +import { projectSkillRuntimeConfig } from "./agent-run-skill-runtime"; import { resolveUserSkillContext } from "./agent-run-user-skills"; import { readMastraChunk } from "./agent-run-utils"; import { resolveAgentToolCredentials } from "./agent-tool-credentials"; import type { LlmCredential } from "./llm-provider"; const MASTRA_FIRST_CHUNK_TIMEOUT_MS = 45_000; -const MASTRA_PROGRESS_TIMEOUT_MS = 11 * 60_000; +const MASTRA_VISIBLE_INACTIVITY_TIMEOUT_MS = 3 * 60_000; +const MASTRA_TOOL_HEARTBEAT_TIMEOUT_MS = 11 * 60_000; +const MASTRA_RUN_DEADLINE_MS = 60 * 60_000; type ProjectSandboxStub = CodeRuntimeContext["sandbox"]; -type MastraOpenedStream = { fullStream: AsyncIterable }; +type MastraOpenedStream = { fullStream: AsyncIterable }; type ResolvedToolCredentials = Awaited>; type ResolvedUserSkillContext = Awaited>; @@ -26,27 +33,27 @@ interface PreparedMastraContext extends ResolvedUserSkillContext { interface ConsumeMastraStreamOptions { abortController: AbortController; abortSignal: AbortSignal; - appendCheckedMastraChunk: (input: StartRunInput, chunk: unknown) => Promise; + appendCheckedMastraChunk: (input: StartRunInput, chunk: AgentChunkType) => Promise; credential: LlmCredential; - hasPendingDecision: () => boolean; input: StartRunInput; logger: ReturnType; setRunStage: (stage: string) => void; stream: MastraOpenedStream; + waitForBrowserTakeover: (signal: AbortSignal) => Promise; } export type MastraStreamOptions = { abortSignal: AbortSignal; - appendCheckedMastraChunk: (input: StartRunInput, chunk: unknown) => Promise; - approvalBroker: ApprovalBroker | undefined; + appendCheckedMastraChunk: (input: StartRunInput, chunk: AgentChunkType) => Promise; artifactRuntime: ArtifactRuntime; env: AgentRunEnv; - hasPendingDecision: () => boolean; input: StartRunInput; logger: ReturnType; modelMessages: ModelMessage[]; sandbox: ProjectSandboxStub; setRunStage: (stage: string) => void; + workspaceResolver: WorkspaceResolver; + waitForBrowserTakeover: (signal: AbortSignal) => Promise; credential: LlmCredential; }; @@ -76,11 +83,11 @@ export async function runMastraStream(options: MastraStreamOptions): Promise { + await projectSkillRuntimeConfig({ + env: options.env, + run: options.input, + sandbox: options.sandbox, + }); const toolCredentials = await resolveAgentToolCredentials({ env: options.env, logger: options.logger, run: options.input, setRunStage: options.setRunStage, }); - const userSkillContext = await resolveUserSkillContext(options.env, options.input.userId); + const userSkillContext = await resolveUserSkillContext( + options.env, + options.input.userId, + options.sandbox, + ); options.logger.info("agent_tool_credentials_resolved", { composioConfigured: Boolean(toolCredentials.composioApiKey), exaConfigured: Boolean(toolCredentials.exaApiKey), firecrawlConfigured: Boolean(toolCredentials.firecrawlApiKey), + googleMediaConfigured: Boolean(toolCredentials.googleMediaApiKey), }); return { ...userSkillContext, toolCredentials }; } @@ -108,7 +125,7 @@ async function openMastraStream( prepared: PreparedMastraContext, abortController: AbortController, ): Promise { - return resolveWithAbortTimeout({ + const opened = await resolveWithAbortTimeout({ abortController, operation: mastra.getAgent("general").stream(options.modelMessages, { abortSignal: abortController.signal, @@ -117,10 +134,35 @@ async function openMastraStream( ...(options.credential.transportProvider === "deepseek" ? { providerOptions: { deepseek: { thinking: { type: "disabled" } } } } : {}), - stopWhen: isLoopFinished(), + ...executionPolicy(options.input.runIntent), }), timeoutMs: MASTRA_FIRST_CHUNK_TIMEOUT_MS, }); + if (opened === "timeout") { + return opened; + } + // Mastra shares a broad ChunkType declaration across agent/workflow outputs; getAgent().stream() + // emits the AgentChunkType branch, which is the only contract accepted past this adapter. + return { fullStream: opened.fullStream as AsyncIterable }; +} + +function executionPolicy(runIntent: StartRunInput["runIntent"]) { + if (runIntent !== "skill-creator") { + return { stopWhen: stepCountIs(50) }; + } + const skillAuthoringTools = [ + "fs_delete", + "fs_list", + "fs_read", + "fs_search", + "fs_write", + "shell_exec", + "skill_create", + ]; + return { + activeTools: skillAuthoringTools, + stopWhen: [hasToolCall("skill_create"), stepCountIs(30)], + }; } function agentRequestContext( @@ -128,40 +170,48 @@ function agentRequestContext( prepared: PreparedMastraContext, ): ReturnType { const { credential, input } = options; - const { toolCredentials, userSkillLoader, userSkills, userSkillStore } = prepared; - return createCodeRequestContext( - { - artifacts: options.artifactRuntime, - sandbox: options.sandbox, - workspaceDir: workspacePathForSlug(input.workspaceSlug), + const { toolCredentials, userSkillLoader, userSkills } = prepared; + const isSkillCreator = input.runIntent === "skill-creator"; + const codeRuntime: CodeRuntimeContext = { + artifacts: options.artifactRuntime, + ensureWorkspace: async () => { + const workspace = await options.workspaceResolver(); + codeRuntime.workspaceDir = workspace.workspaceDir; + return workspace; }, - { - agentDisplayName: input.agentDisplayName, - anthropicApiKey: credential.transportProvider === "anthropic" ? credential.apiKey : undefined, - approvalBroker: options.approvalBroker, - composioApiKey: toolCredentials.composioApiKey, - composioConnectedAccounts: toolCredentials.composioConnectedAccounts, - composioQuotaMeter: toolCredentials.composioQuotaMeter, - composioUserId: toolCredentials.composioUserId, - deepseekApiKey: credential.transportProvider === "deepseek" ? credential.apiKey : undefined, - exaApiKey: toolCredentials.exaApiKey, - firecrawlApiKey: toolCredentials.firecrawlApiKey, - globalMemory: input.globalMemory, - googleApiKey: credential.transportProvider === "google" ? credential.apiKey : undefined, - llmProvider: credential.transportProvider, - masterInstructions: input.masterInstructions, - modelId: credential.transportModelId, - openaiApiKey: credential.transportProvider === "openai" ? credential.apiKey : undefined, - openrouterApiKey: - credential.transportProvider === "openrouter" ? credential.apiKey : undefined, - projectMode: input.projectMode, - runId: input.runId, - taskMessage: input.messageText, - userSkillLoader, - userSkills, - userSkillStore, - }, - ); + sandbox: options.sandbox, + ...(isSkillCreator + ? { workspaceDir: "/workspace" } + : input.workspaceSlug + ? { workspaceDir: workspacePathForSlug(input.workspaceSlug) } + : {}), + }; + return createCodeRequestContext(codeRuntime, { + agentDisplayName: input.agentDisplayName, + anthropicApiKey: credential.transportProvider === "anthropic" ? credential.apiKey : undefined, + composioApiKey: toolCredentials.composioApiKey, + composioConnectedAccounts: toolCredentials.composioConnectedAccounts, + composioQuotaMeter: toolCredentials.composioQuotaMeter, + composioUserId: toolCredentials.composioUserId, + deepseekApiKey: credential.transportProvider === "deepseek" ? credential.apiKey : undefined, + exaApiKey: toolCredentials.exaApiKey, + firecrawlApiKey: toolCredentials.firecrawlApiKey, + globalMemory: input.globalMemory, + googleApiKey: + credential.transportProvider === "google" + ? credential.apiKey + : toolCredentials.googleMediaApiKey, + llmProvider: credential.transportProvider, + modelId: credential.transportModelId, + openaiApiKey: credential.transportProvider === "openai" ? credential.apiKey : undefined, + openrouterApiKey: credential.transportProvider === "openrouter" ? credential.apiKey : undefined, + projectMode: input.projectMode, + runIntent: input.runIntent, + runId: input.runId, + taskMessage: input.messageText, + userSkillLoader, + userSkills, + }); } function linkedAbortController(runAbortSignal: AbortSignal): { @@ -189,11 +239,11 @@ async function consumeOpenedMastraStream(options: ConsumeMastraStreamOptions): P }); options.setRunStage("Streaming model response."); const iterator = options.stream.fullStream[Symbol.asyncIterator](); + await options.waitForBrowserTakeover(options.abortSignal); const firstChunk = await readMastraChunk( iterator, MASTRA_FIRST_CHUNK_TIMEOUT_MS, options.abortController, - options.hasPendingDecision, ); if (firstChunk === "timeout") { await iterator.return?.(); @@ -206,9 +256,9 @@ async function consumeOpenedMastraStream(options: ConsumeMastraStreamOptions): P abortController: options.abortController, appendCheckedMastraChunk: options.appendCheckedMastraChunk, firstChunk, - hasPendingDecision: options.hasPendingDecision, input: options.input, iterator, + waitForBrowserTakeover: options.waitForBrowserTakeover, }); if (streamResult === "timeout-before-visible") { await iterator.return?.(); @@ -234,36 +284,87 @@ function modelStreamTimeoutError(): APIError { async function appendMastraStreamChunks(options: { abortController: AbortController; - appendCheckedMastraChunk: (input: StartRunInput, chunk: unknown) => Promise; - firstChunk: IteratorResult; - hasPendingDecision: () => boolean; + appendCheckedMastraChunk: (input: StartRunInput, chunk: AgentChunkType) => Promise; + firstChunk: IteratorResult; input: StartRunInput; - iterator: AsyncIterator; + iterator: AsyncIterator; + waitForBrowserTakeover: (signal: AbortSignal) => Promise; }): Promise<"completed" | "timeout-after-visible" | "timeout-before-visible"> { let hasVisibleChunk = false; - const firstVisibleChunkDeadline = Date.now() + MASTRA_FIRST_CHUNK_TIMEOUT_MS; + let activityDeadline = Date.now() + MASTRA_FIRST_CHUNK_TIMEOUT_MS; + let runDeadline = Date.now() + MASTRA_RUN_DEADLINE_MS; + const pendingToolCalls = new Set(); if (options.firstChunk.done) { return "completed"; } + ({ activityDeadline, runDeadline } = await extendDeadlinesForTakeover( + options, + activityDeadline, + runDeadline, + )); + updatePendingToolCalls(pendingToolCalls, options.firstChunk.value); hasVisibleChunk = (await options.appendCheckedMastraChunk(options.input, options.firstChunk.value)) > 0; + activityDeadline = nextActivityDeadline(hasVisibleChunk, pendingToolCalls.size > 0); for (;;) { - const timeoutMs = hasVisibleChunk - ? MASTRA_PROGRESS_TIMEOUT_MS - : Math.max(1, firstVisibleChunkDeadline - Date.now()); - const nextChunk = await readMastraChunk( - options.iterator, - timeoutMs, - options.abortController, - options.hasPendingDecision, - ); + ({ activityDeadline, runDeadline } = await extendDeadlinesForTakeover( + options, + activityDeadline, + runDeadline, + )); + const timeoutMs = Math.max(1, Math.min(activityDeadline, runDeadline) - Date.now()); + const nextChunk = await readMastraChunk(options.iterator, timeoutMs, options.abortController); if (nextChunk === "timeout") { return hasVisibleChunk ? "timeout-after-visible" : "timeout-before-visible"; } if (nextChunk.done) { return "completed"; } + ({ activityDeadline, runDeadline } = await extendDeadlinesForTakeover( + options, + activityDeadline, + runDeadline, + )); + updatePendingToolCalls(pendingToolCalls, nextChunk.value); const appendedCount = await options.appendCheckedMastraChunk(options.input, nextChunk.value); hasVisibleChunk = appendedCount > 0 || hasVisibleChunk; + if (appendedCount > 0 || pendingToolCalls.size > 0) { + activityDeadline = nextActivityDeadline(hasVisibleChunk, pendingToolCalls.size > 0); + } + } +} + +async function extendDeadlinesForTakeover( + options: { + abortController: AbortController; + waitForBrowserTakeover: (signal: AbortSignal) => Promise; + }, + activityDeadline: number, + runDeadline: number, +): Promise<{ activityDeadline: number; runDeadline: number }> { + const pausedMs = await options.waitForBrowserTakeover(options.abortController.signal); + return { + activityDeadline: activityDeadline + pausedMs, + runDeadline: runDeadline + pausedMs, + }; +} + +function nextActivityDeadline(hasVisibleChunk: boolean, hasPendingTool: boolean): number { + if (hasPendingTool) { + return Date.now() + MASTRA_TOOL_HEARTBEAT_TIMEOUT_MS; + } + return ( + Date.now() + + (hasVisibleChunk ? MASTRA_VISIBLE_INACTIVITY_TIMEOUT_MS : MASTRA_FIRST_CHUNK_TIMEOUT_MS) + ); +} + +function updatePendingToolCalls(pendingToolCalls: Set, chunk: AgentChunkType): void { + if (chunk.type === "tool-call") { + pendingToolCalls.add(chunk.payload.toolCallId); + return; + } + if (chunk.type === "tool-result" || chunk.type === "tool-error") { + pendingToolCalls.delete(chunk.payload.toolCallId); } } diff --git a/apps/agent-worker/src/durable-objects/agent-run-message-persistence.ts b/apps/agent-worker/src/durable-objects/agent-run-message-persistence.ts index 41c55a37..6926021a 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-message-persistence.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-message-persistence.ts @@ -1,15 +1,25 @@ import { createDb, createThreadMessage, withUserContext } from "@cheatcode/db"; import { createLogger, type Logger } from "@cheatcode/observability"; -import type { UIMessagePart } from "@cheatcode/types"; -import { AgentRunId, ThreadId, UserId } from "@cheatcode/types"; -import type { UIMessageChunk } from "ai"; import { - isMessagePartRow, - type MessagePartRow, - parseSequencedChunk, -} from "../streaming/ui-message-stream"; + AgentRunId, + fragmentMessagePart, + serializedMessagePartsBytes, + ThreadId, + TRANSCRIPT_SEGMENT_MAX_PARTS_BYTES, + type UIMessagePart, + UserId, +} from "@cheatcode/types"; +import type { UIMessageChunk } from "ai"; +import { type MessagePartRow, parseSequencedChunk } from "../streaming/ui-message-stream"; import type { AgentRunEnv } from "./agent-run-env"; -import { deleteRunStateValues, getRunStateValue, setRunStateValue } from "./agent-run-storage"; +import { + deleteRunStateValues, + getRunStateTimestamp, + getRunStateValue, + readAgentRunMessagePartPage, + setRunStateValue, +} from "./agent-run-storage"; +import { transcriptPartFromChunk } from "./agent-run-transcript-chunks"; const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; @@ -26,13 +36,7 @@ const PENDING_MESSAGE_ATTEMPT_KEY = "pending_assistant_message_attempt"; const PENDING_MESSAGE_RETRY_AT_KEY = "pending_assistant_message_retry_at"; const MIN_MESSAGE_RETRY_MS = 5_000; const MAX_MESSAGE_RETRY_MS = 5 * 60 * 1_000; -const MESSAGE_PERSISTENCE_PAGE_SIZE = 100; - -interface TextPartDraft { - chunks: string[]; - index: number; - isClosed: boolean; -} +const TEXT_PERSISTENCE_SLICE_CHARACTERS = 16 * 1024; export async function persistOrQueueAssistantMessage( input: PersistAssistantMessageInput, @@ -84,21 +88,27 @@ async function persistAssistantMessage({ if (!UUID_PATTERN.test(runId) || !UUID_PATTERN.test(threadId)) { return true; } - const parts = assistantPartsFromPages((lastSeq) => readRowsPage(ctx, lastSeq)); - if (parts.length === 0) { - return true; - } - const { db, close } = createDb(env.HYPERDRIVE); + const createdAt = transcriptCreatedAt(ctx); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { - await withUserContext(db, UserId(userId), (tx) => - createThreadMessage(tx, { - agentRunId: AgentRunId(runId), - parts, - role: "assistant", - threadId: ThreadId(threadId), - userId: UserId(userId), - }), - ); + const writer = new AssistantTranscriptWriter(async (parts, segment, isFinal) => { + await withUserContext(db, UserId(userId), (tx) => + createThreadMessage(tx, { + agentRunId: AgentRunId(runId), + agentRunSegment: segment, + agentRunSegmentFinal: isFinal, + createdAt, + parts, + role: "assistant", + threadId: ThreadId(threadId), + userId: UserId(userId), + }), + ); + }); + await writeAssistantTranscript(writer, (lastSeq) => readRowsPage(ctx, lastSeq)); return true; } catch (error) { logger.error("assistant_message_persist_failed", { @@ -116,15 +126,17 @@ async function persistAssistantMessage({ } function readRowsPage(ctx: DurableObjectState, lastSeq: number): MessagePartRow[] { - const rows: unknown[] = ctx.storage.sql - .exec( - `SELECT seq, payload_json FROM message_part - WHERE seq > ? ORDER BY seq LIMIT ?`, - lastSeq, - MESSAGE_PERSISTENCE_PAGE_SIZE, - ) - .toArray(); - return rows.filter(isMessagePartRow); + return readAgentRunMessagePartPage(ctx, lastSeq); +} + +function transcriptCreatedAt(ctx: DurableObjectState): Date { + const stored = getRunStateTimestamp(ctx, "completed_at"); + if (stored !== null) { + return new Date(stored); + } + const fallback = Date.now(); + setRunStateValue(ctx, "completed_at", String(fallback)); + return new Date(fallback); } function storedRunIdentity( @@ -158,11 +170,11 @@ function messageRetryDelay(attempt: number): number { return Math.min(MAX_MESSAGE_RETRY_MS, MIN_MESSAGE_RETRY_MS * 2 ** Math.min(attempt, 6)); } -function assistantPartsFromPages( +async function writeAssistantTranscript( + writer: AssistantTranscriptWriter, readRowsPage: (lastSeq: number) => MessagePartRow[], -): UIMessagePart[] { - const output: UIMessagePart[] = []; - const textParts = new Map(); +): Promise { + const assembler = new AssistantPartAssembler(writer); let cursor = 0; for (;;) { const rows = readRowsPage(cursor); @@ -172,126 +184,155 @@ function assistantPartsFromPages( for (const row of rows) { const sequenced = parseSequencedChunk(row); cursor = sequenced.seq; - appendChunkPart(sequenced.chunk, output, textParts); + await assembler.append(sequenced.seq, sequenced.chunk); } } - finalizeTextParts(output, textParts); - return output.filter((part) => part.type !== "text" || textPartHasContent(part)); + await assembler.finish(); + await writer.finish(); } -function appendChunkPart( - chunk: UIMessageChunk, - output: UIMessagePart[], - textParts: Map, -): void { - if (chunk.type === "text-start") { - ensureTextPart(chunk.id, output, textParts); - return; - } - if (chunk.type === "text-delta") { - const draft = ensureTextPart(chunk.id, output, textParts); - draft.chunks.push(chunk.delta); - draft.isClosed = false; - return; - } - if (chunk.type === "text-end") { - const draft = ensureTextPart(chunk.id, output, textParts); - closeTextPart(output, draft); - return; +type PersistSegment = (parts: UIMessagePart[], segment: number, isFinal: boolean) => Promise; + +class AssistantTranscriptWriter { + private currentBytes = 2; + private currentParts: UIMessagePart[] = []; + private nextSegment = 0; + private pendingParts: UIMessagePart[] | null = null; + + public constructor(private readonly persist: PersistSegment) {} + + public async append(part: UIMessagePart, partId: string): Promise { + const candidates = + serializedPartBytes(part) + 2 > TRANSCRIPT_SEGMENT_MAX_PARTS_BYTES + ? fragmentMessagePart(part, partId) + : [part]; + for (const candidate of candidates) { + await this.appendBounded(candidate); + } } - if (chunk.type === "start-step") { - output.push({ type: "step-start" }); - return; + + public async finish(): Promise { + await this.stageCurrent(); + if (this.pendingParts) { + await this.persist(this.pendingParts, this.nextSegment, true); + this.pendingParts = null; + } } - if (isPersistableDataChunk(chunk) || isPersistableDisplayChunk(chunk)) { - output.push(uiMessagePartFromChunk(chunk)); + + private async appendBounded(part: UIMessagePart): Promise { + const partBytes = serializedPartBytes(part); + const separatorBytes = this.currentParts.length === 0 ? 0 : 1; + if (this.currentBytes + separatorBytes + partBytes > TRANSCRIPT_SEGMENT_MAX_PARTS_BYTES) { + await this.stageCurrent(); + } + if (partBytes + 2 > TRANSCRIPT_SEGMENT_MAX_PARTS_BYTES) { + throw new RangeError("Transcript fragment exceeds the segment byte bound."); + } + this.currentBytes += (this.currentParts.length === 0 ? 0 : 1) + partBytes; + this.currentParts.push(part); } -} -function ensureTextPart( - id: string, - output: UIMessagePart[], - textParts: Map, -): TextPartDraft { - const existing = textParts.get(id); - if (existing) { - return existing; + private async stageCurrent(): Promise { + if (this.currentParts.length === 0) { + return; + } + if (this.pendingParts) { + await this.persist(this.pendingParts, this.nextSegment, false); + this.nextSegment += 1; + } + this.pendingParts = this.currentParts; + this.currentParts = []; + this.currentBytes = 2; } - const draft = { chunks: [], index: output.length, isClosed: false }; - textParts.set(id, draft); - output.push({ type: "text", text: "", state: "streaming" }); - return draft; } -function closeTextPart(output: UIMessagePart[], draft: TextPartDraft): void { - const text = draft.chunks.join(""); - output[draft.index] = { type: "text", text, state: "done" }; - draft.chunks = [text]; - draft.isClosed = true; -} +class AssistantPartAssembler { + private bufferedTextCharacters = 0; + private readonly bufferedTextChunks: string[] = []; + private openTextId: string | null = null; + private textPartSeq = 0; -function finalizeTextParts(output: UIMessagePart[], textParts: Map): void { - for (const draft of textParts.values()) { - if (!draft.isClosed) { - closeTextPart(output, draft); + public constructor(private readonly writer: AssistantTranscriptWriter) {} + + public async append(seq: number, chunk: UIMessageChunk): Promise { + if (chunk.type === "text-start") { + await this.switchTextPart(chunk.id, seq); + return; + } + if (chunk.type === "text-delta") { + await this.switchTextPart(chunk.id, seq); + await this.appendText(chunk.delta); + return; + } + if (chunk.type === "text-end") { + await this.flushText(); + this.openTextId = null; + return; + } + await this.flushText(); + const part = partFromChunk(chunk); + if (part) { + await this.writer.append(part, String(seq)); } } -} - -function isPersistableDataChunk( - chunk: UIMessageChunk, -): chunk is UIMessageChunk & { type: `data-${string}` } { - const value = chunkRecord(chunk); - return ( - typeof value["type"] === "string" && - value["type"].startsWith("data-") && - value["type"] !== "data-seq" && - value["transient"] !== true - ); -} -function isPersistableDisplayChunk(chunk: UIMessageChunk): boolean { - return chunk.type === "file" || chunk.type === "source-url" || chunk.type === "source-document"; -} + public async finish(): Promise { + await this.flushText(); + } -function uiMessagePartFromChunk(chunk: UIMessageChunk): UIMessagePart { - const value = chunkRecord(chunk); - if (chunk.type === "file") { - return { - type: chunk.type, - mediaType: value["mediaType"], - url: value["url"], - ...(typeof value["filename"] === "string" ? { filename: value["filename"] } : {}), - }; + private async switchTextPart(id: string, seq: number): Promise { + if (this.openTextId === id) { + return; + } + await this.flushText(); + this.openTextId = id; + this.textPartSeq = seq; } - if (chunk.type === "source-url") { - return { - type: chunk.type, - sourceId: value["sourceId"], - url: value["url"], - ...(typeof value["title"] === "string" ? { title: value["title"] } : {}), - }; + + private async appendText(value: string): Promise { + let offset = 0; + while (offset < value.length) { + const end = safeTextSliceEnd(value, offset); + const chunk = value.slice(offset, end); + if ( + this.bufferedTextCharacters > 0 && + this.bufferedTextCharacters + chunk.length > TEXT_PERSISTENCE_SLICE_CHARACTERS + ) { + await this.flushText(); + } + this.bufferedTextChunks.push(chunk); + this.bufferedTextCharacters += chunk.length; + offset = end; + } } - if (chunk.type === "source-document") { - return { - type: chunk.type, - mediaType: value["mediaType"], - sourceId: value["sourceId"], - title: value["title"], - ...(typeof value["filename"] === "string" ? { filename: value["filename"] } : {}), - }; + + private async flushText(): Promise { + if (this.bufferedTextCharacters === 0) { + return; + } + const text = this.bufferedTextChunks.join(""); + this.bufferedTextChunks.length = 0; + this.bufferedTextCharacters = 0; + await this.writer.append({ state: "done", text, type: "text" }, `text-${this.textPartSeq}`); } - return { - type: chunk.type, - data: value["data"], - ...(typeof value["id"] === "string" ? { id: value["id"] } : {}), - }; } -function chunkRecord(chunk: UIMessageChunk): Record { - return Object(chunk) as Record; +function partFromChunk(chunk: UIMessageChunk): UIMessagePart | null { + return transcriptPartFromChunk(chunk); } -function textPartHasContent(part: UIMessagePart): boolean { - return typeof part["text"] === "string" && part["text"].trim().length > 0; +function serializedPartBytes(part: UIMessagePart): number { + return serializedMessagePartsBytes([part]) - 2; +} + +function safeTextSliceEnd(value: string, offset: number): number { + const candidate = Math.min(value.length, offset + TEXT_PERSISTENCE_SLICE_CHARACTERS); + if (candidate === value.length) { + return candidate; + } + const previous = value.charCodeAt(candidate - 1); + const next = value.charCodeAt(candidate); + return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff + ? candidate - 1 + : candidate; } diff --git a/apps/agent-worker/src/durable-objects/agent-run-metrics.ts b/apps/agent-worker/src/durable-objects/agent-run-metrics.ts index d8dc79f5..800935c3 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-metrics.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-metrics.ts @@ -25,7 +25,7 @@ export function emitStoredAgentRunMetric( env: AgentRunEnv, input: AgentRunMetricInput, ): void { - if (input.status === "running" || input.status === "paused") { + if (input.status === "running") { return; } const now = Date.now(); diff --git a/apps/agent-worker/src/durable-objects/agent-run-model-persistence.ts b/apps/agent-worker/src/durable-objects/agent-run-model-persistence.ts index a2b0b41b..32039ea6 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-model-persistence.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-model-persistence.ts @@ -21,7 +21,10 @@ interface PersistAgentRunLogicalModelInput { export async function persistAgentRunLogicalModel( input: PersistAgentRunLogicalModelInput, ): Promise { - const dbHandle = createDb(input.env.HYPERDRIVE); + const dbHandle = createDb(input.env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: input.env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { const updated = await withUserContext(dbHandle.db, UserId(input.userId), (db) => updateAgentRunLogicalModelId(db, { diff --git a/apps/agent-worker/src/durable-objects/agent-run-output.ts b/apps/agent-worker/src/durable-objects/agent-run-output.ts index 59c312cc..f05636f8 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-output.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-output.ts @@ -1,25 +1,30 @@ +import type { AgentChunkType } from "@cheatcode/agent-core"; import type { UIMessageChunk } from "ai"; import { createSeqChunk, - isMessagePartRow, type MessagePartRow, parseSequencedChunk, } from "../streaming/ui-message-stream"; import { emitRunAbandoned } from "./agent-run-abandonment"; import type { AgentRunEnv } from "./agent-run-env"; import { emitFirstVisibleChunkMetric } from "./agent-run-performance"; -import { appendAgentRunMessagePart } from "./agent-run-storage"; +import { appendAgentRunMessagePart, readAgentRunMessagePartPage } from "./agent-run-storage"; +import { boundedAgentRunChunks, serializedChunkBytes } from "./agent-run-transcript-chunks"; import { mastraChunkToUiChunks } from "./mastra-stream-chunks"; import { hasActiveRun } from "./run-state"; -const STREAM_REPLAY_PAGE_SIZE = 100; -const STREAM_SUBSCRIBER_HIGH_WATER_MARK = 256; -const ANSWER_SEGMENT_BREAK_TYPES = new Set(["data-tool", "data-thinking"]); +const MAX_ACTIVE_STREAMS = 8; +const STREAM_SUBSCRIBER_HIGH_WATER_MARK_BYTES = 256 * 1024; +const ANSWER_SEGMENT_BREAK_TYPES = new Set(["data-tool"]); -type Subscriber = { controller: ReadableStreamDefaultController }; +type Subscriber = { + controller: ReadableStreamDefaultController; + release: () => void; +}; interface ResumeStreamState { cursor: number; + isReleased: boolean; pendingRows: MessagePartRow[]; subscriber: Subscriber | undefined; } @@ -33,8 +38,10 @@ interface AgentRunOutputOptions { } export class AgentRunOutput { + private activeStreamCount = 0; private answerSegmentCount = 0; private lastVisibleWasAnswerText = false; + private nextOutputEvent = 0; private openAnswerSegmentId: string | null = null; private sawArtifact = false; private readonly subscribers = new Set(); @@ -45,21 +52,34 @@ export class AgentRunOutput { this.openAnswerSegmentId = null; this.answerSegmentCount = 0; this.lastVisibleWasAnswerText = false; + this.nextOutputEvent = 0; this.sawArtifact = false; } - public resume(lastSeq: number): ReadableStream { + public hasStreamCapacity(): boolean { + return this.activeStreamCount < MAX_ACTIVE_STREAMS; + } + + public resume(lastSeq: number): ReadableStream | null { + if (!this.hasStreamCapacity()) { + return null; + } + this.activeStreamCount += 1; const state: ResumeStreamState = { cursor: lastSeq, + isReleased: false, pendingRows: [], subscriber: undefined, }; return new ReadableStream( { - pull: (controller) => this.pullResumeStream(controller, state), + pull: (controller) => this.pullResumeStreamSafely(controller, state), cancel: () => this.cancelResumeStream(state), }, - { highWaterMark: STREAM_SUBSCRIBER_HIGH_WATER_MARK }, + { + highWaterMark: STREAM_SUBSCRIBER_HIGH_WATER_MARK_BYTES, + size: serializedChunkBytes, + }, ); } @@ -72,17 +92,10 @@ export class AgentRunOutput { } private replayRowsPage(lastSeq: number): MessagePartRow[] { - const rows: unknown[] = this.options.ctx.storage.sql - .exec( - "SELECT seq, payload_json FROM message_part WHERE seq > ? ORDER BY seq LIMIT ?", - lastSeq, - STREAM_REPLAY_PAGE_SIZE, - ) - .toArray(); - return rows.filter(isMessagePartRow); + return readAgentRunMessagePartPage(this.options.ctx, lastSeq); } - public async appendMastraChunk(chunk: unknown): Promise { + public async appendMastraChunk(chunk: AgentChunkType): Promise { let appendedCount = 0; for (const uiChunk of mastraChunkToUiChunks(chunk)) { appendedCount += await this.appendAnswerSegmented(uiChunk); @@ -100,13 +113,15 @@ export class AgentRunOutput { await this.appendAnswerSegmented({ type: "text-delta", id: "answer", delta: closing }); } - public async ensureAnswerSegmentEnded(): Promise { + public async ensureAnswerSegmentEnded(options?: { + allowAfterCancelRequest?: boolean; + }): Promise { if (this.openAnswerSegmentId === null) { return 0; } const id = this.openAnswerSegmentId; this.openAnswerSegmentId = null; - await this.append({ type: "text-end", id }); + await this.append({ type: "text-end", id }, options); return 1; } @@ -118,6 +133,14 @@ export class AgentRunOutput { return; } this.trackClosingSignals(chunk); + const fragmentId = `event-${this.nextOutputEvent}`; + this.nextOutputEvent += 1; + for (const bounded of boundedAgentRunChunks(chunk, fragmentId)) { + this.appendBounded(bounded); + } + } + + private appendBounded(chunk: UIMessageChunk): void { const sequencedChunk = { chunk, seq: appendAgentRunMessagePart(this.options.ctx, chunk), @@ -125,18 +148,42 @@ export class AgentRunOutput { emitFirstVisibleChunkMetric(this.options.ctx, this.options.env, chunk); for (const subscriber of [...this.subscribers]) { if ((subscriber.controller.desiredSize ?? 1) <= 0) { - this.subscribers.delete(subscriber); - subscriber.controller.error(new Error("Agent stream subscriber fell behind.")); + this.errorSubscriber(subscriber, new Error("Agent stream subscriber fell behind.")); continue; } - this.write(subscriber.controller, sequencedChunk); + try { + this.write(subscriber.controller, sequencedChunk); + } catch (error) { + this.errorSubscriber(subscriber, error); + } } } public closeSubscribers(): void { - for (const subscriber of this.subscribers) { - subscriber.controller.close(); - this.subscribers.delete(subscriber); + for (const subscriber of [...this.subscribers]) { + try { + subscriber.controller.close(); + } catch { + // A canceled stream may close between snapshotting and termination. + } finally { + this.releaseSubscriber(subscriber); + } + } + } + + private pullResumeStreamSafely( + controller: ReadableStreamDefaultController, + state: ResumeStreamState, + ): void { + try { + this.pullResumeStream(controller, state); + } catch (error) { + this.releaseStream(state); + try { + controller.error(error); + } catch { + // The consumer may have canceled while replay storage was being read. + } } } @@ -171,25 +218,62 @@ export class AgentRunOutput { state: ResumeStreamState, ): void { if (!hasActiveRun(this.options.getStatus()) && !this.options.isTerminalizing()) { - controller.close(); + this.releaseStream(state); + try { + controller.close(); + } catch { + // A terminal stream can race with consumer cancellation. + } return; } - const subscriber = { controller }; + const subscriber: Subscriber = { + controller, + release: () => { + if (state.subscriber === subscriber) { + state.subscriber = undefined; + } + this.releaseStream(state); + }, + }; state.subscriber = subscriber; this.subscribers.add(subscriber); } private cancelResumeStream(state: ResumeStreamState): void { - if (!state.subscriber) { - return; + const subscriber = state.subscriber; + if (subscriber) { + this.subscribers.delete(subscriber); + state.subscriber = undefined; } - this.subscribers.delete(state.subscriber); - state.subscriber = undefined; - if (this.subscribers.size === 0) { + this.releaseStream(state); + if (subscriber && this.subscribers.size === 0) { emitRunAbandoned(this.options.ctx, this.options.env); } } + private releaseSubscriber(subscriber: Subscriber): void { + this.subscribers.delete(subscriber); + subscriber.release(); + } + + private errorSubscriber(subscriber: Subscriber, error: unknown): void { + try { + subscriber.controller.error(error); + } catch { + // Controller termination is best-effort; the stream slot must still be released. + } finally { + this.releaseSubscriber(subscriber); + } + } + + private releaseStream(state: ResumeStreamState): void { + if (state.isReleased) { + return; + } + state.isReleased = true; + this.activeStreamCount = Math.max(0, this.activeStreamCount - 1); + } + private async appendAnswerSegmented(uiChunk: UIMessageChunk): Promise { if (uiChunk.type === "text-delta") { this.lastVisibleWasAnswerText = true; diff --git a/apps/agent-worker/src/durable-objects/agent-run-path.ts b/apps/agent-worker/src/durable-objects/agent-run-path.ts index 729c8507..5eeec505 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-path.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-path.ts @@ -1,5 +1,5 @@ import type { createLogger } from "@cheatcode/observability"; -import type { CodeRuntimeContext } from "@cheatcode/sandbox-contracts"; +import type { CodeRuntimeContext, WorkspaceResolver } from "@cheatcode/sandbox-contracts"; import type { UIMessageChunk } from "ai"; import { restartMobilePreview, runAppBuilder, warmSandbox } from "./agent-run-app-builder"; import type { AgentRunEnv } from "./agent-run-env"; @@ -18,38 +18,34 @@ export interface AgentRunPathOptions { sandbox: ProjectSandboxStub; setRunStage: (stage: string) => void; streamDriverDeps: StreamDriverDeps; + workspaceResolver: WorkspaceResolver; } +type ProjectBoundStartRunInput = StartRunInput & { + projectId: string; + workspaceSlug: string; +}; + +type ProjectBoundAgentRunPathOptions = AgentRunPathOptions & { + input: ProjectBoundStartRunInput; +}; + export async function executeAgentRunPath( options: AgentRunPathOptions, ): Promise<"completed" | "continue"> { - await ensureProjectWorkspaceDir(options); if (isAppBuilderMode(options.input.projectMode)) { - return executeAppBuilderPath(options); + await options.workspaceResolver(); + return executeAppBuilderPath({ + ...options, + input: requireProjectBinding(options.input), + }); } await streamMastraRunWithFallback(options.streamDriverDeps, options); return options.isCanceled() ? "completed" : "continue"; } -async function ensureProjectWorkspaceDir(options: AgentRunPathOptions): Promise { - if (!options.sandbox.exec) { - return; - } - try { - await options.sandbox.exec({ - command: ["mkdir", "-p", `/workspace/${options.input.workspaceSlug}`], - timeoutMs: 15_000, - }); - } catch (error) { - options.logger.warn("workspace_dir_ensure_failed", { - error, - workspaceSlug: options.input.workspaceSlug, - }); - } -} - async function executeAppBuilderPath( - options: AgentRunPathOptions, + options: ProjectBoundAgentRunPathOptions, ): Promise<"completed" | "continue"> { await warmSandbox(options.sandbox, options.logger); if (options.isCanceled()) { @@ -65,6 +61,7 @@ async function executeAppBuilderPath( input: options.input, logger: options.logger, sandbox: options.sandbox, + workspaceResolver: options.workspaceResolver, }); if (options.isCanceled()) { return "completed"; @@ -73,7 +70,9 @@ async function executeAppBuilderPath( return "continue"; } -async function restartMobilePreviewIfNeeded(options: AgentRunPathOptions): Promise { +async function restartMobilePreviewIfNeeded( + options: ProjectBoundAgentRunPathOptions, +): Promise { if (options.input.projectMode !== "app-builder-mobile") { return; } @@ -86,6 +85,13 @@ async function restartMobilePreviewIfNeeded(options: AgentRunPathOptions): Promi } } +function requireProjectBinding(input: StartRunInput): ProjectBoundStartRunInput { + if (!input.projectId || !input.workspaceSlug) { + throw new Error("Workspace resolver completed without a project binding."); + } + return input as ProjectBoundStartRunInput; +} + function isAppBuilderMode(mode: StartRunInput["projectMode"]): boolean { return mode === "app-builder" || mode === "app-builder-mobile"; } diff --git a/apps/agent-worker/src/durable-objects/agent-run-responses.ts b/apps/agent-worker/src/durable-objects/agent-run-responses.ts new file mode 100644 index 00000000..9318005d --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-responses.ts @@ -0,0 +1,63 @@ +import { APIError } from "@cheatcode/observability"; + +/** Requests that are required to finish or erase work admitted before draining. */ +export function isAgentRunDrainContinuation(request: Request): boolean { + if (request.method !== "POST") { + return false; + } + const pathname = new URL(request.url).pathname; + return ( + pathname === "/workflow/execute" || + pathname === "/workflow/failed" || + pathname === "/workflow/rollover" || + pathname === "/delete-all" + ); +} + +export function agentRunReleaseGateResponse(releaseGate: "closed" | "draining"): Response { + const response = new APIError(503, "unavailable_maintenance", "Release is in progress", { + details: { releaseGate, worker: "agent" }, + retriable: true, + }).toResponse(requestId()); + response.headers.set("Cache-Control", "no-store"); + response.headers.set("Retry-After", "5"); + return response; +} + +export function deletedAgentRunResponse(): Response { + return new APIError(410, "not_found_run", "Run state was permanently deleted", { + retriable: false, + }).toResponse(requestId()); +} + +export function absentAgentRunOkResponse(): Response { + return Response.json({ ok: true }); +} + +export function absentAgentRunWorkflowResponse(): Response { + return Response.json({ outcome: "deleted", status: "deleted" }); +} + +export function agentRunStreamCapacityResponse(): Response { + return new APIError(429, "rate_limit_exceeded", "Too many agent stream subscribers", { + hint: "Close another view of this run, then reconnect with the last received sequence.", + retriable: true, + }).toResponse(requestId()); +} + +export async function agentRunWorkflowResponse( + operation: () => Promise, +): Promise { + try { + return await operation(); + } catch (error) { + if (error instanceof APIError) { + return error.toResponse(requestId()); + } + throw error; + } +} + +function requestId(): string { + return `req_${crypto.randomUUID().replaceAll("-", "")}`; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-schemas.ts b/apps/agent-worker/src/durable-objects/agent-run-schemas.ts index 0cd65051..14936af5 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-schemas.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-schemas.ts @@ -1,14 +1,17 @@ -import { CatalogModelIdSchema, LogicalModelIdSchema, ProjectModeSchema } from "@cheatcode/types"; +import { + CatalogModelIdSchema, + LogicalModelIdSchema, + ProjectModeSchema, + RunIntentSchema, +} from "@cheatcode/types"; import { z } from "zod"; export const StartRunInputSchema = z .object({ runId: z.string().uuid(), threadId: z.string().uuid(), - projectId: z.string().uuid(), - // Immutable /workspace subfolder for this project in the per-user "computer" sandbox. Every - // run has a project (ensureProjectForRun creates it before the run), so this is always set. - workspaceSlug: z.string().min(1).max(64), + projectId: z.string().uuid().optional(), + workspaceSlug: z.string().min(1).max(64).optional(), sandboxName: z.string().min(1), userId: z.string().uuid(), messageText: z.string().min(1), @@ -16,13 +19,16 @@ export const StartRunInputSchema = z // Whether `model` was pinned by the request or project settings (vs Auto). Gates // automatic provider fallback so a pinned model is never silently replaced. modelExplicit: z.boolean(), + runIntent: RunIntentSchema.optional(), projectMode: ProjectModeSchema.default("general"), isFirstRun: z.boolean().default(false), - masterInstructions: z.string().trim().min(1).max(20_000).optional(), agentDisplayName: z.string().trim().min(1).max(80).optional(), globalMemory: z.string().trim().min(1).max(8_000).optional(), disabledModels: z.array(CatalogModelIdSchema).max(16).default([]), importRepoUrl: z.string().trim().url().max(300).optional(), }) - .strict(); + .strict() + .refine((value) => Boolean(value.projectId) === Boolean(value.workspaceSlug), { + message: "projectId and workspaceSlug must be supplied together", + }); export type StartRunInput = z.infer; diff --git a/apps/agent-worker/src/durable-objects/agent-run-skill-runtime.ts b/apps/agent-worker/src/durable-objects/agent-run-skill-runtime.ts new file mode 100644 index 00000000..2dfe4b83 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-skill-runtime.ts @@ -0,0 +1,53 @@ +import { mintSkillRuntimeCapability, type SkillRuntimeScope } from "@cheatcode/auth"; +import { resolveWorkerSecret } from "@cheatcode/env"; +import type { SandboxLike } from "@cheatcode/sandbox-contracts"; +import type { AgentRunEnv } from "./agent-run-env"; +import type { StartRunInput } from "./agent-run-schemas"; + +const SKILL_RUNTIME_CONFIG_PATH = "/workspace/.cheatcode/runtime/skill-runtime-config.json"; +const RUN_SCOPES: readonly SkillRuntimeScope[] = [ + "events:write", + "integrations:execute", + "skills:read", + "skills:write", +]; + +/** Projects the run-bound capability consumed by copied skill package scripts. */ +export async function projectSkillRuntimeConfig(input: { + env: AgentRunEnv; + run: StartRunInput; + sandbox: SandboxLike; +}): Promise { + if (!input.sandbox.writeFile) { + throw new Error("Sandbox does not support the skill runtime config projection."); + } + const secret = await resolveWorkerSecret(input.env.SKILL_RUNTIME_TOKEN_SECRET); + if (!secret) { + throw new Error("SKILL_RUNTIME_TOKEN_SECRET is not configured."); + } + const capability = await mintSkillRuntimeCapability({ + ...(input.run.projectId ? { projectId: input.run.projectId } : {}), + runId: input.run.runId, + scopes: RUN_SCOPES, + secret, + userId: input.run.userId, + }); + await input.sandbox.writeFile({ + content: `${JSON.stringify( + { + accessToken: capability.token, + backendBaseUrl: input.env.SKILL_RUNTIME_BASE_URL.replace(/\/+$/u, ""), + deliveryChannel: "web", + expiresAt: capability.expiresAt, + ...(input.run.projectId ? { projectId: input.run.projectId } : {}), + runId: input.run.runId, + sandboxContext: "project", + v: 1, + }, + null, + 2, + )}\n`, + encoding: "utf8", + path: SKILL_RUNTIME_CONFIG_PATH, + }); +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-status-payload.ts b/apps/agent-worker/src/durable-objects/agent-run-status-payload.ts index 5085acf4..3afa506d 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-status-payload.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-status-payload.ts @@ -1,4 +1,3 @@ -import { pendingApprovalSnapshot } from "./agent-run-approvals"; import { getRunStateValue, readStoredRunSnapshot } from "./agent-run-storage"; import { type AgentRunSnapshotStatus, summarizeAgentRunStorage } from "./run-summary"; @@ -9,13 +8,11 @@ interface StatusPayloadInput { export function agentRunStatusPayload(input: StatusPayloadInput): unknown | null { const summary = statusSummary(input); - const pending = pendingApprovalSnapshot(input.ctx); - const pendingApproval = pending ? { pendingApproval: pending } : {}; const stored = readStoredRunSnapshot(input.ctx); if (!stored) { return null; } - return { ...stored, ...pendingApproval, ok: true, status: input.status, summary }; + return { ...stored, ok: true, status: input.status, summary }; } function statusSummary(input: StatusPayloadInput): string { diff --git a/apps/agent-worker/src/durable-objects/agent-run-status-persistence.ts b/apps/agent-worker/src/durable-objects/agent-run-status-persistence.ts index 45ff47e3..8397f26d 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-status-persistence.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-status-persistence.ts @@ -1,22 +1,65 @@ import { createDb, updateAgentRunStatus, withUserContext } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; import { createLogger } from "@cheatcode/observability"; import { AgentRunId, UserId } from "@cheatcode/types"; import { z } from "zod"; -import { deleteRunStateValues, getRunStateValue, setRunStateValue } from "./agent-run-storage"; +import type { AgentRunEnv } from "./agent-run-env"; +import { pendingAssistantMessageRetryAt } from "./agent-run-message-persistence"; +import { emitStoredAgentRunMetric } from "./agent-run-metrics"; +import { + deleteRunStateValues, + getRunStateValue, + isAgentRunDeleted, + setRunStateValue, +} from "./agent-run-storage"; -export type PersistableRunStatus = "running" | "paused" | "completed" | "failed" | "canceled"; +export type PersistableRunStatus = "running" | "completed" | "failed" | "canceled"; interface AgentRunStatusPersistenceEnv { + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; HYPERDRIVE: Hyperdrive; } export interface PersistAgentRunStatusInput { - error?: { message: string; type: string }; + artifactsQuiesced: boolean; runId: string; status: PersistableRunStatus; userId: string; } +export function isTerminalPersistableRunStatus( + status: PersistableRunStatus, +): status is Extract { + return status === "canceled" || status === "completed" || status === "failed"; +} + +export async function persistSerializedAgentRunStatus( + ctx: DurableObjectState, + env: AgentRunEnv, + input: PersistAgentRunStatusInput, + serialize: (operation: () => Promise) => Promise, + armAlarm: () => Promise, +): Promise { + if (isAgentRunDeleted(ctx)) { + return; + } + await serialize(async () => { + if (isAgentRunDeleted(ctx)) { + return; + } + emitStoredAgentRunMetric(ctx, env, input); + if ( + isTerminalPersistableRunStatus(input.status) && + pendingAssistantMessageRetryAt(ctx) !== Number.POSITIVE_INFINITY + ) { + deferAgentRunStatus(ctx, input); + return; + } + await persistOrQueueAgentRunStatus(ctx, env, input); + }); + await armAlarm(); +} + const PENDING_STATUS_KEY = "pending_db_status"; const PENDING_STATUS_RETRY_AT_KEY = "pending_db_status_retry_at"; const MIN_STATUS_RETRY_MS = 5_000; @@ -25,12 +68,9 @@ const MAX_STATUS_RETRY_MS = 5 * 60 * 1000; const PendingStatusSchema = z .object({ attempt: z.number().int().nonnegative(), - error: z - .object({ message: z.string().max(2_000), type: z.string().max(200) }) - .strict() - .optional(), + artifactsQuiesced: z.boolean(), runId: z.string().uuid(), - status: z.enum(["running", "paused", "completed", "failed", "canceled"]), + status: z.enum(["running", "completed", "failed", "canceled"]), userId: z.string().uuid(), }) .strict(); @@ -41,11 +81,14 @@ async function persistAgentRunStatus( env: AgentRunStatusPersistenceEnv, input: PersistAgentRunStatusInput, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { const updated = await withUserContext(db, UserId(input.userId), (tx) => updateAgentRunStatus(tx, { - ...(input.error ? { error: input.error } : {}), + artifactsQuiesced: input.artifactsQuiesced, runId: AgentRunId(input.runId), status: input.status, userId: UserId(input.userId), @@ -76,7 +119,7 @@ async function persistAgentRunStatus( } } -export async function persistOrQueueAgentRunStatus( +async function persistOrQueueAgentRunStatus( ctx: DurableObjectState, env: AgentRunStatusPersistenceEnv, input: PersistAgentRunStatusInput, @@ -85,6 +128,11 @@ export async function persistOrQueueAgentRunStatus( clearPendingStatus(ctx); return; } + deferAgentRunStatus(ctx, input); +} + +/** Keep Postgres nonterminal until an earlier durable transcript outbox has flushed. */ +function deferAgentRunStatus(ctx: DurableObjectState, input: PersistAgentRunStatusInput): void { const previous = readPendingStatus(ctx); queuePendingStatus(ctx, input, (previous?.attempt ?? -1) + 1); } @@ -133,7 +181,7 @@ function readPendingStatus(ctx: DurableObjectState): PendingStatus | null { function statusInputFromPending(pending: PendingStatus): PersistAgentRunStatusInput { return { - ...(pending.error ? { error: pending.error } : {}), + artifactsQuiesced: pending.artifactsQuiesced, runId: pending.runId, status: pending.status, userId: pending.userId, diff --git a/apps/agent-worker/src/durable-objects/agent-run-storage.ts b/apps/agent-worker/src/durable-objects/agent-run-storage.ts index 57292a70..c20422e6 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-storage.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-storage.ts @@ -1,32 +1,79 @@ +import { + assertExactSqliteSchema, + assertSqliteRowCountPreserved, + type ExpectedSqliteObject, + setCurrentSqliteStorageVersion, +} from "@cheatcode/durable-storage"; import { type LogicalModelId, LogicalModelIdSchema, PRODUCTION_DEFAULT_MODEL_ID, } from "@cheatcode/types"; import type { UIMessageChunk } from "ai"; -import { isSeqRow } from "../streaming/ui-message-stream"; +import { isMessagePartRow, isSeqRow, type MessagePartRow } from "../streaming/ui-message-stream"; +import { + AGENT_RUN_MESSAGE_PART_MAX_BYTES, + serializedChunkBytes, +} from "./agent-run-transcript-chunks"; -const DEFAULT_AGENT_NAME = "general"; +const DELETION_TOMBSTONE_KEY = "deletion_tombstone"; +const OWNER_USER_ID_KEY = "owner_user_id"; const RESOLVED_LOGICAL_MODEL_ID_KEY = "resolved_logical_model_id"; -const CURRENT_RUN_COLUMNS = [ - "id", - "thread_id", - "project_id", - "user_id", - "status", - "model_id", - "agent_name", - "created_at", - "started_at", - "completed_at", +const RUN_STATUS_VALUES_SQL = "'pending','running','completed','failed','canceled'"; + +interface ExpectedColumn { + defaultValue: string | null; + isNotNull: boolean; + isPrimaryKey: boolean; + name: string; + type: string; +} + +const RUN_COLUMNS = [ + column("id", "TEXT", true, true), + column("status", "TEXT", true), + column("model_id", "TEXT", true), + column("created_at", "INTEGER", true), + column("started_at", "INTEGER"), + column("completed_at", "INTEGER"), +] as const; +const MESSAGE_PART_COLUMNS = [ + column("seq", "INTEGER", false, true), + column("part_type", "TEXT", true), + column("payload_json", "TEXT", true), +] as const; +const MESSAGE_PART_PAGE_MAX_BYTES = 256 * 1024; +const MESSAGE_PART_PAGE_MAX_ROWS = 32; +const RUN_STATE_COLUMNS = [ + column("key", "TEXT", true, true), + column("value", "TEXT", true), ] as const; +const RUN_TABLE_SQL = `CREATE TABLE run ( + id TEXT PRIMARY KEY CHECK (length(id) = 36), + status TEXT NOT NULL CHECK (status IN (${RUN_STATUS_VALUES_SQL})), + model_id TEXT NOT NULL CHECK (length(model_id) BETWEEN 1 AND 200), + created_at INTEGER NOT NULL CHECK (created_at >= 0), + started_at INTEGER CHECK (started_at IS NULL OR started_at >= created_at), + completed_at INTEGER CHECK (completed_at IS NULL OR completed_at >= created_at) +) STRICT`; +const MESSAGE_PART_TABLE_SQL = `CREATE TABLE message_part ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + part_type TEXT NOT NULL CHECK (length(part_type) BETWEEN 1 AND 100), + payload_json TEXT NOT NULL CHECK (length(cast(payload_json AS blob)) <= ${AGENT_RUN_MESSAGE_PART_MAX_BYTES}) +) STRICT`; +const RUN_STATE_TABLE_SQL = `CREATE TABLE run_state ( + key TEXT PRIMARY KEY CHECK (length(key) BETWEEN 1 AND 256), + value TEXT NOT NULL CHECK (length(cast(value AS blob)) <= 1048576) +) STRICT`; +const AGENT_RUN_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [ + { name: "message_part", sql: MESSAGE_PART_TABLE_SQL, tableName: "message_part", type: "table" }, + { name: "run", sql: RUN_TABLE_SQL, tableName: "run", type: "table" }, + { name: "run_state", sql: RUN_STATE_TABLE_SQL, tableName: "run_state", type: "table" }, +]; export interface StoredRunIdentity { plannedLogicalModelId: LogicalModelId; - projectId: string; runId: string; - threadId: string; - userId: string; } export interface StoredRunSnapshot { @@ -37,13 +84,69 @@ export interface StoredRunSnapshot { modelId: LogicalModelId; runId: string; startedAt: number | null; - status: "canceled" | "completed" | "failed" | "paused" | "running"; + status: "canceled" | "completed" | "failed" | "running"; } export function initializeAgentRunStorage(ctx: DurableObjectState): void { - ensureCurrentRunTable(ctx); - createMessagePartTable(ctx); - createRunStateTable(ctx); + reconcileRunTable(ctx); + reconcileMessagePartTable(ctx); + reconcileRunStateTable(ctx); + normalizeRunStateStatus(ctx); + setCurrentSqliteStorageVersion(ctx); + assertAgentRunStorage(ctx); +} + +/** Read-only presence probe; unlike initialization this does not create a stored object. */ +export function hasAgentRunStorage(ctx: DurableObjectState): boolean { + return ( + ctx.storage.sql + .exec( + "SELECT 1 AS present FROM sqlite_schema WHERE type = 'table' AND name = 'run_state' LIMIT 1", + ) + .toArray().length > 0 + ); +} + +/** Force-normalizes dormant run objects while the production release gate is closed. */ +export function reconcileAgentRunStorage(ctx: DurableObjectState): void { + prepareAgentRunRebuild(ctx); + ctx.storage.transactionSync(() => { + rebuildAgentRunTables(ctx); + normalizeRunStateStatus(ctx); + removePersistedArtifactCapabilities(ctx); + }); + setCurrentSqliteStorageVersion(ctx); + assertAgentRunStorage(ctx); +} + +function prepareAgentRunRebuild(ctx: DurableObjectState): void { + const sources = [ + { columns: RUN_COLUMNS, create: () => createRunTable(ctx, "run"), table: "run" }, + { + columns: MESSAGE_PART_COLUMNS, + create: () => createMessagePartTable(ctx, "message_part"), + table: "message_part", + }, + { + columns: RUN_STATE_COLUMNS, + create: () => createRunStateTable(ctx, "run_state"), + table: "run_state", + }, + ] as const; + for (const source of sources) { + const columns = tableColumns(ctx, source.table); + if (columns.length === 0) { + source.create(); + continue; + } + if (!source.columns.every(({ name }) => hasColumn(columns, name))) { + throw new Error(`Unsupported AgentRun ${source.table} schema; refusing lossy evolution.`); + } + } +} + +export function assertAgentRunStorage(ctx: DurableObjectState): void { + assertExactSqliteSchema(ctx, AGENT_RUN_STORAGE_SCHEMA); } export function getRunStateTimestamp(ctx: DurableObjectState, key: string): number | null { @@ -69,25 +172,70 @@ export function setRunStateValue(ctx: DurableObjectState, key: string, value: st ctx.storage.sql.exec("INSERT OR REPLACE INTO run_state (key, value) VALUES (?, ?)", key, value); } +/** Permanently claims this run-keyed object for deletion before any async cleanup yields. */ +export function claimAgentRunDeletion(ctx: DurableObjectState, userId: string): boolean { + return ctx.storage.transactionSync(() => { + const ownerUserId = getRunStateValue(ctx, OWNER_USER_ID_KEY); + if (ownerUserId && ownerUserId !== userId) { + return false; + } + if (!ownerUserId) { + setRunStateValue(ctx, OWNER_USER_ID_KEY, userId); + } + if (!getRunStateValue(ctx, DELETION_TOMBSTONE_KEY)) { + setRunStateValue(ctx, DELETION_TOMBSTONE_KEY, new Date().toISOString()); + } + return true; + }); +} + +export function isAgentRunDeleted(ctx: DurableObjectState): boolean { + return getRunStateValue(ctx, DELETION_TOMBSTONE_KEY) !== undefined; +} + export function deleteRunStateValues(ctx: DurableObjectState, keys: string[]): void { for (const key of keys) { ctx.storage.sql.exec("DELETE FROM run_state WHERE key = ?", key); } } +function removePersistedArtifactCapabilities(ctx: DurableObjectState): void { + const rows = ctx.storage.sql + .exec("SELECT seq, payload_json FROM message_part WHERE part_type = 'data-artifact'") + .toArray(); + for (const row of rows) { + if ( + !isRecord(row) || + typeof row["seq"] !== "number" || + typeof row["payload_json"] !== "string" + ) { + throw new Error("Invalid stored artifact transcript row; refusing lossy reconciliation."); + } + const parsed = JSON.parse(row["payload_json"]) as unknown; + if (!isRecord(parsed) || parsed["type"] !== "data-artifact" || !isRecord(parsed["data"])) { + throw new Error("Invalid stored artifact payload; refusing lossy reconciliation."); + } + if (!("downloadUrl" in parsed["data"])) { + continue; + } + const data = { ...parsed["data"], downloadUrl: undefined }; + ctx.storage.sql.exec( + "UPDATE message_part SET payload_json = ? WHERE seq = ?", + JSON.stringify({ ...parsed, data }), + row["seq"], + ); + } +} + export function upsertRunRow(ctx: DurableObjectState, input: StoredRunIdentity): void { const now = Date.now(); ctx.storage.sql.exec( `INSERT OR REPLACE INTO run ( - id, thread_id, project_id, user_id, status, model_id, agent_name, created_at, started_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + id, status, model_id, created_at, started_at + ) VALUES (?, ?, ?, ?, ?)`, input.runId, - input.threadId, - input.projectId, - input.userId, "running", input.plannedLogicalModelId, - DEFAULT_AGENT_NAME, now, now, ); @@ -117,7 +265,7 @@ export function getResolvedRunLogicalModelId(ctx: DurableObjectState): LogicalMo export function updateRunRowStatus( ctx: DurableObjectState, - status: "canceled" | "completed" | "failed" | "paused" | "running", + status: "canceled" | "completed" | "failed" | "running", ): void { const runId = getRunStateValue(ctx, "run_id"); if (!runId) { @@ -140,17 +288,14 @@ export function updateRunRowStatus( } export function appendAgentRunMessagePart(ctx: DurableObjectState, chunk: UIMessageChunk): number { + const payload = JSON.stringify(chunk); + if (serializedChunkBytes(chunk) > AGENT_RUN_MESSAGE_PART_MAX_BYTES) { + throw new RangeError("Agent run message part exceeds the durable per-event byte bound."); + } ctx.storage.sql.exec( - `INSERT INTO message_part ( - message_id, role, part_type, part_id, payload_json, transient, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?)`, - "assistant", - "assistant", + "INSERT INTO message_part (part_type, payload_json) VALUES (?, ?)", chunk.type, - chunkPartId(chunk), - JSON.stringify(chunk), - isTransientChunk(chunk) ? 1 : 0, - Date.now(), + payload, ); const row = ctx.storage.sql.exec("SELECT last_insert_rowid() AS seq").toArray()[0]; if (!isSeqRow(row)) { @@ -159,6 +304,58 @@ export function appendAgentRunMessagePart(ctx: DurableObjectState, chunk: UIMess return row.seq; } +export function readAgentRunMessagePartPage( + ctx: DurableObjectState, + lastSeq: number, +): MessagePartRow[] { + assertNextMessagePartBound(ctx, lastSeq); + const rows: unknown[] = ctx.storage.sql + .exec( + `WITH candidates AS ( + SELECT seq, payload_json + FROM message_part + WHERE seq > ? + ORDER BY seq + LIMIT ? + ), sized AS ( + SELECT seq, payload_json, + sum(length(cast(payload_json AS blob))) OVER (ORDER BY seq) AS cumulative_bytes + FROM candidates + ) + SELECT seq, payload_json + FROM sized + WHERE cumulative_bytes <= ? + ORDER BY seq`, + lastSeq, + MESSAGE_PART_PAGE_MAX_ROWS, + MESSAGE_PART_PAGE_MAX_BYTES, + ) + .toArray(); + if (!rows.every(isMessagePartRow)) { + throw new TypeError("Transcript storage returned a malformed message-part row."); + } + return rows; +} + +function assertNextMessagePartBound(ctx: DurableObjectState, lastSeq: number): void { + const row = firstRecord( + ctx.storage.sql + .exec( + `SELECT length(cast(payload_json AS blob)) AS payload_bytes + FROM message_part + WHERE seq > ? + ORDER BY seq + LIMIT 1`, + lastSeq, + ) + .toArray(), + ); + const payloadBytes = row?.["payload_bytes"]; + if (typeof payloadBytes === "number" && payloadBytes > AGENT_RUN_MESSAGE_PART_MAX_BYTES) { + throw new RangeError("Stored transcript event exceeds the supported durable byte bound."); + } +} + export function readStoredRunSnapshot(ctx: DurableObjectState): StoredRunSnapshot | null { const rows = ctx.storage.sql .exec( @@ -189,91 +386,156 @@ export function readStoredRunSnapshot(ctx: DurableObjectState): StoredRunSnapsho }; } -function createRunTable(ctx: DurableObjectState, tableName = "run"): void { - if (tableName !== "run" && tableName !== "run_current") { - throw new Error("Invalid run table name."); +function reconcileRunTable(ctx: DurableObjectState): void { + const columns = tableColumns(ctx, "run"); + if (columns.length === 0) { + createRunTable(ctx, "run"); + return; } + if (hasExactColumns(columns, RUN_COLUMNS) && hasCurrentRunTableSql(ctx)) { + return; + } + if (!RUN_COLUMNS.every(({ name }) => hasColumn(columns, name))) { + throw new Error("Unsupported AgentRun run schema; refusing lossy evolution."); + } + ctx.storage.transactionSync(() => { + ctx.storage.sql.exec("DROP TABLE IF EXISTS run_next"); + createRunTable(ctx, "run_next"); + copyRunRows(ctx, "run", "run_next"); + assertSqliteRowCountPreserved(ctx, "run", "run_next"); + ctx.storage.sql.exec("DROP TABLE run"); + ctx.storage.sql.exec("ALTER TABLE run_next RENAME TO run"); + }); +} + +function createRunTable(ctx: DurableObjectState, table: "run" | "run_next"): void { + ctx.storage.sql.exec(RUN_TABLE_SQL.replace("CREATE TABLE run", `CREATE TABLE ${table}`)); +} + +function reconcileMessagePartTable(ctx: DurableObjectState): void { + const columns = tableColumns(ctx, "message_part"); + if (columns.length === 0) { + createMessagePartTable(ctx, "message_part"); + return; + } + if (hasExactColumns(columns, MESSAGE_PART_COLUMNS)) { + return; + } + if (!MESSAGE_PART_COLUMNS.every(({ name }) => hasColumn(columns, name))) { + throw new Error("Unsupported AgentRun message schema; refusing lossy evolution."); + } + ctx.storage.transactionSync(() => { + ctx.storage.sql.exec("DROP TABLE IF EXISTS message_part_next"); + createMessagePartTable(ctx, "message_part_next"); + const names = MESSAGE_PART_COLUMNS.map(({ name }) => name).join(", "); + ctx.storage.sql.exec( + `INSERT INTO message_part_next (${names}) SELECT ${names} FROM message_part`, + ); + assertSqliteRowCountPreserved(ctx, "message_part", "message_part_next"); + ctx.storage.sql.exec("DROP TABLE message_part"); + ctx.storage.sql.exec("ALTER TABLE message_part_next RENAME TO message_part"); + }); +} + +function createMessagePartTable( + ctx: DurableObjectState, + table: "message_part" | "message_part_next", +): void { ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS ${tableName} ( - id TEXT PRIMARY KEY, - thread_id TEXT NOT NULL, - project_id TEXT NOT NULL, - user_id TEXT NOT NULL, - status TEXT NOT NULL CHECK (status IN ('pending','running','paused','completed','failed','canceled')), - model_id TEXT NOT NULL, - agent_name TEXT NOT NULL, - created_at INTEGER NOT NULL, - started_at INTEGER, - completed_at INTEGER - )`, + MESSAGE_PART_TABLE_SQL.replace("CREATE TABLE message_part", `CREATE TABLE ${table}`), ); } -function ensureCurrentRunTable(ctx: DurableObjectState): void { - const columns = ctx.storage.sql.exec("PRAGMA table_info(run)").toArray(); +function reconcileRunStateTable(ctx: DurableObjectState): void { + const columns = tableColumns(ctx, "run_state"); if (columns.length === 0) { - createRunTable(ctx); + createRunStateTable(ctx, "run_state"); return; } - const names = columns - .map((row) => firstRecord([row])?.["name"]) - .filter((name): name is string => typeof name === "string"); - const isCurrent = - names.length === CURRENT_RUN_COLUMNS.length && - CURRENT_RUN_COLUMNS.every((name) => names.includes(name)); - if (isCurrent) { + if (hasExactColumns(columns, RUN_STATE_COLUMNS)) { return; } - const canPreserve = CURRENT_RUN_COLUMNS.every((name) => names.includes(name)); + if (!RUN_STATE_COLUMNS.every(({ name }) => hasColumn(columns, name))) { + throw new Error("Unsupported AgentRun state schema; refusing lossy evolution."); + } ctx.storage.transactionSync(() => { - ctx.storage.sql.exec("DROP TABLE IF EXISTS run_current"); - createRunTable(ctx, "run_current"); - if (canPreserve) { - const columnList = CURRENT_RUN_COLUMNS.join(", "); - ctx.storage.sql.exec(`INSERT INTO run_current (${columnList}) SELECT ${columnList} FROM run`); - } - ctx.storage.sql.exec("DROP TABLE run"); - ctx.storage.sql.exec("ALTER TABLE run_current RENAME TO run"); + ctx.storage.sql.exec("DROP TABLE IF EXISTS run_state_next"); + createRunStateTable(ctx, "run_state_next"); + ctx.storage.sql.exec( + "INSERT INTO run_state_next (key, value) SELECT key, value FROM run_state", + ); + assertSqliteRowCountPreserved(ctx, "run_state", "run_state_next"); + ctx.storage.sql.exec("DROP TABLE run_state"); + ctx.storage.sql.exec("ALTER TABLE run_state_next RENAME TO run_state"); }); } -function createMessagePartTable(ctx: DurableObjectState): void { +function createRunStateTable(ctx: DurableObjectState, table: "run_state" | "run_state_next"): void { ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS message_part ( - seq INTEGER PRIMARY KEY AUTOINCREMENT, - message_id TEXT NOT NULL, - role TEXT NOT NULL, - part_type TEXT NOT NULL, - part_id TEXT, - payload_json TEXT NOT NULL, - transient INTEGER NOT NULL DEFAULT 0, - created_at INTEGER NOT NULL - )`, + RUN_STATE_TABLE_SQL.replace("CREATE TABLE run_state", `CREATE TABLE ${table}`), ); - ctx.storage.sql.exec("CREATE INDEX IF NOT EXISTS idx_part_msg ON message_part(message_id)"); } -function createRunStateTable(ctx: DurableObjectState): void { +function rebuildAgentRunTables(ctx: DurableObjectState): void { + for (const table of ["run", "message_part", "run_state"] as const) { + ctx.storage.sql.exec(`DROP TABLE IF EXISTS ${table}_reconcile_source`); + ctx.storage.sql.exec(`ALTER TABLE ${table} RENAME TO ${table}_reconcile_source`); + } + ctx.storage.sql.exec(RUN_TABLE_SQL); + ctx.storage.sql.exec(MESSAGE_PART_TABLE_SQL); + ctx.storage.sql.exec(RUN_STATE_TABLE_SQL); + copyRunRows(ctx, "run_reconcile_source", "run"); + ctx.storage.sql.exec( + `INSERT INTO message_part (seq, part_type, payload_json) + SELECT seq, part_type, payload_json FROM message_part_reconcile_source`, + ); ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS run_state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - )`, + "INSERT INTO run_state (key, value) SELECT key, value FROM run_state_reconcile_source", ); + for (const table of ["run", "message_part", "run_state"] as const) { + assertSqliteRowCountPreserved(ctx, `${table}_reconcile_source`, table); + ctx.storage.sql.exec(`DROP TABLE ${table}_reconcile_source`); + } } -function chunkPartId(chunk: UIMessageChunk): string | null { - const value = chunkRecord(chunk); - const id = value["id"] ?? value["toolCallId"] ?? value["sourceId"] ?? null; - return typeof id === "string" ? id : null; +function tableColumns(ctx: DurableObjectState, table: "message_part" | "run" | "run_state") { + return ctx.storage.sql.exec(`PRAGMA table_info(${table})`).toArray(); } -function isTransientChunk(chunk: UIMessageChunk): boolean { - return chunkRecord(chunk)["transient"] === true; +function hasExactColumns(rows: unknown[], expected: readonly ExpectedColumn[]): boolean { + return ( + rows.length === expected.length && + expected.every((value, index) => { + const row = rows[index]; + return ( + isRecord(row) && + row["cid"] === index && + row["name"] === value.name && + row["type"] === value.type && + row["notnull"] === Number(value.isNotNull) && + row["pk"] === Number(value.isPrimaryKey) && + row["dflt_value"] === value.defaultValue + ); + }) + ); } -function chunkRecord(chunk: UIMessageChunk): Record { - return Object(chunk) as Record; +function hasColumn(rows: unknown[], name: string): boolean { + return rows.some((row) => isRecord(row) && row["name"] === name); +} + +function column( + name: string, + type: string, + isNotNull = false, + isPrimaryKey = false, + defaultValue: string | null = null, +): ExpectedColumn { + return { defaultValue, isNotNull, isPrimaryKey, name, type }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; } function readMessageStats(ctx: DurableObjectState): { lastSeq: number; messageCount: number } { @@ -317,20 +579,59 @@ function integerColumn(row: Record, key: string): number | null function runStatusColumn( row: Record, key: string, -): "canceled" | "completed" | "failed" | "paused" | "running" | null { +): "canceled" | "completed" | "failed" | "running" | null { const value = stringColumn(row, key); - if ( - value === "canceled" || - value === "completed" || - value === "failed" || - value === "paused" || - value === "running" - ) { + if (value === "canceled" || value === "completed" || value === "failed" || value === "running") { return value; } return null; } +function hasCurrentRunTableSql(ctx: DurableObjectState): boolean { + const row = firstRecord( + ctx.storage.sql + .exec("SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'run'") + .toArray(), + ); + const sql = row?.["sql"]; + return typeof sql === "string" && compactSql(sql) === compactSql(RUN_TABLE_SQL); +} + +function compactSql(sql: string): string { + return sql.replace(/\s+/g, " ").trim().toLowerCase(); +} + +function copyRunRows( + ctx: DurableObjectState, + source: "run" | "run_reconcile_source", + target: "run" | "run_next", +): void { + ctx.storage.sql.exec( + `INSERT INTO ${target} (id, status, model_id, created_at, started_at, completed_at) + SELECT id, + CASE WHEN status IN (${RUN_STATUS_VALUES_SQL}) THEN status ELSE 'canceled' END, + model_id, + created_at, + started_at, + CASE + WHEN status IN (${RUN_STATUS_VALUES_SQL}) THEN completed_at + ELSE COALESCE(completed_at, started_at, created_at) + END + FROM ${source}`, + ); +} + +function normalizeRunStateStatus(ctx: DurableObjectState): void { + const status = getRunStateValue(ctx, "status"); + if (!status || ["running", "completed", "failed", "canceled"].includes(status)) { + return; + } + setRunStateValue(ctx, "status", "canceled"); + if (!getRunStateValue(ctx, "completed_at")) { + setRunStateValue(ctx, "completed_at", String(Date.now())); + } +} + function parseLogicalModelId(value: string | null | undefined): LogicalModelId | undefined { const parsed = LogicalModelIdSchema.safeParse(value); return parsed.success ? parsed.data : undefined; diff --git a/apps/agent-worker/src/durable-objects/agent-run-stream-driver.ts b/apps/agent-worker/src/durable-objects/agent-run-stream-driver.ts index 2c02f4c0..355c2791 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-stream-driver.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-stream-driver.ts @@ -1,9 +1,17 @@ -import type { ApprovalBroker } from "@cheatcode/agent-core"; +import type { AgentChunkType } from "@cheatcode/agent-core"; import { type createLogger, emitUserEvent } from "@cheatcode/observability"; -import type { ArtifactRuntime, CodeRuntimeContext } from "@cheatcode/sandbox-contracts"; -import { FALLBACK_MODEL_ID, type LogicalModelId } from "@cheatcode/types"; +import type { + ArtifactRuntime, + CodeRuntimeContext, + WorkspaceResolver, +} from "@cheatcode/sandbox-contracts"; +import { + FALLBACK_MODEL_ID, + type LogicalModelId, + type ModelFallbackData, + ModelFallbackDataSchema, +} from "@cheatcode/types"; import type { ModelMessage, UIMessageChunk } from "ai"; -import { appendModelFallbackTransition, offerModelFallback } from "./agent-run-approvals"; import { loadThreadModelContext } from "./agent-run-conversation"; import type { AgentRunEnv } from "./agent-run-env"; import { runMastraStream } from "./agent-run-mastra-stream"; @@ -21,17 +29,16 @@ type ProjectSandboxStub = CodeRuntimeContext["sandbox"]; /** Thin DO closures the stream driver needs. */ export interface StreamDriverDeps { append: (chunk: UIMessageChunk) => Promise; - appendCheckedMastraChunk: (input: StartRunInput, chunk: unknown) => Promise; + appendCheckedMastraChunk: (input: StartRunInput, chunk: AgentChunkType) => Promise; createArtifactRuntime: (input: StartRunInput) => ArtifactRuntime; - createBroker: () => ApprovalBroker; env: AgentRunEnv; - hasPendingDecision: () => boolean; persistLogicalModel: ( input: StartRunInput, logicalModelId: LogicalModelId, logger: ReturnType, ) => Promise; setRunStage: (stage: string) => void; + waitForBrowserTakeover: (signal: AbortSignal) => Promise; } interface StreamRunParams { @@ -40,6 +47,7 @@ interface StreamRunParams { input: StartRunInput; logger: ReturnType; sandbox: ProjectSandboxStub; + workspaceResolver: WorkspaceResolver; } type PreparedStreamRunParams = StreamRunParams & { modelMessages: ModelMessage[] }; @@ -50,8 +58,7 @@ interface StreamAttemptState { /** * Runs the Mastra stream against the primary BYOK provider, falling back to the - * OpenAI default through the interactive consent flow on a provider - * rate-limit or provider-balance failure. + * OpenAI default on a provider rate-limit or provider-balance failure. */ export async function streamMastraRunWithFallback( deps: StreamDriverDeps, @@ -106,15 +113,6 @@ async function handleFallback( throw error; } const fallbackReason = classifyFallbackReason(error); - const decision = await offerModelFallback({ - broker: deps.createBroker(), - fromModel: primaryCredential.logicalModelId, - reason: fallbackReason, - toModel: fallbackCredential.logicalModelId, - }); - if (decision.decision === "deny") { - throw error; - } params.logger.warn("llm_provider_fallback_started", { fromLogicalModelId: primaryCredential.logicalModelId, fromTransportProvider: primaryCredential.transportProvider, @@ -158,15 +156,32 @@ async function streamMastraRun( attemptState.hasVisibleOutput = attemptState.hasVisibleOutput || appendedCount > 0; return appendedCount; }, - approvalBroker: deps.createBroker(), artifactRuntime: deps.createArtifactRuntime(params.input), credential, env: deps.env, - hasPendingDecision: deps.hasPendingDecision, input: params.input, logger: params.logger, modelMessages: params.modelMessages, sandbox: params.sandbox, setRunStage: deps.setRunStage, + waitForBrowserTakeover: deps.waitForBrowserTakeover, + workspaceResolver: params.workspaceResolver, + }); +} + +async function appendModelFallbackTransition(params: { + append: (chunk: UIMessageChunk) => Promise; + fromModel: LogicalModelId; + reason: ModelFallbackData["reason"]; + toModel: LogicalModelId; +}): Promise { + await params.append({ + data: ModelFallbackDataSchema.parse({ + v: 1, + fromModel: params.fromModel, + reason: params.reason, + toModel: params.toModel, + }), + type: "data-model-fallback", }); } diff --git a/apps/agent-worker/src/durable-objects/agent-run-transcript-chunks.ts b/apps/agent-worker/src/durable-objects/agent-run-transcript-chunks.ts new file mode 100644 index 00000000..9da93464 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-transcript-chunks.ts @@ -0,0 +1,100 @@ +import { fragmentMessagePart, parseMessagePart, type UIMessagePart } from "@cheatcode/types"; +import type { UIMessageChunk } from "ai"; + +export const AGENT_RUN_MESSAGE_PART_MAX_BYTES = 64 * 1024; +const TEXT_CHUNK_MAX_CHARACTERS = 8 * 1024; + +/** Normalizes one event into lossless SQLite/stream units with a fixed per-event byte bound. */ +export function* boundedAgentRunChunks( + chunk: UIMessageChunk, + fragmentId: string, +): Generator { + if (serializedChunkBytes(chunk) <= AGENT_RUN_MESSAGE_PART_MAX_BYTES) { + yield chunk; + return; + } + if (chunk.type === "text-delta") { + yield* boundedTextDeltaChunks(chunk); + return; + } + const part = transcriptPartFromChunk(chunk); + if (!part) { + throw new RangeError(`UI stream event ${chunk.type} exceeds the per-event byte bound.`); + } + for (const fragment of fragmentMessagePart(part, fragmentId)) { + const fragmentChunk = fragment as UIMessageChunk; + assertChunkBound(fragmentChunk); + yield fragmentChunk; + } +} + +export function transcriptPartFromChunk(chunk: UIMessageChunk): UIMessagePart | null { + if (isPersistableDataChunk(chunk)) { + return uiMessagePartFromChunk(chunk); + } + return null; +} + +export function serializedChunkBytes(chunk: UIMessageChunk): number { + return new TextEncoder().encode(JSON.stringify(chunk)).byteLength; +} + +function* boundedTextDeltaChunks( + chunk: Extract, +): Generator { + let offset = 0; + while (offset < chunk.delta.length) { + const end = safeSliceEnd(chunk.delta, offset, TEXT_CHUNK_MAX_CHARACTERS); + const bounded = { ...chunk, delta: chunk.delta.slice(offset, end) }; + assertChunkBound(bounded); + yield bounded; + offset = end; + } +} + +function assertChunkBound(chunk: UIMessageChunk): void { + if (serializedChunkBytes(chunk) > AGENT_RUN_MESSAGE_PART_MAX_BYTES) { + throw new RangeError(`Normalized UI stream event ${chunk.type} exceeds the byte bound.`); + } +} + +function isPersistableDataChunk( + chunk: UIMessageChunk, +): chunk is UIMessageChunk & { type: `data-${string}` } { + const value = chunkRecord(chunk); + return ( + typeof value["type"] === "string" && + value["type"].startsWith("data-") && + value["type"] !== "data-seq" && + value["transient"] !== true + ); +} + +function uiMessagePartFromChunk(chunk: UIMessageChunk): UIMessagePart { + const value = chunkRecord(chunk); + return validatedMessagePart({ + type: chunk.type, + data: value["data"], + ...(typeof value["id"] === "string" ? { id: value["id"] } : {}), + }); +} + +function validatedMessagePart(value: unknown): UIMessagePart { + return parseMessagePart(value); +} + +function chunkRecord(chunk: UIMessageChunk): Record { + return Object(chunk) as Record; +} + +function safeSliceEnd(value: string, offset: number, maxCharacters: number): number { + const candidate = Math.min(value.length, offset + maxCharacters); + if (candidate === value.length) { + return candidate; + } + const previous = value.charCodeAt(candidate - 1); + const next = value.charCodeAt(candidate); + return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff + ? candidate - 1 + : candidate; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-user-skills.ts b/apps/agent-worker/src/durable-objects/agent-run-user-skills.ts index 2c1b7f0f..1ab9b09c 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-user-skills.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-user-skills.ts @@ -1,85 +1,150 @@ -import type { UserSkillLoader, UserSkillRuntime, UserSkillStore } from "@cheatcode/agent-core"; +import type { UserSkillDefinition, UserSkillLoader, UserSkillRuntime } from "@cheatcode/agent-core"; import { createDb, getUserSkillByName, - listUserSkillSummaries, + listUserSkillRecords, + type UserSkillRecord, upsertUserSkill, withUserContext, } from "@cheatcode/db"; +import type { SandboxLike } from "@cheatcode/sandbox-contracts"; import { UserId } from "@cheatcode/types"; +import { resolveUserSkillMirror, userSkillSlug, writeUserSkillMirror } from "../user-skill-files"; +import { readUserSkillPackage, writeUserSkillPackageMirror } from "../user-skill-packages"; import type { AgentRunEnv } from "./agent-run-env"; export interface ResolvedUserSkillContext { userSkills: UserSkillRuntime[]; userSkillLoader: UserSkillLoader; - userSkillStore: UserSkillStore; } -const DEFAULT_SKILL_CATEGORY = "Builder & Apps"; - /** * Loads the user's custom skills for the run (so the agent can `skill_invoke` them) - * and builds the `skill_create` persistence store the Skill Creator path uses. Each - * is a request-scoped capability injected into the Mastra request context. + * and builds the request-scoped loader used by `skill_invoke`. */ export async function resolveUserSkillContext( env: AgentRunEnv, userIdRaw: string, + sandbox: SandboxLike, ): Promise { const userId = UserId(userIdRaw); - const userSkills = await readUserSkills(env, userId); + const skillRecords = await readUserSkills(env, userId); + await projectUserSkillPackages(env, userId, sandbox, skillRecords); + const userSkills = skillRecords.map(runtimeSkillSummary); const userSkillLoader: UserSkillLoader = { - load: async (name) => { - const { db, close } = createDb(env.HYPERDRIVE); - try { - const skill = await withUserContext(db, userId, (tx) => - getUserSkillByName(tx, userId, name), - ); - return skill - ? { - body: skill.body, - category: skill.category, - description: skill.description, - name: skill.name, - } - : null; - } finally { - await close(); - } - }, - }; - const userSkillStore: UserSkillStore = { - save: async (skill) => { - const { db, close } = createDb(env.HYPERDRIVE); - try { - await withUserContext(db, userId, (tx) => - upsertUserSkill(tx, { - body: skill.body, - category: skill.category ?? DEFAULT_SKILL_CATEGORY, - description: skill.description, - name: skill.name, - tags: skill.tags ?? [], - userId, - }), - ); - } finally { - await close(); - } - }, + load: async (name) => loadUserSkill(env, userId, sandbox, name), }; - return { userSkills, userSkillLoader, userSkillStore }; + return { userSkills, userSkillLoader }; +} + +async function loadUserSkill( + env: AgentRunEnv, + userId: UserId, + sandbox: SandboxLike, + name: string, +): Promise { + const skill = await readUserSkill(env, userId, name); + if (!skill) return null; + const resolution = await resolveUserSkillMirror(sandbox, skill); + const resolved = + resolution.kind === "promote" + ? await promoteUserSkillMirror(env, userId, sandbox, skill, resolution.mirror) + : skill; + return runtimeSkill(resolved); } -async function readUserSkills(env: AgentRunEnv, userId: UserId): Promise { - const { db, close } = createDb(env.HYPERDRIVE); +async function readUserSkills(env: AgentRunEnv, userId: UserId): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { - const rows = await withUserContext(db, userId, (tx) => listUserSkillSummaries(tx, userId)); - return rows.map((row) => ({ - category: row.category, - description: row.description, - name: row.name, - })); + return await withUserContext(db, userId, (tx) => listUserSkillRecords(tx, userId)); } finally { await close(); } } + +async function projectUserSkillPackages( + env: AgentRunEnv, + userId: UserId, + sandbox: SandboxLike, + skills: UserSkillRecord[], +): Promise { + for (const skill of skills) { + const packageValue = await readUserSkillPackage(env.R2_OUTPUTS, userId, skill.id); + if (packageValue) { + await writeUserSkillPackageMirror(sandbox, skill, packageValue); + } else { + await writeUserSkillMirror(sandbox, skill); + } + } +} + +async function readUserSkill( + env: AgentRunEnv, + userId: UserId, + name: string, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + return await withUserContext(db, userId, (tx) => getUserSkillByName(tx, userId, name)); + } finally { + await close(); + } +} + +async function promoteUserSkillMirror( + env: AgentRunEnv, + userId: UserId, + sandbox: SandboxLike, + skill: UserSkillRecord, + mirror: { + body: string; + category: string; + description: string; + tags: string[]; + }, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const updated = await withUserContext(db, userId, (tx) => + upsertUserSkill(tx, { + body: mirror.body, + category: mirror.category, + description: mirror.description, + name: skill.name, + tags: mirror.tags, + userId, + }), + ); + await writeUserSkillMirror(sandbox, updated); + return updated; + } finally { + await close(); + } +} + +function runtimeSkill(skill: UserSkillRecord): UserSkillDefinition { + return { + body: skill.body, + category: skill.category, + description: skill.description, + name: skill.name, + rootPath: `/workspace/.cheatcode/skills/${userSkillSlug(skill.name)}`, + }; +} + +function runtimeSkillSummary(skill: UserSkillRecord): UserSkillRuntime { + return { + category: skill.category, + description: skill.description, + name: skill.name, + }; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-utils.ts b/apps/agent-worker/src/durable-objects/agent-run-utils.ts index d91e63de..6b59b85b 100644 --- a/apps/agent-worker/src/durable-objects/agent-run-utils.ts +++ b/apps/agent-worker/src/durable-objects/agent-run-utils.ts @@ -1,10 +1,11 @@ +import type { AgentChunkType } from "@cheatcode/agent-core"; import { APIError } from "@cheatcode/observability"; import { resolveWithAbortTimeout } from "./abort-timeout"; -export type MastraChunkRead = IteratorResult | "timeout"; +export type MastraChunkRead = IteratorResult | "timeout"; export function missingInternalUserResponse( - surface: "approval" | "cancel" | "delete-all" | "status" | "streams", + surface: "browser takeover" | "cancel" | "delete-all" | "status" | "streams", ): Response { return new APIError(401, "auth_token_missing", "Missing internal user header", { hint: `Call AgentRun ${surface} through agent-worker.`, @@ -13,17 +14,15 @@ export function missingInternalUserResponse( } export async function readMastraChunk( - iterator: AsyncIterator, + iterator: AsyncIterator, timeoutMs?: number, abortController?: AbortController, - extendWhile?: () => boolean, ): Promise { if (!timeoutMs) { return iterator.next(); } return resolveWithAbortTimeout({ abortController: abortController ?? new AbortController(), - ...(extendWhile ? { extendWhile } : {}), operation: iterator.next(), timeoutMs, }); diff --git a/apps/agent-worker/src/durable-objects/agent-run-workflow-controller.ts b/apps/agent-worker/src/durable-objects/agent-run-workflow-controller.ts new file mode 100644 index 00000000..994e8654 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-workflow-controller.ts @@ -0,0 +1,621 @@ +import { APIError, createLogger } from "@cheatcode/observability"; +import type { AgentRunEnv } from "./agent-run-env"; +import type { StartRunInput } from "./agent-run-schemas"; +import { + deleteRunStateValues, + getRunStateTimestamp, + getRunStateValue, + isAgentRunDeleted, + setRunStateValue, +} from "./agent-run-storage"; +import { admitAgentRunWorkflow } from "./agent-run-workflow"; +import { agentRunExecutionEpochResponse } from "./agent-run-workflow-epoch"; +import { + AGENT_RUN_EXECUTION_EPOCH_MS, + AGENT_RUN_EXECUTION_LEASE_GRACE_MS, + AGENT_RUN_WORKFLOW_ADMITTED_KEY, + AGENT_RUN_WORKFLOW_EXECUTION_STARTED_KEY, + AGENT_RUN_WORKFLOW_GENERATION_KEY, + AGENT_RUN_WORKFLOW_ID_KEY, + AGENT_RUN_WORKFLOW_INPUT_HASH_KEY, + AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY, + AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY, + AGENT_RUN_WORKFLOW_RETRY_AT_KEY, + AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY, + AGENT_RUN_WORKFLOW_RETRY_BASE_MS, + AGENT_RUN_WORKFLOW_RETRY_MAX_MS, + type AgentRunWorkflowCallbackInput, + type AgentRunWorkflowFailureInput, + type AgentRunWorkflowPayload, + AgentRunWorkflowPayloadSchema, + type AgentRunWorkflowRolloverResult, + agentRunWorkflowInputHash, + agentRunWorkflowInstanceId, +} from "./agent-run-workflow-protocol"; +import { hasActiveRun } from "./run-state"; + +interface AgentRunWorkflowControllerDeps { + armAlarm: () => Promise; + ctx: DurableObjectState; + env: AgentRunEnv; + finalizeOwnershipFailure: (message: string) => Promise; + getStatus: () => string | undefined; + run: (input: StartRunInput, abortController: AbortController) => Promise; +} + +/** Durable Workflow admission and retry-safe execution ownership for one run-keyed DO. */ +export class AgentRunWorkflowController { + private activeAbortController: AbortController | undefined; + private activeRunPromise: Promise | undefined; + + public constructor(private readonly deps: AgentRunWorkflowControllerDeps) {} + + public createAdmission(input: StartRunInput): Promise { + return this.payloadFor(input); + } + + /** Ensures a crash immediately after the run claim cannot strand pending admission. */ + public armAdmissionRecovery(): Promise { + return this.deps.ctx.storage.setAlarm(Date.now() + AGENT_RUN_WORKFLOW_RETRY_BASE_MS); + } + + /** Atomically couples the DO run claim to its pending durable Workflow owner. */ + public claimAdmission(payload: AgentRunWorkflowPayload, claimRun: () => void): void { + const parsed = AgentRunWorkflowPayloadSchema.parse(payload); + const workflowId = agentRunWorkflowInstanceId(parsed.input.runId, parsed.generation); + this.deps.ctx.storage.transactionSync(() => { + claimRun(); + this.writePendingAdmission(parsed, workflowId); + }); + } + + public async admit(payload: AgentRunWorkflowPayload): Promise { + const parsed = AgentRunWorkflowPayloadSchema.parse(payload); + const workflowId = agentRunWorkflowInstanceId(parsed.input.runId, parsed.generation); + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true") { + this.assertExpectedIdentity(parsed.inputHash, workflowId, parsed.generation); + await this.ensureAdmissionLease(); + return; + } + this.assertPendingAdmission(parsed, workflowId); + if (getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_AT_KEY) === null) { + await this.deps.armAlarm(); + return; + } + await this.deps.armAlarm(); + try { + await this.admitPendingPayload(parsed, workflowId); + } catch (error) { + await this.recordAdmissionFailure(); + throw error; + } + } + + public async reconcileAdmission(): Promise { + if (!getRunStateValue(this.deps.ctx, "run_id")) { + return; + } + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true") { + await this.ensureAdmissionLease(); + return; + } + const payload = this.pendingPayload(); + const expectedId = agentRunWorkflowInstanceId(payload.input.runId, payload.generation); + if (getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_AT_KEY) === null) { + await this.deps.armAlarm(); + return; + } + try { + await this.admitPendingPayload(payload, expectedId); + } catch (error) { + await this.recordAdmissionFailure(); + throw error; + } + } + + /** Alarm-owned retry for a start whose deterministic Workflow admission was ambiguous. */ + public async recoverPendingAdmission(): Promise { + if ( + !hasActiveRun(this.deps.getStatus()) || + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true" + ) { + return false; + } + const retryAt = getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_AT_KEY); + if (retryAt === null || retryAt > Date.now()) { + return false; + } + try { + const payload = this.pendingPayload(); + await this.admitPendingPayload( + payload, + agentRunWorkflowInstanceId(payload.input.runId, payload.generation), + ); + return true; + } catch (error) { + const runId = getRunStateValue(this.deps.ctx, "run_id"); + createLogger(runId ? { runId } : {}).warn("agent_run_workflow_admission_retry_failed", { + error, + }); + return this.rearmAdmissionFailure(); + } + } + + public async executeEpoch(input: AgentRunWorkflowCallbackInput): Promise { + if (isAgentRunDeleted(this.deps.ctx)) { + return Response.json({ outcome: "deleted", status: "deleted" }); + } + const status = this.deps.getStatus(); + if (!hasActiveRun(status)) { + return Response.json({ outcome: "terminal", status: status ?? "unknown" }); + } + const identity = await this.promoteExecutionOwner(input); + if (identity.outcome !== "current") { + return Response.json({ outcome: identity.outcome, status: identity.status }); + } + if (!this.activeRunPromise) { + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_EXECUTION_STARTED_KEY)) { + return this.failInterruptedExecution(); + } + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_EXECUTION_STARTED_KEY, String(Date.now())); + this.startExecution(input.input); + } + const runPromise = this.activeRunPromise; + if (!runPromise) { + throw new Error("AgentRun execution promise disappeared after admission."); + } + // The Workflow response is the durable owner. waitUntil only bridges the short + // checkpoint gap before the next Workflow epoch attaches to this same promise. + this.deps.ctx.waitUntil(runPromise); + await this.renewLease(); + return agentRunExecutionEpochResponse({ + getStatus: this.deps.getStatus, + isDeleted: () => isAgentRunDeleted(this.deps.ctx), + runPromise, + }); + } + + public async failWorkflow(input: AgentRunWorkflowFailureInput): Promise { + const status = this.deps.getStatus(); + if (isAgentRunDeleted(this.deps.ctx) || !hasActiveRun(status)) { + return Response.json({ ok: true }); + } + if (this.callbackGeneration(input) === "stale") { + return Response.json({ ok: true }); + } + this.abort(new Error("AgentRun Workflow ownership failed")); + await this.join(); + await this.deps.finalizeOwnershipFailure(input.message); + return Response.json({ ok: true }); + } + + public async reserveSuccessor(input: AgentRunWorkflowCallbackInput): Promise { + if (isAgentRunDeleted(this.deps.ctx)) { + return Response.json({ outcome: "deleted", status: "deleted" }); + } + const status = this.deps.getStatus(); + if (!hasActiveRun(status)) { + return Response.json({ outcome: "terminal", status: status ?? "unknown" }); + } + await this.assertPayloadIdentity(input); + const result = this.reserveSuccessorState(input); + if (result.outcome === "reserved") { + const runPromise = this.activeRunPromise; + if (runPromise !== undefined) { + this.deps.ctx.waitUntil(runPromise); + } + await this.deps.armAlarm(); + } + return Response.json(result); + } + + public async handleExpiredLease(): Promise { + const expiresAt = getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY); + if (!hasActiveRun(this.deps.getStatus()) || expiresAt === null || expiresAt > Date.now()) { + return false; + } + this.abort(new Error("AgentRun Workflow execution lease expired")); + await this.join(); + await this.deps.finalizeOwnershipFailure( + "Durable execution ownership was interrupted. Send the prompt again to retry.", + ); + return true; + } + + public abort(reason: Error): void { + const runPromise = this.activeRunPromise; + this.activeAbortController?.abort(reason); + if (runPromise !== undefined) { + this.deps.ctx.waitUntil(runPromise.catch(() => undefined)); + } + } + + public async join(): Promise { + await this.activeRunPromise?.catch(() => undefined); + } + + private async payloadFor(input: StartRunInput): Promise { + const storedRunId = getRunStateValue(this.deps.ctx, "run_id"); + const generation = storedRunId === input.runId ? this.currentGeneration() : 0; + return AgentRunWorkflowPayloadSchema.parse({ + generation, + input, + inputHash: await agentRunWorkflowInputHash(input), + }); + } + + private async promoteExecutionOwner(input: AgentRunWorkflowCallbackInput): Promise<{ + outcome: "continued" | "current" | "deleted" | "terminal"; + status: string; + }> { + await this.assertPayloadIdentity(input); + if (isAgentRunDeleted(this.deps.ctx)) { + return { outcome: "deleted", status: "deleted" }; + } + const status = this.deps.getStatus(); + if (!status || !hasActiveRun(status)) { + return { outcome: "terminal", status: status ?? "unknown" }; + } + if (this.callbackGeneration(input) === "stale") { + return { outcome: "continued", status }; + } + this.promoteAdmission(input.inputHash, input.workflowInstanceId, input.generation); + return { outcome: "current", status }; + } + + private async assertPayloadIdentity(input: AgentRunWorkflowPayload): Promise { + if ((await agentRunWorkflowInputHash(input.input)) !== input.inputHash) { + throw ownershipConflict("AgentRun Workflow input hash mismatch."); + } + if (input.input.runId !== getRunStateValue(this.deps.ctx, "run_id")) { + throw ownershipConflict("AgentRun Workflow run identity mismatch."); + } + } + + private assertExpectedIdentity( + inputHash: string, + workflowInstanceId: string, + generation: number, + ): void { + if ( + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) !== "true" || + !this.hasStoredIdentity(inputHash, workflowInstanceId, generation) + ) { + throw ownershipConflict("AgentRun Workflow ownership identity mismatch."); + } + } + + private assertStoredIdentity( + inputHash: string, + workflowInstanceId: string, + generation: number, + ): void { + if (!this.hasStoredIdentity(inputHash, workflowInstanceId, generation)) { + throw ownershipConflict("AgentRun Workflow ownership identity mismatch."); + } + } + + private promoteAdmission( + inputHash: string, + workflowInstanceId: string, + generation: number, + ): void { + this.deps.ctx.storage.transactionSync(() => { + this.assertStoredIdentity(inputHash, workflowInstanceId, generation); + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) !== "true") { + const pending = AgentRunWorkflowPayloadSchema.safeParse( + safeJson(getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY)), + ); + if ( + !pending.success || + pending.data.generation !== generation || + pending.data.inputHash !== inputHash + ) { + throw ownershipConflict("AgentRun Workflow pending admission state is invalid."); + } + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY, "true"); + deleteRunStateValues(this.deps.ctx, [ + AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY, + AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY, + AGENT_RUN_WORKFLOW_RETRY_AT_KEY, + ]); + } + }); + } + + private hasStoredIdentity( + inputHash: string, + workflowInstanceId: string, + generation: number, + ): boolean { + return ( + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ID_KEY) === workflowInstanceId && + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_INPUT_HASH_KEY) === inputHash && + this.currentGeneration() === generation + ); + } + + private writePendingAdmission( + payload: AgentRunWorkflowPayload, + workflowInstanceId: string, + ): void { + const now = Date.now(); + deleteRunStateValues(this.deps.ctx, [AGENT_RUN_WORKFLOW_ADMITTED_KEY]); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_GENERATION_KEY, String(payload.generation)); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ID_KEY, workflowInstanceId); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_INPUT_HASH_KEY, payload.inputHash); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY, JSON.stringify(payload)); + this.resetAdmissionTimers(now); + } + + private assertPendingAdmission( + payload: AgentRunWorkflowPayload, + workflowInstanceId: string, + ): void { + this.assertStoredIdentity(payload.inputHash, workflowInstanceId, payload.generation); + const pending = this.pendingPayload(); + if (pending.generation !== payload.generation || pending.inputHash !== payload.inputHash) { + throw ownershipConflict("AgentRun Workflow pending admission identity mismatch."); + } + } + + private resetAdmissionTimers(now: number): void { + setRunStateValue( + this.deps.ctx, + AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY, + String(now + AGENT_RUN_EXECUTION_EPOCH_MS + AGENT_RUN_EXECUTION_LEASE_GRACE_MS), + ); + setRunStateValue( + this.deps.ctx, + AGENT_RUN_WORKFLOW_RETRY_AT_KEY, + String(now + AGENT_RUN_WORKFLOW_RETRY_BASE_MS), + ); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY, "0"); + } + + private callbackGeneration(input: { + generation: number; + inputHash: string; + workflowInstanceId: string; + }): "current" | "stale" { + const runId = getRunStateValue(this.deps.ctx, "run_id"); + if ( + !runId || + input.workflowInstanceId !== agentRunWorkflowInstanceId(runId, input.generation) || + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_INPUT_HASH_KEY) !== input.inputHash + ) { + throw ownershipConflict("AgentRun Workflow callback identity mismatch."); + } + const currentGeneration = this.currentGeneration(); + if ( + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ID_KEY) !== + agentRunWorkflowInstanceId(runId, currentGeneration) + ) { + throw ownershipConflict("AgentRun Workflow current generation identity is inconsistent."); + } + if (currentGeneration > input.generation) { + return "stale"; + } + if (currentGeneration < input.generation) { + throw ownershipConflict("AgentRun Workflow callback is from an unreserved generation."); + } + this.assertStoredIdentity(input.inputHash, input.workflowInstanceId, input.generation); + return "current"; + } + + private reserveSuccessorState( + input: AgentRunWorkflowCallbackInput, + ): AgentRunWorkflowRolloverResult { + return this.deps.ctx.storage.transactionSync(() => { + if (isAgentRunDeleted(this.deps.ctx)) { + return { outcome: "deleted", status: "deleted" }; + } + const currentStatus = this.deps.getStatus(); + if (!currentStatus || !hasActiveRun(currentStatus)) { + return { outcome: "terminal", status: currentStatus ?? "unknown" }; + } + if (this.callbackGeneration(input) === "stale") { + return this.replayedReservation(input, currentStatus); + } + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) !== "true") { + throw ownershipConflict("AgentRun Workflow rollover owner is not promoted."); + } + const payload = this.successorPayload(input); + const workflowInstanceId = agentRunWorkflowInstanceId( + payload.input.runId, + payload.generation, + ); + this.writePendingAdmission(payload, workflowInstanceId); + return { outcome: "reserved", payload, status: currentStatus, workflowInstanceId }; + }); + } + + private replayedReservation( + input: AgentRunWorkflowCallbackInput, + status: string, + ): AgentRunWorkflowRolloverResult { + const payload = this.successorPayload(input); + if (this.currentGeneration() !== payload.generation) { + return { outcome: "continued", status }; + } + const workflowInstanceId = agentRunWorkflowInstanceId(payload.input.runId, payload.generation); + this.assertStoredIdentity(payload.inputHash, workflowInstanceId, payload.generation); + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true") { + return { outcome: "continued", status }; + } + const pending = this.pendingPayload(); + if (pending.generation !== payload.generation || pending.inputHash !== payload.inputHash) { + throw ownershipConflict("AgentRun Workflow successor reservation is inconsistent."); + } + return { outcome: "reserved", payload, status, workflowInstanceId }; + } + + private successorPayload(input: AgentRunWorkflowCallbackInput): AgentRunWorkflowPayload { + const generation = input.generation + 1; + if (!Number.isSafeInteger(generation)) { + throw ownershipUnavailable("AgentRun Workflow generation cannot be represented safely."); + } + return AgentRunWorkflowPayloadSchema.parse({ + generation, + input: input.input, + inputHash: input.inputHash, + }); + } + + private currentGeneration(): number { + const raw = getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_GENERATION_KEY); + const generation = raw === undefined ? Number.NaN : Number(raw); + if (!Number.isSafeInteger(generation) || generation < 0) { + throw ownershipConflict("AgentRun Workflow generation state is invalid."); + } + return generation; + } + + private pendingPayload(): AgentRunWorkflowPayload { + const raw = getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY); + const parsed = AgentRunWorkflowPayloadSchema.safeParse(safeJson(raw)); + if (!parsed.success) { + throw ownershipUnavailable("AgentRun Workflow admission state is incomplete."); + } + return parsed.data; + } + + private async admitPendingPayload( + payload: AgentRunWorkflowPayload, + expectedId: string, + ): Promise { + this.assertStoredIdentity(payload.inputHash, expectedId, payload.generation); + const admittedId = await admitAgentRunWorkflow(this.deps.env, payload); + if (admittedId !== expectedId) { + throw ownershipConflict("AgentRun Workflow admitted an unexpected instance id."); + } + if ( + !isAgentRunDeleted(this.deps.ctx) && + hasActiveRun(this.deps.getStatus()) && + this.hasStoredIdentity(payload.inputHash, admittedId, payload.generation) + ) { + deleteRunStateValues(this.deps.ctx, [ + AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY, + AGENT_RUN_WORKFLOW_RETRY_AT_KEY, + ]); + await this.renewLease(); + } + } + + private async recordAdmissionFailure(): Promise { + if ( + isAgentRunDeleted(this.deps.ctx) || + !hasActiveRun(this.deps.getStatus()) || + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true" + ) { + return; + } + this.scheduleAdmissionRetry(); + await this.deps.armAlarm(); + } + + private async rearmAdmissionFailure(): Promise { + if (isAgentRunDeleted(this.deps.ctx) || !hasActiveRun(this.deps.getStatus())) { + return true; + } + if (getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_ADMITTED_KEY) === "true") { + await this.deps.armAlarm(); + return true; + } + const expiresAt = getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY); + if (expiresAt === null) { + this.resetAdmissionTimers(Date.now()); + } else if (expiresAt <= Date.now()) { + return false; + } + this.scheduleAdmissionRetry(); + await this.deps.armAlarm(); + return true; + } + + private scheduleAdmissionRetry(): void { + this.deps.ctx.storage.transactionSync(() => { + const now = Date.now(); + const rawAttempt = Number( + getRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY) ?? "0", + ); + const attempt = Number.isSafeInteger(rawAttempt) && rawAttempt >= 0 ? rawAttempt : 0; + const delay = Math.min( + AGENT_RUN_WORKFLOW_RETRY_BASE_MS * 2 ** Math.min(attempt, 10), + AGENT_RUN_WORKFLOW_RETRY_MAX_MS, + ); + const expiresAt = getRunStateTimestamp( + this.deps.ctx, + AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY, + ); + const failSafeAt = + expiresAt ?? now + AGENT_RUN_EXECUTION_EPOCH_MS + AGENT_RUN_EXECUTION_LEASE_GRACE_MS; + const retryAt = Math.min(now + delay, failSafeAt); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY, String(attempt + 1)); + setRunStateValue(this.deps.ctx, AGENT_RUN_WORKFLOW_RETRY_AT_KEY, String(retryAt)); + }); + } + + private startExecution(input: StartRunInput): void { + const abortController = new AbortController(); + const runPromise = this.deps.run(input, abortController); + this.activeAbortController = abortController; + this.activeRunPromise = runPromise; + void runPromise + .finally(() => { + if (this.activeRunPromise === runPromise) { + this.activeAbortController = undefined; + this.activeRunPromise = undefined; + } + }) + .catch(() => undefined); + } + + private failInterruptedExecution(): Response { + const failure = this.deps.finalizeOwnershipFailure( + "Durable execution was interrupted before it finished. Send the prompt again to retry.", + ); + return agentRunExecutionEpochResponse({ + getStatus: this.deps.getStatus, + isDeleted: () => isAgentRunDeleted(this.deps.ctx), + runPromise: failure, + }); + } + + private async renewLease(): Promise { + setRunStateValue( + this.deps.ctx, + AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY, + String(Date.now() + AGENT_RUN_EXECUTION_EPOCH_MS + AGENT_RUN_EXECUTION_LEASE_GRACE_MS), + ); + await this.deps.armAlarm(); + } + + private async ensureAdmissionLease(): Promise { + if (getRunStateTimestamp(this.deps.ctx, AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY) === null) { + await this.renewLease(); + return; + } + await this.deps.armAlarm(); + } +} + +function safeJson(value: string | undefined): unknown { + if (!value) return null; + try { + return JSON.parse(value) as unknown; + } catch { + return null; + } +} + +function ownershipConflict(message: string): APIError { + return new APIError(409, "conflict_state_invalid", message, { retriable: false }); +} + +function ownershipUnavailable(message: string): APIError { + return new APIError(503, "unavailable_maintenance", message, { + hint: "Retry the same run admission request.", + retriable: true, + }); +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-workflow-epoch.ts b/apps/agent-worker/src/durable-objects/agent-run-workflow-epoch.ts new file mode 100644 index 00000000..2de4da3a --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-workflow-epoch.ts @@ -0,0 +1,75 @@ +import { + AGENT_RUN_EXECUTION_EPOCH_MS, + AGENT_RUN_EXECUTION_HEARTBEAT_MS, + type AgentRunWorkflowEpochResult, +} from "./agent-run-workflow-protocol"; + +interface ExecutionEpochOptions { + getStatus: () => string | undefined; + isDeleted: () => boolean; + runPromise: Promise; +} + +/** Keeps the Workflow-to-DO caller attached, then yields so Workflow can checkpoint ownership. */ +export function agentRunExecutionEpochResponse(options: ExecutionEpochOptions): Response { + const encoder = new TextEncoder(); + let cancelStream = (): void => undefined; + const stream = new ReadableStream({ + cancel: () => cancelStream(), + start(controller) { + let isClosed = false; + const heartbeat = setInterval(() => { + if (isClosed) return; + try { + controller.enqueue(encoder.encode(" ")); + } catch { + cleanup(); + } + }, AGENT_RUN_EXECUTION_HEARTBEAT_MS); + const epoch = setTimeout(() => finish("continue"), AGENT_RUN_EXECUTION_EPOCH_MS); + + const cleanup = (): void => { + if (isClosed) return; + isClosed = true; + clearInterval(heartbeat); + clearTimeout(epoch); + }; + const finish = (outcome: AgentRunWorkflowEpochResult["outcome"]): void => { + if (isClosed) return; + try { + controller.enqueue( + encoder.encode( + JSON.stringify({ + outcome, + status: options.getStatus() ?? "unknown", + } satisfies AgentRunWorkflowEpochResult), + ), + ); + cleanup(); + controller.close(); + } catch { + cleanup(); + } + }; + cancelStream = cleanup; + void options.runPromise.then( + () => finish(options.isDeleted() ? "deleted" : "terminal"), + (error: unknown) => { + if (isClosed) return; + cleanup(); + try { + controller.error(error); + } catch { + // The Workflow caller can cancel between promise settlement and delivery. + } + }, + ); + }, + }); + return new Response(stream, { + headers: { + "Cache-Control": "no-store", + "Content-Type": "application/json; charset=utf-8", + }, + }); +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-workflow-protocol.ts b/apps/agent-worker/src/durable-objects/agent-run-workflow-protocol.ts new file mode 100644 index 00000000..c84ee5f7 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-workflow-protocol.ts @@ -0,0 +1,117 @@ +import { z } from "zod"; +import { StartRunInputSchema } from "./agent-run-schemas"; + +export const AGENT_RUN_WORKFLOW_ADMITTED_KEY = "workflow_admitted"; +export const AGENT_RUN_WORKFLOW_EXECUTION_STARTED_KEY = "workflow_execution_started"; +export const AGENT_RUN_WORKFLOW_GENERATION_KEY = "workflow_generation"; +export const AGENT_RUN_WORKFLOW_ID_KEY = "workflow_id"; +export const AGENT_RUN_WORKFLOW_INPUT_HASH_KEY = "workflow_input_hash"; +export const AGENT_RUN_WORKFLOW_LEASE_EXPIRES_AT_KEY = "workflow_lease_expires_at"; +export const AGENT_RUN_WORKFLOW_PENDING_INPUT_KEY = "workflow_pending_input"; +export const AGENT_RUN_WORKFLOW_RETRY_ATTEMPT_KEY = "workflow_retry_attempt"; +export const AGENT_RUN_WORKFLOW_RETRY_AT_KEY = "workflow_retry_at"; + +export const AGENT_RUN_EXECUTION_EPOCH_MS = 4 * 60 * 1_000; +export const AGENT_RUN_EXECUTION_LEASE_GRACE_MS = 90 * 1_000; +export const AGENT_RUN_EXECUTION_HEARTBEAT_MS = 15 * 1_000; +export const AGENT_RUN_WORKFLOW_MAX_RESPONSE_BYTES = 8 * 1_024; +export const AGENT_RUN_WORKFLOW_ROLLOVER_MAX_RESPONSE_BYTES = 256 * 1_024; +export const AGENT_RUN_WORKFLOW_EXECUTION_RETRY_LIMIT = 5; +export const AGENT_RUN_WORKFLOW_FAILURE_RETRY_LIMIT = 5; +export const AGENT_RUN_WORKFLOW_ROLLOVER_RETRY_LIMIT = 5; +// This is a generation boundary, not a run cap. The successor keeps the same +// semantic run while resetting Workflow step and subrequest accounting. +export const AGENT_RUN_WORKFLOW_ROLLOVER_EPOCHS = 1_000; +export const AGENT_RUN_WORKFLOW_RETRY_BASE_MS = 5_000; +export const AGENT_RUN_WORKFLOW_RETRY_MAX_MS = 60_000; + +const CLOUDFLARE_WORKFLOW_DEFAULT_SUBREQUEST_LIMIT = 10_000; +const EXECUTION_SUBREQUESTS_PER_ATTEMPT = 1; +const FAILURE_SUBREQUESTS_PER_ATTEMPT = 1; +const ROLLOVER_RESERVATION_SUBREQUESTS_PER_ATTEMPT = 1; +// A colliding successor creation can create, get, inspect, and restart the exact +// instance before one retry attempt returns. +const SUCCESSOR_CREATION_SUBREQUESTS_PER_ATTEMPT = 4; +const AGENT_RUN_WORKFLOW_MAX_SUBREQUESTS = + AGENT_RUN_WORKFLOW_ROLLOVER_EPOCHS * + (AGENT_RUN_WORKFLOW_EXECUTION_RETRY_LIMIT + 1) * + EXECUTION_SUBREQUESTS_PER_ATTEMPT + + (AGENT_RUN_WORKFLOW_ROLLOVER_RETRY_LIMIT + 1) * ROLLOVER_RESERVATION_SUBREQUESTS_PER_ATTEMPT + + (AGENT_RUN_WORKFLOW_ROLLOVER_RETRY_LIMIT + 1) * SUCCESSOR_CREATION_SUBREQUESTS_PER_ATTEMPT + + (AGENT_RUN_WORKFLOW_FAILURE_RETRY_LIMIT + 1) * FAILURE_SUBREQUESTS_PER_ATTEMPT; + +if (AGENT_RUN_WORKFLOW_MAX_SUBREQUESTS > CLOUDFLARE_WORKFLOW_DEFAULT_SUBREQUEST_LIMIT) { + throw new Error("AgentRun Workflow rollover exceeds its Cloudflare subrequest budget."); +} + +const Sha256HexSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const WorkflowGenerationSchema = z.number().int().nonnegative().safe(); + +export const AgentRunWorkflowPayloadSchema = z + .object({ + generation: WorkflowGenerationSchema, + input: StartRunInputSchema, + inputHash: Sha256HexSchema, + }) + .strict(); + +export type AgentRunWorkflowPayload = z.infer; + +export const AgentRunWorkflowCallbackInputSchema = AgentRunWorkflowPayloadSchema.extend({ + workflowInstanceId: z.string().min(1).max(100), +}).strict(); + +export type AgentRunWorkflowCallbackInput = z.infer; + +export const AgentRunWorkflowFailureInputSchema = z + .object({ + generation: WorkflowGenerationSchema, + inputHash: Sha256HexSchema, + message: z.string().trim().min(1).max(500), + workflowInstanceId: z.string().min(1).max(100), + }) + .strict(); + +export type AgentRunWorkflowFailureInput = z.infer; + +export const AgentRunWorkflowEpochResultSchema = z + .object({ + outcome: z.enum(["continue", "continued", "deleted", "terminal"]), + status: z.string().min(1).max(32), + }) + .strict(); + +export type AgentRunWorkflowEpochResult = z.infer; + +const AgentRunWorkflowRolloverTerminalResultSchema = z + .object({ + outcome: z.enum(["continued", "deleted", "terminal"]), + status: z.string().min(1).max(32), + }) + .strict(); + +const AgentRunWorkflowRolloverReservedResultSchema = z + .object({ + outcome: z.literal("reserved"), + payload: AgentRunWorkflowPayloadSchema, + status: z.string().min(1).max(32), + workflowInstanceId: z.string().min(1).max(100), + }) + .strict(); + +export const AgentRunWorkflowRolloverResultSchema = z.union([ + AgentRunWorkflowRolloverTerminalResultSchema, + AgentRunWorkflowRolloverReservedResultSchema, +]); + +export type AgentRunWorkflowRolloverResult = z.infer; + +export function agentRunWorkflowInstanceId(runId: string, generation: number): string { + return `agent-run-${runId}-${generation}`; +} + +export async function agentRunWorkflowInputHash(input: unknown): Promise { + const bytes = new TextEncoder().encode(JSON.stringify(StartRunInputSchema.parse(input))); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-workflow.ts b/apps/agent-worker/src/durable-objects/agent-run-workflow.ts new file mode 100644 index 00000000..c80df66f --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-workflow.ts @@ -0,0 +1,255 @@ +import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers"; +import { NonRetryableError } from "cloudflare:workflows"; +import { readBoundedResponseJson, readBoundedResponseText } from "@cheatcode/observability"; +import type { AgentRun } from "./agent-run"; +import { + AGENT_RUN_WORKFLOW_EXECUTION_RETRY_LIMIT, + AGENT_RUN_WORKFLOW_FAILURE_RETRY_LIMIT, + AGENT_RUN_WORKFLOW_MAX_RESPONSE_BYTES, + AGENT_RUN_WORKFLOW_ROLLOVER_EPOCHS, + AGENT_RUN_WORKFLOW_ROLLOVER_MAX_RESPONSE_BYTES, + AGENT_RUN_WORKFLOW_ROLLOVER_RETRY_LIMIT, + type AgentRunWorkflowEpochResult, + AgentRunWorkflowEpochResultSchema, + type AgentRunWorkflowPayload, + AgentRunWorkflowPayloadSchema, + type AgentRunWorkflowRolloverResult, + AgentRunWorkflowRolloverResultSchema, + agentRunWorkflowInputHash, + agentRunWorkflowInstanceId, +} from "./agent-run-workflow-protocol"; + +const EXECUTION_EPOCH_STEP = { + // Keep every retry gap inside the DO's short waitUntil bridge. A longer + // backoff could evict the sole in-memory coroutine between ownership calls. + retries: { + limit: AGENT_RUN_WORKFLOW_EXECUTION_RETRY_LIMIT, + delay: "5 seconds", + backoff: "constant", + }, + timeout: "5 minutes", +} as const; +const FAILURE_STEP = { + retries: { + limit: AGENT_RUN_WORKFLOW_FAILURE_RETRY_LIMIT, + delay: "10 seconds", + backoff: "exponential", + }, + timeout: "2 minutes", +} as const; +const ROLLOVER_STEP = { + retries: { + limit: AGENT_RUN_WORKFLOW_ROLLOVER_RETRY_LIMIT, + delay: "5 seconds", + backoff: "constant", + }, + timeout: "2 minutes", +} as const; + +interface AgentRunWorkflowEnv extends AgentRunWorkflowBindings { + AGENT_RUN: DurableObjectNamespace; +} + +export interface AgentRunWorkflowBindings { + AGENT_RUN_WORKFLOW: Workflow; + CHEATCODE_RELEASE_GATE: "closed" | "draining" | "open"; +} + +/** Durable owner for one semantic AgentRun; execution is renewed in bounded, retry-safe epochs. */ +export class AgentRunWorkflow extends WorkflowEntrypoint< + AgentRunWorkflowEnv, + AgentRunWorkflowPayload +> { + public override async run( + event: Readonly>, + step: WorkflowStep, + ): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + throw new NonRetryableError( + "AgentRun Workflow is fenced by a closed release", + "AgentRunReleaseGateClosed", + ); + } + const payload = await parseWorkflowPayload(event.payload); + try { + for (let epoch = 0; epoch < AGENT_RUN_WORKFLOW_ROLLOVER_EPOCHS; epoch += 1) { + const result = await step.do("hold AgentRun execution epoch", EXECUTION_EPOCH_STEP, () => + executeEpoch(this.env, event.instanceId, payload), + ); + if (result.outcome !== "continue") { + return result; + } + } + return continueAgentRunGeneration(this.env, step, event.instanceId, payload); + } catch (error) { + await step.do("terminalize lost AgentRun ownership", FAILURE_STEP, () => + terminalizeOwnershipFailure(this.env, event.instanceId, payload), + ); + throw error; + } + } +} + +export async function admitAgentRunWorkflow( + env: AgentRunWorkflowBindings, + payload: AgentRunWorkflowPayload, +): Promise { + if (env.CHEATCODE_RELEASE_GATE === "closed") { + throw new Error("AgentRun Workflow admission is fenced by a closed release."); + } + const id = agentRunWorkflowInstanceId(payload.input.runId, payload.generation); + try { + const instance = await env.AGENT_RUN_WORKFLOW.create({ + id, + params: payload, + retention: { errorRetention: "30 days", successRetention: "1 day" }, + }); + return instance.id; + } catch (createError) { + return reuseAgentRunWorkflow(env.AGENT_RUN_WORKFLOW, id, createError); + } +} + +async function reuseAgentRunWorkflow( + workflow: Workflow, + id: string, + createError: unknown, +): Promise { + try { + const instance = await workflow.get(id); + const { status } = await instance.status(); + if (status === "unknown") { + throw createError; + } + if (status === "errored" || status === "terminated") { + await instance.restart(); + } + return instance.id; + } catch { + throw createError; + } +} + +async function continueAgentRunGeneration( + env: AgentRunWorkflowEnv, + step: WorkflowStep, + workflowInstanceId: string, + payload: AgentRunWorkflowPayload, +): Promise { + const reservation = await step.do("reserve AgentRun successor", ROLLOVER_STEP, () => + reserveAgentRunSuccessor(env, workflowInstanceId, payload), + ); + if (reservation.outcome !== "reserved") { + return AgentRunWorkflowEpochResultSchema.parse(reservation); + } + await step.do("create AgentRun successor", ROLLOVER_STEP, async () => { + const admittedId = await admitAgentRunWorkflow(env, reservation.payload); + if (admittedId !== reservation.workflowInstanceId) { + throw new Error("AgentRun Workflow successor admitted an unexpected instance id."); + } + return { workflowInstanceId: admittedId }; + }); + return AgentRunWorkflowEpochResultSchema.parse({ + outcome: "continued", + status: reservation.status, + }); +} + +async function reserveAgentRunSuccessor( + env: AgentRunWorkflowEnv, + workflowInstanceId: string, + payload: AgentRunWorkflowPayload, +): Promise { + const stub = env.AGENT_RUN.get(env.AGENT_RUN.idFromName(payload.input.runId)); + const response = await stub.fetch("https://agent-run.internal/workflow/rollover", { + body: JSON.stringify({ ...payload, workflowInstanceId }), + method: "POST", + }); + if (!response.ok) { + const detail = await readBoundedResponseText( + response, + AGENT_RUN_WORKFLOW_ROLLOVER_MAX_RESPONSE_BYTES, + "AgentRun Workflow rollover", + ); + const message = `AgentRun Workflow rollover returned HTTP ${response.status}: ${detail.slice(0, 300)}`; + if (response.status >= 400 && response.status < 500) { + throw new NonRetryableError(message, "AgentRunRolloverContractError"); + } + throw new Error(message); + } + return AgentRunWorkflowRolloverResultSchema.parse( + await readBoundedResponseJson( + response, + AGENT_RUN_WORKFLOW_ROLLOVER_MAX_RESPONSE_BYTES, + "AgentRun Workflow rollover", + ), + ); +} + +async function parseWorkflowPayload(value: unknown): Promise { + const parsed = AgentRunWorkflowPayloadSchema.safeParse(value); + if (!parsed.success) { + throw new NonRetryableError("Invalid AgentRun Workflow payload", "AgentRunPayloadError"); + } + if ((await agentRunWorkflowInputHash(parsed.data.input)) !== parsed.data.inputHash) { + throw new NonRetryableError( + "AgentRun Workflow payload hash mismatch", + "AgentRunPayloadHashError", + ); + } + return parsed.data; +} + +async function executeEpoch( + env: AgentRunWorkflowEnv, + workflowInstanceId: string, + payload: AgentRunWorkflowPayload, +): Promise { + const stub = env.AGENT_RUN.get(env.AGENT_RUN.idFromName(payload.input.runId)); + const response = await stub.fetch("https://agent-run.internal/workflow/execute", { + body: JSON.stringify({ ...payload, workflowInstanceId }), + method: "POST", + }); + if (!response.ok) { + const detail = await readBoundedResponseText( + response, + AGENT_RUN_WORKFLOW_MAX_RESPONSE_BYTES, + "AgentRun execution epoch", + ); + const message = `AgentRun execution epoch returned HTTP ${response.status}: ${detail.slice(0, 300)}`; + if (response.status >= 400 && response.status < 500) { + throw new NonRetryableError(message, "AgentRunExecutionContractError"); + } + throw new Error(message); + } + return AgentRunWorkflowEpochResultSchema.parse( + await readBoundedResponseJson( + response, + AGENT_RUN_WORKFLOW_MAX_RESPONSE_BYTES, + "AgentRun execution epoch", + ), + ); +} + +async function terminalizeOwnershipFailure( + env: AgentRunWorkflowEnv, + workflowInstanceId: string, + payload: AgentRunWorkflowPayload, +): Promise<{ ok: true }> { + const stub = env.AGENT_RUN.get(env.AGENT_RUN.idFromName(payload.input.runId)); + const response = await stub.fetch("https://agent-run.internal/workflow/failed", { + body: JSON.stringify({ + inputHash: payload.inputHash, + generation: payload.generation, + message: "Durable AgentRun execution ownership failed.", + workflowInstanceId, + }), + method: "POST", + }); + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + throw new Error(`AgentRun ownership terminalization returned HTTP ${response.status}`); + } + await response.body?.cancel().catch(() => undefined); + return { ok: true }; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run-workspace.ts b/apps/agent-worker/src/durable-objects/agent-run-workspace.ts new file mode 100644 index 00000000..3df5b962 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/agent-run-workspace.ts @@ -0,0 +1,119 @@ +import { + createDb, + materializeThreadProject, + withUserContext, + workspacePathForSlug, +} from "@cheatcode/db"; +import { APIError, type createLogger } from "@cheatcode/observability"; +import type { + CodeRuntimeContext, + WorkspaceBinding, + WorkspaceResolver, +} from "@cheatcode/sandbox-contracts"; +import { ThreadId, UserId } from "@cheatcode/types"; +import type { UIMessageChunk } from "ai"; +import type { AgentRunEnv } from "./agent-run-env"; +import type { StartRunInput } from "./agent-run-schemas"; + +interface WorkspaceResolverInput { + append: (chunk: UIMessageChunk) => Promise; + env: AgentRunEnv; + input: StartRunInput; + logger: ReturnType; + sandbox: CodeRuntimeContext["sandbox"]; +} + +/** Request-scoped resolver shared by every workspace-backed tool in one agent run. */ +export function createRunWorkspaceResolver(input: WorkspaceResolverInput): WorkspaceResolver { + let pending: Promise | null = null; + return () => { + pending ??= resolveWorkspace(input).catch((error: unknown) => { + pending = null; + throw error; + }); + return pending; + }; +} + +async function resolveWorkspace(input: WorkspaceResolverInput): Promise { + if (input.input.projectId && input.input.workspaceSlug) { + await ensureWorkspaceDirectory(input, input.input.workspaceSlug); + return binding(input.input.projectId, input.input.workspaceSlug); + } + const result = await materializeWorkspaceProject(input); + if (result.type === "thread-not-found") { + throw new APIError(404, "not_found_thread", "Thread not found", { retriable: false }); + } + if (result.type === "project-read-only") { + throw new APIError(403, "permission_plan_required", "Project is read-only after downgrade", { + details: { archiveAfter: result.archiveAfter?.toISOString() ?? null }, + retriable: false, + }); + } + if (result.type === "project-limit-reached") { + throw new APIError(403, "permission_plan_required", "Active project limit reached", { + details: { limit: result.limit, used: result.used }, + hint: "Upgrade your plan or archive an existing project before creating workspace files.", + retriable: false, + }); + } + const project = result.project; + input.input.projectId = project.id; + input.input.workspaceSlug = project.workspaceSlug; + await ensureWorkspaceDirectory(input, project.workspaceSlug); + if (result.type === "created") { + await input.append({ + data: { projectId: project.id, projectName: project.name, v: 1 }, + type: "data-project-created", + }); + } + input.logger.info("agent_workspace_materialized", { + projectId: project.id, + workspaceSlug: project.workspaceSlug, + }); + return binding(project.id, project.workspaceSlug); +} + +async function materializeWorkspaceProject(input: WorkspaceResolverInput) { + const userId = UserId(input.input.userId); + const { db, close } = createDb(input.env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: input.env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + return await withUserContext(db, userId, (tx) => + materializeThreadProject(tx, { + threadId: ThreadId(input.input.threadId), + userId, + }), + ); + } finally { + await close(); + } +} + +async function ensureWorkspaceDirectory( + input: WorkspaceResolverInput, + workspaceSlug: string, +): Promise { + if (!input.sandbox.exec) { + return; + } + const result = await input.sandbox.exec({ + command: ["mkdir", "-p", workspacePathForSlug(workspaceSlug)], + timeoutMs: 15_000, + }); + if (!result.success) { + throw new APIError(503, "sandbox_failed_to_start", "Could not prepare the project workspace", { + retriable: true, + }); + } +} + +function binding(projectId: string, workspaceSlug: string): WorkspaceBinding { + return { + projectId, + workspaceDir: workspacePathForSlug(workspaceSlug), + workspaceSlug, + }; +} diff --git a/apps/agent-worker/src/durable-objects/agent-run.ts b/apps/agent-worker/src/durable-objects/agent-run.ts index e46c92cc..0f2c5754 100644 --- a/apps/agent-worker/src/durable-objects/agent-run.ts +++ b/apps/agent-worker/src/durable-objects/agent-run.ts @@ -1,9 +1,14 @@ import { DurableObject } from "cloudflare:workers"; +import type { AgentChunkType } from "@cheatcode/agent-core"; import { APIError, createLogger } from "@cheatcode/observability"; -import type { ArtifactRuntime, CodeRuntimeContext } from "@cheatcode/sandbox-contracts"; +import type { + ArtifactRuntime, + CodeRuntimeContext, + WorkspaceResolver, +} from "@cheatcode/sandbox-contracts"; import { AgentRunId, - ApprovalDecisionResponseSchema, + type InternalDurableObjectStorageRequest, ProjectId, RunStatusSnapshotSchema, ThreadId, @@ -11,14 +16,9 @@ import { } from "@cheatcode/types"; import type { UIMessageChunk } from "ai"; import { createAgentStreamResponse } from "../streaming/ui-message-stream"; -import { emitRunAbandoned } from "./agent-run-abandonment"; -import { - type ApprovalDecisionInput, - armAgentRunAlarm, - RunApprovalController, - type RunIdentity, -} from "./agent-run-approvals"; +import { armAgentRunAlarm, armClosedAgentRunAlarm } from "./agent-run-alarm"; import { storeAgentArtifact } from "./agent-run-artifacts"; +import { AgentRunBrowserTakeover } from "./agent-run-browser-takeover"; import { emitMastraChunkTelemetry } from "./agent-run-chunk-telemetry"; import type { AgentRunEnv } from "./agent-run-env"; import { handleAgentRunRequest } from "./agent-run-http"; @@ -28,82 +28,134 @@ import { persistOrQueueAssistantMessage, retryPendingAssistantMessage, } from "./agent-run-message-persistence"; -import { emitStoredAgentRunMetric } from "./agent-run-metrics"; import { persistAgentRunLogicalModel } from "./agent-run-model-persistence"; import { AgentRunOutput } from "./agent-run-output"; import { executeAgentRunPath } from "./agent-run-path"; +import { + absentAgentRunOkResponse, + absentAgentRunWorkflowResponse, + agentRunReleaseGateResponse, + agentRunStreamCapacityResponse, + agentRunWorkflowResponse, + deletedAgentRunResponse, + isAgentRunDrainContinuation, +} from "./agent-run-responses"; import { resolveAgentRunRetentionAction } from "./agent-run-retention"; import type { StartRunInput } from "./agent-run-schemas"; import { agentRunStatusPayload } from "./agent-run-status-payload"; import { + isTerminalPersistableRunStatus, type PersistableRunStatus, pendingStatusRetryAt, - persistOrQueueAgentRunStatus, + persistSerializedAgentRunStatus, retryPendingAgentRunStatus, } from "./agent-run-status-persistence"; import { + claimAgentRunDeletion, getRunStateTimestamp, getRunStateValue, + hasAgentRunStorage, initializeAgentRunStorage, + isAgentRunDeleted, setRunStateValue, updateRunRowStatus, upsertRunRow, } from "./agent-run-storage"; import type { StreamDriverDeps } from "./agent-run-stream-driver"; +import { AgentRunWorkflowController } from "./agent-run-workflow-controller"; +import { createRunWorkspaceResolver } from "./agent-run-workspace"; +import { reconcileAgentRunStorageRequest } from "./durable-storage-reconciliation"; import { mastraChunkError, normalizeMastraStreamError } from "./mastra-stream-chunks"; import { hasActiveRun } from "./run-state"; import { type AgentRunSnapshotStatus, snapshotAgentRunStatus } from "./run-summary"; type ProjectSandboxStub = CodeRuntimeContext["sandbox"]; type TerminalRunStatus = "canceled" | "completed" | "failed"; +interface RunIdentity { + runId: string; + threadId: string; + userId: string; +} export class AgentRun extends DurableObject { - private activeRunAbortController: AbortController | undefined; - private activeRunPromise: Promise | undefined; + private alarmExecutionPromise: Promise | undefined; private cancelRequested = false; private deletionInProgress = false; - private readonly approvals: RunApprovalController; + private ownershipLost = false; private readonly output: AgentRunOutput; + private readonly browserTakeover: AgentRunBrowserTakeover; private requestAdmissionTail: Promise = Promise.resolve(); private statusPersistenceChain: Promise = Promise.resolve(); private terminalTransitionOpen = false; private terminalTransitionPromise: Promise | undefined; private terminalTransitionStatus: TerminalRunStatus | undefined; - + private readonly workflow: AgentRunWorkflowController; public constructor(ctx: DurableObjectState, env: AgentRunEnv) { super(ctx, env); this.output = new AgentRunOutput({ ctx: this.ctx, env: this.env, getStatus: () => this.getStatus(), - isCanceled: () => this.isRunCanceled(), + isCanceled: () => this.isExecutionStopped(), isTerminalizing: () => this.terminalTransitionOpen, }); - this.approvals = new RunApprovalController({ - append: (chunk, options) => this.append(chunk, options), - armAlarm: () => this.armAlarm(), + this.browserTakeover = new AgentRunBrowserTakeover({ ctx: this.ctx, - currentStatus: () => this.getStatus(), env: this.env, - finalizeUnrecoverable: () => this.finalizeUnrecoverableApproval(), - identity: () => this.runIdentity(), - isCanceled: () => this.isRunCanceled(), - setRunStatus: (status) => this.setRunStatus(status), + getOwnerUserId: () => this.getOwnerUserId(), + getStatus: () => this.getStatus(), }); - this.ctx.blockConcurrencyWhile(async () => { - initializeAgentRunStorage(this.ctx); + this.workflow = new AgentRunWorkflowController({ + armAlarm: () => this.armAlarm(), + ctx: this.ctx, + env: this.env, + finalizeOwnershipFailure: (message) => this.finalizeOwnershipFailure(message), + getStatus: () => this.getStatus(), + run: (input, abortController) => this.run(input, abortController), }); } public override async alarm(): Promise { + if (!hasAgentRunStorage(this.ctx)) { + await this.ctx.storage.deleteAlarm(); + return; + } + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + await armClosedAgentRunAlarm(this.ctx, this.getStatus()); + return; + } + const execution = this.handleAlarm(); + this.alarmExecutionPromise = execution; + try { + await execution; + } finally { + if (this.alarmExecutionPromise === execution) { + this.alarmExecutionPromise = undefined; + } + } + } + + public reconcileStorageSchema(value: InternalDurableObjectStorageRequest) { + return reconcileAgentRunStorageRequest(this.ctx, this.env, value); + } + private async handleAlarm(): Promise { + if (isAgentRunDeleted(this.ctx)) { + await this.ctx.storage.deleteAlarm(); + return; + } if (!getRunStateValue(this.ctx, "run_id")) { await this.ctx.storage.deleteAlarm(); return; } - await this.serializeStatusPersistence(() => retryPendingAgentRunStatus(this.ctx, this.env)); - await retryPendingAssistantMessage(this.ctx, this.env); - if (await this.approvals.handleAlarmIfDue()) { + if (await this.workflow.recoverPendingAdmission()) { + return; + } + if (await this.workflow.handleExpiredLease()) { return; } + await retryPendingAssistantMessage(this.ctx, this.env); + if (pendingAssistantMessageRetryAt(this.ctx) === Number.POSITIVE_INFINITY) { + await this.serializeStatusPersistence(() => retryPendingAgentRunStatus(this.ctx, this.env)); + } if ( pendingAssistantMessageRetryAt(this.ctx) !== Number.POSITIVE_INFINITY || pendingStatusRetryAt(this.ctx) !== Number.POSITIVE_INFINITY @@ -116,8 +168,8 @@ export class AgentRun extends DurableObject { now: Date.now(), }); if (action === "delete-all") { + this.deletionInProgress = true; await this.ctx.storage.deleteAll(); - await this.ctx.storage.deleteAlarm(); return; } if (action === "clear-messages") { @@ -127,20 +179,48 @@ export class AgentRun extends DurableObject { } public override fetch(request: Request): Promise { - // Reserve FIFO admission synchronously, before request-body parsing yields. - // A presence probe delivered after /start can therefore never observe the - // object before that start has either parsed and claimed state or failed. - const response = this.requestAdmissionTail.then(() => - handleAgentRunRequest(request, { - approval: (userId, body) => this.approval(userId, body), - cancel: (userId) => this.cancel(userId), - deleteAll: (userId) => this.deleteAllState(userId), - finalizeDetachedRun: () => this.finalizeDetachedRun(), - resume: (userId, lastSeq) => this.resume(userId, lastSeq), - start: (input) => this.start(input), - status: (userId) => this.status(userId), - }), - ); + const releaseGate = this.env.CHEATCODE_RELEASE_GATE; + if ( + releaseGate === "closed" || + (releaseGate === "draining" && !isAgentRunDrainContinuation(request)) + ) { + return Promise.resolve(agentRunReleaseGateResponse(releaseGate)); + } + // FIFO admission makes /start settle before a later presence probe observes the object. + const response = this.requestAdmissionTail.then(() => { + const hasStorage = hasAgentRunStorage(this.ctx); + return handleAgentRunRequest(request, { + browserTakeoverResume: (userId, takeoverId) => + hasStorage ? this.browserTakeover.resume(userId, takeoverId) : absentAgentRunOkResponse(), + browserTakeoverStart: (userId) => + hasStorage ? this.browserTakeover.start(userId) : absentAgentRunOkResponse(), + browserTakeoverStatus: (userId) => + hasStorage ? this.browserTakeover.status(userId) : absentAgentRunOkResponse(), + cancel: (userId) => (hasStorage ? this.cancel(userId) : absentAgentRunOkResponse()), + deleteAll: (userId) => + hasStorage ? this.deleteAllState(userId) : absentAgentRunOkResponse(), + executeWorkflow: (input) => + hasStorage + ? agentRunWorkflowResponse(() => this.workflow.executeEpoch(input)) + : absentAgentRunWorkflowResponse(), + failWorkflow: (input) => + hasStorage + ? agentRunWorkflowResponse(() => this.workflow.failWorkflow(input)) + : absentAgentRunOkResponse(), + rolloverWorkflow: (input) => + hasStorage + ? agentRunWorkflowResponse(() => this.workflow.reserveSuccessor(input)) + : absentAgentRunWorkflowResponse(), + resume: (userId, lastSeq) => + hasStorage ? this.resume(userId, lastSeq) : new Response(null, { status: 204 }), + start: (input) => { + if (!hasStorage) initializeAgentRunStorage(this.ctx); + return this.start(input); + }, + status: (userId) => + hasStorage ? this.status(userId) : new Response(null, { status: 204 }), + }); + }); this.requestAdmissionTail = response.then( () => undefined, () => undefined, @@ -148,19 +228,13 @@ export class AgentRun extends DurableObject { return response; } - private start(input: StartRunInput): Response { + private async start(input: StartRunInput): Promise { + if (this.deletionInProgress || isAgentRunDeleted(this.ctx)) { + return deletedAgentRunResponse(); + } const storedRunId = getRunStateValue(this.ctx, "run_id"); if (storedRunId === input.runId) { - if (this.getOwnerUserId() !== input.userId) { - return new APIError(403, "permission_denied", "Run ownership mismatch", { - hint: "Open the thread from the account that started the active run.", - retriable: false, - }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); - } - return createAgentStreamResponse({ - status: hasActiveRun(this.getStatus()) ? 202 : 200, - stream: this.output.resume(0), - }); + return this.resumeExistingStart(input); } if (storedRunId || hasActiveRun(this.getStatus())) { return new APIError(409, "conflict_run_already_active", "An agent run is already active", { @@ -168,23 +242,52 @@ export class AgentRun extends DurableObject { retriable: false, }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); } - this.resetForNewRun(); + if (!this.output.hasStreamCapacity()) { + return agentRunStreamCapacityResponse(); + } + const admission = await this.workflow.createAdmission(input); + await this.workflow.armAdmissionRecovery(); this.cancelRequested = false; - this.setRunIdentity(input); - this.setOwnerUserId(input.userId); - this.setStatus("running"); + this.ownershipLost = false; + this.workflow.claimAdmission(admission, () => { + this.setRunIdentity(input); + this.setOwnerUserId(input.userId); + this.setStatus("running"); + }); + await this.workflow.admit(admission); const stream = this.output.resume(0); - const abortController = new AbortController(); - this.activeRunAbortController = abortController; - const runPromise = this.run(input, abortController); - this.activeRunPromise = runPromise; - this.ctx.waitUntil(runPromise); + if (!stream) { + return agentRunStreamCapacityResponse(); + } return createAgentStreamResponse({ status: 202, stream, }); } + private async resumeExistingStart(input: StartRunInput): Promise { + if (this.getOwnerUserId() !== input.userId) { + return new APIError(403, "permission_denied", "Run ownership mismatch", { + hint: "Open the thread from the account that started the active run.", + retriable: false, + }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); + } + if (!this.output.hasStreamCapacity()) { + return agentRunStreamCapacityResponse(); + } + if (hasActiveRun(this.getStatus())) { + await this.workflow.admit(await this.workflow.createAdmission(input)); + } + const stream = this.output.resume(0); + if (!stream) { + return agentRunStreamCapacityResponse(); + } + return createAgentStreamResponse({ + status: hasActiveRun(this.getStatus()) ? 202 : 200, + stream, + }); + } + private resume(userId: string, lastSeq: number): Response { const ownerUserId = this.getOwnerUserId(); if (!ownerUserId && !this.output.hasReplayRows(lastSeq) && !hasActiveRun(this.getStatus())) { @@ -199,12 +302,11 @@ export class AgentRun extends DurableObject { if (!this.output.hasReplayRows(lastSeq) && !hasActiveRun(this.getStatus())) { return new Response(null, { status: 204 }); } - return createAgentStreamResponse({ - stream: this.output.resume(lastSeq), - }); + const stream = this.output.resume(lastSeq); + return stream ? createAgentStreamResponse({ stream }) : agentRunStreamCapacityResponse(); } - private status(userId: string): Response { + private async status(userId: string): Promise { const runId = getRunStateValue(this.ctx, "run_id"); if (!runId) { return new Response(null, { status: 204 }); @@ -216,6 +318,9 @@ export class AgentRun extends DurableObject { retriable: false, }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); } + if (hasActiveRun(this.getStatus())) { + await this.workflow.reconcileAdmission(); + } const status = this.snapshotStatus(); const payload = agentRunStatusPayload({ ctx: this.ctx, status }); if (!payload) { @@ -228,24 +333,25 @@ export class AgentRun extends DurableObject { } private async deleteAllState(userId: string): Promise { - const ownerUserId = this.getOwnerUserId(); - if (ownerUserId && ownerUserId !== userId) { + if (!claimAgentRunDeletion(this.ctx, userId)) { return new APIError(403, "permission_denied", "Run ownership mismatch", { retriable: false, }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); } this.deletionInProgress = true; - const activeRunPromise = this.activeRunPromise; + const alarmExecutionPromise = this.alarmExecutionPromise; const terminalTransitionPromise = this.terminalTransitionPromise; this.cancelRequested = true; - this.activeRunAbortController?.abort(new Error("run state deleted")); - await this.approvals.cancelPending(); + this.workflow.abort(new Error("run state deleted")); // Join the canceled coroutine so a late terminal-status write cannot recreate erased state. - await activeRunPromise?.catch(() => undefined); - await terminalTransitionPromise?.catch(() => undefined); + await Promise.all([ + this.workflow.join(), + alarmExecutionPromise?.catch(() => undefined), + terminalTransitionPromise?.catch(() => undefined), + ]); + await this.statusPersistenceChain; this.output.closeSubscribers(); await this.ctx.storage.deleteAll(); - await this.ctx.storage.deleteAlarm(); return Response.json({ ok: true }); } @@ -260,15 +366,15 @@ export class AgentRun extends DurableObject { if (!hasActiveRun(this.getStatus())) { return Response.json({ ok: true }); } - await this.finalizeTerminal("canceled", () => this.commitCancellation()); + await this.finalizeTerminal("canceled", () => this.commitCancellation(), true); return Response.json({ ok: true }); } private async commitCancellation(): Promise { this.cancelRequested = true; - this.activeRunAbortController?.abort(new Error("run canceled")); + this.workflow.abort(new Error("run canceled")); try { - await this.approvals.cancelPending(); + await this.workflow.join(); await this.append( { type: "data-error", @@ -295,67 +401,77 @@ export class AgentRun extends DurableObject { }); } } finally { - await this.persistStoredRunStatus("canceled", { - message: "Run canceled by user.", - type: "run_canceled", - }); - } - } - - private async run(input: StartRunInput, abortController: AbortController): Promise { - try { - await executeAgentRunLifecycle( + await this.persistStoredRunStatus( + "canceled", { - append: (chunk) => this.append(chunk), - ctx: this.ctx, - env: this.env, - executeRunPath: (runInput, sandbox, logger, signal) => - this.executeRunPath(runInput, sandbox, logger, signal), - finalizeTerminal: (status, operation) => this.finalizeTerminal(status, operation), - isCanceled: () => this.isRunCanceled(), - output: this.output, - persistRunStatus: (runInput, status, error) => - this.persistRunStatus(runInput, status, error), - setRunStage: (stage) => this.setRunStage(stage), + message: "Run canceled by user.", + type: "run_canceled", }, - input, - abortController, + true, ); - } finally { - if (this.activeRunAbortController === abortController) { - this.activeRunAbortController = undefined; - this.activeRunPromise = undefined; - } } } + private async run(input: StartRunInput, abortController: AbortController): Promise { + await executeAgentRunLifecycle( + { + append: (chunk) => this.append(chunk), + ctx: this.ctx, + env: this.env, + cleanupBrowserTakeover: () => this.browserTakeover.cleanup(), + executeRunPath: (runInput, sandbox, logger, signal) => + this.executeRunPath(runInput, sandbox, logger, signal), + finalizeTerminal: (status, operation) => this.finalizeTerminal(status, operation, true), + isCanceled: () => this.isExecutionStopped(), + output: this.output, + persistRunStatus: (runInput, status, error) => + this.persistRunStatusById({ + artifactsQuiesced: isTerminalPersistableRunStatus(status), + ...(error ? { error } : {}), + runId: runInput.runId, + status, + userId: runInput.userId, + }), + setRunStage: (stage) => this.setRunStage(stage), + }, + input, + abortController, + ); + } + private async executeRunPath( input: StartRunInput, sandbox: ProjectSandboxStub, logger: ReturnType, abortSignal: AbortSignal, ): Promise<"completed" | "continue"> { + const workspaceResolver = createRunWorkspaceResolver({ + append: (chunk) => this.append(chunk), + env: this.env, + input, + logger, + sandbox, + }); return executeAgentRunPath({ abortSignal, append: (chunk) => this.append(chunk), env: this.env, input, - isCanceled: () => this.isRunCanceled(), + isCanceled: () => this.isExecutionStopped(), logger, sandbox, setRunStage: (stage) => this.setRunStage(stage), - streamDriverDeps: this.streamDriverDeps(), + streamDriverDeps: this.streamDriverDeps(workspaceResolver), + workspaceResolver, }); } - private streamDriverDeps(): StreamDriverDeps { + private streamDriverDeps(workspaceResolver: WorkspaceResolver): StreamDriverDeps { return { append: (chunk) => this.append(chunk), appendCheckedMastraChunk: (input, chunk) => this.appendCheckedMastraChunk(input, chunk), - createArtifactRuntime: (input) => this.createArtifactRuntime(input), - createBroker: () => this.approvals.createBroker(), + createArtifactRuntime: (input) => this.createArtifactRuntime(input, workspaceResolver), env: this.env, - hasPendingDecision: () => this.approvals.hasPendingDecision(), persistLogicalModel: (input, logicalModelId, logger) => persistAgentRunLogicalModel({ ctx: this.ctx, @@ -366,10 +482,14 @@ export class AgentRun extends DurableObject { userId: input.userId, }), setRunStage: (stage) => this.setRunStage(stage), + waitForBrowserTakeover: (signal) => this.browserTakeover.wait(signal), }; } - private async appendCheckedMastraChunk(input: StartRunInput, chunk: unknown): Promise { + private async appendCheckedMastraChunk( + input: StartRunInput, + chunk: AgentChunkType, + ): Promise { const streamError = mastraChunkError(chunk); if (streamError) { throw normalizeMastraStreamError(streamError); @@ -378,19 +498,24 @@ export class AgentRun extends DurableObject { return this.output.appendMastraChunk(chunk); } - private createArtifactRuntime(input: StartRunInput): ArtifactRuntime { + private createArtifactRuntime( + input: StartRunInput, + workspaceResolver: WorkspaceResolver, + ): ArtifactRuntime { return { - put: async (artifact) => - storeAgentArtifact({ + put: async (artifact) => { + const workspace = await workspaceResolver(); + return storeAgentArtifact({ artifact, env: this.env, input: { - projectId: ProjectId(input.projectId), + projectId: ProjectId(workspace.projectId), runId: AgentRunId(input.runId), threadId: ThreadId(input.threadId), userId: UserId(input.userId), }, - }), + }); + }, }; } @@ -409,7 +534,10 @@ export class AgentRun extends DurableObject { return snapshotAgentRunStatus(this.getStatus()); } - private setStatus(status: "running" | "paused" | "completed" | "failed" | "canceled"): void { + private setStatus(status: "running" | "completed" | "failed" | "canceled"): void { + if (isAgentRunDeleted(this.ctx)) { + return; + } this.ctx.storage.sql.exec( "INSERT OR REPLACE INTO run_state (key, value) VALUES ('status', ?)", status, @@ -420,13 +548,10 @@ export class AgentRun extends DurableObject { } } - /** Flips both the DO run state and the Postgres row. */ - private async setRunStatus(status: "paused" | "running"): Promise { - this.setStatus(status); - await this.persistStoredRunStatus(status); - } - private armAlarm(): Promise { + if (isAgentRunDeleted(this.ctx)) { + return this.ctx.storage.deleteAlarm(); + } return armAgentRunAlarm(this.ctx); } @@ -440,44 +565,24 @@ export class AgentRun extends DurableObject { return { runId, threadId, userId }; } - private async approval(userId: string, body: ApprovalDecisionInput): Promise { - if (this.getOwnerUserId() !== userId) { - return new APIError(403, "permission_denied", "Run ownership mismatch", { - hint: "Open the thread from the account that started the run.", - retriable: false, - }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); - } - try { - const result = await this.approvals.applyDecision({ - approvalId: body.approvalId, - decision: body.decision, - ...(body.reason ? { reason: body.reason } : {}), - }); - return Response.json(ApprovalDecisionResponseSchema.parse(result)); - } catch (error) { - if (error instanceof APIError) { - return error.toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); - } - throw error; + private async finalizeOwnershipFailure(message: string): Promise { + if (isAgentRunDeleted(this.ctx)) { + return; } + this.ownershipLost = true; + await this.finalizeTerminal("failed", () => this.commitOwnershipFailure(message), true); } - /** DO eviction mid-approval: fail the run deterministically. */ - private async finalizeUnrecoverableApproval(): Promise { - await this.finalizeTerminal("failed", () => this.commitUnrecoverableApproval()); - } - - private async commitUnrecoverableApproval(): Promise { - await this.append({ - type: "data-error", - data: { - v: 1, - code: "approval_unrecoverable", - message: "Run could not recover the pending approval after a restart. Start a new run.", - retriable: false, + private async commitOwnershipFailure(message: string): Promise { + await this.append( + { + type: "data-error", + data: { v: 1, code: "run_interrupted", message, retriable: true }, }, - }); - await this.append({ type: "finish", finishReason: "error" }); + { allowAfterCancelRequest: true }, + ); + await this.output.ensureAnswerSegmentEnded({ allowAfterCancelRequest: true }); + await this.append({ type: "finish", finishReason: "error" }, { allowAfterCancelRequest: true }); const identity = this.runIdentity(); if (identity) { await persistOrQueueAssistantMessage({ @@ -486,60 +591,21 @@ export class AgentRun extends DurableObject { logger: createLogger({ runId: identity.runId, userId: identity.userId }), ...identity, }); - await this.persistRunStatusById({ - error: { message: "Pending approval was unrecoverable.", type: "approval_unrecoverable" }, - runId: identity.runId, - status: "failed", - userId: identity.userId, - }); - } - } - - /** A persisted running state without its execution controller can never make progress. */ - private async finalizeDetachedRun(): Promise { - if (this.getStatus() !== "running" || this.activeRunAbortController) { - return; - } - const identity = this.runIdentity(); - if (!identity) { - return; } - await this.finalizeTerminal("failed", () => this.commitDetachedRun(identity)); - } - - private async commitDetachedRun(identity: RunIdentity): Promise { - emitRunAbandoned(this.ctx, this.env); - const message = "This run was interrupted before it finished. Send the prompt again to retry."; - await this.append({ - type: "data-error", - data: { v: 1, code: "run_interrupted", message, retriable: true }, - }); - await this.append({ type: "finish", finishReason: "error" }); - await persistOrQueueAssistantMessage({ - ctx: this.ctx, - env: this.env, - logger: createLogger({ runId: identity.runId, userId: identity.userId }), - ...identity, - }); - await this.persistRunStatusById({ - error: { message, type: "run_interrupted" }, - runId: identity.runId, - status: "failed", - userId: identity.userId, - }); + await this.persistStoredRunStatus("failed", { message, type: "run_interrupted" }, true); } private setRunIdentity(input: StartRunInput): void { setRunStateValue(this.ctx, "run_id", input.runId); setRunStateValue(this.ctx, "thread_id", input.threadId); - setRunStateValue(this.ctx, "project_id", input.projectId); + setRunStateValue(this.ctx, "sandbox_name", input.sandboxName); + if (input.projectId) { + setRunStateValue(this.ctx, "project_id", input.projectId); + } if (input.isFirstRun) setRunStateValue(this.ctx, "is_first_run", "true"); upsertRunRow(this.ctx, { plannedLogicalModelId: input.model, - projectId: input.projectId, runId: input.runId, - threadId: input.threadId, - userId: input.userId, }); } @@ -552,69 +618,64 @@ export class AgentRun extends DurableObject { } private setRunStage(stage: string): void { + if (isAgentRunDeleted(this.ctx)) { + return; + } setRunStateValue(this.ctx, "run_stage", stage); } private isRunCanceled(): boolean { - return this.cancelRequested || this.getStatus() === "canceled"; + return isAgentRunDeleted(this.ctx) || this.cancelRequested || this.getStatus() === "canceled"; } - private resetForNewRun(): void { - this.deletionInProgress = false; - this.terminalTransitionOpen = false; - this.terminalTransitionPromise = undefined; - this.terminalTransitionStatus = undefined; - this.ctx.storage.sql.exec("DELETE FROM message_part"); - this.ctx.storage.sql.exec("DELETE FROM run"); - this.ctx.storage.sql.exec("DELETE FROM run_state"); - } - - private async persistRunStatus( - input: StartRunInput, - status: PersistableRunStatus, - error?: { message: string; type: string }, - ): Promise { - await this.persistRunStatusById({ - ...(error ? { error } : {}), - runId: input.runId, - status, - userId: input.userId, - }); + private isExecutionStopped(): boolean { + return this.isRunCanceled() || this.ownershipLost; } private async persistStoredRunStatus( status: PersistableRunStatus, error?: { message: string; type: string }, + artifactsQuiesced = false, ): Promise { const runId = getRunStateValue(this.ctx, "run_id"); const userId = this.getOwnerUserId(); if (!runId || !userId) { return; } - await this.persistRunStatusById({ ...(error ? { error } : {}), runId, status, userId }); + await this.persistRunStatusById({ + artifactsQuiesced, + ...(error ? { error } : {}), + runId, + status, + userId, + }); } private async persistRunStatusById(input: { + artifactsQuiesced: boolean; error?: { message: string; type: string }; runId: string; status: PersistableRunStatus; userId: string; }): Promise { - await this.serializeStatusPersistence(async () => { - emitStoredAgentRunMetric(this.ctx, this.env, input); - await persistOrQueueAgentRunStatus(this.ctx, this.env, input); - }); - await this.armAlarm(); + await persistSerializedAgentRunStatus( + this.ctx, + this.env, + input, + (operation) => this.serializeStatusPersistence(operation), + () => this.armAlarm(), + ); } private async finalizeTerminal( status: TerminalRunStatus, operation: () => Promise, + artifactsQuiesced: boolean, ): Promise { if (!this.tryCommitTerminal(status)) { return false; } - const transition = this.performTerminalTransition(status, operation); + const transition = this.performTerminalTransition(status, operation, artifactsQuiesced); this.terminalTransitionPromise = transition; try { await transition; @@ -629,11 +690,12 @@ export class AgentRun extends DurableObject { private async performTerminalTransition( status: TerminalRunStatus, operation: () => Promise, + artifactsQuiesced: boolean, ): Promise { try { await operation(); } catch (error) { - await this.persistTerminalFallback(status, error); + await this.persistTerminalFallback(status, error, artifactsQuiesced); throw error; } finally { this.terminalTransitionOpen = false; @@ -655,16 +717,22 @@ export class AgentRun extends DurableObject { return true; } - private async persistTerminalFallback(status: TerminalRunStatus, error: unknown): Promise { + private async persistTerminalFallback( + status: TerminalRunStatus, + error: unknown, + artifactsQuiesced: boolean, + ): Promise { const runId = getRunStateValue(this.ctx, "run_id"); const logger = createLogger(runId ? { runId } : {}); logger.error("agent_terminal_finalize_failed", { error, terminalStatus: status }); - await this.persistStoredRunStatus(status).catch((persistError: unknown) => { - logger.error("agent_terminal_fallback_persist_failed", { - error: persistError, - terminalStatus: status, - }); - }); + await this.persistStoredRunStatus(status, undefined, artifactsQuiesced).catch( + (persistError: unknown) => { + logger.error("agent_terminal_fallback_persist_failed", { + error: persistError, + terminalStatus: status, + }); + }, + ); } private async serializeStatusPersistence(operation: () => Promise): Promise { diff --git a/apps/agent-worker/src/durable-objects/agent-tool-credentials.ts b/apps/agent-worker/src/durable-objects/agent-tool-credentials.ts index 2d3b3a4e..4d37dffc 100644 --- a/apps/agent-worker/src/durable-objects/agent-tool-credentials.ts +++ b/apps/agent-worker/src/durable-objects/agent-tool-credentials.ts @@ -3,10 +3,14 @@ import type { AgentRunEnv } from "./agent-run-env"; import type { StartRunInput } from "./agent-run-schemas"; import type { ComposioRuntimeCredentials } from "./composio-provider"; import { resolveComposioRuntimeCredentials } from "./composio-provider"; +import type { MediaCredentials } from "./media-provider"; +import { resolveMediaCredentials } from "./media-provider"; import type { ResearchCredentials } from "./research-provider"; import { resolveResearchCredentials } from "./research-provider"; -export type AgentToolCredentials = ComposioRuntimeCredentials & ResearchCredentials; +export type AgentToolCredentials = ComposioRuntimeCredentials & + MediaCredentials & + ResearchCredentials; export async function resolveAgentToolCredentials(input: { env: AgentRunEnv; @@ -22,8 +26,11 @@ export async function resolveAgentToolCredentials(input: { input.run, input.logger, ); + input.setRunStage("Resolving media providers."); + const mediaCredentials = await resolveMediaCredentials(input.env, input.run, input.logger); return { ...composioCredentials, + ...mediaCredentials, ...researchCredentials, }; } diff --git a/apps/agent-worker/src/durable-objects/app-builder-template.ts b/apps/agent-worker/src/durable-objects/app-builder-template.ts index 0f0a0c3f..41408940 100644 --- a/apps/agent-worker/src/durable-objects/app-builder-template.ts +++ b/apps/agent-worker/src/durable-objects/app-builder-template.ts @@ -40,7 +40,7 @@ body { export function appBuilderPageSource(messageText: string): string { return `const cards = [ ["Gateway", "Clerk auth and rate limits route the request."], - ["AgentRun", "Durable Object stores resumable stream parts."], + ["Agent session", "Durable Object stores resumable stream parts."], ["Sandbox", "Daytona serves this live preview."], ]; diff --git a/apps/agent-worker/src/durable-objects/composio-provider.ts b/apps/agent-worker/src/durable-objects/composio-provider.ts index 1989edc5..57ae1830 100644 --- a/apps/agent-worker/src/durable-objects/composio-provider.ts +++ b/apps/agent-worker/src/durable-objects/composio-provider.ts @@ -7,8 +7,8 @@ import { entitlementCacheFromValues, quotaPeriodEndFor } from "@cheatcode/billin import { createDb, type DatabaseHandle, - findEntitlementByUserId, - listUserIntegrations, + findAgentEntitlementByUserId, + listAgentIntegrations, withUserContext, } from "@cheatcode/db"; import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; @@ -26,6 +26,7 @@ import { closeDatabaseBestEffort } from "./db-close"; interface ComposioProviderEnv { COMPOSIO_API_KEY?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; HYPERDRIVE: Hyperdrive; QUOTA_TRACKER: DurableObjectNamespace; } @@ -53,14 +54,17 @@ export async function resolveComposioRuntimeCredentials( logger: ReturnType, ): Promise { const apiKey = await readOptionalComposioApiKey(env, logger); - const dbHandle = createDb(env.HYPERDRIVE); + const dbHandle = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { const userId = UserId(input.userId); const state = await withUserContext(dbHandle.db, userId, async (db) => { // A user-context transaction owns one pg client. Keep its queries sequential instead of // pretending to parallelize them through the same connection. - const integrations = await listUserIntegrations(db, userId); - const entitlement = await findEntitlementByUserId(db, userId); + const integrations = await listAgentIntegrations(db, userId); + const entitlement = await findAgentEntitlementByUserId(db, userId); const resolvedEntitlement = entitlementCacheFromValues(entitlement ?? { tier: "free" }); return { connectedAccounts: connectedAccountsFromRows(integrations), @@ -88,7 +92,7 @@ export async function resolveComposioRuntimeCredentials( } function connectedAccountsFromRows( - rows: Awaited>, + rows: Awaited>, ): ComposioConnectedAccounts { const connectedAccounts: ComposioConnectedAccounts = {}; for (const row of rows) { diff --git a/apps/agent-worker/src/durable-objects/durable-storage-reconciliation.ts b/apps/agent-worker/src/durable-objects/durable-storage-reconciliation.ts new file mode 100644 index 00000000..9eda3f98 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/durable-storage-reconciliation.ts @@ -0,0 +1,44 @@ +import { + assertStorageReconciliationRequest, + reconcileExactSqliteStorage, + storageSchemaEvidence, +} from "@cheatcode/durable-storage"; +import type { + InternalDurableObjectStorageRequest, + InternalDurableObjectStorageResponse, +} from "@cheatcode/types"; +import type { AgentRunEnv } from "./agent-run-env"; +import { assertAgentRunStorage, reconcileAgentRunStorage } from "./agent-run-storage"; +import type { ProjectSandboxEnv } from "./project-sandbox-lifecycle-support"; +import { + assertProjectSandboxStorage, + reconcileProjectSandboxStorage, +} from "./project-sandbox-workspace-state"; + +export function reconcileAgentRunStorageRequest( + ctx: DurableObjectState, + env: AgentRunEnv, + value: InternalDurableObjectStorageRequest, +): InternalDurableObjectStorageResponse { + const input = assertStorageReconciliationRequest(ctx, env, value, "AgentRun"); + reconcileExactSqliteStorage( + input.mode, + () => assertAgentRunStorage(ctx), + () => reconcileAgentRunStorage(ctx), + ); + return storageSchemaEvidence(input); +} + +export function reconcileProjectSandboxStorageRequest( + ctx: DurableObjectState, + env: ProjectSandboxEnv, + value: InternalDurableObjectStorageRequest, +): InternalDurableObjectStorageResponse { + const input = assertStorageReconciliationRequest(ctx, env, value, "ProjectSandbox"); + reconcileExactSqliteStorage( + input.mode, + () => assertProjectSandboxStorage(ctx), + () => reconcileProjectSandboxStorage(ctx), + ); + return storageSchemaEvidence(input); +} diff --git a/apps/agent-worker/src/durable-objects/llm-provider.ts b/apps/agent-worker/src/durable-objects/llm-provider.ts index f2f35eb1..1913eb68 100644 --- a/apps/agent-worker/src/durable-objects/llm-provider.ts +++ b/apps/agent-worker/src/durable-objects/llm-provider.ts @@ -19,8 +19,9 @@ import { import { closeDatabaseBestEffort } from "./db-close"; interface LlmProviderEnv { - HYPERDRIVE: Hyperdrive; + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; DEEPSEEK_PLATFORM_API_KEY?: WorkerSecret; + HYPERDRIVE: Hyperdrive; } interface LlmProviderInput { @@ -173,7 +174,10 @@ async function resolveProviderKey( logger: ReturnType, platformFallback: PlatformFallbackContext, ): Promise { - const dbHandle = createDb(env.HYPERDRIVE); + const dbHandle = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); const brandedUserId = UserId(userId); try { const resolved = await withUserContext(dbHandle.db, brandedUserId, (db) => diff --git a/apps/agent-worker/src/durable-objects/mastra-stream-chunks.ts b/apps/agent-worker/src/durable-objects/mastra-stream-chunks.ts index 44e9bfd1..71cd3674 100644 --- a/apps/agent-worker/src/durable-objects/mastra-stream-chunks.ts +++ b/apps/agent-worker/src/durable-objects/mastra-stream-chunks.ts @@ -1,82 +1,169 @@ +import type { AgentChunkType } from "@cheatcode/agent-core"; +import { type ArtifactKind, ArtifactKindSchema } from "@cheatcode/types/artifacts"; +import { TOOL_CAPABILITIES } from "@cheatcode/types/capabilities"; import type { UIMessageChunk } from "ai"; const ANSWER_TEXT_ID = "answer"; -const SANDBOX_TOOL_NAMES = new Set([ - "browser_act", - "browser_extract", - "browser_observe", - "browser_open", - "browser_screenshot", - "data_chart", - "docs_generate_docx", - "docs_generate_pdf", - "docs_generate_slides", - "docs_generate_xlsx", - "fs_delete", - "fs_list", - "fs_read", - "fs_search", - "fs_write", - "git_clone", - "git_commit", - "git_push", - "git_status", - "runCode", - "shell_exec", - "shell_kill_process", - "shell_start_process", - "shell_terminal", - "start_dev_server", -]); +const SANDBOX_TOOL_NAMES = capabilityNameSet("usesSandbox"); +const ARTIFACT_TOOL_NAMES = capabilityNameSet("producesArtifact"); -const ARTIFACT_TOOL_NAMES = new Set([ - "browser_screenshot", - "data_chart", - "docs_generate_docx", - "docs_generate_pdf", - "docs_generate_slides", - "docs_generate_xlsx", -]); +type ToolCallPayload = Extract["payload"]; +type ToolResultPayload = Extract["payload"]; +type ToolErrorPayload = Extract["payload"]; +type VisibleAgentChunk = Extract< + AgentChunkType, + { type: "text-delta" | "tool-call" | "tool-error" | "tool-result" } +>; +type NonVisibleAgentChunk = Exclude; +type PrivateOutputAgentChunk = Extract< + NonVisibleAgentChunk, + { + type: + | "file" + | "reasoning-delta" + | "reasoning-end" + | "reasoning-signature" + | "reasoning-start" + | "redacted-reasoning" + | "source"; + } +>; +type ControlAgentChunk = Exclude; -export function mastraChunkToUiChunks(chunk: unknown): UIMessageChunk[] { - const record = asRecord(chunk); - const chunkType = stringField(record, "type"); +export function mastraChunkToUiChunks(chunk: AgentChunkType): UIMessageChunk[] { + switch (chunk.type) { + case "text-delta": + return textDeltaChunks(chunk.payload.text); + case "tool-call": + return toolCallChunks(chunk.payload); + case "tool-result": + return toolResultChunks(chunk.payload); + case "tool-error": + return isSandboxTool(chunk.payload) ? [sandboxStatusChunk("ready")] : []; + default: + return nonVisibleMastraChunk(chunk); + } +} - if (chunkType === "text-delta") { - return textDeltaChunks(record); +function nonVisibleMastraChunk(chunk: NonVisibleAgentChunk): UIMessageChunk[] { + switch (chunk.type) { + // Reasoning is intentionally private; binary/provider-source output must cross the + // bounded artifact tools instead of entering the transcript directly. + case "file": + case "reasoning-delta": + case "reasoning-end": + case "reasoning-signature": + case "reasoning-start": + case "redacted-reasoning": + case "source": + return []; + default: + return controlMastraChunk(chunk); } +} - if (chunkType === "tool-call") { - return toolCallChunks(record); +function controlMastraChunk(chunk: ControlAgentChunk): UIMessageChunk[] { + switch (chunk.type) { + // These lifecycle/control chunks either have a dedicated Cheatcode channel or carry no + // user-visible transcript data. Listing them makes a future Mastra union addition fail CI. + case "abort": + case "background-task-cancelled": + case "background-task-completed": + case "background-task-failed": + case "background-task-output": + case "background-task-progress": + case "background-task-resumed": + case "background-task-running": + case "background-task-started": + case "background-task-suspended": + case "error": + case "finish": + case "goal": + case "is-task-complete": + case "object": + case "object-result": + case "raw": + case "response-metadata": + case "start": + case "step-finish": + case "step-output": + case "step-start": + case "text-end": + case "text-start": + case "tool-call-delta": + case "tool-call-input-streaming-end": + case "tool-call-input-streaming-start": + case "tool-call-suspended": + case "tool-output": + case "tripwire": + case "watch": + return []; + default: + return []; } +} - if (chunkType === "tool-result" && isSandboxToolChunk(record)) { - const payload = chunkPayload(record); +function toolResultChunks(payload: ToolResultPayload): UIMessageChunk[] { + if (payload.toolName === "skill_create") { + const skill = skillProposedChunkFromResult(payload.result); + return skill ? [skill] : []; + } + if (isSandboxTool(payload)) { const chunks = [sandboxStatusChunk("ready")]; - if (isArtifactToolChunk(record)) { - const artifact = artifactChunkFromPayload(payload); + if (isArtifactTool(payload)) { + const artifact = artifactChunkFromResult(payload.result); if (artifact) { chunks.push(artifact); } } return chunks; } - - if (chunkType === "tool-result" && isArtifactToolChunk(record)) { - const payload = chunkPayload(record); - const artifact = artifactChunkFromPayload(payload); + if (isArtifactTool(payload)) { + const artifact = artifactChunkFromResult(payload.result); return artifact ? [artifact] : []; } - return []; } -export function mastraChunkError(chunk: unknown): unknown | null { - const record = asRecord(chunk); - if (stringField(record, "type") !== "error") { +function skillProposedChunkFromResult(result: unknown): UIMessageChunk | undefined { + const record = asRecord(result); + const name = stringField(record, "name"); + const description = stringField(record, "description"); + const body = stringField(record, "body"); + const category = stringField(record, "category"); + const proposalId = stringField(record, "proposalId"); + const slug = stringField(record, "slug"); + const tags = stringArrayField(record, "tags"); + if ( + record["proposed"] !== true || + !name || + !description || + !body || + !category || + !proposalId || + !slug || + !tags + ) { + return undefined; + } + return { + type: "data-skill-proposed", + data: { body, category, description, name, proposalId, slug, tags, v: 1 }, + }; +} + +function stringArrayField(record: Record, key: string): string[] | undefined { + const value = record[key]; + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : undefined; +} + +export function mastraChunkError(chunk: AgentChunkType): unknown | null { + if (chunk.type !== "error") { return null; } - return record["error"] ?? new Error("Unknown Mastra stream error."); + return chunk.payload.error ?? new Error("Unknown Mastra stream error."); } export function normalizeMastraStreamError(error: unknown): Error { @@ -104,12 +191,7 @@ export function normalizeMastraStreamError(error: unknown): Error { return normalized; } -function textDeltaChunks(record: Record): UIMessageChunk[] { - const payload = chunkPayload(record); - const text = - stringField(payload, "text") || - stringField(payload, "textDelta") || - stringField(payload, "delta"); +function textDeltaChunks(text: string): UIMessageChunk[] { if (text.length === 0) { return []; } @@ -128,28 +210,22 @@ const MAX_TOOL_INPUT_STRING = 256; // Surface every tool call as a transcript row (Cheatcode parity). Sandbox tools also drive // the Computer-panel status; non-sandbox tools only get the row. -function toolCallChunks(record: Record): UIMessageChunk[] { - const payload = chunkPayload(record); - const toolName = stringField(payload, "toolName"); - if (!toolName) { - return []; - } - const chunks: UIMessageChunk[] = [toolActivityChunk(payload, toolName)]; - if (SANDBOX_TOOL_NAMES.has(toolName)) { +function toolCallChunks(payload: ToolCallPayload): UIMessageChunk[] { + const chunks: UIMessageChunk[] = [toolActivityChunk(payload)]; + if (SANDBOX_TOOL_NAMES.has(payload.toolName)) { chunks.push(sandboxStatusChunk("starting")); } return chunks; } -function toolActivityChunk(payload: Record, toolName: string): UIMessageChunk { - const toolCallId = stringField(payload, "toolCallId"); +function toolActivityChunk(payload: ToolCallPayload): UIMessageChunk { const input = toolInputFromPayload(payload); return { type: "data-tool", data: { v: 1, - toolName, - ...(toolCallId ? { toolCallId } : {}), + toolCallId: payload.toolCallId, + toolName: payload.toolName, ...(input ? { input } : {}), }, }; @@ -157,14 +233,10 @@ function toolActivityChunk(payload: Record, toolName: string): // Keep the persisted part small: only scalar args, capped count + string length. The // transcript row needs the path/command/url/query, not the full (possibly huge) payload. -function toolInputFromPayload( - payload: Record, -): Record | undefined { - for (const key of ["args", "input", "toolInput", "arguments"]) { - const raw = asRecord(payload[key]); - if (Object.keys(raw).length > 0) { - return truncateToolInput(raw); - } +function toolInputFromPayload(payload: ToolCallPayload): Record | undefined { + const input = asRecord(payload.args); + if (Object.keys(input).length > 0) { + return truncateToolInput(input); } return undefined; } @@ -211,65 +283,44 @@ function stringField(record: Record, key: string): string { return typeof value === "string" ? value : ""; } -function isSandboxToolChunk(record: Record): boolean { - const payload = chunkPayload(record); - return SANDBOX_TOOL_NAMES.has(stringField(payload, "toolName")); +function isSandboxTool(payload: ToolResultPayload | ToolErrorPayload): boolean { + return SANDBOX_TOOL_NAMES.has(payload.toolName); } -function isArtifactToolChunk(record: Record): boolean { - const payload = chunkPayload(record); - return ARTIFACT_TOOL_NAMES.has(stringField(payload, "toolName")); +function isArtifactTool(payload: ToolResultPayload): boolean { + return ARTIFACT_TOOL_NAMES.has(payload.toolName); } -function chunkPayload(record: Record): Record { - const payload = asRecord(record["payload"]); - return Object.keys(payload).length > 0 ? payload : record; -} - -function artifactChunkFromPayload(payload: Record): UIMessageChunk | undefined { - const artifact = artifactRecordFromPayload(payload); +function artifactChunkFromResult(result: unknown): UIMessageChunk | undefined { + const artifact = artifactRecordFromResult(result); const outputId = stringField(artifact, "outputId"); const kind = artifactKind(artifact); - const downloadUrl = stringField(artifact, "downloadUrl"); const mimeType = stringField(artifact, "mimeType"); const filename = stringField(artifact, "filename"); const sizeBytes = numberField(artifact, "sizeBytes"); - if (!outputId || !kind || !downloadUrl || !mimeType) { + if (!outputId || !kind || !mimeType || !filename || sizeBytes === undefined) { return undefined; } return { type: "data-artifact", data: { v: 1, - downloadUrl, - ...(filename ? { filename } : {}), + filename, kind, mimeType, outputId, - ...(sizeBytes !== undefined ? { sizeBytes } : {}), + sizeBytes, }, }; } -function artifactRecordFromPayload(payload: Record): Record { - const output = asRecord(payload["output"]); - if (stringField(output, "downloadUrl")) { - return output; - } - const outputArtifact = asRecord(output["artifact"]); - if (stringField(outputArtifact, "downloadUrl")) { - return outputArtifact; - } - const outputResultArtifact = artifactFromResultList(output["results"]); - if (outputResultArtifact) { - return outputResultArtifact; - } - const result = asRecord(payload["result"]); - if (stringField(result, "downloadUrl")) { +function artifactRecordFromResult(value: unknown): Record { + const result = asRecord(value); + if (stringField(result, "outputId")) { return result; } const resultArtifact = asRecord(result["artifact"]); - if (stringField(resultArtifact, "downloadUrl")) { + if (stringField(resultArtifact, "outputId")) { return resultArtifact; } const resultListArtifact = artifactFromResultList(result["results"]); @@ -285,7 +336,7 @@ function artifactFromResultList(value: unknown): Record | undef } for (const item of value.slice(0, 10)) { const artifact = asRecord(asRecord(item)["artifact"]); - if (stringField(artifact, "downloadUrl")) { + if (stringField(artifact, "outputId")) { return artifact; } } @@ -297,20 +348,13 @@ function numberField(record: Record, key: string): number | und return typeof value === "number" && Number.isFinite(value) ? value : undefined; } -function artifactKind( - record: Record, -): "audio" | "docx" | "image" | "pdf" | "slide" | "video" | "xlsx" | undefined { - const value = stringField(record, "kind"); - if ( - value === "audio" || - value === "docx" || - value === "image" || - value === "pdf" || - value === "slide" || - value === "video" || - value === "xlsx" - ) { - return value; - } - return undefined; +function artifactKind(record: Record): ArtifactKind | undefined { + const parsed = ArtifactKindSchema.safeParse(stringField(record, "kind")); + return parsed.success ? parsed.data : undefined; +} + +function capabilityNameSet(flag: "producesArtifact" | "usesSandbox"): ReadonlySet { + return new Set( + TOOL_CAPABILITIES.filter((capability) => capability[flag]).map((capability) => capability.name), + ); } diff --git a/apps/agent-worker/src/durable-objects/media-provider.ts b/apps/agent-worker/src/durable-objects/media-provider.ts new file mode 100644 index 00000000..d708a5d3 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/media-provider.ts @@ -0,0 +1,46 @@ +import { getProviderKey } from "@cheatcode/byok"; +import { createDb, type DatabaseHandle, withUserContext } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; +import type { createLogger } from "@cheatcode/observability"; +import { UserId } from "@cheatcode/types"; +import { closeDatabaseBestEffort } from "./db-close"; + +interface MediaProviderEnv { + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; + HYPERDRIVE: Hyperdrive; +} + +interface MediaProviderInput { + userId: string; +} + +export interface MediaCredentials { + googleMediaApiKey?: string | undefined; +} + +export async function resolveMediaCredentials( + env: MediaProviderEnv, + input: MediaProviderInput, + logger: ReturnType, +): Promise { + const dbHandle = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const googleMediaApiKey = await withUserContext(dbHandle.db, UserId(input.userId), (db) => + getProviderKey(db, "google"), + ); + logger.info("byok_media_provider_key_checked", { google: Boolean(googleMediaApiKey) }); + return googleMediaApiKey ? { googleMediaApiKey } : {}; + } finally { + await closeMediaDatabase(dbHandle, logger); + } +} + +async function closeMediaDatabase( + dbHandle: DatabaseHandle, + logger: ReturnType, +): Promise { + await closeDatabaseBestEffort({ dbHandle, logger, operation: "resolve_media_credentials" }); +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-code-server.ts b/apps/agent-worker/src/durable-objects/project-sandbox-code-server.ts index 2f4f7f85..f8f2a827 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-code-server.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-code-server.ts @@ -7,9 +7,16 @@ export const CODE_SERVER_SETTINGS_MARKER = "/home/node/.local/share/code-server/user-data/.cheatcode-settings-v6"; export const CODE_SERVER_START_TIMEOUT_MS = 120_000; -export function codeServerFolderUrl(rawUrl: string, folderPath: string): string { +export function codeServerFolderUrl( + rawUrl: string, + folderPath: string, + initialFilePath?: string, +): string { const url = new URL(rawUrl); url.searchParams.set("folder", folderPath); + if (initialFilePath) { + url.searchParams.set("cc_open_file", initialFilePath); + } return url.toString(); } diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-content-support.ts b/apps/agent-worker/src/durable-objects/project-sandbox-content-support.ts index b29d0d2d..f9721620 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-content-support.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-content-support.ts @@ -106,10 +106,6 @@ if archive_size > max_output_bytes: export { PROJECT_ARCHIVE_MAX_OUTPUT_BYTES }; -export function isSingleWorkspaceSegment(slug: string): boolean { - return slug.length > 0 && !slug.includes("/") && slug !== "." && slug !== ".."; -} - export function lowercaseExtension(path: string): string { const filename = basename(path).toLowerCase(); const dot = filename.lastIndexOf("."); diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-content.ts b/apps/agent-worker/src/durable-objects/project-sandbox-content.ts index 2fc398b0..141735de 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-content.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-content.ts @@ -28,7 +28,6 @@ import { encodeBase64, imageMimeType, isOfficePreviewExtension, - isSingleWorkspaceSegment, lowercaseExtension, MAX_PREVIEW_BYTES, PREVIEW_DIR, @@ -50,10 +49,14 @@ import { shellQuote, timeoutSeconds, } from "./project-sandbox-process-support"; -import { ProjectSandboxProcesses } from "./project-sandbox-processes"; import { type ProjectArchiveInput, ProjectArchiveInputSchema, + type ProjectBrowserTakeoverInput, + ProjectBrowserTakeoverInputSchema, + type ProjectBrowserTakeoverResult, + type ProjectBrowserTakeoverStopInput, + ProjectBrowserTakeoverStopInputSchema, type ProjectCleanupWorkspaceInput, ProjectCleanupWorkspaceInputSchema, type ProjectCodeServerInput, @@ -78,13 +81,17 @@ import { type ProjectWriteFileInput, ProjectWriteFileInputSchema, } from "./project-sandbox-runtime"; +import { ProjectSandboxWorkspaceTransition } from "./project-sandbox-workspace-transition"; const PREVIEW_STATUS_PROBE_TIMEOUT_MS = 3_000; const PREVIEW_WAKE_TIMEOUT_MS = 90_000; const SIGNED_PREVIEW_TTL_SECONDS = 60 * 60; const SANDBOX_READ_FILE_MAX_BYTES = 1024 * 1024; +const BROWSER_TAKEOVER_PORT_MIN = 60_000; +const BROWSER_TAKEOVER_PORT_MAX = 60_999; +const BROWSER_TAKEOVER_SCRIPT = "/opt/cheatcode/start-browser-takeover.sh"; -export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { +export abstract class ProjectSandboxContent extends ProjectSandboxWorkspaceTransition { public downloadProjectArchive(input: ProjectArchiveInput): Promise { return this.downloadProjectArchiveForRpc(input, () => undefined); } @@ -244,6 +251,48 @@ export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { return { token: link.token, url: link.url }; } + public async exposeBrowserTakeover( + input: ProjectBrowserTakeoverInput, + ): Promise { + const parsed = ProjectBrowserTakeoverInputSchema.parse(input); + const browserDriver = await this.processRecord(browserDriverProcessId(parsed.runId)); + if (!browserDriver) { + throw new APIError(409, "conflict_state_invalid", "No live browser session is available", { + hint: "Let Cheatcode open a website before taking over the browser.", + retriable: true, + }); + } + const processId = browserTakeoverProcessId(parsed.runId); + const port = await this.allocateProcessPort({ + maxPort: BROWSER_TAKEOVER_PORT_MAX, + minPort: BROWSER_TAKEOVER_PORT_MIN, + processId, + }); + const password = crypto.randomUUID().replaceAll("-", ""); + await this.startProcess({ + command: ["sh", BROWSER_TAKEOVER_SCRIPT], + env: { TAKEOVER_PASSWORD: password, TAKEOVER_PORT: String(port) }, + keepAliveTimeoutMs: parsed.expiresInSeconds * 1_000, + maxRestarts: 0, + processId, + restartOnFailure: false, + waitForPort: { path: "/vnc.html", port, timeoutMs: 30_000 }, + }); + const id = await this.ensureSandbox(); + const signed = await this.client().getSignedPreviewUrl(id, port, parsed.expiresInSeconds); + const url = noVncSessionUrl(signed.url, password); + return { + expiresAt: new Date(Date.now() + parsed.expiresInSeconds * 1_000).toISOString(), + takeoverId: parsed.takeoverId, + url, + }; + } + + public async stopBrowserTakeover(input: ProjectBrowserTakeoverStopInput): Promise { + const parsed = ProjectBrowserTakeoverStopInputSchema.parse(input); + await this.killProcess({ processId: browserTakeoverProcessId(parsed.runId) }); + } + public async exposeCodeServer(input: ProjectCodeServerInput): Promise<{ expiresAt: string; port: number; @@ -257,9 +306,6 @@ export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { parsed.workspacePath === WORKSPACE_DIR ? await this.ensureCodeServerDisplayFolder(id, parsed.workspacePath) : parsed.workspacePath; - if (parsed.initialFilePath) { - await this.openCodeServerFile(id, parsed.initialFilePath).catch(() => undefined); - } await this.client() .getPreviewLink(id, CODE_SERVER_PORT) .catch(() => undefined); @@ -268,11 +314,12 @@ export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { port: CODE_SERVER_PORT, sandboxId: id, secret: await this.previewSecret(), + useSubdomain: true, }); return { expiresAt: built.expiresAt, port: CODE_SERVER_PORT, - url: codeServerFolderUrl(built.url, displayFolder), + url: codeServerFolderUrl(built.url, displayFolder, parsed.initialFilePath), workspacePath: parsed.workspacePath, }; } @@ -343,20 +390,21 @@ export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { return { running, state: runtime.state }; } - public async cleanupProjectWorkspace(input: ProjectCleanupWorkspaceInput): Promise { - const { workspaceSlug } = ProjectCleanupWorkspaceInputSchema.parse(input); - const id = await this.existingSandboxId(); - if (!id) { - return; - } - const slot = `${APP_PREVIEW_SLOT_PREFIX}${workspaceSlug}`; - const port = (await this.portAllocation()).ports[workspaceSlug]; - await this.deleteProcessRecord(id, slot); - if (port !== undefined) { - await this.deleteProcessesOnPort(id, port, slot); + public cleanupProjectWorkspace(input: ProjectCleanupWorkspaceInput): Promise { + const parsed = ProjectCleanupWorkspaceInputSchema.parse(input); + return this.deleteProjectWorkspace(parsed, () => + this.performProjectWorkspaceCleanup(parsed.workspaceSlug), + ); + } + + private async performProjectWorkspaceCleanup(workspaceSlug: string): Promise { + const id = await this.ensureExistingSandboxStarted(); + await super.killAllProcesses(); + if (id) { + await this.terminateUntrackedSandboxProcesses(id); + await this.removeWorkspaceFolder(id, workspaceSlug); } await this.freeProjectPort(workspaceSlug); - await this.removeWorkspaceFolder(id, workspaceSlug); } private async mobileExpoProxy( @@ -546,35 +594,37 @@ export abstract class ProjectSandboxContent extends ProjectSandboxProcesses { return probe?.exitCode === 0 ? CODE_SERVER_DISPLAY_DIR : workspacePath; } - private async openCodeServerFile(id: string, path: string): Promise { - await this.client().execute(id, { - command: [ - "CODE_SERVER_USER_DATA_DIR=/home/node/.local/share/code-server/user-data", - "CODE_SERVER_EXTENSIONS_DIR=/home/node/.local/share/code-server/extensions", - "code-server", - "--user-data-dir /home/node/.local/share/code-server/user-data", - "--extensions-dir /home/node/.local/share/code-server/extensions", - "--reuse-window", - shellQuote(path), - ">/tmp/cheatcode-code-server-open-file.log 2>&1 || true", - ].join(" "), - timeout: 15, - }); - } - private async removeWorkspaceFolder(id: string, workspaceSlug: string): Promise { - if (!isSingleWorkspaceSegment(workspaceSlug)) { - return; + try { + await this.client().deleteFilePath(id, `${WORKSPACE_DIR}/${workspaceSlug}`, true); + } catch (error) { + throw this.toUpstreamError(error, "Project workspace removal failed."); } - await this.client() - .execute(id, { - command: `rm -rf ${shellQuote(`${WORKSPACE_DIR}/${workspaceSlug}`)}`, - timeout: timeoutSeconds(60_000), - }) - .catch(() => undefined); } } +function browserDriverProcessId(runId: string): string { + return `cheatcode-browser-driver-${safeProcessSuffix(runId)}`; +} + +function browserTakeoverProcessId(runId: string): string { + return `cheatcode-browser-takeover-${safeProcessSuffix(runId)}`; +} + +function safeProcessSuffix(value: string): string { + return value.replaceAll(/[^A-Za-z0-9_-]/g, "-").slice(0, 120); +} + +function noVncSessionUrl(signedUrl: string, password: string): string { + const url = new URL(signedUrl); + url.pathname = `${url.pathname.replace(/\/?$/u, "/")}vnc.html`; + url.searchParams.set("autoconnect", "1"); + url.searchParams.set("password", password); + url.searchParams.set("reconnect", "1"); + url.searchParams.set("resize", "remote"); + return url.toString(); +} + function isDaytonaResponseTooLarge(error: unknown): boolean { return error instanceof DaytonaApiError && error.code === "daytona_response_too_large"; } diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-daytona-identity.ts b/apps/agent-worker/src/durable-objects/project-sandbox-daytona-identity.ts new file mode 100644 index 00000000..bcc509d1 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-daytona-identity.ts @@ -0,0 +1,118 @@ +import type { DaytonaSandbox } from "@cheatcode/tools-code"; + +const APP_LABEL = "cheatcode"; + +export function canonicalSandboxLabels(input: { + sandboxName: string; + snapshot: string; + volumeId: string; + volumeName: string; +}): Record { + return { + app: APP_LABEL, + role: "canonical", + sandboxId: input.sandboxName, + sandboxOwner: input.sandboxName, + snapshot: input.snapshot, + workspaceVolumeId: input.volumeId, + workspaceVolumeName: input.volumeName, + }; +} + +export function candidateSandboxLabels(input: { + sandboxName: string; + snapshot: string; + upgradeId: string; + volumeId: string; + volumeName: string; +}): Record { + return { + app: APP_LABEL, + role: "candidate", + sandboxOwner: input.sandboxName, + snapshot: input.snapshot, + upgradeId: input.upgradeId, + workspaceVolumeId: input.volumeId, + workspaceVolumeName: input.volumeName, + }; +} + +export function retiredSandboxLabels(input: { + sandbox: DaytonaSandbox; + sandboxName: string; + upgradeId: string; +}): Record { + return { + app: APP_LABEL, + role: "retired", + sandboxOwner: input.sandboxName, + snapshot: input.sandbox.snapshot, + upgradeId: input.upgradeId, + ...(input.sandbox.labels["workspaceVolumeId"] + ? { workspaceVolumeId: input.sandbox.labels["workspaceVolumeId"] } + : {}), + ...(input.sandbox.labels["workspaceVolumeName"] + ? { workspaceVolumeName: input.sandbox.labels["workspaceVolumeName"] } + : {}), + }; +} + +export function isCanonicalSandbox(sandbox: DaytonaSandbox, sandboxName: string): boolean { + return sandbox.labels["app"] === APP_LABEL && sandbox.labels["sandboxId"] === sandboxName; +} + +export function isDesiredCanonicalSandbox( + sandbox: DaytonaSandbox, + input: { sandboxName: string; snapshot: string; volumeId?: string; volumeName: string }, +): boolean { + const volumeId = input.volumeId ?? sandbox.labels["workspaceVolumeId"]; + return ( + isCanonicalSandbox(sandbox, input.sandboxName) && + sandbox.labels["role"] === "canonical" && + sandbox.snapshot === input.snapshot && + sandbox.labels["snapshot"] === input.snapshot && + typeof volumeId === "string" && + volumeId.length > 0 && + sandbox.labels["workspaceVolumeId"] === volumeId && + sandbox.labels["workspaceVolumeName"] === input.volumeName && + hasWorkspaceMount(sandbox, volumeId, input.sandboxName) + ); +} + +export function isUpgradeCandidate( + sandbox: DaytonaSandbox, + input: { + sandboxName: string; + snapshot: string; + upgradeId: string; + volumeId: string; + volumeName: string; + }, +): boolean { + return ( + sandbox.labels["app"] === APP_LABEL && + sandbox.labels["role"] === "candidate" && + sandbox.labels["sandboxOwner"] === input.sandboxName && + sandbox.labels["snapshot"] === input.snapshot && + sandbox.labels["upgradeId"] === input.upgradeId && + sandbox.labels["workspaceVolumeId"] === input.volumeId && + sandbox.labels["workspaceVolumeName"] === input.volumeName && + sandbox.snapshot === input.snapshot && + hasWorkspaceMount(sandbox, input.volumeId, input.sandboxName) + ); +} + +function hasWorkspaceMount( + sandbox: DaytonaSandbox, + volumeId: string, + sandboxName: string, +): boolean { + return ( + sandbox.volumes?.some( + (volume) => + volume.volumeId === volumeId && + volume.mountPath === "/workspace" && + volume.subpath === sandboxName, + ) === true + ); +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-files.ts b/apps/agent-worker/src/durable-objects/project-sandbox-files.ts index f5d18818..ef8009c7 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-files.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-files.ts @@ -67,7 +67,7 @@ function toFileEntry(info: DaytonaFileInfo, parentDir: string, root: string): Fi relativePath: relativePath(root, path), type: info.isDir ? "directory" : "file", size: info.size, - modifiedAt: info.modifiedAt ?? info.modTime ?? new Date(0).toISOString(), + modifiedAt: info.modifiedAt, }; } diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-identity-state.ts b/apps/agent-worker/src/durable-objects/project-sandbox-identity-state.ts new file mode 100644 index 00000000..eba364c6 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-identity-state.ts @@ -0,0 +1,74 @@ +import { APIError } from "@cheatcode/observability"; +import { z } from "zod"; + +const SANDBOX_OWNER_USER_ID_KEY = "sandbox_owner_user_id"; +const SANDBOX_NAME_KEY = "sandbox_name"; +const OwnerUserIdSchema = z.string().uuid(); + +export class ProjectSandboxIdentityState { + private cachedOwnerUserId: string | null = null; + private cachedSandboxName: string | undefined; + + constructor(private readonly ctx: DurableObjectState) {} + + public async initialize(): Promise { + const fromId = this.ctx.id.name; + const [storedName, storedOwnerUserId] = await Promise.all([ + this.ctx.storage.get(SANDBOX_NAME_KEY), + this.ctx.storage.get(SANDBOX_OWNER_USER_ID_KEY), + ]); + if (fromId) { + this.cachedSandboxName = fromId; + } else if (typeof storedName === "string") { + this.cachedSandboxName = storedName; + } + if (storedOwnerUserId !== undefined) { + this.cachedOwnerUserId = OwnerUserIdSchema.parse(storedOwnerUserId); + } + } + + public async registerOwner(userId: string, sandboxName?: string): Promise { + const resolvedSandboxName = this.sandboxName(); + if (sandboxName && resolvedSandboxName !== sandboxName) { + throw new APIError(403, "permission_denied", "Sandbox identity mismatch", { + retriable: false, + }); + } + const parsedUserId = OwnerUserIdSchema.parse(userId); + const existingUserId = this.cachedOwnerUserId; + if (existingUserId && existingUserId !== parsedUserId) { + throw new APIError(403, "permission_denied", "Sandbox ownership mismatch", { + retriable: false, + }); + } + if (existingUserId === parsedUserId) { + return; + } + await this.ctx.storage.put({ + [SANDBOX_NAME_KEY]: resolvedSandboxName, + [SANDBOX_OWNER_USER_ID_KEY]: parsedUserId, + }); + this.cachedOwnerUserId = parsedUserId; + this.cachedSandboxName = resolvedSandboxName; + } + + public ownerUserId(): string | null { + return this.cachedOwnerUserId; + } + + public hasRegisteredOwner(): boolean { + return this.cachedOwnerUserId !== null; + } + + public clearRegisteredOwner(): void { + this.cachedOwnerUserId = null; + } + + public sandboxName(): string { + const name = this.cachedSandboxName ?? this.ctx.id.name; + if (!name) { + throw new Error("ProjectSandbox must be addressed with idFromName()."); + } + return name; + } +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle-support.ts b/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle-support.ts new file mode 100644 index 00000000..490cec42 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle-support.ts @@ -0,0 +1,77 @@ +import type { WorkerSecret } from "@cheatcode/env"; +import { APIError } from "@cheatcode/observability"; +import { DaytonaApiError, type DaytonaSandbox } from "@cheatcode/tools-code"; +import { z } from "zod"; + +export interface ProjectSandboxEnv { + CHEATCODE_RELEASE_GATE: "closed" | "draining" | "open"; + CHEATCODE_RELEASE_SHA?: string; + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; + DAYTONA_API_KEY: WorkerSecret; + DAYTONA_API_URL: string; + DAYTONA_TARGET: string; + DAYTONA_SANDBOX_SNAPSHOT: string; + DAYTONA_WORKSPACE_VOLUME: string; + HYPERDRIVE: Hyperdrive; + DAYTONA_ORG_ID?: string; + DAYTONA_PREVIEW_HOST_SUFFIXES?: string; + PREVIEW_TOKEN_SECRET: WorkerSecret; + PREVIEW_HOSTNAME: string; + QUOTA_TRACKER: DurableObjectNamespace; + R2_AUDIT: R2Bucket; +} + +export const ACCOUNT_DELETION_TOMBSTONE_KEY = "account_deletion_tombstone"; +export const DAYTONA_ID_KEY = "daytona_sandbox_id"; +export const RUN_LEASES_KEY = "run_leases"; +export const DEFAULT_IDLE_STOP_MIN = 30; +export const AUTO_ARCHIVE_MIN = 1_440; +export const NEVER_AUTO_DELETE = -1; +export const KEEPALIVE_ALARM_MS = 4 * 60 * 1_000; +export const STALE_RUN_LEASE_MS = 20 * 60 * 1_000; +export const STARTED_REVERIFY_MS = 30_000; +export const ENSURE_STARTED_ATTEMPTS = 30; +export const ENSURE_STARTED_DELAY_MS = 2_000; +export const RunLeasesSchema = z + .array(z.object({ runId: z.string(), startedMs: z.number() }).strict()) + .default([]); + +export function isDaytonaNameConflictError(error: unknown): boolean { + if (!(error instanceof DaytonaApiError) || error.status !== 409) { + return false; + } + const message = error.message.toLowerCase(); + return message.includes("already exists") || message.includes("conflict"); +} + +export function isStartableState(state: string): boolean { + return state === "stopped" || state === "archived"; +} + +export function accountSandboxDeletedError(): APIError { + return new APIError( + 410, + "conflict_state_invalid", + "Sandbox account state is unavailable after deletion started", + { retriable: false }, + ); +} + +export function uniqueSandboxes(sandboxes: DaytonaSandbox[]): DaytonaSandbox[] { + return [...new Map(sandboxes.map((sandbox) => [sandbox.id, sandbox])).values()]; +} + +export function parseSandboxJson(value: string | null | undefined): unknown { + try { + return JSON.parse(value ?? "") as unknown; + } catch { + return null; + } +} + +export function sandboxReleaseGateError(): APIError { + return new APIError(503, "unavailable_maintenance", "Release is in progress", { + details: { releaseGate: "closed", worker: "agent" }, + retriable: true, + }); +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle.ts b/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle.ts index b8dcbb41..a45a446c 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-lifecycle.ts @@ -1,5 +1,5 @@ import { DurableObject } from "cloudflare:workers"; -import { PreviewHostnameSchema, resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; +import { PreviewHostnameSchema, resolveWorkerSecret } from "@cheatcode/env"; import { APIError, createLogger } from "@cheatcode/observability"; import { DaytonaApiError, @@ -9,6 +9,22 @@ import { } from "@cheatcode/tools-code"; import { z } from "zod"; import { type SandboxExecAuditEntry, writeExecAudit } from "./project-sandbox-audit"; +import { ProjectSandboxIdentityState } from "./project-sandbox-identity-state"; +import { + ACCOUNT_DELETION_TOMBSTONE_KEY, + accountSandboxDeletedError, + DAYTONA_ID_KEY, + DEFAULT_IDLE_STOP_MIN, + KEEPALIVE_ALARM_MS, + type ProjectSandboxEnv, + parseSandboxJson, + RUN_LEASES_KEY, + RunLeasesSchema, + STALE_RUN_LEASE_MS, + STARTED_REVERIFY_MS, + sandboxReleaseGateError, + uniqueSandboxes, +} from "./project-sandbox-lifecycle-support"; import { beginSandboxUsageBestEffort, clearSandboxMeterState, @@ -17,45 +33,20 @@ import { type SandboxMeteringContext, setSandboxQuotaPeriod, } from "./project-sandbox-metering"; +import { assertProjectSandboxOwnerActive } from "./project-sandbox-owner-admission"; +import { ProjectSandboxProvisioning } from "./project-sandbox-provisioning"; +import type { + ParsedProjectCleanupWorkspaceInput, + ProjectSandboxRuntimeState, +} from "./project-sandbox-runtime"; +import { clearWorkspaceCommand } from "./project-sandbox-snapshot-scripts"; import { - isDestroyed, - isFailedState, - scrubPersistedProcessEnvironments, - sleep, -} from "./project-sandbox-process-support"; -import type { ProjectSandboxRuntimeState } from "./project-sandbox-runtime"; - -export interface ProjectSandboxEnv { - DAYTONA_API_KEY: WorkerSecret; - DAYTONA_API_URL: string; - DAYTONA_TARGET: string; - DAYTONA_SANDBOX_SNAPSHOT: string; - DAYTONA_ORG_ID?: string; - DAYTONA_PREVIEW_HOST_SUFFIXES?: string; - PREVIEW_TOKEN_SECRET: WorkerSecret; - PREVIEW_HOSTNAME: string; - QUOTA_TRACKER: DurableObjectNamespace; - R2_AUDIT: R2Bucket; -} + initializeProjectSandboxStorage, + openProjectSandboxWorkspaceState, + ProjectSandboxWorkspaceState, +} from "./project-sandbox-workspace-state"; -const SANDBOX_OWNER_USER_ID_KEY = "sandbox_owner_user_id"; -const ACCOUNT_DELETION_TOMBSTONE_KEY = "account_deletion_tombstone"; -const DAYTONA_ID_KEY = "daytona_sandbox_id"; -const RUN_LEASES_KEY = "run_leases"; -const SANDBOX_NAME_KEY = "sandbox_name"; -const DEFAULT_IDLE_STOP_MIN = 30; -const AUTO_ARCHIVE_MIN = 1_440; -const NEVER_AUTO_DELETE = -1; -const KEEPALIVE_ALARM_MS = 4 * 60 * 1_000; -const STALE_RUN_LEASE_MS = 20 * 60 * 1_000; -const STARTED_REVERIFY_MS = 30_000; -const ENSURE_STARTED_ATTEMPTS = 30; -const ENSURE_STARTED_DELAY_MS = 2_000; -const DURABLE_DELETE_BATCH_SIZE = 128; -const OwnerUserIdSchema = z.string().uuid(); -const RunLeasesSchema = z - .array(z.object({ runId: z.string(), startedMs: z.number() }).strict()) - .default([]); +const ClearWorkspaceEvidenceSchema = z.object({ cleared: z.literal(true) }).strict(); export abstract class ProjectSandboxLifecycle extends DurableObject { private accountDeletionCompleted = false; @@ -63,41 +54,34 @@ export abstract class ProjectSandboxLifecycle extends DurableObject | undefined; private activeOperationCount = 0; private readonly activeOperationDrainWaiters = new Set<() => void>(); + private activeWorkspaceTransitionId: string | null = null; private daytonaClient: DaytonaClient | undefined; private daytonaId: string | undefined; private sandboxMutationTail: Promise = Promise.resolve(); private startedVerifiedAtMs = 0; - private cachedSandboxName: string | undefined; - private snapshotDriftLoggedFor: string | undefined; + private readonly identityState: ProjectSandboxIdentityState; + private readonly provisioning: ProjectSandboxProvisioning; + private workspaceStateValue: ProjectSandboxWorkspaceState | undefined; constructor(ctx: DurableObjectState, env: ProjectSandboxEnv) { super(ctx, env); - void ctx.blockConcurrencyWhile(async () => { - const fromId = ctx.id.name; - if ((await ctx.storage.get(ACCOUNT_DELETION_TOMBSTONE_KEY)) === true) { - this.accountDeletionInProgress = true; - this.cachedSandboxName = fromId; - return; - } - const stored = await ctx.storage.get(SANDBOX_NAME_KEY); - if (fromId) { - this.cachedSandboxName = fromId; - if (stored !== fromId) { - await ctx.storage.put(SANDBOX_NAME_KEY, fromId); - } - } else if (typeof stored === "string") { - this.cachedSandboxName = stored; - } - await scrubPersistedProcessEnvironments(ctx.storage); + this.identityState = new ProjectSandboxIdentityState(ctx); + this.provisioning = new ProjectSandboxProvisioning({ + cachedSandboxId: async () => this.daytonaId ?? this.storedDaytonaId(), + env, + sandboxName: () => this.sandboxName(), + toUpstreamError: (error, fallback) => this.toUpstreamError(error, fallback), }); + if (env.CHEATCODE_RELEASE_GATE !== "closed") { + this.workspaceStateValue = openProjectSandboxWorkspaceState(ctx); + void ctx.blockConcurrencyWhile(() => this.initializeIdentityState()); + } } - /** - * Atomically fences the user-scoped sandbox before draining in-flight RPCs and - * deleting its external and durable state. The tombstone is intentionally kept - * after cleanup so an evicted object can never recreate the account sandbox. - */ public deleteAccountState(): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } if (this.accountDeletionCompleted) { return Promise.resolve(); } @@ -105,8 +89,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { @@ -118,11 +101,145 @@ export abstract class ProjectSandboxLifecycle extends DurableObject(operation: () => Promise): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + return this.withActiveOperation(null, operation, false, true); + } + protected withActiveOwnerRegistration( + userId: string, + operation: () => Promise, + ): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + if (this.identityState.hasRegisteredOwner()) { + return this.withActiveOperation(null, operation); + } + let release: (() => void) | undefined; + try { + release = this.acquireActiveSandboxOperation(undefined, true); + return assertProjectSandboxOwnerActive(this.env, userId) + .then(() => { + if (this.accountDeletionInProgress) { + throw accountSandboxDeletedError(); + } + return operation().then((result) => { + if (this.accountDeletionInProgress) { + throw accountSandboxDeletedError(); + } + this.ensureWorkspaceState(); + return result; + }); + }) + .finally(release); + } catch (error) { + release?.(); + return Promise.reject(error); + } + } + protected withActiveSharedWorkspaceMutation(operation: () => Promise): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + return this.withActiveOperation( + null, + async () => { + await this.workspaceState.waitForWorkspaceDrain(); + return operation(); + }, + true, + ); + } + protected withActiveWorkspaceTransition( + transitionId: string, + operation: () => Promise, + ): Promise { + if (this.env.CHEATCODE_RELEASE_GATE !== "closed") { + return Promise.reject( + new APIError( + 409, + "conflict_state_invalid", + "Workspace transitions require the closed release gate", + { retriable: false }, + ), + ); + } + return this.initializeIdentityState().then(() => + this.runActiveWorkspaceTransition(transitionId, operation), + ); + } + private runActiveWorkspaceTransition( + transitionId: string, + operation: () => Promise, + ): Promise { + let releaseSandbox: (() => void) | undefined; + let releaseWorkspace: (() => void) | undefined; + try { + const workspaceState = this.openWorkspaceState(); + releaseWorkspace = workspaceState + ? workspaceState.acquireTransitionMutation(transitionId) + : this.acquireTransientWorkspaceTransition(transitionId); + return Promise.all([ + this.waitForActiveSandboxOperations(), + workspaceState?.waitForWorkspaceDrain() ?? Promise.resolve(), + ]) + .then(() => { + releaseSandbox = this.acquireActiveSandboxOperation(transitionId, true); + return operation(); + }) + .finally(() => { + releaseSandbox?.(); + releaseWorkspace?.(); + }); + } catch (error) { + releaseWorkspace?.(); + releaseSandbox?.(); + return Promise.reject(error); + } + } + + private acquireTransientWorkspaceTransition(transitionId: string): () => void { + if (this.activeWorkspaceTransitionId !== null) { + throw new APIError(409, "conflict_state_invalid", "Workspace maintenance is in progress", { + retriable: true, + }); + } + this.activeWorkspaceTransitionId = transitionId; + let isReleased = false; + return () => { + if (isReleased) { + return; + } + isReleased = true; + if (this.activeWorkspaceTransitionId === transitionId) { + this.activeWorkspaceTransitionId = null; + } + }; + } + protected withActiveProjectWorkspaceOperation( + workspaceScope: string | readonly string[] | null, + operation: () => Promise, + ): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + return this.withActiveOperation(workspaceScope, operation, false, true); + } + private withActiveOperation( + workspaceScope: string | readonly string[] | null, + operation: () => Promise, + isSharedMutation = false, + shouldLeaseUnknownWorkspace = false, + ): Promise { let release: (() => void) | undefined; try { - release = this.acquireActiveSandboxOperation(); + release = this.acquireActiveOperation( + workspaceScope, + isSharedMutation, + shouldLeaseUnknownWorkspace, + ); return operation().finally(release); } catch (error) { release?.(); @@ -130,13 +247,22 @@ export abstract class ProjectSandboxLifecycle extends DurableObject void) => Promise, + ): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + return this.withActiveStreamingOperation(workspaceScope, operation); + } + private withActiveStreamingOperation( + workspaceScope: string | readonly string[] | null, operation: (release: () => void) => Promise, ): Promise { let release: (() => void) | undefined; try { - release = this.acquireActiveSandboxOperation(); + release = this.acquireActiveOperation(workspaceScope, false, true); return operation(release).catch((error: unknown) => { release?.(); throw error; @@ -147,26 +273,65 @@ export abstract class ProjectSandboxLifecycle extends DurableObject Promise): Promise { - return this.accountDeletionInProgress + return this.env.CHEATCODE_RELEASE_GATE === "closed" || + this.accountDeletionInProgress || + !this.identityState.hasRegisteredOwner() ? Promise.resolve() : this.withActiveSandboxOperation(operation); } - public async registerOwner(userId: string, sandboxName?: string): Promise { - if (sandboxName && this.cachedSandboxName !== sandboxName) { - this.cachedSandboxName = sandboxName; - await this.ctx.storage.put(SANDBOX_NAME_KEY, sandboxName); - } - const parsedUserId = OwnerUserIdSchema.parse(userId); - const existingUserId = await this.ownerUserId(); - if (existingUserId && existingUserId !== parsedUserId) { - throw new APIError(403, "permission_denied", "Sandbox ownership mismatch", { - retriable: false, - }); + private async initializeIdentityState(): Promise { + const isAccountDeleted = (await this.ctx.storage.get(ACCOUNT_DELETION_TOMBSTONE_KEY)) === true; + if (isAccountDeleted) { + this.accountDeletionInProgress = true; } - await this.ctx.storage.put(SANDBOX_OWNER_USER_ID_KEY, parsedUserId); + await this.identityState.initialize(); + } + + private get workspaceState(): ProjectSandboxWorkspaceState { + return this.ensureWorkspaceState(); + } + + private ensureWorkspaceState(): ProjectSandboxWorkspaceState { + if (!this.workspaceStateValue) { + initializeProjectSandboxStorage(this.ctx); + this.workspaceStateValue = new ProjectSandboxWorkspaceState(this.ctx); + } + return this.workspaceStateValue; + } + + private openWorkspaceState(): ProjectSandboxWorkspaceState | undefined { + this.workspaceStateValue ??= openProjectSandboxWorkspaceState(this.ctx); + return this.workspaceStateValue; + } + + protected deleteProjectWorkspace( + input: ParsedProjectCleanupWorkspaceInput, + cleanup: () => Promise, + ): Promise { + return this.workspaceState.deleteWorkspace(input, cleanup); + } + + protected withActiveProjectWorkspaceCleanup(operation: () => Promise): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return Promise.reject(sandboxReleaseGateError()); + } + let release: (() => void) | undefined; + try { + // Cleanup itself must not take a workspace lease: its durable tombstone + // blocks new work, then it drains every existing lease before killing + // sandbox-wide processes. + release = this.acquireActiveSandboxOperation(undefined, false, true); + return operation().finally(release); + } catch (error) { + release?.(); + return Promise.reject(error); + } + } + + public async registerOwner(userId: string, sandboxName?: string): Promise { + await this.identityState.registerOwner(userId, sandboxName); } public async setQuotaPeriod(periodEndIso: string): Promise { @@ -290,8 +455,6 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { - // Persist the fence before any external call. If the object is evicted during - // Daytona cleanup, its next incarnation still rejects operational RPCs. await this.ctx.storage.put(ACCOUNT_DELETION_TOMBSTONE_KEY, true); await this.waitForActiveSandboxOperations(); await this.withSandboxMutation(async () => { @@ -305,8 +468,33 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { const client = await this.ensureClient(); try { - const sandbox = await this.findExistingSandbox(client); - if (sandbox) { + const canonical = await this.provisioning.findExisting(client); + const owned = await this.provisioning.findOwned(client); + const sandboxes = uniqueSandboxes(canonical ? [canonical, ...owned] : owned); + const volumeSandbox = sandboxes.find( + (sandbox) => sandbox.labels["workspaceVolumeName"] === this.env.DAYTONA_WORKSPACE_VOLUME, + ); + if (volumeSandbox) { + if (!(await this.provisioning.ensureStarted(client, volumeSandbox))) { + throw new APIError(502, "upstream_sandbox_failed", "Daytona workspace disappeared", { + retriable: true, + }); + } + const cleared = await client.execute(volumeSandbox.id, { + command: clearWorkspaceCommand(), + timeout: 480, + }); + if ( + cleared.exitCode !== 0 || + !ClearWorkspaceEvidenceSchema.safeParse(parseSandboxJson(cleared.result)).success + ) { + throw new APIError(502, "upstream_sandbox_failed", "Daytona workspace deletion failed", { + details: { sandboxId: this.sandboxName() }, + retriable: true, + }); + } + } + for (const sandbox of sandboxes) { await client.deleteSandbox(sandbox.id); } } catch (error) { @@ -323,16 +511,9 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { - for (;;) { - const keys = [ - ...(await this.ctx.storage.list({ limit: DURABLE_DELETE_BATCH_SIZE })).keys(), - ].filter((key) => key !== ACCOUNT_DELETION_TOMBSTONE_KEY); - if (keys.length === 0) { - break; - } - await this.ctx.storage.delete(keys); - } - await this.ctx.storage.deleteAlarm(); + await this.ctx.storage.deleteAll(); + this.workspaceStateValue = undefined; + this.identityState.clearRegisteredOwner(); this.clearCachedSandbox(); } @@ -356,10 +537,22 @@ export abstract class ProjectSandboxLifecycle extends DurableObject void { + private acquireActiveSandboxOperation( + transitionId?: string, + allowUnregisteredOwner = false, + allowWorkspaceCleanup = false, + ): () => void { if (this.accountDeletionInProgress) { throw accountSandboxDeletedError(); } + if (!allowUnregisteredOwner && !this.identityState.hasRegisteredOwner()) { + throw accountSandboxDeletedError(); + } + const workspaceState = + allowUnregisteredOwner && !this.identityState.hasRegisteredOwner() + ? this.workspaceStateValue + : this.ensureWorkspaceState(); + workspaceState?.assertOperationAllowed(transitionId, allowWorkspaceCleanup); this.activeOperationCount += 1; let isReleased = false; return () => { @@ -371,6 +564,33 @@ export abstract class ProjectSandboxLifecycle extends DurableObject void { + const releaseSandbox = this.acquireActiveSandboxOperation(); + let releaseWorkspace: (() => void) | undefined; + try { + const workspaceSlugs = + typeof workspaceScope === "string" ? [workspaceScope] : (workspaceScope ?? []); + releaseWorkspace = isSharedMutation + ? this.workspaceState.acquireSharedMutation() + : workspaceSlugs.length > 0 + ? this.workspaceState.acquire(workspaceSlugs) + : shouldLeaseUnknownWorkspace + ? this.workspaceState.acquireUnscoped() + : undefined; + } catch (error) { + releaseSandbox(); + throw error; + } + return () => { + releaseWorkspace?.(); + releaseSandbox(); + }; + } + protected client(): DaytonaClient { if (!this.daytonaClient) { throw new Error("Daytona client accessed before initialization."); @@ -400,6 +620,15 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { return this.withSandboxMutation(async () => { if (this.daytonaId && Date.now() - this.startedVerifiedAtMs < STARTED_REVERIFY_MS) { @@ -409,6 +638,28 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { + return this.withSandboxMutation(async () => { + const client = await this.ensureClient(); + let existing: DaytonaSandbox | null; + try { + existing = await this.provisioning.findExisting(client); + if (!existing) { + return null; + } + if (!(await this.provisioning.ensureStarted(client, existing))) { + return null; + } + } catch (error) { + throw this.toUpstreamError(error, "Daytona sandbox cleanup startup failed."); + } + this.daytonaId = existing.id; + await this.ctx.storage.put(DAYTONA_ID_KEY, existing.id); + this.startedVerifiedAtMs = Date.now(); + return existing.id; + }); + } + private async withSandboxMutation(operation: () => Promise): Promise { const previous = this.sandboxMutationTail; let release = (): void => undefined; @@ -428,13 +679,17 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { const client = await this.ensureClient(); try { - const existing = await this.findExistingSandbox(client); + const existing = await this.provisioning.findExisting(client); if (existing) { this.daytonaId = existing.id; return existing.id; @@ -456,7 +711,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { return { env: this.env, - ownerUserId: await this.ownerUserId(), + ownerUserId: this.identityState.ownerUserId(), sandboxId: this.sandboxName(), storage: this.ctx.storage, }; @@ -477,11 +732,7 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { @@ -502,171 +753,11 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { - const name = this.sandboxName(); - return (await this.findExistingSandbox(client)) ?? this.createSandbox(client, name); - } - - private async createSandbox(client: DaytonaClient, name: string): Promise { - try { - const created = await client.createSandbox({ - name, - snapshot: this.env.DAYTONA_SANDBOX_SNAPSHOT, - target: this.env.DAYTONA_TARGET, - user: "node", - labels: { - app: "cheatcode", - sandboxId: name, - snapshot: this.env.DAYTONA_SANDBOX_SNAPSHOT, - }, - autoStopInterval: DEFAULT_IDLE_STOP_MIN, - autoArchiveInterval: AUTO_ARCHIVE_MIN, - autoDeleteInterval: NEVER_AUTO_DELETE, - }); - this.assertSandboxIdentity(created); - this.observeSnapshotDrift(created); - return created; - } catch (error) { - if (isDaytonaNameConflictError(error)) { - const existing = await this.findExistingSandboxAfterCreateConflict(client, name); - if (existing) { - return existing; - } - } - throw this.toUpstreamError(error, "Daytona sandbox failed to start."); - } - } - - private async findExistingSandboxAfterCreateConflict( - client: DaytonaClient, - name: string, - ): Promise { - const byLabel = await this.findSandboxByLabels(client); - if (byLabel) { - return byLabel; - } - const byName = await client.getSandbox(name); - if (byName && !isDestroyed(byName)) { - this.assertSandboxIdentity(byName); - this.observeSnapshotDrift(byName); - return byName; - } - return null; - } - - private async findExistingSandbox(client: DaytonaClient): Promise { - const cachedId = this.daytonaId ?? (await this.storedDaytonaId()); - if (cachedId) { - const existing = await client.getSandbox(cachedId); - if (existing && !isDestroyed(existing)) { - this.assertSandboxIdentity(existing); - this.observeSnapshotDrift(existing); - return existing; - } - } - return this.findSandboxByLabels(client); - } - - private async findSandboxByLabels(client: DaytonaClient): Promise { - const byLabel = await client.listSandboxesByLabels({ - app: "cheatcode", - sandboxId: this.sandboxName(), - }); - const live = byLabel.filter((sandbox) => !isDestroyed(sandbox)); - if (live.length > 1) { - throw new APIError(409, "conflict_state_invalid", "Multiple active sandboxes found", { - details: { daytonaIds: live.map((sandbox) => sandbox.id), sandboxId: this.sandboxName() }, - hint: "Resolve the duplicate Daytona sandboxes explicitly before retrying.", - retriable: false, - }); - } - const sandbox = live[0] ?? null; - if (sandbox) { - this.assertSandboxIdentity(sandbox); - this.observeSnapshotDrift(sandbox); - } - return sandbox; - } - - private assertSandboxIdentity(sandbox: DaytonaSandbox): void { - const name = this.sandboxName(); - if ( - sandbox.name === name && - sandbox.labels["app"] === "cheatcode" && - sandbox.labels["sandboxId"] === name - ) { - return; - } - throw new APIError(409, "conflict_state_invalid", "Daytona sandbox identity mismatch", { - details: { actualName: sandbox.name, daytonaId: sandbox.id, expectedName: name }, - hint: "Inspect the sandbox labels and durable object binding before retrying.", - retriable: false, - }); - } - - private observeSnapshotDrift(sandbox: DaytonaSandbox): void { - const expected = this.env.DAYTONA_SANDBOX_SNAPSHOT; - if (sandbox.snapshot === expected) { - return; - } - const signature = `${sandbox.id}:${sandbox.snapshot}:${expected}`; - if (this.snapshotDriftLoggedFor === signature) { - return; - } - this.snapshotDriftLoggedFor = signature; - createLogger().warn("sandbox_snapshot_drift", { - actualSnapshot: sandbox.snapshot, - daytonaId: sandbox.id, - expectedSnapshot: expected, - sandboxId: this.sandboxName(), - state: sandbox.state, - }); - } - - private async ensureStarted(client: DaytonaClient, sandbox: DaytonaSandbox): Promise { - if (sandbox.state === "started") { - return; - } - if (sandbox.state === "stopped" || sandbox.state === "archived") { - await client.startSandbox(sandbox.id).catch((error: unknown) => { - throw this.toUpstreamError(error, "Daytona sandbox failed to start."); - }); - } - for (let attempt = 0; attempt < ENSURE_STARTED_ATTEMPTS; attempt += 1) { - const current = await client.getSandbox(sandbox.id); - if (current?.state === "started") { - return; - } - if (current && isFailedState(current.state)) { - throw new APIError( - 502, - "upstream_sandbox_failed", - `Daytona sandbox in state ${current.state}`, - { details: { sandboxId: this.sandboxName(), state: current.state }, retriable: true }, - ); - } - await sleep(ENSURE_STARTED_DELAY_MS); - } - throw new APIError( - 504, - "upstream_sandbox_failed", - "Daytona sandbox did not reach started state", - { - retriable: true, - }, - ); - } - private async storedDaytonaId(): Promise { const value = await this.ctx.storage.get(DAYTONA_ID_KEY); return typeof value === "string" ? value : null; } - private async ownerUserId(): Promise { - const value = await this.ctx.storage.get(SANDBOX_OWNER_USER_ID_KEY); - return typeof value === "string" ? value : null; - } - private async runLeases(): Promise> { return RunLeasesSchema.parse((await this.ctx.storage.get(RUN_LEASES_KEY)) ?? []); } @@ -691,24 +782,6 @@ export abstract class ProjectSandboxLifecycle extends DurableObject { + const parsedUserId = UserId(userId); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const isActive = await withUserContext(db, parsedUserId, (transaction) => + isUserAccountActive(transaction, parsedUserId), + ); + if (!isActive) { + throw accountSandboxDeletedError(); + } + } finally { + await close(); + } +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-preview.ts b/apps/agent-worker/src/durable-objects/project-sandbox-preview.ts index 308bd7a3..57c3802b 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-preview.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-preview.ts @@ -15,6 +15,9 @@ export interface BuildPreviewUrlInput { // Mobile (Expo web) previews use the clean-subdomain URL form because Expo Router derives its // route from window.location and each project runs Metro on its own port. isMobile?: boolean; + // Embedded services such as Code Server exchange messages with their parent. + // Give them their final origin up front instead of a temporary local handoff origin. + useSubdomain?: boolean; } export interface BuiltPreviewUrl { @@ -40,9 +43,10 @@ export async function buildPreviewUrl(input: BuildPreviewUrlInput): Promise` path prefix // yields "Unmatched Route". The local proxy routes this by Host, matching prod's subdomain // routing. Web (Next.js) previews keep the path form, which they tolerate. - const url = input.isMobile - ? `http://${host}${path}` - : `http://${hostname}/__sandbox/${encodePreviewHost(host)}${path}`; + const url = + input.isMobile || input.useSubdomain + ? `http://${host}${path}` + : `http://${hostname}/__sandbox/${encodePreviewHost(host)}${path}`; return { expiresAt: new Date(capability.expiresAt).toISOString(), url }; } const url = `https://${host}${path}`; diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-process-cleanup.ts b/apps/agent-worker/src/durable-objects/project-sandbox-process-cleanup.ts new file mode 100644 index 00000000..60ddb39a --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-process-cleanup.ts @@ -0,0 +1,143 @@ +/** Kills every sandbox process whose real cwd is the exact project directory or a child. */ +export const WORKSPACE_PROCESS_TERMINATION_SCRIPT = ` +import os +import signal +import stat +import sys +import time + +root = sys.argv[1] +try: + metadata = os.lstat(root) +except FileNotFoundError: + metadata = None +if metadata is not None and stat.S_ISLNK(metadata.st_mode): + print("Project workspace is a symbolic link", file=sys.stderr) + raise SystemExit(2) + +root = os.path.realpath(root) if metadata is not None else os.path.abspath(root) +prefix = root + os.sep +self_pid = os.getpid() +self_uid = os.getuid() + +def matching_pids(): + matches = [] + for name in os.listdir("/proc"): + if not name.isdigit(): + continue + pid = int(name) + if pid == self_pid: + continue + try: + if os.stat(f"/proc/{pid}").st_uid != self_uid: + continue + cwd = os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, ProcessLookupError): + continue + except OSError as error: + raise RuntimeError(f"Could not inspect cwd for process {pid}: {error}") from error + if cwd.endswith(" (deleted)"): + cwd = cwd.removesuffix(" (deleted)") + if cwd == root or cwd.startswith(prefix): + matches.append(pid) + return matches + +def send(pids, sig): + denied = [] + for pid in pids: + try: + os.kill(pid, sig) + except ProcessLookupError: + continue + except PermissionError: + denied.append(pid) + return denied + +denied = send(matching_pids(), signal.SIGTERM) +deadline = time.monotonic() + 3 +remaining = matching_pids() +while remaining and time.monotonic() < deadline: + time.sleep(0.1) + remaining = matching_pids() +denied.extend(send(remaining, signal.SIGKILL)) +time.sleep(0.1) +survivors = matching_pids() +if denied or survivors: + print(f"Could not terminate workspace processes: denied={denied} survivors={survivors}", file=sys.stderr) + raise SystemExit(1) +`; + +/** Kills every same-user sandbox process except the cleanup command's control-plane ancestry. */ +export const SANDBOX_PROCESS_TERMINATION_SCRIPT = ` +import os +import signal +import sys +import time + +self_pid = os.getpid() +self_uid = os.getuid() + +def process_fields(pid): + try: + with open(f"/proc/{pid}/stat", "r", encoding="utf-8") as handle: + fields = handle.read().rsplit(")", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): + return None + if len(fields) < 2: + raise RuntimeError(f"Could not parse process metadata for {pid}") + return fields + +def parent_pid(pid): + fields = process_fields(pid) + return 0 if fields is None else int(fields[1]) + +protected = {self_pid} +ancestor = self_pid +while ancestor > 1: + ancestor = parent_pid(ancestor) + if ancestor <= 0: + break + protected.add(ancestor) + +def matching_pids(): + matches = [] + for name in os.listdir("/proc"): + if not name.isdigit(): + continue + pid = int(name) + if pid in protected: + continue + try: + fields = process_fields(pid) + if fields is not None and fields[0] != "Z" and os.stat(f"/proc/{pid}").st_uid == self_uid: + matches.append(pid) + except (FileNotFoundError, ProcessLookupError): + continue + except OSError as error: + raise RuntimeError(f"Could not inspect process {pid}: {error}") from error + return matches + +def send(pids, sig): + denied = [] + for pid in pids: + try: + os.kill(pid, sig) + except ProcessLookupError: + continue + except PermissionError: + denied.append(pid) + return denied + +denied = send(matching_pids(), signal.SIGTERM) +deadline = time.monotonic() + 3 +remaining = matching_pids() +while remaining and time.monotonic() < deadline: + time.sleep(0.1) + remaining = matching_pids() +denied.extend(send(remaining, signal.SIGKILL)) +time.sleep(0.1) +survivors = matching_pids() +if denied or survivors: + print(f"Could not terminate sandbox processes: denied={denied} survivors={survivors}", file=sys.stderr) + raise SystemExit(1) +`; diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-process-support.ts b/apps/agent-worker/src/durable-objects/project-sandbox-process-support.ts index b718d1ac..177b9e03 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-process-support.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-process-support.ts @@ -1,5 +1,5 @@ import { APIError } from "@cheatcode/observability"; -import type { DaytonaSandbox } from "@cheatcode/tools-code"; +import { type DaytonaSandbox, WorkspacePathSchema } from "@cheatcode/tools-code"; import { z } from "zod"; import type { ProjectStartProcessInputSchema } from "./project-sandbox-runtime"; @@ -10,11 +10,16 @@ export const PORT_ALLOC_KEY = "port_alloc"; export const PROCESS_PORT_ALLOC_KEY = "process_port_alloc"; const PROCESS_PORT_RESERVATION_TTL_MS = 6 * 60 * 60 * 1_000; export const PROC_PREFIX = "proc:"; +export const MAX_TRACKED_PROCESSES = 32; const WEB_PORT_BASE = 5173; export const ProcessRecordSchema = z .object({ - sessionId: z.string(), + sessionId: z + .string() + .min(1) + .max(250) + .regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u), cmdId: z.string(), command: z.string(), port: z.number().optional(), @@ -22,7 +27,7 @@ export const ProcessRecordSchema = z keepAliveTimeoutMs: z.number().int().nonnegative().optional(), maxRestarts: z.number().int().nonnegative().optional(), restartOnFailure: z.boolean().optional(), - cwd: z.string(), + cwd: WorkspacePathSchema, startedAtMs: z.number().int().nonnegative().optional(), }) .strict(); @@ -34,6 +39,29 @@ export type ProcessPolicy = Pick< "keepAliveTimeoutMs" | "maxRestarts" | "restartOnFailure" >; +export class ProcessMutationQueue { + private tail: Promise = Promise.resolve(); + + public async run(operation: () => Promise): Promise { + const previous = this.tail; + let release = (): void => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + const queued = previous.catch(() => undefined).then(() => gate); + this.tail = queued; + await previous.catch(() => undefined); + try { + return await operation(); + } finally { + release(); + if (this.tail === queued) { + this.tail = Promise.resolve(); + } + } + } +} + export const PortAllocationSchema = z .object({ webNext: z.number().int().positive().default(WEB_PORT_BASE), @@ -41,7 +69,6 @@ export const PortAllocationSchema = z ports: z.record(z.string(), z.number().int().positive()).default({}), }) .strict(); -export type PortAllocation = z.infer; export const ProcessPortReservationsSchema = z .record( @@ -61,14 +88,6 @@ export function timeoutSeconds(timeoutMs: number | undefined): number { } export function assertValidProcessStart(input: ParsedProcessStartInput): void { - if (input.waitForPort && !input.processId) { - throw new APIError( - 400, - "invalid_request_body", - "Port-bound processes require a stable process ID.", - { retriable: false }, - ); - } if ((input.maxRestarts ?? 0) > 0 && input.restartOnFailure !== true) { throw new APIError(400, "invalid_request_body", "maxRestarts requires restartOnFailure.", { retriable: false, @@ -208,24 +227,6 @@ export function shellQuote(arg: string): string { return `'${arg.replaceAll("'", "'\\''")}'`; } -export async function scrubPersistedProcessEnvironments( - storage: DurableObjectStorage, -): Promise { - const records = await storage.list({ prefix: PROC_PREFIX }); - for (const [key, value] of records) { - if (typeof value !== "object" || value === null || !("env" in value)) { - continue; - } - const { env: _removedEnvironment, ...scrubbed } = value as Record; - const parsed = ProcessRecordSchema.safeParse(scrubbed); - if (parsed.success) { - await storage.put(key, parsed.data); - } else { - await storage.delete(key); - } - } -} - export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-processes.ts b/apps/agent-worker/src/durable-objects/project-sandbox-processes.ts index e24ffa45..1189233d 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-processes.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-processes.ts @@ -11,20 +11,24 @@ import { sandboxExecProcessName } from "./project-sandbox-audit"; import { WORKSPACE_DIR } from "./project-sandbox-content-support"; import { ProjectSandboxLifecycle } from "./project-sandbox-lifecycle"; import { recordSandboxUsageBestEffort } from "./project-sandbox-metering"; +import { + SANDBOX_PROCESS_TERMINATION_SCRIPT, + WORKSPACE_PROCESS_TERMINATION_SCRIPT, +} from "./project-sandbox-process-cleanup"; import { emptyConsoleSnapshot, sliceProcessLogs } from "./project-sandbox-process-logs"; import { assertValidProcessStart, ENV_FILE_DIR, firstAvailablePort, + MAX_TRACKED_PROCESSES, type NamedProcessRecord, type ParsedProcessStartInput, PORT_ALLOC_KEY, - type PortAllocation, PortAllocationSchema, PROC_PREFIX, PROCESS_PORT_ALLOC_KEY, + ProcessMutationQueue, type ProcessPolicy, - type ProcessPortReservations, ProcessPortReservationsSchema, type ProcessRecord, ProcessRecordSchema, @@ -46,6 +50,8 @@ import { ProjectAllocateProcessPortInputSchema, type ProjectExecInput, ProjectExecInputSchema, + type ProjectGetPortInput, + ProjectGetPortInputSchema, type ProjectKillProcessInput, ProjectKillProcessInputSchema, type ProjectReadDevServerLogsInput, @@ -55,6 +61,7 @@ import { type ProjectStartProcessInput, ProjectStartProcessInputSchema, } from "./project-sandbox-runtime"; +import { writeSandboxRuntimeManifest } from "./project-sandbox-runtime-manifest"; const DEFAULT_EXEC_TIMEOUT_MS = 60_000; @@ -65,11 +72,15 @@ interface ProjectSandboxStatus { } export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { + private readonly processMutations = new ProcessMutationQueue(); + public async ensureReady(): Promise { const result = await this.runCode({ code: "print('ready')", language: "python" }); + const id = await this.existingSandboxId(); + if (id) await this.syncRuntimeManifestBestEffort(id); return { - healthy: result.success === true, - ping: result.stdout?.trim() ?? "", + healthy: result.success, + ping: result.stdout.trim(), sandboxId: this.sandboxName(), }; } @@ -148,10 +159,16 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { } public async startProcess(input: ProjectStartProcessInput): Promise { + return this.processMutations.run(() => this.startProcessExclusive(input)); + } + + private async startProcessExclusive( + input: ProjectStartProcessInput, + ): Promise { const parsed = ProjectStartProcessInputSchema.parse(input); assertValidProcessStart(parsed); const id = await this.ensureSandbox(); - const name = parsed.processId ?? `process-${crypto.randomUUID()}`; + const name = parsed.processId; const sessionId = `cc-${name}`; await this.prepareProcessSlot(id, name, parsed); const cwd = parsed.cwd ?? WORKSPACE_DIR; @@ -161,78 +178,111 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { maxRestarts: parsed.maxRestarts ?? 0, restartOnFailure: parsed.restartOnFailure ?? false, }; - const exec = await this.launchSessionProcess( - id, - sessionId, - name, - cwd, - supervisedProcessCommand(rawCommand, policy), - parsed.env, - parsed.stdin, - ); - const record = processRecordFromLaunch(parsed, policy, { - cmdId: exec.cmdId ?? sessionId, + const provisionalRecord = processRecordFromLaunch(parsed, policy, { + cmdId: sessionId, command: rawCommand, cwd, sessionId, }); + await this.persistProcessOwnershipIntent(name, provisionalRecord); + let exec: DaytonaSessionExecResponse; + try { + exec = await this.launchSessionProcess( + id, + sessionId, + name, + cwd, + supervisedProcessCommand(rawCommand, policy), + parsed.env, + parsed.stdin, + ); + } catch (error) { + await this.cleanupLaunchedProcess(id, sessionId, name); + await this.syncRuntimeManifestBestEffort(id); + throw error; + } + const record = { ...provisionalRecord, cmdId: exec.cmdId ?? sessionId }; await this.persistStartedProcess(id, name, record, parsed.waitForPort); + await this.syncRuntimeManifestBestEffort(id); await recordSandboxUsageBestEffort(await this.meteringContext()); return { command: record.command, id: name, status: "running" }; } public async allocateProjectPort(input: ProjectAllocatePortInput): Promise { const parsed = ProjectAllocatePortInputSchema.parse(input); - const allocation = await this.portAllocation(); - const existing = allocation.ports[parsed.projectId]; - if (existing !== undefined) { - return existing; - } - const used = new Set(Object.values(allocation.ports)); - let candidate = parsed.stack === "mobile" ? allocation.mobileNext : allocation.webNext; - while (used.has(candidate)) { - candidate += 1; - } - allocation.ports[parsed.projectId] = candidate; - if (parsed.stack === "mobile") { - allocation.mobileNext = candidate + 1; - } else { - allocation.webNext = candidate + 1; - } - await this.ctx.storage.put(PORT_ALLOC_KEY, allocation); - return candidate; + return this.ctx.storage.transaction(async (transaction) => { + const allocation = PortAllocationSchema.parse((await transaction.get(PORT_ALLOC_KEY)) ?? {}); + const existing = allocation.ports[parsed.projectId]; + if (existing !== undefined) { + return existing; + } + const used = new Set(Object.values(allocation.ports)); + let candidate = parsed.stack === "mobile" ? allocation.mobileNext : allocation.webNext; + while (used.has(candidate)) { + candidate += 1; + } + allocation.ports[parsed.projectId] = candidate; + if (parsed.stack === "mobile") { + allocation.mobileNext = candidate + 1; + } else { + allocation.webNext = candidate + 1; + } + await transaction.put(PORT_ALLOC_KEY, allocation); + return candidate; + }); + } + + public async getProjectPort(input: ProjectGetPortInput): Promise { + const parsed = ProjectGetPortInputSchema.parse(input); + const allocation = PortAllocationSchema.parse( + (await this.ctx.storage.get(PORT_ALLOC_KEY)) ?? {}, + ); + return allocation.ports[parsed.projectId] ?? null; } public async allocateProcessPort(input: ProjectAllocateProcessPortInput): Promise { const parsed = ProjectAllocateProcessPortInputSchema.parse(input); - const now = Date.now(); - let reservations = await this.processPortReservations(); - const records = await this.ctx.storage.list({ prefix: PROC_PREFIX }); - reservations = pruneExpiredProcessPortReservations(reservations, records, now); - const used = usedProcessPorts(reservations, records, parsed.processId); - const existing = reservations[parsed.processId]; - if ( - existing && - existing.port >= parsed.minPort && - existing.port <= parsed.maxPort && - !used.has(existing.port) - ) { - reservations[parsed.processId] = { ...existing, reservedAtMs: now }; - await this.ctx.storage.put(PROCESS_PORT_ALLOC_KEY, reservations); - return existing.port; - } - const port = firstAvailablePort(used, parsed.minPort, parsed.maxPort); - if (port === null) { - throw new APIError(503, "sandbox_failed_to_start", "No sandbox process port is available.", { - retriable: true, - }); - } - reservations[parsed.processId] = { port, reservedAtMs: now }; - await this.ctx.storage.put(PROCESS_PORT_ALLOC_KEY, reservations); - return port; + return this.ctx.storage.transaction(async (transaction) => { + const now = Date.now(); + let reservations = ProcessPortReservationsSchema.parse( + (await transaction.get(PROCESS_PORT_ALLOC_KEY)) ?? {}, + ); + const records = await transaction.list({ prefix: PROC_PREFIX }); + reservations = pruneExpiredProcessPortReservations(reservations, records, now); + const used = usedProcessPorts(reservations, records, parsed.processId); + const existing = reservations[parsed.processId]; + if ( + existing && + existing.port >= parsed.minPort && + existing.port <= parsed.maxPort && + !used.has(existing.port) + ) { + reservations[parsed.processId] = { ...existing, reservedAtMs: now }; + await transaction.put(PROCESS_PORT_ALLOC_KEY, reservations); + return existing.port; + } + const port = firstAvailablePort(used, parsed.minPort, parsed.maxPort); + if (port === null) { + throw new APIError( + 503, + "sandbox_failed_to_start", + "No sandbox process port is available.", + { + retriable: true, + }, + ); + } + reservations[parsed.processId] = { port, reservedAtMs: now }; + await transaction.put(PROCESS_PORT_ALLOC_KEY, reservations); + return port; + }); } public async killAllProcesses(): Promise { + return this.processMutations.run(() => this.killAllProcessesExclusive()); + } + + private async killAllProcessesExclusive(): Promise { const id = await this.existingSandboxId(); const records = await this.ctx.storage.list({ prefix: PROC_PREFIX }); let killed = 0; @@ -245,21 +295,30 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { await this.ctx.storage.delete(key); } } + if (id) { + await this.client().deleteFilePath(id, ENV_FILE_DIR, true); + } await this.ctx.storage.delete(PROCESS_PORT_ALLOC_KEY); + if (id) await this.syncRuntimeManifestBestEffort(id); return killed; } public async killProcess(input: ProjectKillProcessInput): Promise { const parsed = ProjectKillProcessInputSchema.parse(input); - const record = await this.processRecord(parsed.processId); + return this.processMutations.run(() => this.killProcessExclusive(parsed.processId)); + } + + private async killProcessExclusive(processId: string): Promise { + const record = await this.processRecord(processId); const id = record ? await this.existingSandboxId() : null; if (record && id) { - await this.deleteProcessRecord(id, parsed.processId); + await this.deleteProcessRecord(id, processId); } else { - await this.ctx.storage.delete(`${PROC_PREFIX}${parsed.processId}`); - await this.releaseProcessPort(parsed.processId); + await this.ctx.storage.delete(`${PROC_PREFIX}${processId}`); + await this.releaseProcessPort(processId); } - return { processId: parsed.processId, status: "killed", success: true }; + if (id) await this.syncRuntimeManifestBestEffort(id); + return { processId, status: "killed", success: true }; } public async readDevServerLogs( @@ -276,6 +335,7 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { if (isMissingDaytonaProcessError(error)) { await this.ctx.storage.delete(`${PROC_PREFIX}${process.name}`); await this.releaseProcessPort(process.name); + await this.syncRuntimeManifestBestEffort(id); return null; } throw this.toUpstreamError(error, "Sandbox console read failed."); @@ -304,9 +364,7 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { restartEnv?: Record, ): Promise { const sessionId = record.sessionId || `cc-${name}`; - await this.client() - .deleteSession(id, sessionId) - .catch(() => undefined); + await this.client().deleteSession(id, sessionId); const exec = await this.launchSessionProcess( id, sessionId, @@ -321,8 +379,10 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { cmdId: exec.cmdId ?? sessionId, startedAtMs: Date.now(), } satisfies ProcessRecord); + await this.syncRuntimeManifestBestEffort(id); } catch (error) { await this.cleanupLaunchedProcess(id, sessionId, name); + await this.syncRuntimeManifestBestEffort(id); throw error; } } @@ -394,15 +454,63 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { ): Promise { const record = await this.processRecord(name); if (record) { + await this.client().deleteSession(id, record.sessionId); await this.deleteSessionEnvironment(id, record.sessionId); - await this.client() - .deleteSession(id, record.sessionId) - .catch(() => undefined); } - await this.ctx.storage.delete(`${PROC_PREFIX}${name}`); if (!keepPortReservation) { await this.releaseProcessPort(name); } + await this.ctx.storage.delete(`${PROC_PREFIX}${name}`); + } + + protected async terminateUntrackedWorkspaceProcesses( + id: string, + workspaceSlug: string, + ): Promise { + const workspacePath = `${WORKSPACE_DIR}/${workspaceSlug}`; + const result = await this.client() + .execute(id, { + command: `python3 -c ${shellQuote(WORKSPACE_PROCESS_TERMINATION_SCRIPT)} ${shellQuote(workspacePath)}`, + cwd: WORKSPACE_DIR, + timeout: timeoutSeconds(15_000), + }) + .catch((error: unknown) => { + throw this.toUpstreamError(error, "Project workspace process termination failed."); + }); + if (result.exitCode !== 0) { + throw new APIError( + 502, + "upstream_sandbox_failed", + "Project workspace processes could not be terminated.", + { + details: { output: (result.result ?? "").slice(-1_000), workspaceSlug }, + retriable: true, + }, + ); + } + } + + protected async terminateUntrackedSandboxProcesses(id: string): Promise { + const result = await this.client() + .execute(id, { + command: `python3 -c ${shellQuote(SANDBOX_PROCESS_TERMINATION_SCRIPT)}`, + cwd: WORKSPACE_DIR, + timeout: timeoutSeconds(15_000), + }) + .catch((error: unknown) => { + throw this.toUpstreamError(error, "Sandbox process termination failed."); + }); + if (result.exitCode !== 0) { + throw new APIError( + 502, + "upstream_sandbox_failed", + "Sandbox processes could not be terminated.", + { + details: { output: (result.result ?? "").slice(-1_000) }, + retriable: true, + }, + ); + } } protected async deleteProcessesOnPort( @@ -421,14 +529,16 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { } protected async freeProjectPort(workspaceSlug: string): Promise { - const allocation = await this.portAllocation(); - if (allocation.ports[workspaceSlug] === undefined) { - return; - } - const ports = Object.fromEntries( - Object.entries(allocation.ports).filter(([slug]) => slug !== workspaceSlug), - ); - await this.ctx.storage.put(PORT_ALLOC_KEY, { ...allocation, ports }); + await this.ctx.storage.transaction(async (transaction) => { + const allocation = PortAllocationSchema.parse((await transaction.get(PORT_ALLOC_KEY)) ?? {}); + if (allocation.ports[workspaceSlug] === undefined) { + return; + } + const ports = Object.fromEntries( + Object.entries(allocation.ports).filter(([slug]) => slug !== workspaceSlug), + ); + await transaction.put(PORT_ALLOC_KEY, { ...allocation, ports }); + }); } private async prepareProcessSlot( @@ -436,15 +546,51 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { name: string, input: ParsedProcessStartInput, ): Promise { - if (!input.processId && !input.waitForPort) { - return; - } await this.deleteProcessRecord(id, name, true); + await this.ensureProcessCapacity(id); if (input.waitForPort) { await this.deleteProcessesOnPort(id, input.waitForPort.port, name); } } + private async ensureProcessCapacity(id: string): Promise { + let records = await this.ctx.storage.list({ prefix: PROC_PREFIX }); + if (records.size < MAX_TRACKED_PROCESSES) { + return; + } + await this.pruneCompletedProcessRecords(id, records); + records = await this.ctx.storage.list({ prefix: PROC_PREFIX }); + if (records.size >= MAX_TRACKED_PROCESSES) { + throw new APIError( + 429, + "sandbox_process_limit_reached", + "The sandbox has no available managed process slot.", + { + hint: "Stop an existing managed process or reuse its stable process ID.", + retriable: false, + }, + ); + } + } + + private async pruneCompletedProcessRecords( + id: string, + records: Map, + ): Promise { + for (const [key, value] of records) { + const parsed = ProcessRecordSchema.safeParse(value); + if (!parsed.success) { + await this.deleteProcessRecord(id, key.slice(PROC_PREFIX.length)); + continue; + } + const session = await this.client().getSession(id, parsed.data.sessionId); + const command = session?.commands.find((candidate) => candidate.id === parsed.data.cmdId); + if (session === null || typeof command?.exitCode === "number") { + await this.deleteProcessRecord(id, key.slice(PROC_PREFIX.length)); + } + } + } + private async persistStartedProcess( id: string, name: string, @@ -465,6 +611,15 @@ export abstract class ProjectSandboxProcesses extends ProjectSandboxLifecycle { } } + private async persistProcessOwnershipIntent(name: string, record: ProcessRecord): Promise { + try { + await this.ctx.storage.put(`${PROC_PREFIX}${name}`, record); + } catch (error) { + await this.releaseProcessPort(name); + throw error; + } + } + private async buildSessionCommand( id: string, sessionId: string, @@ -518,28 +673,32 @@ cd ${shellQuote(cwd)} && ${rawCommand}`; } return execution; } catch (error) { + await this.client().deleteSession(id, sessionId); await this.deleteSessionEnvironment(id, sessionId); - await this.client() - .deleteSession(id, sessionId) - .catch(() => undefined); await this.releaseProcessPort(name); throw error; } } private async cleanupLaunchedProcess(id: string, sessionId: string, name: string): Promise { + await this.client().deleteSession(id, sessionId); await this.deleteSessionEnvironment(id, sessionId); - await this.client() - .deleteSession(id, sessionId) - .catch(() => undefined); - await this.ctx.storage.delete(`${PROC_PREFIX}${name}`); await this.releaseProcessPort(name); + await this.ctx.storage.delete(`${PROC_PREFIX}${name}`); + } + + private async syncRuntimeManifestBestEffort(id: string): Promise { + const records = await this.ctx.storage.list({ prefix: PROC_PREFIX }); + await writeSandboxRuntimeManifest(this.client(), id, records).catch((error: unknown) => { + createLogger().warn("sandbox_runtime_manifest_sync_failed", { + error, + sandboxId: this.sandboxName(), + }); + }); } private async deleteSessionEnvironment(id: string, sessionId: string): Promise { - await this.client() - .deleteFilePath(id, `${ENV_FILE_DIR}/${sessionId}.env`, false) - .catch(() => undefined); + await this.client().deleteFilePath(id, `${ENV_FILE_DIR}/${sessionId}.env`, false); } private async throwIfProcessExited( @@ -599,25 +758,19 @@ cd ${shellQuote(cwd)} && ${rawCommand}`; return exact ? [{ name, record: exact }] : []; } - protected async portAllocation(): Promise { - return PortAllocationSchema.parse((await this.ctx.storage.get(PORT_ALLOC_KEY)) ?? {}); - } - - private async processPortReservations(): Promise { - return ProcessPortReservationsSchema.parse( - (await this.ctx.storage.get(PROCESS_PORT_ALLOC_KEY)) ?? {}, - ); - } - private async releaseProcessPort(processId: string): Promise { - const reservations = await this.processPortReservations(); - if (reservations[processId] === undefined) { - return; - } - await this.ctx.storage.put( - PROCESS_PORT_ALLOC_KEY, - withoutProcessReservation(reservations, processId), - ); + await this.ctx.storage.transaction(async (transaction) => { + const reservations = ProcessPortReservationsSchema.parse( + (await transaction.get(PROCESS_PORT_ALLOC_KEY)) ?? {}, + ); + if (reservations[processId] === undefined) { + return; + } + await transaction.put( + PROCESS_PORT_ALLOC_KEY, + withoutProcessReservation(reservations, processId), + ); + }); } } diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-provisioning.ts b/apps/agent-worker/src/durable-objects/project-sandbox-provisioning.ts new file mode 100644 index 00000000..0e6065e1 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-provisioning.ts @@ -0,0 +1,277 @@ +import { APIError } from "@cheatcode/observability"; +import { + DaytonaApiError, + type DaytonaClient, + type DaytonaSandbox, + type DaytonaVolume, +} from "@cheatcode/tools-code"; +import { + canonicalSandboxLabels, + isCanonicalSandbox, + isDesiredCanonicalSandbox, +} from "./project-sandbox-daytona-identity"; +import { + AUTO_ARCHIVE_MIN, + DEFAULT_IDLE_STOP_MIN, + ENSURE_STARTED_ATTEMPTS, + ENSURE_STARTED_DELAY_MS, + isDaytonaNameConflictError, + isStartableState, + NEVER_AUTO_DELETE, + type ProjectSandboxEnv, +} from "./project-sandbox-lifecycle-support"; +import { isDestroyed, isFailedState, sleep } from "./project-sandbox-process-support"; + +const WORKSPACE_MOUNT_PATH = "/workspace"; +const VOLUME_READY_ATTEMPTS = 60; +const VOLUME_READY_DELAY_MS = 2_000; + +interface ProjectSandboxProvisioningInput { + cachedSandboxId: () => Promise; + env: ProjectSandboxEnv; + sandboxName: () => string; + toUpstreamError: (error: unknown, fallback: string) => APIError; +} + +/** Resolves one canonical Daytona sandbox and brings it to a started state. */ +export class ProjectSandboxProvisioning { + public constructor(private readonly input: ProjectSandboxProvisioningInput) {} + + public async resolve(client: DaytonaClient): Promise { + const name = this.input.sandboxName(); + const resolved = (await this.findExisting(client)) ?? (await this.create(client, name)); + if (this.isDesired(resolved)) { + return resolved; + } + throw new APIError( + 503, + "unavailable_maintenance", + "Sandbox requires release-scoped snapshot reconciliation", + { + details: { + actualSnapshot: resolved.snapshot, + expectedSnapshot: this.input.env.DAYTONA_SANDBOX_SNAPSHOT, + sandboxId: name, + }, + retriable: false, + }, + ); + } + + public async findExisting(client: DaytonaClient): Promise { + const cachedId = await this.input.cachedSandboxId(); + if (cachedId) { + const existing = await client.getSandbox(cachedId); + if ( + existing && + !isDestroyed(existing) && + isCanonicalSandbox(existing, this.input.sandboxName()) + ) { + return existing; + } + } + return this.findByLabels(client); + } + + public async findOwned(client: DaytonaClient): Promise { + const sandboxName = this.input.sandboxName(); + const owned = await client.listSandboxesByLabels({ + app: "cheatcode", + sandboxOwner: sandboxName, + }); + return owned.filter( + (sandbox) => !isDestroyed(sandbox) && sandbox.labels["sandboxOwner"] === sandboxName, + ); + } + + public async ensureStarted(client: DaytonaClient, sandbox: DaytonaSandbox): Promise { + if (sandbox.state === "started") { + return true; + } + let hasRequestedStart = await this.startIfPossible(client, sandbox); + for (let attempt = 0; attempt < ENSURE_STARTED_ATTEMPTS; attempt += 1) { + const current = await client.getSandbox(sandbox.id); + if (!current) { + return false; + } + if (current.state === "started") { + return true; + } + if (!hasRequestedStart) { + hasRequestedStart = await this.startIfPossible(client, current); + } + if (isFailedState(current.state)) { + throw new APIError( + 502, + "upstream_sandbox_failed", + `Daytona sandbox in state ${current.state}`, + { + details: { sandboxId: this.input.sandboxName(), state: current.state }, + retriable: true, + }, + ); + } + await sleep(ENSURE_STARTED_DELAY_MS); + } + throw new APIError( + 504, + "upstream_sandbox_failed", + "Daytona sandbox did not reach started state", + { retriable: true }, + ); + } + + public async ensureWorkspaceVolume(client: DaytonaClient): Promise { + const name = this.input.env.DAYTONA_WORKSPACE_VOLUME; + let volume = await client.getVolumeByName(name); + if (!volume) { + try { + volume = await client.createVolume(name); + } catch (error) { + if (!(error instanceof DaytonaApiError) || error.status !== 409) { + throw this.input.toUpstreamError(error, "Daytona workspace volume creation failed."); + } + volume = await client.getVolumeByName(name); + } + } + if (!volume) { + throw new APIError(502, "upstream_sandbox_failed", "Daytona workspace volume disappeared", { + retriable: true, + }); + } + return this.waitForReadyVolume(client, volume); + } + + public isDesired(sandbox: DaytonaSandbox): boolean { + return isDesiredCanonicalSandbox(sandbox, { + sandboxName: this.input.sandboxName(), + snapshot: this.input.env.DAYTONA_SANDBOX_SNAPSHOT, + volumeName: this.input.env.DAYTONA_WORKSPACE_VOLUME, + }); + } + + private async create(client: DaytonaClient, name: string): Promise { + try { + const volume = await this.ensureWorkspaceVolume(client); + const created = await client.createSandbox({ + name, + snapshot: this.input.env.DAYTONA_SANDBOX_SNAPSHOT, + target: this.input.env.DAYTONA_TARGET, + user: "node", + labels: canonicalSandboxLabels({ + sandboxName: name, + snapshot: this.input.env.DAYTONA_SANDBOX_SNAPSHOT, + volumeId: volume.id, + volumeName: volume.name, + }), + volumes: [{ mountPath: WORKSPACE_MOUNT_PATH, subpath: name, volumeId: volume.id }], + autoStopInterval: DEFAULT_IDLE_STOP_MIN, + autoArchiveInterval: AUTO_ARCHIVE_MIN, + autoDeleteInterval: NEVER_AUTO_DELETE, + }); + this.assertIdentity(created); + return created; + } catch (error) { + if (isDaytonaNameConflictError(error)) { + const existing = await this.findAfterCreateConflict(client, name); + if (existing) { + return existing; + } + } + throw this.input.toUpstreamError(error, "Daytona sandbox failed to start."); + } + } + + private async findAfterCreateConflict( + client: DaytonaClient, + name: string, + ): Promise { + const byLabel = await this.findByLabels(client); + if (byLabel) { + return byLabel; + } + const byName = await client.getSandbox(name); + if (byName && !isDestroyed(byName)) { + this.assertIdentity(byName); + return byName; + } + return null; + } + + private async findByLabels(client: DaytonaClient): Promise { + const sandboxName = this.input.sandboxName(); + const byLabel = await client.listSandboxesByLabels({ + app: "cheatcode", + sandboxId: sandboxName, + }); + const live = byLabel.filter((sandbox) => !isDestroyed(sandbox)); + if (live.length > 1) { + throw new APIError(409, "conflict_state_invalid", "Multiple active sandboxes found", { + details: { daytonaIds: live.map((sandbox) => sandbox.id), sandboxId: sandboxName }, + hint: "Resolve the duplicate Daytona sandboxes explicitly before retrying.", + retriable: false, + }); + } + const listed = live[0]; + if (!listed) { + return null; + } + const sandbox = await client.getSandbox(listed.id); + if (!sandbox || isDestroyed(sandbox)) { + return null; + } + this.assertIdentity(sandbox); + return sandbox; + } + + public assertIdentity(sandbox: DaytonaSandbox): void { + const name = this.input.sandboxName(); + if (isCanonicalSandbox(sandbox, name)) { + return; + } + throw new APIError(409, "conflict_state_invalid", "Daytona sandbox identity mismatch", { + details: { actualName: sandbox.name, daytonaId: sandbox.id, expectedName: name }, + hint: "Inspect the sandbox labels and durable object binding before retrying.", + retriable: false, + }); + } + + private async waitForReadyVolume( + client: DaytonaClient, + initial: DaytonaVolume, + ): Promise { + let volume = initial; + for (let attempt = 0; attempt < VOLUME_READY_ATTEMPTS; attempt += 1) { + if (volume.state === "ready") { + return volume; + } + if (volume.state === "error") { + throw new APIError(502, "upstream_sandbox_failed", "Daytona workspace volume failed", { + details: { volumeId: volume.id }, + retriable: false, + }); + } + await sleep(VOLUME_READY_DELAY_MS); + const current = await client.getVolumeByName(volume.name); + if (!current) { + throw new APIError(502, "upstream_sandbox_failed", "Daytona workspace volume disappeared", { + retriable: true, + }); + } + volume = current; + } + throw new APIError(504, "upstream_sandbox_failed", "Daytona workspace volume was not ready", { + retriable: true, + }); + } + + private async startIfPossible(client: DaytonaClient, sandbox: DaytonaSandbox): Promise { + if (!isStartableState(sandbox.state)) { + return false; + } + await client.startSandbox(sandbox.id).catch((error: unknown) => { + throw this.input.toUpstreamError(error, "Daytona sandbox failed to start."); + }); + return true; + } +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-runtime-manifest.ts b/apps/agent-worker/src/durable-objects/project-sandbox-runtime-manifest.ts new file mode 100644 index 00000000..88c75a45 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-runtime-manifest.ts @@ -0,0 +1,87 @@ +import type { DaytonaClient } from "@cheatcode/tools-code"; +import { z } from "zod"; +import { + APP_PREVIEW_SLOT_PREFIX, + PROC_PREFIX, + ProcessRecordSchema, +} from "./project-sandbox-process-support"; + +const RUNTIME_DIRECTORY = "/workspace/.cheatcode"; +const SANDBOX_RUNTIME_MANIFEST_PATH = `${RUNTIME_DIRECTORY}/runtime.json`; + +const RuntimeProjectSchema = z + .object({ + cwd: z.string().min(1), + isMobile: z.boolean(), + port: z.number().int().positive().max(65_535).nullable(), + processId: z.string().min(1), + startupCommands: z.array(z.string().min(1)).max(4), + }) + .strict(); + +const SandboxRuntimeManifestSchema = z + .object({ + generatedAt: z.string().datetime(), + projects: z.record(z.string(), RuntimeProjectSchema), + source: z.literal("durable-object-process-state"), + version: z.literal(1), + }) + .strict(); + +export async function writeSandboxRuntimeManifest( + client: DaytonaClient, + sandboxId: string, + records: Map, +): Promise { + const manifest = buildSandboxRuntimeManifest(records); + const temporaryPath = `${SANDBOX_RUNTIME_MANIFEST_PATH}.tmp-${crypto.randomUUID()}`; + await client.createFolder(sandboxId, RUNTIME_DIRECTORY, "0700"); + await client.uploadFile( + sandboxId, + temporaryPath, + new TextEncoder().encode(`${JSON.stringify(manifest, null, 2)}\n`), + ); + const moved = await client.execute(sandboxId, { + command: `mv -f ${shellQuote(temporaryPath)} ${shellQuote(SANDBOX_RUNTIME_MANIFEST_PATH)}`, + timeout: 10, + }); + if (moved.exitCode !== 0) { + await client.deleteFilePath(sandboxId, temporaryPath, false).catch(() => undefined); + throw new Error("Could not publish the sandbox runtime projection."); + } +} + +function buildSandboxRuntimeManifest( + records: Map, +): z.infer { + const projects: Record> = {}; + for (const [key, value] of [...records.entries()].sort(([left], [right]) => + left.localeCompare(right), + )) { + const processId = key.slice(PROC_PREFIX.length); + if (!key.startsWith(PROC_PREFIX) || !processId.startsWith(APP_PREVIEW_SLOT_PREFIX)) { + continue; + } + const parsed = ProcessRecordSchema.safeParse(value); + if (!parsed.success) continue; + const workspaceSlug = processId.slice(APP_PREVIEW_SLOT_PREFIX.length); + if (!/^[a-z0-9][a-z0-9-]{0,119}$/u.test(workspaceSlug)) continue; + projects[workspaceSlug] = { + cwd: parsed.data.cwd, + isMobile: parsed.data.isMobile ?? false, + port: parsed.data.port ?? null, + processId, + startupCommands: [parsed.data.command], + }; + } + return SandboxRuntimeManifestSchema.parse({ + generatedAt: new Date().toISOString(), + projects, + source: "durable-object-process-state", + version: 1, + }); +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-runtime.ts b/apps/agent-worker/src/durable-objects/project-sandbox-runtime.ts index 20b6dbd7..b11783e4 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox-runtime.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox-runtime.ts @@ -1,14 +1,24 @@ import { EnvironmentVariablesSchema } from "@cheatcode/sandbox-contracts"; import { WorkspaceFilePathSchema, WorkspacePathSchema } from "@cheatcode/tools-code"; +import { ProjectId } from "@cheatcode/types"; import { z } from "zod"; -const CommandArgvSchema = z.array(z.string().min(1)).min(1).max(128); +const CommandArgvSchema = z.array(z.string().min(1).max(8_192)).min(1).max(128); const ProcessIdSchema = z .string() .min(1) .max(200) .regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/u, "Process IDs may contain letters, numbers, . _ : -."); +export const ProjectWorkspaceSlugSchema = z + .string() + .min(1) + .max(64) + .regex( + /^[a-z0-9]+(?:-[a-z0-9]+)*$/u, + "Workspace slugs may contain lowercase letters, numbers, and single hyphens.", + ); + export const ProjectRunCodeInputSchema = z .object({ language: z.enum(["python", "javascript"]), @@ -40,7 +50,7 @@ export const ProjectStartProcessInputSchema = ProjectExecInputSchema.extend({ .max(24 * 60 * 60 * 1000) .optional(), maxRestarts: z.number().int().min(0).max(25).optional(), - processId: ProcessIdSchema.optional(), + processId: ProcessIdSchema, restartOnFailure: z.boolean().optional(), waitForPort: z .object({ @@ -124,6 +134,12 @@ export const ProjectAllocatePortInputSchema = z }) .strict(); +export const ProjectGetPortInputSchema = z + .object({ + projectId: ProjectWorkspaceSlugSchema, + }) + .strict(); + export const ProjectAllocateProcessPortInputSchema = z .object({ maxPort: z.number().int().min(1_024).max(65_535), @@ -145,7 +161,7 @@ export const ProjectWakePreviewInputSchema = z // Which project's dev server to wake — its ProcessRecord slot is keyed by the project's // workspaceSlug (matching the start_dev_server tool + app-builder paths). Absent for a // project-less chat, where there is no dev server to revive. - workspaceSlug: z.string().min(1).max(200).optional(), + workspaceSlug: ProjectWorkspaceSlugSchema.optional(), }) .strict(); @@ -154,7 +170,7 @@ export const ProjectWakePreviewInputSchema = z // Always provided: only a project chat calls this, and every project owns a workspace slug. export const ProjectPreviewStatusInputSchema = z .object({ - workspaceSlug: z.string().min(1).max(200), + workspaceSlug: ProjectWorkspaceSlugSchema, }) .strict(); @@ -169,25 +185,39 @@ export const ProjectSignedPreviewUrlInputSchema = z }) .strict(); +export const ProjectBrowserTakeoverInputSchema = z + .object({ + expiresInSeconds: z + .number() + .int() + .min(60) + .max(10 * 60), + runId: z.string().uuid(), + takeoverId: z.string().uuid(), + }) + .strict(); + +export const ProjectBrowserTakeoverStopInputSchema = z + .object({ runId: z.string().uuid() }) + .strict(); + // Per-project teardown inside the shared per-user sandbox: names ONE project's workspace folder // (/workspace/) whose dev server, port, and folder should be reclaimed — without // ever touching the shared sandbox itself. export const ProjectCleanupWorkspaceInputSchema = z .object({ - workspaceSlug: z.string().min(1).max(200), + projectId: z.string().uuid().toLowerCase().transform(ProjectId), + workspaceSlug: ProjectWorkspaceSlugSchema, }) - .strict(); + .strict() + .refine( + (input) => input.workspaceSlug.endsWith(`-${input.projectId.toLowerCase()}`), + "Workspace slug does not belong to the requested project.", + ); export const ProjectArchiveInputSchema = z .object({ - workspaceSlug: z - .string() - .min(1) - .max(200) - .refine( - (slug) => !slug.includes("/") && slug !== "." && slug !== "..", - "Workspace slug must be a single path segment.", - ), + workspaceSlug: ProjectWorkspaceSlugSchema, }) .strict(); @@ -201,11 +231,19 @@ export type ProjectSearchFilesInput = z.input; export type ProjectKillProcessInput = z.input; export type ProjectAllocatePortInput = z.input; +export type ProjectGetPortInput = z.input; export type ProjectAllocateProcessPortInput = z.input; export type ProjectCodeServerInput = z.input; export type ProjectWakePreviewInput = z.input; export type ProjectPreviewStatusInput = z.input; export type ProjectSignedPreviewUrlInput = z.input; +export type ProjectBrowserTakeoverInput = z.input; +export type ProjectBrowserTakeoverStopInput = z.input; +export interface ProjectBrowserTakeoverResult { + expiresAt: string; + takeoverId: string; + url: string; +} export type ProjectArchiveInput = z.input; /** Result of waking a preview: the (possibly restarted) dev-server preview URL + liveness. */ @@ -226,6 +264,32 @@ export interface ProjectSandboxRuntimeState { sandboxId?: string; } export type ProjectCleanupWorkspaceInput = z.input; +export type ParsedProjectCleanupWorkspaceInput = z.output< + typeof ProjectCleanupWorkspaceInputSchema +>; + +/** Returns the immutable project folder segment for a canonical /workspace path. */ +export function workspaceSlugFromPath(path: string | undefined): string | null { + if (!path) { + return null; + } + const segments: string[] = []; + for (const segment of path.split("/")) { + if (!segment || segment === ".") { + continue; + } + if (segment === "..") { + segments.pop(); + continue; + } + segments.push(segment); + } + if (segments[0] !== "workspace" || segments.length < 2) { + return null; + } + const parsed = ProjectWorkspaceSlugSchema.safeParse(segments[1]); + return parsed.success ? parsed.data : null; +} function shellQuote(arg: string): string { return `'${arg.replaceAll("'", "'\\''")}'`; diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-scripts.ts b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-scripts.ts new file mode 100644 index 00000000..94f4db7b --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-scripts.ts @@ -0,0 +1,396 @@ +import { shellQuote } from "./project-sandbox-process-support"; + +export const SNAPSHOT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024; + +export function prepareWorkspaceArchiveCommand(upgradeId: string): string { + return pythonCommand(PREPARE_WORKSPACE_ARCHIVE_SCRIPT, { + chunkBytes: SNAPSHOT_TRANSFER_CHUNK_BYTES, + upgradeId, + }); +} + +export function verifyWorkspaceArchiveCommand(input: { + archiveDigest: string; + archiveSize: number; + chunkCount: number; + treeDigest: string; + upgradeId: string; +}): string { + return pythonCommand(VERIFY_WORKSPACE_ARCHIVE_SCRIPT, input); +} + +export function digestWorkspaceCommand(): string { + return pythonCommand(DIGEST_WORKSPACE_SCRIPT, {}); +} + +export function verifyTransferChunkCommand(input: { + digest: string; + path: string; + size: number; +}): string { + return pythonCommand(VERIFY_TRANSFER_CHUNK_SCRIPT, input); +} + +export function clearWorkspaceCommand(): string { + return pythonCommand(CLEAR_WORKSPACE_SCRIPT, {}); +} + +function pythonCommand(script: string, payload: unknown): string { + const encoded = btoa(JSON.stringify(payload)); + return `python3 -c ${shellQuote(script)} ${shellQuote(encoded)}`; +} + +const TREE_DIGEST_FUNCTIONS = ` +def raise_walk_error(error): + raise error + +def tree_entries(root): + values = [] + for current, directories, files in os.walk(root, topdown=True, onerror=raise_walk_error, followlinks=False): + directories.sort() + files.sort() + for name in directories + files: + path = os.path.join(current, name) + relative = os.path.relpath(path, root).replace(os.sep, "/") + values.append((relative, path)) + values.sort(key=lambda value: value[0]) + return values + +def tree_digest(root): + digest = hashlib.sha256() + for relative, path in tree_entries(root): + metadata = os.lstat(path) + mode = stat.S_IMODE(metadata.st_mode) + if stat.S_ISDIR(metadata.st_mode): + header = [relative, "directory", mode] + elif stat.S_ISREG(metadata.st_mode): + header = [relative, "file", mode, metadata.st_size] + elif stat.S_ISLNK(metadata.st_mode): + header = [relative, "symlink", mode, os.readlink(path)] + else: + raise RuntimeError(f"Unsupported workspace entry: {relative}") + encoded = json.dumps(header, ensure_ascii=True, separators=(",", ":")).encode("ascii") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + if stat.S_ISREG(metadata.st_mode): + with open(path, "rb") as source: + while True: + block = source.read(1024 * 1024) + if not block: + break + digest.update(block) + return digest.hexdigest() +`; + +const PREPARE_WORKSPACE_ARCHIVE_SCRIPT = ` +import base64 +import hashlib +import json +import os +import shutil +import stat +import sys +import tarfile + +${TREE_DIGEST_FUNCTIONS} + +payload = json.loads(base64.b64decode(sys.argv[1]).decode("utf-8")) +upgrade_id = payload["upgradeId"] +chunk_bytes = payload["chunkBytes"] +if not isinstance(upgrade_id, str) or len(upgrade_id) != 32 or any(c not in "0123456789abcdef" for c in upgrade_id): + raise RuntimeError("Invalid snapshot upgrade identity") +if not isinstance(chunk_bytes, int) or chunk_bytes < 1: + raise RuntimeError("Invalid snapshot transfer chunk size") + +root = "/workspace" +base = f"/tmp/cheatcode-snapshot-upgrade/{upgrade_id}" +chunks_path = os.path.join(base, "chunks") +os.makedirs(root, exist_ok=True) +shutil.rmtree(base, ignore_errors=True) +os.makedirs(chunks_path, mode=0o700) + +class ChunkWriter: + def __init__(self, directory, chunk_size): + self.archive_digest = hashlib.sha256() + self.archive_size = 0 + self.chunk_count = 0 + self.chunk_size = chunk_size + self.current = None + self.current_size = 0 + self.directory = directory + + def write(self, data): + view = memoryview(data) + total = len(view) + while view: + if self.current is None: + path = os.path.join(self.directory, f"chunk-{self.chunk_count:012d}") + self.current = open(path, "wb") + self.current_size = 0 + self.chunk_count += 1 + take = min(len(view), self.chunk_size - self.current_size) + block = view[:take] + self.current.write(block) + self.archive_digest.update(block) + self.archive_size += take + self.current_size += take + view = view[take:] + if self.current_size == self.chunk_size: + self.current.close() + self.current = None + return total + + def flush(self): + if self.current is not None: + self.current.flush() + + def close(self): + if self.current is not None: + self.current.close() + self.current = None + +digest = tree_digest(root) +writer = ChunkWriter(chunks_path, chunk_bytes) +with tarfile.open(fileobj=writer, mode="w|", dereference=False) as archive: + for relative, path in tree_entries(root): + archive.add(path, arcname=relative, recursive=False) +writer.close() + +print(json.dumps({ + "archiveDigest": writer.archive_digest.hexdigest(), + "archiveSize": writer.archive_size, + "chunkCount": writer.chunk_count, + "treeDigest": digest, +}, separators=(",", ":"))) +`; + +const DIGEST_WORKSPACE_SCRIPT = ` +import base64 +import hashlib +import json +import os +import stat +import sys + +${TREE_DIGEST_FUNCTIONS} + +json.loads(base64.b64decode(sys.argv[1]).decode("utf-8")) +os.makedirs("/workspace", exist_ok=True) +print(json.dumps({"treeDigest": tree_digest("/workspace")}, separators=(",", ":"))) +`; + +const VERIFY_TRANSFER_CHUNK_SCRIPT = ` +import base64 +import hashlib +import json +import os +import sys + +payload = json.loads(base64.b64decode(sys.argv[1]).decode("utf-8")) +path = payload["path"] +if not isinstance(path, str) or not path.startswith("/tmp/cheatcode-snapshot-upgrade/"): + raise RuntimeError("Invalid snapshot transfer path") +digest = hashlib.sha256() +size = 0 +with open(path, "rb") as source: + while True: + block = source.read(1024 * 1024) + if not block: + break + digest.update(block) + size += len(block) +verified = size == payload["size"] and digest.hexdigest() == payload["digest"] +print(json.dumps({"verified": verified}, separators=(",", ":"))) +raise SystemExit(0 if verified else 4) +`; + +const CLEAR_WORKSPACE_SCRIPT = ` +import base64 +import json +import os +import shutil + +json.loads(base64.b64decode(__import__("sys").argv[1]).decode("utf-8")) +root = "/workspace" +os.makedirs(root, exist_ok=True) +for name in os.listdir(root): + path = os.path.join(root, name) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.unlink(path) +print(json.dumps({"cleared": True}, separators=(",", ":"))) +`; + +const VERIFY_WORKSPACE_ARCHIVE_SCRIPT = ` +import base64 +import hashlib +import json +import os +from pathlib import PurePosixPath +import shutil +import sqlite3 +import stat +import sys +import tarfile + +${TREE_DIGEST_FUNCTIONS} + +payload = json.loads(base64.b64decode(sys.argv[1]).decode("utf-8")) +upgrade_id = payload["upgradeId"] +if not isinstance(upgrade_id, str) or len(upgrade_id) != 32 or any(c not in "0123456789abcdef" for c in upgrade_id): + raise RuntimeError("Invalid snapshot upgrade identity") +base = f"/tmp/cheatcode-snapshot-upgrade/{upgrade_id}" +chunks_path = os.path.join(base, "chunks") +directory_modes_path = os.path.join(base, "directory-modes.sqlite") + +def retry_transfer(reason): + print(json.dumps({"reason": reason, "retryTransfer": True}, separators=(",", ":"))) + raise SystemExit(3) + +archive_digest = hashlib.sha256() +archive_size = 0 +try: + for index in range(payload["chunkCount"]): + chunk_path = os.path.join(chunks_path, f"chunk-{index:012d}") + with open(chunk_path, "rb") as source: + while True: + block = source.read(1024 * 1024) + if not block: + break + archive_digest.update(block) + archive_size += len(block) +except FileNotFoundError: + retry_transfer("missing chunk") + +if archive_size != payload["archiveSize"] or archive_digest.hexdigest() != payload["archiveDigest"]: + retry_transfer("archive digest mismatch") + +def safe_relative(value): + path = PurePosixPath(value) + if path.is_absolute() or not path.parts or any(part in ("", ".", "..") for part in path.parts): + raise RuntimeError(f"Unsafe archive path: {value}") + return path + +def validate_link(member): + if not (member.issym() or member.islnk()): + return + link = PurePosixPath(member.linkname) + if link.is_absolute(): + raise RuntimeError(f"Absolute archive link: {member.name}") + combined = link if member.islnk() else PurePosixPath(member.name).parent / link + depth = 0 + for part in combined.parts: + if part in ("", "."): + continue + if part == "..": + depth -= 1 + else: + depth += 1 + if depth < 0: + raise RuntimeError(f"Escaping archive link: {member.name}") + +class ChunkReader: + def __init__(self, directory, count): + self.count = count + self.current = None + self.directory = directory + self.index = 0 + + def read(self, size=-1): + if size is None or size < 0: + size = 1024 * 1024 + output = bytearray() + while len(output) < size and self.index < self.count: + if self.current is None: + path = os.path.join(self.directory, f"chunk-{self.index:012d}") + self.current = open(path, "rb") + block = self.current.read(size - len(output)) + if block: + output.extend(block) + continue + self.current.close() + self.current = None + self.index += 1 + return bytes(output) + + def close(self): + if self.current is not None: + self.current.close() + self.current = None + +def target_path(relative): + parts = safe_relative(relative).parts + current = root + for part in parts[:-1]: + current = os.path.join(current, part) + if os.path.lexists(current): + metadata = os.lstat(current) + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise RuntimeError(f"Unsafe archive parent: {relative}") + else: + os.mkdir(current, mode=0o700) + return os.path.join(root, *parts) + +root = "/workspace" +os.makedirs(root, exist_ok=True) +for name in os.listdir(root): + path = os.path.join(root, name) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.unlink(path) + +if os.path.exists(directory_modes_path): + os.unlink(directory_modes_path) +directory_modes = sqlite3.connect(directory_modes_path) +directory_modes.execute("CREATE TABLE modes (path TEXT PRIMARY KEY, mode INTEGER NOT NULL)") +reader = ChunkReader(chunks_path, payload["chunkCount"]) +with tarfile.open(fileobj=reader, mode="r|") as archive: + for member in archive: + safe_relative(member.name) + validate_link(member) + if not (member.isdir() or member.isreg() or member.issym() or member.islnk()): + raise RuntimeError(f"Unsupported archive member: {member.name}") + target = target_path(member.name) + if member.isdir(): + if os.path.lexists(target) and not os.path.isdir(target): + raise RuntimeError(f"Archive directory collides with a file: {member.name}") + os.makedirs(target, mode=0o700, exist_ok=True) + os.chmod(target, 0o700) + directory_modes.execute("INSERT INTO modes (path, mode) VALUES (?, ?)", (target, member.mode)) + elif member.issym(): + if os.path.lexists(target): + raise RuntimeError(f"Archive symlink destination exists: {member.name}") + os.symlink(member.linkname, target) + elif member.islnk(): + source_path = target_path(member.linkname) + if not os.path.isfile(source_path) or os.path.islink(source_path): + raise RuntimeError(f"Archive hardlink source is unavailable: {member.name}") + with open(source_path, "rb") as source, open(target, "xb") as destination: + shutil.copyfileobj(source, destination, 1024 * 1024) + os.chmod(target, member.mode) + else: + source = archive.extractfile(member) + if source is None: + raise RuntimeError(f"Archive file has no payload: {member.name}") + with source, open(target, "xb") as destination: + shutil.copyfileobj(source, destination, 1024 * 1024) + os.chmod(target, member.mode) +reader.close() +directory_modes.commit() +for target, mode in directory_modes.execute("SELECT path, mode FROM modes ORDER BY length(path) DESC"): + os.chmod(target, mode) +directory_modes.close() +os.unlink(directory_modes_path) + +verified_tree = tree_digest(root) +if verified_tree != payload["treeDigest"]: + print(json.dumps({"reason": "workspace digest mismatch", "retryTransfer": False}, separators=(",", ":"))) + raise SystemExit(4) +print(json.dumps({ + "archiveDigest": archive_digest.hexdigest(), + "retryTransfer": False, + "treeDigest": verified_tree, +}, separators=(",", ":"))) +`; diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-state.ts b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-state.ts new file mode 100644 index 00000000..a1811e59 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-state.ts @@ -0,0 +1,151 @@ +import { z } from "zod"; + +export const SNAPSHOT_RELEASE_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const UPGRADE_ID_PATTERN = /^[0-9a-f]{32}$/u; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/u; + +const SnapshotUpgradeStateBaseSchema = z + .object({ + archiveDigest: z.string().regex(DIGEST_PATTERN).nullable(), + archiveSize: z.number().int().positive().nullable(), + candidateId: z.string().min(1).max(500).nullable(), + chunkCount: z.number().int().positive().nullable(), + needsTransfer: z.boolean(), + nextChunk: z.number().int().nonnegative(), + phase: z.enum([ + "claimed", + "source-prepared", + "candidate-created", + "candidate-verified", + "source-retired", + "candidate-promoted", + "switched", + "completed", + ]), + releaseSha: z.string().regex(SNAPSHOT_RELEASE_SHA_PATTERN), + sandboxName: z.string().min(1).max(100), + sourceId: z.string().min(1).max(500), + sourceSnapshot: z.string().min(1).max(500), + targetSnapshot: z.string().min(1).max(500), + treeDigest: z.string().regex(DIGEST_PATTERN).nullable(), + upgradeId: z.string().regex(UPGRADE_ID_PATTERN), + volumeId: z.string().uuid(), + volumeName: z.string().min(1).max(100), + }) + .strict(); + +export type SnapshotUpgradeState = z.infer; +export const SnapshotUpgradeStateSchema = + SnapshotUpgradeStateBaseSchema.superRefine(validateUpgradeState); + +function validateUpgradeState(state: SnapshotUpgradeState, context: z.RefinementCtx): void { + addStateIssue(context, preparedWithoutDigest(state), "Prepared upgrade has no tree digest."); + addStateIssue(context, incompleteTransferEvidence(state), "Transfer evidence is incomplete."); + addStateIssue(context, transferCursorExceeded(state), "Transfer cursor exceeds its chunk count."); + addStateIssue( + context, + mountedUpgradeHasTransferState(state), + "Mounted-volume upgrade has transfer state.", + ); + addStateIssue( + context, + verifiedCandidateHasMissingChunks(state), + "Verified candidate has incomplete chunks.", + ); + addStateIssue( + context, + claimedUpgradeHasEvidence(state), + "Claimed upgrade contains prepared evidence.", + ); + addStateIssue( + context, + candidatePhaseWithoutId(state), + "Candidate phase has no candidate sandbox.", + ); +} + +function addStateIssue(context: z.RefinementCtx, isInvalid: boolean, message: string): void { + if (isInvalid) context.addIssue({ code: "custom", message }); +} + +function preparedWithoutDigest(state: SnapshotUpgradeState): boolean { + return state.phase !== "claimed" && state.treeDigest === null; +} + +function incompleteTransferEvidence(state: SnapshotUpgradeState): boolean { + return ( + state.needsTransfer && + state.phase !== "claimed" && + (state.archiveDigest === null || state.archiveSize === null || state.chunkCount === null) + ); +} + +function transferCursorExceeded(state: SnapshotUpgradeState): boolean { + return state.needsTransfer && state.chunkCount !== null && state.nextChunk > state.chunkCount; +} + +function mountedUpgradeHasTransferState(state: SnapshotUpgradeState): boolean { + return ( + !state.needsTransfer && + (state.archiveDigest !== null || + state.archiveSize !== null || + state.chunkCount !== null || + state.nextChunk !== 0) + ); +} + +function verifiedCandidateHasMissingChunks(state: SnapshotUpgradeState): boolean { + return ( + state.needsTransfer && + isCandidateVerifiedPhase(state.phase) && + state.nextChunk !== state.chunkCount + ); +} + +function claimedUpgradeHasEvidence(state: SnapshotUpgradeState): boolean { + return ( + state.phase === "claimed" && + (state.archiveDigest !== null || + state.archiveSize !== null || + state.chunkCount !== null || + state.nextChunk !== 0 || + state.treeDigest !== null) + ); +} + +function candidatePhaseWithoutId(state: SnapshotUpgradeState): boolean { + return state.phase !== "claimed" && state.phase !== "source-prepared" && !state.candidateId; +} + +function isCandidateVerifiedPhase(phase: SnapshotUpgradeState["phase"]): boolean { + return ( + phase === "candidate-verified" || + phase === "source-retired" || + phase === "candidate-promoted" || + phase === "switched" || + phase === "completed" + ); +} + +export const ArchiveEvidenceSchema = z + .object({ + archiveDigest: z.string().regex(DIGEST_PATTERN), + archiveSize: z.number().int().positive(), + chunkCount: z.number().int().positive(), + treeDigest: z.string().regex(DIGEST_PATTERN), + }) + .strict(); +export const DigestEvidenceSchema = z + .object({ treeDigest: z.string().regex(DIGEST_PATTERN) }) + .strict(); +export const ChunkEvidenceSchema = z.object({ verified: z.boolean() }).strict(); +export const ArchiveVerificationSchema = z + .object({ + archiveDigest: z.string().regex(DIGEST_PATTERN), + retryTransfer: z.literal(false), + treeDigest: z.string().regex(DIGEST_PATTERN), + }) + .strict(); +export const RetryTransferSchema = z + .object({ reason: z.string().min(1).max(500), retryTransfer: z.literal(true) }) + .strict(); diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-upgrade.ts b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-upgrade.ts new file mode 100644 index 00000000..d227a32f --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-snapshot-upgrade.ts @@ -0,0 +1,724 @@ +import { APIError } from "@cheatcode/observability"; +import { DaytonaApiError, type DaytonaClient, type DaytonaSandbox } from "@cheatcode/tools-code"; +import type { SandboxSnapshotReconciliation } from "@cheatcode/types"; +import type { z } from "zod"; +import { + candidateSandboxLabels, + canonicalSandboxLabels, + isDesiredCanonicalSandbox, + isUpgradeCandidate, + retiredSandboxLabels, +} from "./project-sandbox-daytona-identity"; +import { + AUTO_ARCHIVE_MIN, + DAYTONA_ID_KEY, + DEFAULT_IDLE_STOP_MIN, + NEVER_AUTO_DELETE, + type ProjectSandboxEnv, +} from "./project-sandbox-lifecycle-support"; +import { isDestroyed, sleep } from "./project-sandbox-process-support"; +import type { ProjectSandboxProvisioning } from "./project-sandbox-provisioning"; +import { + digestWorkspaceCommand, + prepareWorkspaceArchiveCommand, + SNAPSHOT_TRANSFER_CHUNK_BYTES, + verifyTransferChunkCommand, + verifyWorkspaceArchiveCommand, +} from "./project-sandbox-snapshot-scripts"; +import { + ArchiveEvidenceSchema, + ArchiveVerificationSchema, + ChunkEvidenceSchema, + DigestEvidenceSchema, + RetryTransferSchema, + SNAPSHOT_RELEASE_SHA_PATTERN, + type SnapshotUpgradeState, + SnapshotUpgradeStateSchema, +} from "./project-sandbox-snapshot-state"; + +const SNAPSHOT_UPGRADE_STATE_KEY = "sandbox_snapshot_upgrade"; +const SNAPSHOT_EXEC_TIMEOUT_SECONDS = 480; +const DELETE_VERIFY_ATTEMPTS = 30; +const DELETE_VERIFY_DELAY_MS = 2_000; +const WORKSPACE_MOUNT_PATH = "/workspace"; + +interface SnapshotUpgradeInput { + adoptSandboxId: (sandboxId: string) => void; + client: DaytonaClient; + ctx: DurableObjectState; + env: ProjectSandboxEnv; + killProcesses: () => Promise; + provisioning: ProjectSandboxProvisioning; + sandboxName: string; + toUpstreamError: (error: unknown, fallback: string) => APIError; +} + +export class ProjectSandboxSnapshotUpgrade { + public constructor(private readonly input: SnapshotUpgradeInput) {} + + public async advance(releaseSha: string): Promise { + if (!SNAPSHOT_RELEASE_SHA_PATTERN.test(releaseSha)) { + throw snapshotInvariant("Snapshot upgrade release identity is invalid."); + } + try { + return await this.advanceExclusive(releaseSha); + } catch (error) { + throw this.input.toUpstreamError(error, "Daytona snapshot reconciliation failed."); + } + } + + private async advanceExclusive(releaseSha: string): Promise { + let state = await this.loadState(); + if (state?.phase === "completed" && state.releaseSha === releaseSha) { + this.assertCurrentContract(state, releaseSha); + return this.completeAndClear(state); + } + if (state?.phase === "completed") { + await this.input.ctx.storage.delete(SNAPSHOT_UPGRADE_STATE_KEY); + state = null; + } + state ??= await this.initializeState(releaseSha); + if (!state) { + return this.currentOrAbsentResult(); + } + this.assertCurrentContract(state, releaseSha); + for (;;) { + const next = await this.advancePhase(state); + if (next === null) { + return upgradingResult(state); + } + state = next; + if (state.phase === "completed") { + return this.completeAndClear(state); + } + } + } + + private async completeAndClear( + state: SnapshotUpgradeState, + ): Promise { + const result = completedResult(state); + await this.input.ctx.storage.delete(SNAPSHOT_UPGRADE_STATE_KEY); + if ((await this.input.ctx.storage.get(SNAPSHOT_UPGRADE_STATE_KEY)) !== undefined) { + throw snapshotInvariant("Completed snapshot upgrade state could not be removed."); + } + return result; + } + + private async initializeState(releaseSha: string): Promise { + const source = await this.input.provisioning.findExisting(this.input.client); + if (!source || isDestroyed(source)) { + return null; + } + if (this.input.provisioning.isDesired(source)) { + return null; + } + const volume = await this.input.provisioning.ensureWorkspaceVolume(this.input.client); + const upgradeId = await upgradeIdentity( + this.input.sandboxName, + source.id, + volume.id, + this.target, + ); + const state = SnapshotUpgradeStateSchema.parse({ + archiveDigest: null, + archiveSize: null, + candidateId: null, + chunkCount: null, + needsTransfer: + source.labels["workspaceVolumeId"] !== volume.id || + source.labels["workspaceVolumeName"] !== volume.name, + nextChunk: 0, + phase: "claimed", + releaseSha, + sandboxName: this.input.sandboxName, + sourceId: source.id, + sourceSnapshot: source.snapshot, + targetSnapshot: this.target, + treeDigest: null, + upgradeId, + volumeId: volume.id, + volumeName: volume.name, + }); + await this.storeState(state); + return state; + } + + private async currentOrAbsentResult(): Promise { + const current = await this.input.provisioning.findExisting(this.input.client); + if (!current || isDestroyed(current)) { + return snapshotResult({ + sourceSnapshot: null, + status: "absent", + targetSnapshot: this.target, + }); + } + if (!this.input.provisioning.isDesired(current)) { + throw snapshotInvariant("Snapshot upgrade state is absent for a noncurrent sandbox."); + } + return snapshotResult({ + sourceSnapshot: current.snapshot, + status: "current", + targetSnapshot: this.target, + }); + } + + private async advancePhase(state: SnapshotUpgradeState): Promise { + switch (state.phase) { + case "claimed": + return this.prepareSource(state); + case "source-prepared": + return this.createCandidate(state); + case "candidate-created": + return this.verifyCandidate(state); + case "candidate-verified": + return this.retireSource(state); + case "source-retired": + return this.promoteCandidate(state); + case "candidate-promoted": + return this.switchDurableIdentity(state); + case "switched": + return this.deleteSource(state); + case "completed": + return state; + } + } + + private async prepareSource(state: SnapshotUpgradeState): Promise { + const source = await this.requireSource(state); + await this.ensureStarted(source, "source"); + await this.input.killProcesses(); + const evidence = state.needsTransfer + ? await this.executeJson( + source.id, + prepareWorkspaceArchiveCommand(state.upgradeId), + ArchiveEvidenceSchema, + ) + : await this.executeJson(source.id, digestWorkspaceCommand(), DigestEvidenceSchema); + const next = SnapshotUpgradeStateSchema.parse({ + ...state, + archiveDigest: "archiveDigest" in evidence ? evidence.archiveDigest : null, + archiveSize: "archiveSize" in evidence ? evidence.archiveSize : null, + chunkCount: "chunkCount" in evidence ? evidence.chunkCount : null, + nextChunk: 0, + phase: "source-prepared", + treeDigest: evidence.treeDigest, + }); + await this.storeState(next); + return next; + } + + private async createCandidate(state: SnapshotUpgradeState): Promise { + const candidate = + (await this.findCandidate(state)) ?? (await this.createCandidateSandbox(state)); + this.assertCandidate(candidate, state); + await this.ensureStarted(candidate, "candidate"); + const next = SnapshotUpgradeStateSchema.parse({ + ...state, + candidateId: candidate.id, + phase: "candidate-created", + }); + await this.storeState(next); + return next; + } + + private async createCandidateSandbox(state: SnapshotUpgradeState): Promise { + try { + return await this.input.client.createSandbox({ + autoArchiveInterval: AUTO_ARCHIVE_MIN, + autoDeleteInterval: NEVER_AUTO_DELETE, + autoStopInterval: DEFAULT_IDLE_STOP_MIN, + labels: candidateSandboxLabels(candidateLabelInput(state)), + name: candidateName(state), + snapshot: state.targetSnapshot, + target: this.input.env.DAYTONA_TARGET, + user: "node", + volumes: [ + { + mountPath: WORKSPACE_MOUNT_PATH, + subpath: this.input.sandboxName, + volumeId: state.volumeId, + }, + ], + }); + } catch (error) { + if (error instanceof DaytonaApiError && error.status === 409) { + const existing = await this.findCandidate(state); + if (existing) return existing; + } + throw error; + } + } + + private async findCandidate(state: SnapshotUpgradeState): Promise { + const matches = ( + await this.input.client.listSandboxesByLabels({ + app: "cheatcode", + role: "candidate", + sandboxOwner: state.sandboxName, + upgradeId: state.upgradeId, + }) + ).filter((sandbox) => !isDestroyed(sandbox)); + if (matches.length > 1) { + throw snapshotInvariant("Multiple live Daytona snapshot candidates were found."); + } + const listed = matches[0]; + if (!listed) return null; + const candidate = await this.input.client.getSandbox(listed.id); + if (!candidate || isDestroyed(candidate)) return null; + this.assertCandidate(candidate, state); + return candidate; + } + + private async verifyCandidate(state: SnapshotUpgradeState): Promise { + const candidate = await this.requireCandidate(state); + await this.ensureStarted(candidate, "candidate"); + if (!state.needsTransfer) { + return this.verifyMountedWorkspace(state, candidate); + } + const transfer = await this.transferNextChunk(state, candidate); + if (!transfer) return null; + if (transfer.nextChunk < requireNumber(state.chunkCount, "chunk count")) return null; + return this.restoreTransferredWorkspace(transfer, candidate); + } + + private async transferNextChunk( + state: SnapshotUpgradeState, + candidate: DaytonaSandbox, + ): Promise { + const chunkCount = requireNumber(state.chunkCount, "chunk count"); + if (state.nextChunk >= chunkCount) return state; + const source = await this.requireSource(state); + await this.ensureStarted(source, "source"); + const sourcePath = transferChunkPath(state.upgradeId, state.nextChunk); + let bytes: Uint8Array; + try { + bytes = await this.input.client.downloadFile( + state.sourceId, + sourcePath, + SNAPSHOT_TRANSFER_CHUNK_BYTES, + ); + } catch (error) { + if (error instanceof DaytonaApiError && error.status === 404) { + await this.resetPreparedSource(state); + return null; + } + throw error; + } + const digest = await sha256Hex(bytes); + await this.input.client.createFolder(candidate.id, transferChunksPath(state.upgradeId), "0700"); + await this.input.client.uploadFile(candidate.id, sourcePath, bytes); + await this.verifyTransferredChunk(candidate.id, sourcePath, bytes.byteLength, digest); + const next = SnapshotUpgradeStateSchema.parse({ ...state, nextChunk: state.nextChunk + 1 }); + await this.storeState(next); + return next; + } + + private async verifyTransferredChunk( + candidateId: string, + path: string, + size: number, + digest: string, + ): Promise { + const result = await this.input.client.execute(candidateId, { + command: verifyTransferChunkCommand({ digest, path, size }), + timeout: SNAPSHOT_EXEC_TIMEOUT_SECONDS, + }); + const evidence = ChunkEvidenceSchema.safeParse(parseJson(result.result)); + if (result.exitCode === 0 && evidence.success && evidence.data.verified) return; + if (result.exitCode === 4 && evidence.success && !evidence.data.verified) { + throw new APIError(502, "upstream_sandbox_failed", "Daytona transfer chunk was corrupted", { + retriable: true, + }); + } + throw snapshotInvariant("Daytona transfer chunk verification failed.", result.result); + } + + private async restoreTransferredWorkspace( + state: SnapshotUpgradeState, + candidate: DaytonaSandbox, + ): Promise { + const result = await this.input.client.execute(candidate.id, { + command: verifyWorkspaceArchiveCommand({ + archiveDigest: requireString(state.archiveDigest, "archive digest"), + archiveSize: requireNumber(state.archiveSize, "archive size"), + chunkCount: requireNumber(state.chunkCount, "chunk count"), + treeDigest: requireString(state.treeDigest, "tree digest"), + upgradeId: state.upgradeId, + }), + timeout: SNAPSHOT_EXEC_TIMEOUT_SECONDS, + }); + if (result.exitCode === 3 && RetryTransferSchema.safeParse(parseJson(result.result)).success) { + const reset = SnapshotUpgradeStateSchema.parse({ ...state, nextChunk: 0 }); + await this.storeState(reset); + return null; + } + const evidence = parseExecutedResult(result, ArchiveVerificationSchema, "workspace restore"); + if ( + evidence.archiveDigest !== state.archiveDigest || + evidence.treeDigest !== state.treeDigest + ) { + throw snapshotInvariant("Candidate workspace digest does not match the source."); + } + return this.markCandidateVerified(state); + } + + private async verifyMountedWorkspace( + state: SnapshotUpgradeState, + candidate: DaytonaSandbox, + ): Promise { + const evidence = await this.executeJson( + candidate.id, + digestWorkspaceCommand(), + DigestEvidenceSchema, + ); + if (evidence.treeDigest !== state.treeDigest) { + throw snapshotInvariant("Mounted workspace changed while the release gate was closed."); + } + return this.markCandidateVerified(state); + } + + private async markCandidateVerified(state: SnapshotUpgradeState): Promise { + const next = SnapshotUpgradeStateSchema.parse({ ...state, phase: "candidate-verified" }); + await this.storeState(next); + return next; + } + + private async retireSource(state: SnapshotUpgradeState): Promise { + const source = await this.input.client.getSandbox(state.sourceId); + if (source && !isDestroyed(source)) { + const expected = retiredSandboxLabels({ + sandbox: source, + sandboxName: this.input.sandboxName, + upgradeId: state.upgradeId, + }); + if (!labelsEqual(source.labels, expected)) { + this.input.provisioning.assertIdentity(source); + await this.input.client.replaceSandboxLabels(source.id, expected); + await this.assertLabels(source.id, expected, "retired source"); + } + } + const next = SnapshotUpgradeStateSchema.parse({ ...state, phase: "source-retired" }); + await this.storeState(next); + return next; + } + + private async promoteCandidate(state: SnapshotUpgradeState): Promise { + const candidateId = requireString(state.candidateId, "candidate id"); + const candidate = await this.input.client.getSandbox(candidateId); + if (!candidate || isDestroyed(candidate)) { + throw snapshotInvariant("Snapshot upgrade candidate disappeared."); + } + const canonical = canonicalSandboxLabels({ + sandboxName: this.input.sandboxName, + snapshot: state.targetSnapshot, + volumeId: state.volumeId, + volumeName: state.volumeName, + }); + const liveCanonical = ( + await this.input.client.listSandboxesByLabels({ + app: "cheatcode", + sandboxId: this.input.sandboxName, + }) + ).filter((sandbox) => !isDestroyed(sandbox)); + if (liveCanonical.some((sandbox) => sandbox.id !== candidate.id)) { + throw snapshotInvariant("Another canonical Daytona sandbox appeared during promotion."); + } + const isAlreadyPromoted = isDesiredCanonicalSandbox(candidate, { + sandboxName: this.input.sandboxName, + snapshot: state.targetSnapshot, + volumeId: state.volumeId, + volumeName: state.volumeName, + }); + if (!isAlreadyPromoted) { + this.assertCandidate(candidate, state); + await this.input.client.replaceSandboxLabels(candidate.id, canonical); + await this.assertLabels(candidate.id, canonical, "promoted candidate"); + } + const promoted = await this.requireDesiredCandidate(state); + const next = SnapshotUpgradeStateSchema.parse({ + ...state, + candidateId: promoted.id, + phase: "candidate-promoted", + }); + await this.storeState(next); + return next; + } + + private async switchDurableIdentity(state: SnapshotUpgradeState): Promise { + const candidateId = requireString(state.candidateId, "candidate id"); + const next = SnapshotUpgradeStateSchema.parse({ ...state, phase: "switched" }); + await this.input.ctx.storage.transaction(async (transaction) => { + await transaction.put(DAYTONA_ID_KEY, candidateId); + await transaction.put(SNAPSHOT_UPGRADE_STATE_KEY, next); + }); + this.input.adoptSandboxId(candidateId); + return next; + } + + private async deleteSource(state: SnapshotUpgradeState): Promise { + const candidateId = requireString(state.candidateId, "candidate id"); + await this.input.client.deleteFilePath(candidateId, transferBasePath(state.upgradeId), true); + const source = await this.input.client.getSandbox(state.sourceId); + if (source && !isDestroyed(source)) { + await this.input.client.deleteFilePath(source.id, transferBasePath(state.upgradeId), true); + await this.input.client.deleteSandbox(source.id); + await this.waitForDeletion(source.id); + } + const next = SnapshotUpgradeStateSchema.parse({ ...state, phase: "completed" }); + await this.storeState(next); + return next; + } + + private async waitForDeletion(sandboxId: string): Promise { + for (let attempt = 0; attempt < DELETE_VERIFY_ATTEMPTS; attempt += 1) { + const current = await this.input.client.getSandbox(sandboxId); + if (!current || isDestroyed(current)) return; + await sleep(DELETE_VERIFY_DELAY_MS); + } + throw new APIError(504, "upstream_sandbox_failed", "Retired Daytona sandbox was not deleted", { + retriable: true, + }); + } + + private async resetPreparedSource(state: SnapshotUpgradeState): Promise { + const reset = SnapshotUpgradeStateSchema.parse({ + ...state, + archiveDigest: null, + archiveSize: null, + chunkCount: null, + nextChunk: 0, + phase: "claimed", + treeDigest: null, + }); + await this.storeState(reset); + } + + private async requireSource(state: SnapshotUpgradeState): Promise { + const source = await this.input.client.getSandbox(state.sourceId); + if (!source || isDestroyed(source)) { + throw snapshotInvariant("Source Daytona sandbox disappeared before promotion."); + } + this.input.provisioning.assertIdentity(source); + return source; + } + + private async requireCandidate(state: SnapshotUpgradeState): Promise { + const id = requireString(state.candidateId, "candidate id"); + const candidate = await this.input.client.getSandbox(id); + if (!candidate || isDestroyed(candidate)) { + throw snapshotInvariant("Snapshot upgrade candidate disappeared."); + } + this.assertCandidate(candidate, state); + return candidate; + } + + private async requireDesiredCandidate(state: SnapshotUpgradeState): Promise { + const id = requireString(state.candidateId, "candidate id"); + const candidate = await this.input.client.getSandbox(id); + if ( + !candidate || + isDestroyed(candidate) || + !isDesiredCanonicalSandbox(candidate, { + sandboxName: this.input.sandboxName, + snapshot: state.targetSnapshot, + volumeId: state.volumeId, + volumeName: state.volumeName, + }) + ) { + throw snapshotInvariant("Promoted Daytona sandbox identity did not converge."); + } + return candidate; + } + + private async ensureStarted(sandbox: DaytonaSandbox, label: string): Promise { + if (!(await this.input.provisioning.ensureStarted(this.input.client, sandbox))) { + throw snapshotInvariant(`Snapshot upgrade ${label} sandbox disappeared.`); + } + } + + private assertCandidate(candidate: DaytonaSandbox, state: SnapshotUpgradeState): void { + if (!isUpgradeCandidate(candidate, candidateLabelInput(state))) { + throw snapshotInvariant("Daytona snapshot candidate identity mismatch."); + } + } + + private async assertLabels( + id: string, + expected: Record, + label: string, + ): Promise { + const current = await this.input.client.getSandbox(id); + if (!current || isDestroyed(current) || !labelsEqual(current.labels, expected)) { + throw snapshotInvariant(`Daytona ${label} labels did not converge.`); + } + } + + private async executeJson( + sandboxId: string, + command: string, + schema: T, + ): Promise> { + const result = await this.input.client.execute(sandboxId, { + command, + timeout: SNAPSHOT_EXEC_TIMEOUT_SECONDS, + }); + return parseExecutedResult(result, schema, "snapshot evidence"); + } + + private assertCurrentContract(state: SnapshotUpgradeState, releaseSha: string): void { + if ( + state.releaseSha !== releaseSha || + state.sandboxName !== this.input.sandboxName || + state.targetSnapshot !== this.target || + state.volumeName !== this.input.env.DAYTONA_WORKSPACE_VOLUME + ) { + throw snapshotInvariant("Pending snapshot upgrade belongs to another release contract."); + } + } + + private async loadState(): Promise { + const value = await this.input.ctx.storage.get(SNAPSHOT_UPGRADE_STATE_KEY); + return value === undefined ? null : SnapshotUpgradeStateSchema.parse(value); + } + + private storeState(state: SnapshotUpgradeState): Promise { + return this.input.ctx.storage.put( + SNAPSHOT_UPGRADE_STATE_KEY, + SnapshotUpgradeStateSchema.parse(state), + ); + } + + private get target(): string { + return this.input.env.DAYTONA_SANDBOX_SNAPSHOT; + } +} + +function parseExecutedResult( + result: { exitCode: number; result?: string | null | undefined }, + schema: T, + label: string, +): z.infer { + if (result.exitCode !== 0) { + throw snapshotInvariant(`Daytona ${label} command failed.`, result.result); + } + const parsed = schema.safeParse(parseJson(result.result)); + if (!parsed.success) { + throw snapshotInvariant(`Daytona ${label} was invalid.`); + } + return parsed.data; +} + +function snapshotResult(input: { + sourceSnapshot: string | null; + status: "absent" | "current"; + targetSnapshot: string; +}): SandboxSnapshotReconciliation { + return { + complete: true, + sourceSnapshot: input.sourceSnapshot, + status: input.status, + targetSnapshot: input.targetSnapshot, + upgradeId: null, + workspaceDigest: null, + }; +} + +function upgradingResult(state: SnapshotUpgradeState): SandboxSnapshotReconciliation { + return { + complete: false, + sourceSnapshot: state.sourceSnapshot, + status: "upgrading", + targetSnapshot: state.targetSnapshot, + upgradeId: state.upgradeId, + workspaceDigest: state.treeDigest, + }; +} + +function completedResult(state: SnapshotUpgradeState): SandboxSnapshotReconciliation { + return { + complete: true, + sourceSnapshot: state.sourceSnapshot, + status: "upgraded", + targetSnapshot: state.targetSnapshot, + upgradeId: state.upgradeId, + workspaceDigest: requireString(state.treeDigest, "tree digest"), + }; +} + +function candidateLabelInput(state: SnapshotUpgradeState) { + return { + sandboxName: state.sandboxName, + snapshot: state.targetSnapshot, + upgradeId: state.upgradeId, + volumeId: state.volumeId, + volumeName: state.volumeName, + }; +} + +function candidateName(state: SnapshotUpgradeState): string { + return `cheatcode-upgrade-${state.upgradeId}`; +} + +function transferBasePath(upgradeId: string): string { + return `/tmp/cheatcode-snapshot-upgrade/${upgradeId}`; +} + +function transferChunksPath(upgradeId: string): string { + return `${transferBasePath(upgradeId)}/chunks`; +} + +function transferChunkPath(upgradeId: string, index: number): string { + return `${transferChunksPath(upgradeId)}/chunk-${String(index).padStart(12, "0")}`; +} + +async function upgradeIdentity( + sandboxName: string, + sourceId: string, + volumeId: string, + targetSnapshot: string, +): Promise { + const digest = await sha256Hex(JSON.stringify([sandboxName, sourceId, targetSnapshot, volumeId])); + return digest.slice(0, 32); +} + +async function sha256Hex(value: string | Uint8Array): Promise { + const source = typeof value === "string" ? new TextEncoder().encode(value) : value; + const bytes = new Uint8Array(source.byteLength); + bytes.set(source); + const digest = await crypto.subtle.digest("SHA-256", bytes.buffer); + return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function labelsEqual(actual: Record, expected: Record): boolean { + const actualEntries = Object.entries(actual).sort(([left], [right]) => left.localeCompare(right)); + const expectedEntries = Object.entries(expected).sort(([left], [right]) => + left.localeCompare(right), + ); + return JSON.stringify(actualEntries) === JSON.stringify(expectedEntries); +} + +function parseJson(value: string | null | undefined): unknown { + try { + return JSON.parse(value ?? "") as unknown; + } catch { + return null; + } +} + +function requireString(value: string | null, label: string): string { + if (!value) throw snapshotInvariant(`Snapshot upgrade ${label} is absent.`); + return value; +} + +function requireNumber(value: number | null, label: string): number { + if (value === null) throw snapshotInvariant(`Snapshot upgrade ${label} is absent.`); + return value; +} + +function snapshotInvariant(message: string, output?: string | null): APIError { + return new APIError(409, "conflict_state_invalid", message, { + ...(output ? { details: { output: output.slice(-1_000) } } : {}), + retriable: false, + }); +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-workspace-state.ts b/apps/agent-worker/src/durable-objects/project-sandbox-workspace-state.ts new file mode 100644 index 00000000..65cb1b72 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-workspace-state.ts @@ -0,0 +1,475 @@ +import { + assertExactSqliteSchema, + assertSqliteRowCountPreserved, + type ExpectedSqliteObject, + setCurrentSqliteStorageVersion, +} from "@cheatcode/durable-storage"; +import { APIError } from "@cheatcode/observability"; +import type { ParsedProjectCleanupWorkspaceInput } from "./project-sandbox-runtime"; + +const CREATE_WORKSPACE_TOMBSTONE_TABLE = `CREATE TABLE IF NOT EXISTS project_workspace_tombstone ( + workspace_slug TEXT PRIMARY KEY CHECK ( + length(workspace_slug) BETWEEN 38 AND 64 + AND workspace_slug = lower(workspace_slug) + AND workspace_slug NOT GLOB '*[^a-z0-9-]*' + AND substr(workspace_slug, 1, 1) <> '-' + AND instr(workspace_slug, '--') = 0 + ), + project_id TEXT NOT NULL UNIQUE CHECK ( + length(project_id) = 36 + AND project_id = lower(project_id) + AND substr(project_id, 9, 1) = '-' + AND substr(project_id, 14, 1) = '-' + AND substr(project_id, 19, 1) = '-' + AND substr(project_id, 24, 1) = '-' + AND length(replace(project_id, '-', '')) = 32 + AND replace(project_id, '-', '') NOT GLOB '*[^0-9a-f]*' + AND substr(workspace_slug, -37) = '-' || project_id + ), + deleted_at INTEGER NOT NULL CHECK (deleted_at BETWEEN 1000000000000 AND 9999999999999), + completed_at INTEGER CHECK ( + completed_at IS NULL + OR ( + completed_at BETWEEN 1000000000000 AND 9999999999999 + AND completed_at >= deleted_at + ) + ) +) STRICT`; +const PROJECT_SANDBOX_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [ + { + name: "project_workspace_tombstone", + sql: CREATE_WORKSPACE_TOMBSTONE_TABLE, + tableName: "project_workspace_tombstone", + type: "table", + }, +]; + +export function initializeProjectSandboxStorage(ctx: DurableObjectState): void { + createWorkspaceStateTables(ctx); + setCurrentSqliteStorageVersion(ctx); + assertProjectSandboxStorage(ctx); +} + +/** Rebuilds workspace fences exactly while every public sandbox operation is closed. */ +export function reconcileProjectSandboxStorage(ctx: DurableObjectState): void { + createWorkspaceStateTables(ctx); + ctx.storage.transactionSync(() => rebuildWorkspaceStateTables(ctx)); + setCurrentSqliteStorageVersion(ctx); + assertProjectSandboxStorage(ctx); +} + +export function assertProjectSandboxStorage(ctx: DurableObjectState): void { + assertExactSqliteSchema(ctx, PROJECT_SANDBOX_STORAGE_SCHEMA); +} + +/** Opens only an already-materialized workspace store; empty activations stay storage-free. */ +export function openProjectSandboxWorkspaceState( + ctx: DurableObjectState, +): ProjectSandboxWorkspaceState | undefined { + const tableCount = ctx.storage.sql + .exec( + `SELECT count(*) AS table_count + FROM sqlite_schema + WHERE type = 'table' + AND name = 'project_workspace_tombstone'`, + ) + .one()["table_count"]; + if (tableCount === 0) { + return undefined; + } + return new ProjectSandboxWorkspaceState(ctx); +} + +/** + * Owns the synchronous fence and in-memory drain state for project workspaces. + * Project tombstones intentionally outlive cleanup while the owning account is active. + */ +export class ProjectSandboxWorkspaceState { + private activeTransitionLeaseId: string | null = null; + private readonly activeCounts = new Map(); + private activeSharedMutationCount = 0; + private activeUnscopedOperationCount = 0; + private readonly cleanupPromises = new Map>(); + private cleanupTail: Promise = Promise.resolve(); + private cleanupInProgressCount = 0; + private readonly sharedDrainWaiters = new Set<() => void>(); + private readonly unscopedDrainWaiters = new Set<() => void>(); + private readonly workspaceDrainWaiters = new Set<() => void>(); + private pendingDeletionCount: number; + + constructor(private readonly ctx: DurableObjectState) { + assertProjectSandboxStorage(ctx); + const pending = ctx.storage.sql + .exec( + `SELECT count(*) AS pending_count + FROM project_workspace_tombstone + WHERE completed_at IS NULL`, + ) + .one()["pending_count"]; + if (typeof pending !== "number" || !Number.isSafeInteger(pending) || pending < 0) { + throw new Error("Project workspace tombstone state is corrupt."); + } + this.pendingDeletionCount = pending; + } + + public acquire(workspaceSlugs: readonly string[]): () => void { + if ( + this.cleanupInProgressCount > 0 || + this.pendingDeletionCount > 0 || + this.activeSharedMutationCount > 0 || + this.activeTransitionLeaseId + ) { + throw cleanupInProgressError(); + } + const uniqueSlugs = [...new Set(workspaceSlugs)]; + for (const workspaceSlug of uniqueSlugs) { + this.assertWorkspaceAvailable(workspaceSlug); + } + for (const workspaceSlug of uniqueSlugs) { + this.activeCounts.set(workspaceSlug, (this.activeCounts.get(workspaceSlug) ?? 0) + 1); + } + let isReleased = false; + return () => { + if (isReleased) { + return; + } + isReleased = true; + for (const workspaceSlug of uniqueSlugs) { + this.release(workspaceSlug); + } + }; + } + + public acquireSharedMutation(): () => void { + if ( + this.cleanupInProgressCount > 0 || + this.pendingDeletionCount > 0 || + this.activeSharedMutationCount > 0 || + this.activeTransitionLeaseId + ) { + throw cleanupInProgressError(); + } + return this.acquireSharedMutationLease(); + } + + public acquireUnscoped(): () => void { + if ( + this.cleanupInProgressCount > 0 || + this.pendingDeletionCount > 0 || + this.activeSharedMutationCount > 0 || + this.activeTransitionLeaseId + ) { + throw cleanupInProgressError(); + } + this.activeUnscopedOperationCount += 1; + let isReleased = false; + return () => { + if (isReleased) { + return; + } + isReleased = true; + this.activeUnscopedOperationCount -= 1; + if (this.activeUnscopedOperationCount === 0) { + for (const resolve of this.unscopedDrainWaiters) { + resolve(); + } + this.unscopedDrainWaiters.clear(); + } + }; + } + + public acquireTransitionMutation(transitionId: string): () => void { + if ( + this.cleanupInProgressCount > 0 || + this.pendingDeletionCount > 0 || + this.activeSharedMutationCount > 0 || + this.activeTransitionLeaseId !== null + ) { + throw cleanupInProgressError(); + } + this.activeTransitionLeaseId = transitionId; + const releaseMutation = this.acquireSharedMutationLease(); + let isReleased = false; + return () => { + if (isReleased) { + return; + } + isReleased = true; + releaseMutation(); + if (this.activeTransitionLeaseId === transitionId) { + this.activeTransitionLeaseId = null; + } + }; + } + + public assertOperationAllowed(transitionId?: string, allowWorkspaceCleanup = false): void { + if (this.activeTransitionLeaseId !== null && this.activeTransitionLeaseId !== transitionId) { + throw sharedMutationInProgressError(); + } + if ( + !allowWorkspaceCleanup && + (this.cleanupInProgressCount > 0 || this.pendingDeletionCount > 0) + ) { + throw cleanupInProgressError(); + } + } + + public assertAccountDeletionAllowed(): void { + if (this.activeTransitionLeaseId) { + throw sharedMutationInProgressError(); + } + } + + private acquireSharedMutationLease(): () => void { + this.activeSharedMutationCount += 1; + let isReleased = false; + return () => { + if (isReleased) { + return; + } + isReleased = true; + this.activeSharedMutationCount -= 1; + if (this.activeSharedMutationCount === 0) { + for (const resolve of this.sharedDrainWaiters) { + resolve(); + } + this.sharedDrainWaiters.clear(); + } + }; + } + + public deleteWorkspace( + input: ParsedProjectCleanupWorkspaceInput, + cleanup: () => Promise, + ): Promise { + if (this.activeTransitionLeaseId) { + throw sharedMutationInProgressError(); + } + if (this.claimDeletion(input)) { + return Promise.resolve(); + } + const existing = this.cleanupPromises.get(input.workspaceSlug); + if (existing) { + return existing; + } + this.cleanupInProgressCount += 1; + const deletion = this.cleanupTail + .catch(() => undefined) + .then(() => this.performDeletion(input, cleanup)); + this.cleanupTail = deletion.catch(() => undefined); + const tracked = deletion.finally(() => { + this.cleanupInProgressCount -= 1; + if (this.cleanupPromises.get(input.workspaceSlug) === tracked) { + this.cleanupPromises.delete(input.workspaceSlug); + } + }); + this.cleanupPromises.set(input.workspaceSlug, tracked); + return tracked; + } + + public async waitForWorkspaceDrain(): Promise { + await Promise.all([this.waitForScopedWorkspaceDrain(), this.waitForUnscopedDrain()]); + } + + private claimDeletion(input: ParsedProjectCleanupWorkspaceInput): boolean { + let isNewClaim = false; + const isCompleted = this.ctx.storage.transactionSync(() => { + const rows = this.ctx.storage.sql + .exec( + `SELECT project_id, workspace_slug, completed_at + FROM project_workspace_tombstone + WHERE workspace_slug = ? OR project_id = ?`, + input.workspaceSlug, + input.projectId, + ) + .toArray(); + if (rows.length > 0) { + return parseExistingClaim(rows, input); + } + this.ctx.storage.sql.exec( + `INSERT INTO project_workspace_tombstone + (workspace_slug, project_id, deleted_at, completed_at) + VALUES (?, ?, ?, NULL)`, + input.workspaceSlug, + input.projectId, + Date.now(), + ); + isNewClaim = true; + return false; + }); + if (isNewClaim) { + this.pendingDeletionCount += 1; + } + return isCompleted; + } + + private assertWorkspaceAvailable(workspaceSlug: string): void { + const tombstone = this.ctx.storage.sql + .exec( + "SELECT 1 FROM project_workspace_tombstone WHERE workspace_slug = ? LIMIT 1", + workspaceSlug, + ) + .toArray(); + if (tombstone.length > 0) { + throw deletedWorkspaceError(workspaceSlug); + } + } + + private async performDeletion( + input: ParsedProjectCleanupWorkspaceInput, + cleanup: () => Promise, + ): Promise { + await Promise.all([ + this.waitForScopedWorkspaceDrain(), + this.waitForSharedDrain(), + this.waitForUnscopedDrain(), + ]); + await cleanup(); + this.markCompleted(input); + } + + private waitForScopedWorkspaceDrain(): Promise { + if (this.activeCounts.size === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.workspaceDrainWaiters.add(resolve); + }); + } + + private markCompleted(input: ParsedProjectCleanupWorkspaceInput): void { + this.ctx.storage.transactionSync(() => { + this.ctx.storage.sql.exec( + `UPDATE project_workspace_tombstone + SET completed_at = COALESCE(completed_at, max(?, deleted_at)) + WHERE workspace_slug = ? AND project_id = ?`, + Date.now(), + input.workspaceSlug, + input.projectId, + ); + const rows = this.ctx.storage.sql + .exec( + `SELECT completed_at FROM project_workspace_tombstone + WHERE workspace_slug = ? AND project_id = ?`, + input.workspaceSlug, + input.projectId, + ) + .toArray(); + if (rows.length !== 1 || typeof rows[0]?.["completed_at"] !== "number") { + throw new Error("Project workspace cleanup completion could not be persisted."); + } + }); + if (this.pendingDeletionCount < 1) { + throw new Error("Project workspace tombstone drain state is corrupt."); + } + this.pendingDeletionCount -= 1; + } + + private waitForSharedDrain(): Promise { + if (this.activeSharedMutationCount === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.sharedDrainWaiters.add(resolve); + }); + } + + private waitForUnscopedDrain(): Promise { + if (this.activeUnscopedOperationCount === 0) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.unscopedDrainWaiters.add(resolve); + }); + } + + private release(workspaceSlug: string): void { + const remaining = (this.activeCounts.get(workspaceSlug) ?? 1) - 1; + if (remaining > 0) { + this.activeCounts.set(workspaceSlug, remaining); + return; + } + this.activeCounts.delete(workspaceSlug); + if (this.activeCounts.size === 0) { + for (const resolve of this.workspaceDrainWaiters) { + resolve(); + } + this.workspaceDrainWaiters.clear(); + } + } +} + +function createWorkspaceStateTables(ctx: DurableObjectState): void { + ctx.storage.sql.exec(CREATE_WORKSPACE_TOMBSTONE_TABLE); +} + +function rebuildWorkspaceStateTables(ctx: DurableObjectState): void { + ctx.storage.sql.exec("DROP TABLE IF EXISTS project_workspace_tombstone_reconcile_source"); + ctx.storage.sql.exec( + "ALTER TABLE project_workspace_tombstone RENAME TO project_workspace_tombstone_reconcile_source", + ); + ctx.storage.sql.exec(canonicalCreateSql(CREATE_WORKSPACE_TOMBSTONE_TABLE)); + copyWorkspaceStateRows(ctx); + ctx.storage.sql.exec("DROP TABLE project_workspace_tombstone_reconcile_source"); + // These are release-control evidence only; project deletion fences live in the tombstone table. + ctx.storage.sql.exec("DROP TABLE IF EXISTS project_workspace_transition"); + ctx.storage.sql.exec("DROP TABLE IF EXISTS project_workspace_retired_slug"); + ctx.storage.sql.exec("DROP TABLE IF EXISTS project_workspace_transition_reconcile_source"); + ctx.storage.sql.exec("DROP TABLE IF EXISTS project_workspace_retired_slug_reconcile_source"); +} + +function copyWorkspaceStateRows(ctx: DurableObjectState): void { + ctx.storage.sql.exec( + `INSERT INTO project_workspace_tombstone + (workspace_slug, project_id, deleted_at, completed_at) + SELECT workspace_slug, project_id, deleted_at, completed_at + FROM project_workspace_tombstone_reconcile_source`, + ); + assertSqliteRowCountPreserved( + ctx, + "project_workspace_tombstone_reconcile_source", + "project_workspace_tombstone", + ); +} + +function canonicalCreateSql(sql: string): string { + return sql.replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE"); +} + +function parseExistingClaim( + rows: Record[], + input: ParsedProjectCleanupWorkspaceInput, +): boolean { + const row = rows.length === 1 ? rows[0] : undefined; + if (row?.["project_id"] !== input.projectId || row["workspace_slug"] !== input.workspaceSlug) { + throw workspaceIdentityConflictError(input); + } + return typeof row["completed_at"] === "number"; +} + +function deletedWorkspaceError(workspaceSlug: string): APIError { + return new APIError(410, "conflict_state_invalid", "Project workspace has been deleted", { + details: { workspaceSlug }, + retriable: false, + }); +} + +function cleanupInProgressError(): APIError { + return new APIError(409, "conflict_state_invalid", "Project workspace cleanup is in progress", { + retriable: true, + }); +} + +function sharedMutationInProgressError(): APIError { + return new APIError(409, "conflict_state_invalid", "Workspace maintenance is in progress", { + retriable: true, + }); +} + +function workspaceIdentityConflictError(input: ParsedProjectCleanupWorkspaceInput): APIError { + return new APIError(409, "conflict_state_invalid", "Project workspace identity mismatch", { + details: { projectId: input.projectId, workspaceSlug: input.workspaceSlug }, + hint: "Refuse the stale cleanup request and inspect the project deletion record.", + retriable: false, + }); +} diff --git a/apps/agent-worker/src/durable-objects/project-sandbox-workspace-transition.ts b/apps/agent-worker/src/durable-objects/project-sandbox-workspace-transition.ts new file mode 100644 index 00000000..bf2c1771 --- /dev/null +++ b/apps/agent-worker/src/durable-objects/project-sandbox-workspace-transition.ts @@ -0,0 +1,611 @@ +import { APIError } from "@cheatcode/observability"; +import { + CanonicalProjectWorkspaceSlugSchema, + canonicalWorkspaceDigest, + type InternalWorkspaceReconciliationBody, + InternalWorkspaceReconciliationBodySchema, + type InternalWorkspaceReconciliationResponse, + type SandboxSnapshotReconciliation, + type WorkspaceTransitionProject, +} from "@cheatcode/types"; +import { z } from "zod"; +import { WORKSPACE_DIR } from "./project-sandbox-content-support"; +import { + APP_PREVIEW_SLOT_PREFIX, + PORT_ALLOC_KEY, + PortAllocationSchema, + PROC_PREFIX, + PROCESS_PORT_ALLOC_KEY, + ProcessPortReservationsSchema, + type ProcessRecord, + ProcessRecordSchema, + pruneExpiredProcessPortReservations, + shellQuote, + timeoutSeconds, +} from "./project-sandbox-process-support"; +import { ProjectSandboxProcesses } from "./project-sandbox-processes"; +import { ProjectWorkspaceSlugSchema, workspaceSlugFromPath } from "./project-sandbox-runtime"; +import { ProjectSandboxSnapshotUpgrade } from "./project-sandbox-snapshot-upgrade"; + +const WorkspaceTransitionScriptResultSchema = z + .object({ present: z.array(CanonicalProjectWorkspaceSlugSchema) }) + .strict(); +const WORKSPACE_TRANSITION_STATE_KEY = "workspace_transition_reconciliation"; +const WorkspaceTransitionStateSchema = z + .object({ + canonicalDigest: z.string().regex(/^[0-9a-f]{64}$/u), + presentSlugs: z.array(CanonicalProjectWorkspaceSlugSchema).max(10_000), + releaseSha: z.string().regex(/^[0-9a-f]{40}$/u), + }) + .strict() + .superRefine((state, context) => { + if ( + state.presentSlugs.some((workspaceSlug, index) => { + const previous = state.presentSlugs[index - 1]; + return previous !== undefined && previous >= workspaceSlug; + }) + ) { + context.addIssue({ + code: "custom", + message: "Workspace presence evidence must be sorted and unique.", + }); + } + }); + +type WorkspaceTransitionState = z.infer; + +interface WorkspaceTransitionIdentity { + canonicalDigest: string; +} + +interface ReconciliationPlan { + affectedProcessNames: Set; + changedWorkspaceSlugs: Set; + staleWorkspaceSlugs: Set; +} + +export abstract class ProjectSandboxWorkspaceTransition extends ProjectSandboxProcesses { + public async prepareWorkspaceTransition( + input: InternalWorkspaceReconciliationBody, + ): Promise { + const parsed = parsePhase(input, "prepare"); + const identity = await transitionIdentity(parsed); + const existing = await this.loadWorkspaceTransitionState(); + if (existing) { + this.assertWorkspaceTransitionState(existing, parsed, identity); + await this.verifyPreparedWorkspaceTransition(parsed.projects, existing.presentSlugs); + await this.verifyStoredWorkspaceState(parsed.projects); + return transitionResult( + parsed, + identity, + "prepared", + pendingSnapshot(this.env.DAYTONA_SANDBOX_SNAPSHOT), + ); + } + const records = await this.loadProcessRecords(); + const allocation = PortAllocationSchema.parse( + (await this.ctx.storage.get(PORT_ALLOC_KEY)) ?? {}, + ); + ProcessPortReservationsSchema.parse((await this.ctx.storage.get(PROCESS_PORT_ALLOC_KEY)) ?? {}); + assertProjectPortTransitionSafe(parsed.projects, allocation.ports); + const plan = reconciliationPlan(parsed.projects, records, allocation.ports); + const id = await this.ensureExistingSandboxStarted(); + await this.terminateTransitionProcesses(id, records, plan); + const present = id ? await this.runWorkspaceTransitionScript(id, parsed.projects) : []; + const counts = await this.reconcileStoredWorkspaceState(parsed.projects, plan); + await this.verifyStoredWorkspaceState(parsed.projects); + await this.ctx.storage.put( + WORKSPACE_TRANSITION_STATE_KEY, + WorkspaceTransitionStateSchema.parse({ + canonicalDigest: identity.canonicalDigest, + presentSlugs: [...present].sort(), + releaseSha: parsed.releaseSha, + }), + ); + return transitionResult( + parsed, + identity, + "prepared", + pendingSnapshot(this.env.DAYTONA_SANDBOX_SNAPSHOT), + counts, + ); + } + + public async finalizeWorkspaceTransition( + input: InternalWorkspaceReconciliationBody, + ): Promise { + const parsed = parsePhase(input, "finalize"); + const identity = await transitionIdentity(parsed); + const state = await this.loadWorkspaceTransitionState(); + if (state) { + this.assertWorkspaceTransitionState(state, parsed, identity); + await this.verifyPreparedWorkspaceTransition(parsed.projects, state.presentSlugs); + } else { + assertCanonicalFinalizationInput(parsed.projects); + const id = await this.ensureExistingSandboxStarted(); + if (id) { + await this.verifyWorkspaceTransitionScript(id, parsed.projects); + } + } + await this.verifyStoredWorkspaceState(parsed.projects); + const snapshot = await this.advanceSnapshotUpgrade(parsed.releaseSha); + if (snapshot.complete) { + await this.clearWorkspaceTransitionState(); + } + return transitionResult(parsed, identity, "completed", snapshot); + } + + private async verifyPreparedWorkspaceTransition( + projects: WorkspaceTransitionProject[], + expectedPresent: string[], + ): Promise { + const id = await this.ensureExistingSandboxStarted(); + if (id) { + await this.verifyWorkspaceTransitionScript(id, projects, expectedPresent); + return; + } + if (expectedPresent.length > 0) { + throw transitionError("Prepared project folders no longer have a Daytona sandbox."); + } + } + + private async loadWorkspaceTransitionState(): Promise { + const value = await this.ctx.storage.get(WORKSPACE_TRANSITION_STATE_KEY); + return value === undefined ? null : WorkspaceTransitionStateSchema.parse(value); + } + + private async clearWorkspaceTransitionState(): Promise { + await this.ctx.storage.delete(WORKSPACE_TRANSITION_STATE_KEY); + if ((await this.ctx.storage.get(WORKSPACE_TRANSITION_STATE_KEY)) !== undefined) { + throw transitionError("Completed workspace transition state could not be removed."); + } + } + + private assertWorkspaceTransitionState( + state: WorkspaceTransitionState, + input: InternalWorkspaceReconciliationBody, + identity: WorkspaceTransitionIdentity, + ): void { + if ( + state.releaseSha !== input.releaseSha || + state.canonicalDigest !== identity.canonicalDigest + ) { + throw transitionError("Pending workspace transition belongs to another release contract."); + } + } + + private async advanceSnapshotUpgrade(releaseSha: string): Promise { + const client = await this.ensureClient(); + return new ProjectSandboxSnapshotUpgrade({ + adoptSandboxId: (sandboxId) => this.adoptDaytonaId(sandboxId), + client, + ctx: this.ctx, + env: this.env, + killProcesses: () => super.killAllProcesses(), + provisioning: this.sandboxProvisioning(), + sandboxName: this.sandboxName(), + toUpstreamError: (error, fallback) => this.toUpstreamError(error, fallback), + }).advance(releaseSha); + } + + private async loadProcessRecords(): Promise> { + const stored = await this.ctx.storage.list({ prefix: PROC_PREFIX }); + const records = new Map(); + for (const [key, value] of stored) { + const parsed = ProcessRecordSchema.safeParse(value); + if (!parsed.success) { + throw transitionError("Stored sandbox process state is invalid."); + } + records.set(key.slice(PROC_PREFIX.length), parsed.data); + } + return records; + } + + private async terminateTransitionProcesses( + id: string | null, + records: Map, + plan: ReconciliationPlan, + ): Promise { + if (id) { + for (const workspaceSlug of [...plan.staleWorkspaceSlugs].sort()) { + await this.terminateUntrackedWorkspaceProcesses(id, workspaceSlug); + } + } + for (const processName of [...plan.affectedProcessNames].sort()) { + if (id && records.has(processName)) { + await this.deleteProcessRecord(id, processName); + } else { + await this.ctx.storage.delete(`${PROC_PREFIX}${processName}`); + } + } + } + + private async reconcileStoredWorkspaceState( + projects: WorkspaceTransitionProject[], + plan: ReconciliationPlan, + ): Promise<{ + processPortReservationsRemoved: number; + processRecordsRemoved: number; + projectPortsRemoved: number; + }> { + const canonical = canonicalWorkspaceSet(projects); + const canonicalForCurrent = new Map( + projects.map((project) => [project.currentWorkspaceSlug, project.canonicalWorkspaceSlug]), + ); + return this.ctx.storage.transaction(async (transaction) => { + const storedAllocation = await transaction.get(PORT_ALLOC_KEY); + const allocation = PortAllocationSchema.parse(storedAllocation ?? {}); + const { ports, removed: projectPortsRemoved } = canonicalProjectPorts( + allocation.ports, + canonical, + canonicalForCurrent, + ); + if (storedAllocation !== undefined || Object.keys(ports).length > 0) { + await transaction.put(PORT_ALLOC_KEY, { ...allocation, ports }); + } + + const records = await transaction.list({ prefix: PROC_PREFIX }); + const storedReservations = await transaction.get(PROCESS_PORT_ALLOC_KEY); + const before = ProcessPortReservationsSchema.parse(storedReservations ?? {}); + const pruned = pruneExpiredProcessPortReservations(before, records, Date.now()); + const reservations = canonicalProcessPortReservations( + pruned, + canonical, + plan.affectedProcessNames, + ); + if (storedReservations !== undefined || Object.keys(reservations).length > 0) { + await transaction.put(PROCESS_PORT_ALLOC_KEY, reservations); + } + return { + processPortReservationsRemoved: + Object.keys(before).length - Object.keys(reservations).length, + processRecordsRemoved: plan.affectedProcessNames.size, + projectPortsRemoved, + }; + }); + } + + private async verifyStoredWorkspaceState(projects: WorkspaceTransitionProject[]): Promise { + const canonical = canonicalWorkspaceSet(projects); + const records = await this.loadProcessRecords(); + for (const [name, record] of records) { + if (processNeedsRemoval(name, record, canonical, new Set())) { + throw transitionError("Noncanonical sandbox process state remains after reconciliation."); + } + } + const allocation = PortAllocationSchema.parse( + (await this.ctx.storage.get(PORT_ALLOC_KEY)) ?? {}, + ); + if (Object.keys(allocation.ports).some((workspaceSlug) => !canonical.has(workspaceSlug))) { + throw transitionError("Noncanonical project port state remains after reconciliation."); + } + const reservations = ProcessPortReservationsSchema.parse( + (await this.ctx.storage.get(PROCESS_PORT_ALLOC_KEY)) ?? {}, + ); + if ( + Object.keys(reservations).some((processId) => { + const slot = previewSlotWorkspaceSlug(processId); + return slot === "invalid" || (slot !== null && !canonical.has(slot)); + }) + ) { + throw transitionError("Noncanonical process port state remains after reconciliation."); + } + } + + private async runWorkspaceTransitionScript( + id: string, + projects: WorkspaceTransitionProject[], + ): Promise { + return this.executeWorkspaceTransitionScript(id, { mode: "prepare", projects }); + } + + private async verifyWorkspaceTransitionScript( + id: string, + projects: WorkspaceTransitionProject[], + expectedPresent?: string[], + ): Promise { + const present = await this.executeWorkspaceTransitionScript(id, { mode: "verify", projects }); + if (expectedPresent) { + assertSameSlugs(present, expectedPresent); + } + } + + private async executeWorkspaceTransitionScript( + id: string, + payload: { + mode: "prepare" | "verify"; + projects: WorkspaceTransitionProject[]; + }, + ): Promise { + const encoded = btoa(JSON.stringify(payload)); + const result = await this.client().execute(id, { + command: `python3 -c ${shellQuote(WORKSPACE_TRANSITION_SCRIPT)} ${shellQuote(encoded)}`, + cwd: WORKSPACE_DIR, + timeout: timeoutSeconds(120_000), + }); + if (result.exitCode !== 0) { + throw transitionError("Project workspace folders could not be reconciled.", result.result); + } + const parsed = WorkspaceTransitionScriptResultSchema.safeParse(parseJson(result.result)); + if (!parsed.success) { + throw transitionError("Project workspace transition returned invalid evidence."); + } + return parsed.data.present; + } +} + +function reconciliationPlan( + projects: WorkspaceTransitionProject[], + records: Map, + projectPorts: Record, +): ReconciliationPlan { + const canonical = canonicalWorkspaceSet(projects); + const changedWorkspaceSlugs = new Set( + projects + .filter((project) => project.currentWorkspaceSlug !== project.canonicalWorkspaceSlug) + .flatMap((project) => [project.currentWorkspaceSlug, project.canonicalWorkspaceSlug]), + ); + const staleWorkspaceSlugs = new Set( + projects + .filter((project) => project.currentWorkspaceSlug !== project.canonicalWorkspaceSlug) + .flatMap((project) => [project.currentWorkspaceSlug, project.canonicalWorkspaceSlug]), + ); + for (const workspaceSlug of Object.keys(projectPorts)) { + if (!canonical.has(workspaceSlug)) { + const parsed = ProjectWorkspaceSlugSchema.safeParse(workspaceSlug); + if (parsed.success) { + staleWorkspaceSlugs.add(parsed.data); + } + } + } + const affectedProcessNames = new Set(); + for (const [name, record] of records) { + const scopes = processWorkspaceSlugs(name, record); + for (const scope of scopes) { + if (scope !== "invalid" && !canonical.has(scope)) { + staleWorkspaceSlugs.add(scope); + } + } + if (processNeedsRemoval(name, record, canonical, changedWorkspaceSlugs)) { + affectedProcessNames.add(name); + } + } + return { affectedProcessNames, changedWorkspaceSlugs, staleWorkspaceSlugs }; +} + +function canonicalProjectPorts( + current: Record, + canonical: ReadonlySet, + canonicalForCurrent: ReadonlyMap, +): { ports: Record; removed: number } { + const ports: Record = {}; + let removed = 0; + for (const [workspaceSlug, port] of Object.entries(current)) { + const target = canonicalForCurrent.get(workspaceSlug) ?? workspaceSlug; + if (!canonical.has(target)) { + removed += 1; + continue; + } + if (ports[target] !== undefined && ports[target] !== port) { + throw transitionError("Canonical project port allocation collides with another port."); + } + if (target !== workspaceSlug) { + removed += 1; + } + ports[target] = port; + } + return { ports, removed }; +} + +function canonicalProcessPortReservations( + current: z.infer, + canonical: ReadonlySet, + affectedProcessNames: ReadonlySet, +): z.infer { + return Object.fromEntries( + Object.entries(current).filter(([processId]) => { + if (affectedProcessNames.has(processId)) { + return false; + } + const slot = previewSlotWorkspaceSlug(processId); + return slot === null || (slot !== "invalid" && canonical.has(slot)); + }), + ); +} + +function assertProjectPortTransitionSafe( + projects: WorkspaceTransitionProject[], + projectPorts: Record, +): void { + const canonical = canonicalWorkspaceSet(projects); + const canonicalForCurrent = new Map( + projects.map((project) => [project.currentWorkspaceSlug, project.canonicalWorkspaceSlug]), + ); + const ports = new Map(); + for (const [workspaceSlug, port] of Object.entries(projectPorts)) { + const target = canonicalForCurrent.get(workspaceSlug) ?? workspaceSlug; + if (!canonical.has(target)) { + continue; + } + const existing = ports.get(target); + if (existing !== undefined && existing !== port) { + throw transitionError("Canonical project port allocation collides with another port."); + } + ports.set(target, port); + } +} + +function processNeedsRemoval( + name: string, + record: ProcessRecord, + canonical: ReadonlySet, + changed: ReadonlySet, +): boolean { + const scopes = processWorkspaceSlugs(name, record); + return scopes.some((scope) => scope === "invalid" || !canonical.has(scope) || changed.has(scope)); +} + +function processWorkspaceSlugs(name: string, record: ProcessRecord): Array { + const scopes: Array = []; + const cwdSlug = workspaceSlugFromPath(record.cwd); + if (cwdSlug) { + scopes.push(cwdSlug); + } else if (record.cwd !== WORKSPACE_DIR && record.cwd.startsWith(`${WORKSPACE_DIR}/`)) { + scopes.push("invalid"); + } + const slotSlug = previewSlotWorkspaceSlug(name); + if (slotSlug !== null && !scopes.includes(slotSlug)) { + scopes.push(slotSlug); + } + return scopes; +} + +function previewSlotWorkspaceSlug(processId: string): string | "invalid" | null { + if (!processId.startsWith(APP_PREVIEW_SLOT_PREFIX)) { + return null; + } + const parsed = ProjectWorkspaceSlugSchema.safeParse( + processId.slice(APP_PREVIEW_SLOT_PREFIX.length), + ); + return parsed.success ? parsed.data : "invalid"; +} + +function canonicalWorkspaceSet(projects: WorkspaceTransitionProject[]): Set { + return new Set(projects.map((project) => project.canonicalWorkspaceSlug)); +} + +function assertCanonicalFinalizationInput(projects: WorkspaceTransitionProject[]): void { + if (projects.some((project) => project.currentWorkspaceSlug !== project.canonicalWorkspaceSlug)) { + throw transitionError("Workspace transition evidence is absent before canonical commit."); + } +} + +async function transitionIdentity( + input: InternalWorkspaceReconciliationBody, +): Promise { + const projects = [...input.projects].sort((left, right) => + left.projectId.localeCompare(right.projectId), + ); + return { + canonicalDigest: await canonicalWorkspaceDigest( + projects.map((project) => project.canonicalWorkspaceSlug), + ), + }; +} + +function parsePhase( + input: InternalWorkspaceReconciliationBody, + phase: "finalize" | "prepare", +): InternalWorkspaceReconciliationBody { + const parsed = InternalWorkspaceReconciliationBodySchema.parse(input); + if (parsed.phase !== phase) { + throw transitionError(`Workspace transition phase must be ${phase}.`); + } + return parsed; +} + +function transitionResult( + input: InternalWorkspaceReconciliationBody, + identity: WorkspaceTransitionIdentity, + transitionPhase: "completed" | "prepared", + snapshot: SandboxSnapshotReconciliation, + counts = { + processPortReservationsRemoved: 0, + processRecordsRemoved: 0, + projectPortsRemoved: 0, + }, +): InternalWorkspaceReconciliationResponse { + return { + canonicalDigest: identity.canonicalDigest, + canonicalWorkspaceCount: input.projects.length, + ...counts, + ok: true, + releaseSha: input.releaseSha, + snapshot, + transitionPhase, + verified: true, + }; +} + +function pendingSnapshot(targetSnapshot: string): SandboxSnapshotReconciliation { + return { + complete: false, + sourceSnapshot: null, + status: "upgrading", + targetSnapshot, + upgradeId: null, + workspaceDigest: null, + }; +} + +function parseJson(value: string | null | undefined): unknown { + try { + return JSON.parse(value ?? "") as unknown; + } catch { + return null; + } +} + +function assertSameSlugs(actual: readonly string[], expected: readonly string[]): void { + if ([...actual].sort().join("\n") !== [...expected].sort().join("\n")) { + throw transitionError("Project workspace folder evidence changed during transition."); + } +} + +function transitionError(message: string, output?: string | null): APIError { + return new APIError(409, "conflict_state_invalid", message, { + ...(output ? { details: { output: output.slice(-1_000) } } : {}), + retriable: false, + }); +} + +const WORKSPACE_TRANSITION_SCRIPT = ` +import base64 +import json +import os +import stat +import sys + +payload = json.loads(base64.b64decode(sys.argv[1]).decode("utf-8")) +root = "/workspace" +projects = payload["projects"] + +def path_for(slug): + return os.path.join(root, slug) + +def kind(path): + try: + metadata = os.lstat(path) + except FileNotFoundError: + return "absent" + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise RuntimeError(f"Workspace path is not a real directory: {path}") + return "directory" + +states = [] +for project in projects: + current = project["currentWorkspaceSlug"] + canonical = project["canonicalWorkspaceSlug"] + current_kind = kind(path_for(current)) + canonical_kind = current_kind if current == canonical else kind(path_for(canonical)) + if current != canonical and current_kind == "directory" and canonical_kind == "directory": + raise RuntimeError(f"Workspace rename destination already exists: {canonical}") + states.append((current, canonical, current_kind, canonical_kind)) + +if payload["mode"] == "prepare": + for current, canonical, current_kind, canonical_kind in states: + if current != canonical and current_kind == "directory" and canonical_kind == "absent": + os.rename(path_for(current), path_for(canonical)) + +present = [] +for project in projects: + current = project["currentWorkspaceSlug"] + canonical = project["canonicalWorkspaceSlug"] + if current != canonical and kind(path_for(current)) != "absent": + raise RuntimeError(f"Noncanonical workspace still exists: {current}") + if kind(path_for(canonical)) == "directory": + present.append(canonical) + +present.sort() +print(json.dumps({"present": present}, separators=(",", ":"))) +`; diff --git a/apps/agent-worker/src/durable-objects/project-sandbox.ts b/apps/agent-worker/src/durable-objects/project-sandbox.ts index a5aae444..64682620 100644 --- a/apps/agent-worker/src/durable-objects/project-sandbox.ts +++ b/apps/agent-worker/src/durable-objects/project-sandbox.ts @@ -1,4 +1,12 @@ +import type { + InternalDurableObjectStorageRequest, + InternalDurableObjectStorageResponse, +} from "@cheatcode/types"; +import { InternalWorkspaceReconciliationBodySchema } from "@cheatcode/types"; +import { reconcileProjectSandboxStorageRequest } from "./durable-storage-reconciliation"; import { ProjectSandboxContent } from "./project-sandbox-content"; +import { APP_PREVIEW_SLOT_PREFIX } from "./project-sandbox-process-support"; +import { ProjectWorkspaceSlugSchema, workspaceSlugFromPath } from "./project-sandbox-runtime"; /** * Public Durable Object facade. Every operational RPC takes an in-memory lease @@ -6,10 +14,16 @@ import { ProjectSandboxContent } from "./project-sandbox-content"; * work without holding blockConcurrencyWhile across Daytona requests. */ export class ProjectSandbox extends ProjectSandboxContent { + public reconcileStorageSchema( + value: InternalDurableObjectStorageRequest, + ): InternalDurableObjectStorageResponse { + return reconcileProjectSandboxStorageRequest(this.ctx, this.env, value); + } + public override registerOwner( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.registerOwner(...args)); + return this.withActiveOwnerRegistration(args[0], () => super.registerOwner(...args)); } public override setQuotaPeriod( @@ -27,6 +41,8 @@ export class ProjectSandbox extends ProjectSandboxContent { public override renewRun( ...args: Parameters ): ReturnType { + // Late cleanup is absorbed by account deletion; workspace transitions reject it + // because end/alarm can change Daytona activity or auto-stop during a rename. return this.withActiveSandboxCleanupSignal(() => super.renewRun(...args)); } @@ -75,93 +91,125 @@ export class ProjectSandbox extends ProjectSandboxContent { public override runCode( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.runCode(...args)); + return this.withActiveProjectWorkspaceOperation(null, () => super.runCode(...args)); } public override exec( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.exec(...args)); + return this.withActiveProjectWorkspaceOperation(null, () => super.exec(...args)); } public override startProcess( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.startProcess(...args)); + return this.withActiveProjectWorkspaceOperation(null, () => super.startProcess(...args)); } public override allocateProjectPort( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.allocateProjectPort(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlug(args[0].projectId), () => + super.allocateProjectPort(...args), + ); + } + + public override getProjectPort( + ...args: Parameters + ): ReturnType { + return this.withActiveProjectWorkspaceOperation(workspaceSlug(args[0].projectId), () => + super.getProjectPort(...args), + ); } public override allocateProcessPort( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.allocateProcessPort(...args)); + return this.withActiveProjectWorkspaceOperation( + workspaceSlugFromProcessId(args[0].processId), + () => super.allocateProcessPort(...args), + ); } public override killAllProcesses( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.killAllProcesses(...args)); + return this.withActiveSharedWorkspaceMutation(() => super.killAllProcesses(...args)); } public override killProcess( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.killProcess(...args)); + return this.withActiveProjectWorkspaceOperation( + workspaceSlugFromProcessId(args[0].processId), + () => super.killProcess(...args), + ); } public override readDevServerLogs( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.readDevServerLogs(...args)); + return this.withActiveProjectWorkspaceOperation( + workspaceSlugFromProcessId(args[0].processId), + () => super.readDevServerLogs(...args), + ); } public override downloadProjectArchive( ...args: Parameters ): ReturnType { - return this.withActiveSandboxStreamingOperation((release) => - super.downloadProjectArchiveForRpc(args[0], release), + return this.withActiveProjectWorkspaceStreamingOperation( + workspaceSlug(args[0].workspaceSlug), + (release) => super.downloadProjectArchiveForRpc(args[0], release), ); } public override readFile( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.readFile(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.readFile(...args), + ); } public override previewFile( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.previewFile(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.previewFile(...args), + ); } public override writeFile( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.writeFile(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.writeFile(...args), + ); } public override listFiles( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.listFiles(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.listFiles(...args), + ); } public override searchFiles( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.searchFiles(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.searchFiles(...args), + ); } public override deleteFile( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.deleteFile(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlugFromPath(args[0].path), () => + super.deleteFile(...args), + ); } public override getSignedPreviewUrl( @@ -170,27 +218,78 @@ export class ProjectSandbox extends ProjectSandboxContent { return this.withActiveSandboxOperation(() => super.getSignedPreviewUrl(...args)); } + public override exposeBrowserTakeover( + ...args: Parameters + ): ReturnType { + return this.withActiveSandboxOperation(() => super.exposeBrowserTakeover(...args)); + } + + public override stopBrowserTakeover( + ...args: Parameters + ): ReturnType { + return this.withActiveSandboxCleanupSignal(() => super.stopBrowserTakeover(...args)); + } + public override exposeCodeServer( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.exposeCodeServer(...args)); + return this.withActiveProjectWorkspaceOperation( + workspaceSlugFromPath(args[0].workspacePath), + () => super.exposeCodeServer(...args), + ); } public override wakePreview( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.wakePreview(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlug(args[0].workspaceSlug), () => + super.wakePreview(...args), + ); } public override projectPreviewStatus( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.projectPreviewStatus(...args)); + return this.withActiveProjectWorkspaceOperation(workspaceSlug(args[0].workspaceSlug), () => + super.projectPreviewStatus(...args), + ); } public override cleanupProjectWorkspace( ...args: Parameters ): ReturnType { - return this.withActiveSandboxOperation(() => super.cleanupProjectWorkspace(...args)); + return this.withActiveProjectWorkspaceCleanup(() => super.cleanupProjectWorkspace(...args)); } + + public override prepareWorkspaceTransition( + ...args: Parameters + ): ReturnType { + return this.withActiveWorkspaceTransition(transitionId(args[0]), () => + super.prepareWorkspaceTransition(...args), + ); + } + + public override finalizeWorkspaceTransition( + ...args: Parameters + ): ReturnType { + return this.withActiveWorkspaceTransition(transitionId(args[0]), () => + super.finalizeWorkspaceTransition(...args), + ); + } +} + +function transitionId(input: unknown): string { + const parsed = InternalWorkspaceReconciliationBodySchema.parse(input); + return `workspace-sandbox-release:${parsed.releaseSha}`; +} + +function workspaceSlug(value: string | undefined): string | null { + const parsed = ProjectWorkspaceSlugSchema.safeParse(value); + return parsed.success ? parsed.data : null; +} + +function workspaceSlugFromProcessId(processId: string | undefined): string | null { + return processId?.startsWith(APP_PREVIEW_SLOT_PREFIX) + ? workspaceSlug(processId.slice(APP_PREVIEW_SLOT_PREFIX.length)) + : null; } diff --git a/apps/agent-worker/src/durable-objects/research-provider.ts b/apps/agent-worker/src/durable-objects/research-provider.ts index 1243c6e7..b310fb1b 100644 --- a/apps/agent-worker/src/durable-objects/research-provider.ts +++ b/apps/agent-worker/src/durable-objects/research-provider.ts @@ -1,10 +1,12 @@ import { getProviderKey } from "@cheatcode/byok"; import { createDb, type DatabaseHandle, withUserContext } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; import type { createLogger } from "@cheatcode/observability"; import { UserId } from "@cheatcode/types"; import { closeDatabaseBestEffort } from "./db-close"; interface ResearchProviderEnv { + DATABASE_CONTEXT_SIGNING_SECRET_AGENT: WorkerSecret; HYPERDRIVE: Hyperdrive; } @@ -22,7 +24,10 @@ export async function resolveResearchCredentials( input: ResearchProviderInput, logger: ReturnType, ): Promise { - const dbHandle = createDb(env.HYPERDRIVE); + const dbHandle = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { const credentials = await withUserContext(dbHandle.db, UserId(input.userId), async (db) => { const exaApiKey = await getProviderKey(db, "exa"); diff --git a/apps/agent-worker/src/durable-objects/run-state.ts b/apps/agent-worker/src/durable-objects/run-state.ts index eed478ed..756afd4b 100644 --- a/apps/agent-worker/src/durable-objects/run-state.ts +++ b/apps/agent-worker/src/durable-objects/run-state.ts @@ -11,5 +11,5 @@ export function parseLastSeqParam(value: string | null): number | null { } export function hasActiveRun(status: string | undefined): boolean { - return status === "running" || status === "paused"; + return status === "running"; } diff --git a/apps/agent-worker/src/durable-objects/run-summary.ts b/apps/agent-worker/src/durable-objects/run-summary.ts index c63440f7..5488aa2a 100644 --- a/apps/agent-worker/src/durable-objects/run-summary.ts +++ b/apps/agent-worker/src/durable-objects/run-summary.ts @@ -1,16 +1,10 @@ import { isMessagePartRow } from "../streaming/ui-message-stream"; -export type AgentRunSnapshotStatus = - | "idle" - | "running" - | "paused" - | "completed" - | "failed" - | "canceled"; +export type AgentRunSnapshotStatus = "idle" | "running" | "completed" | "failed" | "canceled"; const SUMMARY_MAX_LENGTH = 240; const SUMMARY_QUERY_PAGE_SIZE = 100; -const SNAPSHOT_STATUSES = new Set(["running", "paused", "completed", "failed", "canceled"]); +const SNAPSHOT_STATUSES = new Set(["running", "completed", "failed", "canceled"]); export function snapshotAgentRunStatus(status: string | undefined): AgentRunSnapshotStatus { return SNAPSHOT_STATUSES.has(status ?? "") ? (status as AgentRunSnapshotStatus) : "idle"; diff --git a/apps/agent-worker/src/index.ts b/apps/agent-worker/src/index.ts index cb03ae44..4c248b0a 100644 --- a/apps/agent-worker/src/index.ts +++ b/apps/agent-worker/src/index.ts @@ -1,5 +1,6 @@ import { AgentWorkerEnvSchema } from "@cheatcode/env"; import { + APIError, createLogger, emitErrorEvent, emitPerformanceMetric, @@ -7,19 +8,28 @@ import { toAPIError, withErrorHandler, } from "@cheatcode/observability"; -import { normalizeTelemetryPath } from "@cheatcode/types"; +import { + INTERNAL_DATABASE_READINESS_PATH, + INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + normalizeTelemetryPath, +} from "@cheatcode/types"; import { type Context, Hono } from "hono"; import { routePath } from "hono/route"; import { registerAgentRunHttpRoutes } from "./agent-api-run-routes"; import { registerAgentSystemHttpRoutes } from "./agent-api-system-routes"; import type { AgentEnv } from "./agent-env"; +import { registerAgentDatabaseReadinessRoute } from "./database-readiness"; +import { registerAgentDurableObjectStorageRoute } from "./durable-object-storage"; import { AgentRun } from "./durable-objects/agent-run"; +import { AgentRunWorkflow } from "./durable-objects/agent-run-workflow"; import { ProjectSandbox } from "./durable-objects/project-sandbox"; import { formatAgentRouteError } from "./error-handling"; -import { tryHandleLocalPreviewRequest } from "./local-preview"; import { registerSandboxHttpRoutes } from "./sandbox-http-routes"; +import { registerSkillProposalHttpRoutes } from "./skill-proposal-http-routes"; +import { registerSkillRuntimeExecutionRoutes } from "./skill-runtime-execution-routes"; +import { registerSkillRuntimeManagedRoutes } from "./skill-runtime-managed-routes"; -export { AgentRun, ProjectSandbox }; +export { AgentRun, AgentRunWorkflow, ProjectSandbox }; export const agentApp = new Hono<{ Bindings: AgentEnv }>(); @@ -58,29 +68,35 @@ agentApp.use("*", async (c, next) => { agentApp.get("/health", (c) => c.json({ ok: true, + releaseGate: c.env.CHEATCODE_RELEASE_GATE, releaseSha: c.env.CHEATCODE_RELEASE_SHA ?? "development", versionId: c.env.CF_VERSION_METADATA?.id ?? null, worker: "agent", }), ); +registerAgentDatabaseReadinessRoute(agentApp); +registerAgentDurableObjectStorageRoute(agentApp); registerAgentSystemHttpRoutes(agentApp); registerAgentRunHttpRoutes(agentApp); registerSandboxHttpRoutes(agentApp); +registerSkillProposalHttpRoutes(agentApp); +registerSkillRuntimeManagedRoutes(agentApp); +registerSkillRuntimeExecutionRoutes(agentApp); const agentHandler = { async fetch(request: Request, env: AgentEnv, ctx: ExecutionContext): Promise { AgentWorkerEnvSchema.parse(env); const id = request.headers.get("X-Request-Id") ?? requestId(); try { + const releaseGate = agentReleaseGateResponse(request, env, id); + if (releaseGate) { + return releaseGate; + } const requestWithId = isWebSocketUpgrade(request) ? request : new Request(request); if (!isWebSocketUpgrade(requestWithId)) { requestWithId.headers.set("X-Request-Id", id); } - const localPreview = await tryHandleLocalPreviewRequest(requestWithId, env); - if (localPreview) { - return withRequestId(localPreview, id); - } return withRequestId(await agentApp.fetch(requestWithId, env, ctx), id); } catch (error) { const apiError = toAPIError(error); @@ -98,6 +114,64 @@ const agentHandler = { }, }; +const WORKSPACE_RECONCILIATION_PATH = + /^\/internal\/users\/[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/reconcile-workspaces$/u; +const USER_STATE_DELETION_PATH = + /^\/internal\/users\/[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/delete-state$/u; + +function agentReleaseGateResponse( + request: Request, + env: AgentEnv, + id: string, +): Response | undefined { + if (env.CHEATCODE_RELEASE_GATE === "open") { + return undefined; + } + const pathname = new URL(request.url).pathname; + if ( + env.CHEATCODE_RELEASE_GATE === "closed" && + request.method === "POST" && + (pathname === INTERNAL_DATABASE_READINESS_PATH || + pathname === INTERNAL_DURABLE_OBJECT_STORAGE_PATH) + ) { + return undefined; + } + if ( + request.method === "POST" && + ((env.CHEATCODE_RELEASE_GATE === "draining" && USER_STATE_DELETION_PATH.test(pathname)) || + (env.CHEATCODE_RELEASE_GATE === "closed" && WORKSPACE_RECONCILIATION_PATH.test(pathname))) + ) { + return undefined; + } + if (request.method === "GET" && pathname === "/health") { + return withRequestId( + Response.json( + { + ok: true, + releaseGate: env.CHEATCODE_RELEASE_GATE, + releaseSha: env.CHEATCODE_RELEASE_SHA ?? "development", + versionId: env.CF_VERSION_METADATA?.id ?? null, + worker: "agent", + }, + { headers: { "Cache-Control": "no-store" } }, + ), + id, + ); + } + const response = new APIError(503, "unavailable_maintenance", "Release is in progress", { + details: { + releaseGate: env.CHEATCODE_RELEASE_GATE, + releaseSha: env.CHEATCODE_RELEASE_SHA ?? null, + versionId: env.CF_VERSION_METADATA?.id ?? null, + worker: "agent", + }, + retriable: true, + }).toResponse(id); + response.headers.set("Cache-Control", "no-store"); + response.headers.set("Retry-After", "5"); + return response; +} + function logAgentRequestError(error: unknown, requestIdValue: string, route: string): void { const apiError = toAPIError(error); createLogger({ requestId: requestIdValue }).error("agent_request_failed", { diff --git a/apps/agent-worker/src/internal-maintenance.ts b/apps/agent-worker/src/internal-maintenance.ts index 569b4a90..2f19cbba 100644 --- a/apps/agent-worker/src/internal-maintenance.ts +++ b/apps/agent-worker/src/internal-maintenance.ts @@ -1,17 +1,85 @@ -import { verifyInternalMaintenanceRequest } from "@cheatcode/auth"; +import { + assertDistinctHmacSecrets, + assertInternalMaintenanceEnvelope, + verifyInternalMaintenanceRequest, +} from "@cheatcode/auth"; import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; import { APIError } from "@cheatcode/observability"; -export async function verifyAgentMaintenanceRequest(input: { +interface AgentMaintenanceRequestInput { + expectedPathname: string; rawBody: string; request: Request; - secret: WorkerSecret | undefined; -}): Promise { - const secret = await readRequiredSecret(input.secret, "INTERNAL_MAINTENANCE_SECRET"); + secrets: AgentMaintenanceSecretBindings; +} + +interface AgentMaintenanceSecretBindings { + RELEASE_DATABASE_READINESS_SECRET: WorkerSecret; + WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET: WorkerSecret; +} + +export function assertAgentInternalHostname(request: Request): void { + if (new URL(request.url).hostname !== "agent.internal") { + throw new APIError(401, "auth_token_invalid", "Internal agent route requires service binding", { + retriable: false, + }); + } +} + +export function assertAgentLifecycleCapability(request: Request): void { + assertInternalMaintenanceEnvelope(request, { + audience: "agent", + capability: "agent-lifecycle", + issuer: "webhooks", + }); +} + +export function assertAgentDatabaseReadinessCapability(request: Request): void { + assertInternalMaintenanceEnvelope(request, { + audience: "agent", + capability: "database-readiness", + issuer: "gateway", + }); +} + +export function assertAgentDurableObjectStorageCapability(request: Request): void { + assertInternalMaintenanceEnvelope(request, { + audience: "agent", + capability: "durable-object-schema", + issuer: "gateway", + }); +} + +export function verifyAgentLifecycleRequest(input: AgentMaintenanceRequestInput): Promise { + return verifyAgentRequest(input, "agent-lifecycle"); +} + +export function verifyAgentDatabaseReadinessRequest( + input: AgentMaintenanceRequestInput, +): Promise { + return verifyAgentRequest(input, "database-readiness"); +} + +export function verifyAgentDurableObjectStorageRequest( + input: AgentMaintenanceRequestInput, +): Promise { + return verifyAgentRequest(input, "durable-object-schema"); +} + +async function verifyAgentRequest( + input: AgentMaintenanceRequestInput, + capability: "agent-lifecycle" | "database-readiness" | "durable-object-schema", +): Promise { + const secrets = await requireAgentMaintenanceSecrets(input.secrets); await verifyInternalMaintenanceRequest({ + expectedAudience: "agent", + expectedCapability: capability, + expectedIssuer: capability === "agent-lifecycle" ? "webhooks" : "gateway", + expectedMethod: "POST", + expectedPathname: input.expectedPathname, rawBody: input.rawBody, request: input.request, - secret, + secret: capability === "agent-lifecycle" ? secrets.agentLifecycle : secrets.databaseReadiness, }); } @@ -25,27 +93,34 @@ export function parseInternalMaintenanceJson(rawBody: string): unknown { } } -async function readRequiredSecret(secret: WorkerSecret | undefined, name: string): Promise { - const value = await readOptionalSecret(secret, name); - if (!value) { - throw new APIError(503, "unavailable_maintenance", `${name} is not configured`, { - hint: `Set ${name} in the agent Worker environment.`, - retriable: false, - }); +async function requireAgentMaintenanceSecrets(env: AgentMaintenanceSecretBindings): Promise<{ + agentLifecycle: string; + databaseReadiness: string; +}> { + try { + const [agentLifecycle, databaseReadiness] = await Promise.all([ + resolveRequiredSecret(env.WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET), + resolveRequiredSecret(env.RELEASE_DATABASE_READINESS_SECRET), + ]); + assertDistinctHmacSecrets([agentLifecycle, databaseReadiness]); + return { agentLifecycle, databaseReadiness }; + } catch { + throw new APIError( + 503, + "unavailable_maintenance", + "Agent maintenance secrets are unavailable", + { + hint: "Configure two distinct maintenance secrets containing at least 32 UTF-8 bytes.", + retriable: false, + }, + ); } - return value; } -async function readOptionalSecret( - secret: WorkerSecret | undefined, - name: string, -): Promise { - try { - return await resolveWorkerSecret(secret); - } catch { - throw new APIError(503, "unavailable_maintenance", `${name} is unavailable`, { - hint: `Verify the ${name} Cloudflare secret binding.`, - retriable: false, - }); +async function resolveRequiredSecret(binding: WorkerSecret): Promise { + const secret = await resolveWorkerSecret(binding); + if (!secret?.trim()) { + throw new Error("Maintenance secret is missing"); } + return secret; } diff --git a/apps/agent-worker/src/local-preview.ts b/apps/agent-worker/src/local-preview.ts deleted file mode 100644 index e1ab1388..00000000 --- a/apps/agent-worker/src/local-preview.ts +++ /dev/null @@ -1,462 +0,0 @@ -import { - mintPreviewCapability, - PreviewCapabilityError, - type PreviewCapabilityKind, - verifyPreviewCapability, -} from "@cheatcode/auth"; -import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; -import { APIError, readBoundedResponseText } from "@cheatcode/observability"; -import { - CODE_SERVER_PORT, - injectCodeServerParentBridge, - isCodeServerWorkbenchHtml, - MAX_CODE_SERVER_HTML_BYTES, -} from "@cheatcode/preview-bridge"; -import { DaytonaClient } from "@cheatcode/tools-code"; - -interface LocalPreviewEnv { - DAYTONA_API_KEY: WorkerSecret; - DAYTONA_API_URL: string; - DAYTONA_ORG_ID?: string; - DAYTONA_TARGET: string; - PREVIEW_TOKEN_SECRET: WorkerSecret; -} - -interface LocalPreviewTarget { - port: string; - sandboxId: string; -} - -interface LocalPreviewAuthorization { - fromQuery: boolean; -} - -interface ResolvedLocalPreviewOrigin { - authorization: LocalPreviewAuthorization; - originalHost: string; - origin: { - signed: boolean; - token: string; - url: string; - }; - target: LocalPreviewTarget; - url: URL; -} - -interface LocalPreviewRequestContext { - audience: string; - authorization: LocalPreviewAuthorization; - originalHost: string; - secret: string; - target: LocalPreviewTarget; - url: URL; -} - -const DAYTONA_TOKEN_HEADER = "x-daytona-preview-token"; -const DAYTONA_SKIP_WARNING_HEADER = "X-Daytona-Skip-Preview-Warning"; -const FORWARDED_HOST_HEADER = "X-Forwarded-Host"; -const LOCAL_PREVIEW_CLIENT_HOST_HEADER = "X-Cheatcode-Local-Preview-Client-Host"; -const LOCAL_PREVIEW_HOST_SUFFIX = ".localhost"; -const LOCAL_PREVIEW_HOST_PATTERN = /^([a-z0-9-]+)--(\d{1,5})$/; -const PREVIEW_TOKEN_COOKIE = "cc_pt"; -const PREVIEW_TOKEN_QUERY = "__cc_pt"; -const LOCAL_CODE_SERVER_PARENT_ORIGIN = "http://localhost:3000"; - -export async function tryHandleLocalPreviewRequest( - request: Request, - env: LocalPreviewEnv, -): Promise { - const context = await resolveLocalPreviewRequestContext(request, env); - if (!context) return null; - if (context.authorization.fromQuery) { - if (request.method !== "GET" && request.method !== "HEAD") { - throw new APIError( - 400, - "invalid_request_body", - "Preview handoff requires a navigation request", - { retriable: false }, - ); - } - const session = await mintPreviewCapability({ - kind: "session", - secret: context.secret, - target: capabilityTarget(context.audience, context.target), - }); - return localPreviewSessionRedirect( - context.url, - context.originalHost, - localPreviewSessionCookie(session.token, session.expiresAt), - ); - } - const origin = await resolveLocalDaytonaOrigin(context.target, env); - const upstreamUrl = localPreviewUpstreamUrl(origin.url, context.url); - if (isWebSocketUpgrade(request)) { - return fetchLocalPreviewWebSocket(request, upstreamUrl, origin, context.originalHost); - } - return fetchLocalPreviewOrigin( - request, - upstreamUrl, - origin, - context.originalHost, - context.target.port === String(CODE_SERVER_PORT), - ); -} - -export async function resolveLocalPreviewOrigin( - request: Request, - env: LocalPreviewEnv, -): Promise { - const context = await resolveLocalPreviewRequestContext(request, env); - if (!context) { - return null; - } - const origin = await resolveLocalDaytonaOrigin(context.target, env); - return { - authorization: context.authorization, - origin, - originalHost: context.originalHost, - target: context.target, - url: context.url, - }; -} - -async function resolveLocalPreviewRequestContext( - request: Request, - env: LocalPreviewEnv, -): Promise { - const url = new URL(request.url); - const audience = request.headers.get("Host") ?? url.host; - const target = parseLocalPreviewHost(audience); - if (!target) { - return null; - } - const secret = await requireLocalPreviewSecret(env); - const authorization = await authorizeLocalPreview(request, url, target, secret); - return { - audience, - authorization, - originalHost: localPreviewClientHost(request, url), - secret, - target, - url, - }; -} - -async function resolveLocalDaytonaOrigin(target: LocalPreviewTarget, env: LocalPreviewEnv) { - const client = await localDaytonaClient(env); - return { - ...(await client.getPreviewLink(target.sandboxId, Number(target.port))), - signed: false, - }; -} - -function localPreviewClientHost(request: Request, url: URL): string { - return ( - request.headers.get(LOCAL_PREVIEW_CLIENT_HOST_HEADER) ?? request.headers.get("Host") ?? url.host - ); -} - -function parseLocalPreviewHost(host: string): LocalPreviewTarget | null { - const hostname = (host.split(":")[0] ?? "").toLowerCase(); - if (!hostname.endsWith(LOCAL_PREVIEW_HOST_SUFFIX)) { - return null; - } - const label = hostname.slice(0, hostname.length - LOCAL_PREVIEW_HOST_SUFFIX.length); - const match = LOCAL_PREVIEW_HOST_PATTERN.exec(label); - const sandboxId = match?.[1]; - const port = match?.[2]; - if (!sandboxId || !port) { - return null; - } - const portNumber = Number(port); - if (!Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65_535) { - return null; - } - return { port, sandboxId }; -} - -async function authorizeLocalPreview( - request: Request, - url: URL, - target: LocalPreviewTarget, - secret: string, -): Promise { - const queryToken = url.searchParams.get(PREVIEW_TOKEN_QUERY); - const cookieToken = readCookie(request.headers.get("Cookie"), PREVIEW_TOKEN_COOKIE); - const raw = queryToken ?? cookieToken; - if (!raw) { - throw new APIError(401, "auth_token_missing", "Missing preview access token", { - retriable: false, - }); - } - const kind: PreviewCapabilityKind = queryToken ? "handoff" : "session"; - try { - await verifyPreviewCapability({ - expectedKind: kind, - secret, - target: capabilityTarget(request.headers.get("Host") ?? url.host, target), - token: raw, - }); - return { fromQuery: kind === "handoff" }; - } catch (error) { - if (error instanceof PreviewCapabilityError && error.reason === "expired") { - throw new APIError(401, "auth_token_expired", "Preview access token has expired", { - retriable: false, - }); - } - if (error instanceof PreviewCapabilityError) { - throw invalidPreviewToken(); - } - throw error; - } -} - -function capabilityTarget(audience: string, target: LocalPreviewTarget) { - return { - audience, - port: Number(target.port), - sandboxId: target.sandboxId, - }; -} - -async function requireLocalPreviewSecret(env: LocalPreviewEnv): Promise { - const secret = await resolveWorkerSecret(env.PREVIEW_TOKEN_SECRET); - if (!secret) { - throw new APIError(500, "internal_error", "Preview token secret is not configured", { - retriable: false, - }); - } - return secret; -} - -async function localDaytonaClient(env: LocalPreviewEnv): Promise { - const apiKey = await resolveWorkerSecret(env.DAYTONA_API_KEY); - if (!apiKey) { - throw new APIError(502, "upstream_sandbox_failed", "Daytona API key is not configured", { - retriable: false, - }); - } - return new DaytonaClient({ - apiKey, - apiUrl: env.DAYTONA_API_URL, - target: env.DAYTONA_TARGET, - ...(env.DAYTONA_ORG_ID ? { organizationId: env.DAYTONA_ORG_ID } : {}), - }); -} - -async function fetchLocalPreviewOrigin( - request: Request, - upstreamUrl: URL, - origin: { signed: boolean; token: string }, - originalHost: string, - isCodeServer: boolean, -): Promise { - const response = await fetch(upstreamUrl, localPreviewRequestInit(request, origin, originalHost)); - if (!isCodeServer || request.method !== "GET" || !isHtmlResponse(response)) { - return response; - } - const text = await readBoundedResponseText( - response, - MAX_CODE_SERVER_HTML_BYTES, - "Code-server HTML", - ); - const action = daytonaWarningAcceptAction(text); - if (!action) { - return codePreviewHtmlResponse(text, response); - } - const acceptCookie = await acceptDaytonaPreviewWarning(upstreamUrl, action, origin); - const retryResponse = await fetch( - upstreamUrl, - localPreviewRequestInit(request, origin, originalHost, acceptCookie), - ); - if (!isHtmlResponse(retryResponse)) { - return retryResponse; - } - const retryText = await readBoundedResponseText( - retryResponse, - MAX_CODE_SERVER_HTML_BYTES, - "Code-server HTML", - ); - return codePreviewHtmlResponse(retryText, retryResponse); -} - -async function fetchLocalPreviewWebSocket( - request: Request, - upstreamUrl: URL, - origin: { signed: boolean; token: string; url: string }, - originalHost: string, -): Promise { - const wsRequest = new Request(upstreamUrl.toString(), request); - wsRequest.headers.delete("Host"); - wsRequest.headers.delete("Cookie"); - if (!origin.signed) { - wsRequest.headers.set(DAYTONA_TOKEN_HEADER, origin.token); - } - wsRequest.headers.set(DAYTONA_SKIP_WARNING_HEADER, "true"); - wsRequest.headers.set(FORWARDED_HOST_HEADER, originalHost); - const browserOrigin = - request.headers.get("Origin") ?? `${new URL(request.url).protocol}//${originalHost}`; - const browserProtocol = new URL(browserOrigin).protocol.replace(":", ""); - wsRequest.headers.set("Origin", browserOrigin); - wsRequest.headers.set("Forwarded", `host=${originalHost};proto=${browserProtocol}`); - wsRequest.headers.set("X-Forwarded-Proto", browserProtocol); - const response = await fetch(wsRequest); - if (response.webSocket) { - return new Response(null, { status: 101, webSocket: response.webSocket }); - } - return response; -} - -function localPreviewRequestInit( - request: Request, - origin: { signed: boolean; token: string }, - originalHost: string, - cookie?: string | null, -): RequestInit { - const headers = new Headers(request.headers); - headers.delete("Host"); - headers.delete("Cookie"); - if (!origin.signed) { - headers.set(DAYTONA_TOKEN_HEADER, origin.token); - } - headers.set(DAYTONA_SKIP_WARNING_HEADER, "true"); - headers.set(FORWARDED_HOST_HEADER, originalHost); - const browserOrigin = - request.headers.get("Origin") ?? `${new URL(request.url).protocol}//${originalHost}`; - const browserProtocol = new URL(browserOrigin).protocol.replace(":", ""); - headers.set("Forwarded", `host=${originalHost};proto=${browserProtocol}`); - headers.set("X-Forwarded-Proto", browserProtocol); - headers.set("Accept-Encoding", "identity"); - if (cookie) { - headers.set("Cookie", cookie); - } - const init: RequestInit = { - headers, - method: request.method, - redirect: "manual", - }; - if (request.method !== "GET" && request.method !== "HEAD") { - init.body = request.body; - } - return init; -} - -function localPreviewUpstreamUrl(originUrl: string, requestUrl: URL): URL { - const upstreamUrl = new URL(originUrl); - const requestParams = new URLSearchParams(requestUrl.search); - upstreamUrl.pathname = requestUrl.pathname; - for (const [key, value] of requestParams) { - upstreamUrl.searchParams.append(key, value); - } - upstreamUrl.searchParams.delete(PREVIEW_TOKEN_QUERY); - upstreamUrl.searchParams.delete("cc_preview_reload"); - upstreamUrl.searchParams.delete("cc_theme"); - return upstreamUrl; -} - -function isHtmlResponse(response: Response): boolean { - return response.headers.get("Content-Type")?.toLowerCase().includes("text/html") ?? false; -} - -function codePreviewHtmlResponse(html: string, response: Response): Response { - if (!isCodeServerWorkbenchHtml(html)) { - return textResponse(html, response); - } - return textResponse( - injectCodeServerParentBridge(html, LOCAL_CODE_SERVER_PARENT_ORIGIN), - response, - ); -} - -function textResponse(text: string, response: Response): Response { - const headers = new Headers(response.headers); - headers.delete("Content-Encoding"); - headers.delete("Content-Length"); - headers.set("Cache-Control", "no-store"); - return new Response(text, { - headers, - status: response.status, - statusText: response.statusText, - }); -} - -function daytonaWarningAcceptAction(html: string): string | null { - if (!html.includes("Preview URL Warning")) { - return null; - } - const match = / { - let acceptUrl: URL; - try { - acceptUrl = new URL(action, upstreamUrl.origin); - } catch { - return null; - } - if (acceptUrl.origin !== upstreamUrl.origin || acceptUrl.username || acceptUrl.password) { - return null; - } - const headers = new Headers({ [DAYTONA_SKIP_WARNING_HEADER]: "true" }); - if (!origin.signed) { - headers.set(DAYTONA_TOKEN_HEADER, origin.token); - } - const response = await fetch(acceptUrl, { - headers, - method: "POST", - redirect: "manual", - }); - const cookie = response.headers.get("Set-Cookie")?.split(";", 1)[0] ?? null; - await response.body?.cancel().catch(() => undefined); - return cookie; -} - -function localPreviewSessionRedirect(url: URL, originalHost: string, setCookie: string): Response { - const location = new URL(url); - location.host = originalHost; - location.searchParams.delete(PREVIEW_TOKEN_QUERY); - return new Response(null, { - headers: { - "Cache-Control": "private, no-store", - Location: location.toString(), - "Set-Cookie": setCookie, - }, - status: 302, - }); -} - -function localPreviewSessionCookie(token: string, expiresAt: number): string { - const maxAge = Math.max(0, Math.floor((expiresAt - Date.now()) / 1000)); - // Local HTTP cannot use the Secure-required __Host- prefix; retain the same - // host-only, HttpOnly, Strict transport semantics under a dev-only name. - return `${PREVIEW_TOKEN_COOKIE}=${token}; HttpOnly; Path=/; Max-Age=${maxAge}; SameSite=Strict`; -} - -function isWebSocketUpgrade(request: Request): boolean { - return (request.headers.get("Upgrade") ?? "").toLowerCase() === "websocket"; -} - -function invalidPreviewToken(): APIError { - return new APIError(401, "auth_token_invalid", "Invalid preview access token", { - retriable: false, - }); -} - -function readCookie(cookieHeader: string | null, name: string): string | null { - if (!cookieHeader) { - return null; - } - for (const cookie of cookieHeader.split(";")) { - const trimmed = cookie.trim(); - const separator = trimmed.indexOf("="); - if (separator !== -1 && trimmed.slice(0, separator) === name) { - return trimmed.slice(separator + 1); - } - } - return null; -} diff --git a/apps/agent-worker/src/output-download.ts b/apps/agent-worker/src/output-download.ts index 64fc459d..75b3838f 100644 --- a/apps/agent-worker/src/output-download.ts +++ b/apps/agent-worker/src/output-download.ts @@ -1,22 +1,30 @@ import { APIError } from "@cheatcode/observability"; +import { + type OutputDownloadUrlResponse, + OutputDownloadUrlResponseSchema, + UserId, + type UserId as UserIdType, +} from "@cheatcode/types"; import { z } from "zod"; const DEFAULT_OUTPUT_DOWNLOAD_BASE_URL = "https://gateway.trycheatcode.com"; const OUTPUT_DOWNLOAD_TTL_SECONDS = 60 * 60; - -export const OutputIdSchema = z.string().uuid(); +const MINIMUM_SIGNING_SECRET_BYTES = 32; +const MAXIMUM_SIGNING_SECRET_BYTES = 1_024; export const OutputDownloadQuerySchema = z .object({ expires: z.coerce.number().int().positive(), sig: z.string().min(32).max(256), + userId: z.string().uuid().transform(UserId), }) .strict(); -export interface CreateSignedOutputDownloadUrlInput { +export interface CreateOutputDownloadCapabilityInput { baseUrl?: string | undefined; outputId: string; secret: string | undefined; + userId: UserIdType; } export interface VerifySignedOutputDownloadInput { @@ -25,16 +33,18 @@ export interface VerifySignedOutputDownloadInput { outputId: string; secret: string | undefined; signature: string; + userId: UserIdType; } -export async function createSignedOutputDownloadUrl( - input: CreateSignedOutputDownloadUrlInput, -): Promise { +export async function createOutputDownloadCapability( + input: CreateOutputDownloadCapabilityInput, +): Promise { const expires = Math.floor(Date.now() / 1000) + OUTPUT_DOWNLOAD_TTL_SECONDS; const signature = await signOutputDownload({ expires, outputId: input.outputId, secret: requiredSigningSecret(input.secret), + userId: input.userId, }); const url = new URL( `/v1/outputs/${input.outputId}/download`, @@ -42,7 +52,11 @@ export async function createSignedOutputDownloadUrl( ); url.searchParams.set("expires", String(expires)); url.searchParams.set("sig", signature); - return url.toString(); + url.searchParams.set("userId", input.userId); + return OutputDownloadUrlResponseSchema.parse({ + downloadUrl: url.toString(), + expiresAt: new Date(expires * 1_000).toISOString(), + }); } export async function verifySignedOutputDownload( @@ -56,6 +70,7 @@ export async function verifySignedOutputDownload( expires: input.expires, outputId: input.outputId, secret: requiredSigningSecret(input.secret), + userId: input.userId, }); return constantTimeEqual(expected, input.signature); } @@ -92,9 +107,10 @@ function invalidDownloadBaseUrl(): APIError { function requiredSigningSecret(value: string | undefined): string { const trimmed = value?.trim(); - if (!trimmed) { + const size = trimmed ? new TextEncoder().encode(trimmed).byteLength : 0; + if (!trimmed || size < MINIMUM_SIGNING_SECRET_BYTES || size > MAXIMUM_SIGNING_SECRET_BYTES) { throw new APIError(500, "internal_error", "Artifact download signing is not configured", { - hint: "Set OUTPUT_DOWNLOAD_SIGNING_SECRET on cheatcode-agent.", + hint: "Set OUTPUT_DOWNLOAD_SIGNING_SECRET to a distinct 32-byte-or-longer secret.", retriable: false, }); } @@ -105,6 +121,7 @@ async function signOutputDownload(input: { expires: number; outputId: string; secret: string; + userId: UserIdType; }): Promise { const key = await crypto.subtle.importKey( "raw", @@ -113,7 +130,12 @@ async function signOutputDownload(input: { false, ["sign"], ); - const payload = `${input.outputId}.${input.expires}`; + const payload = [ + "cheatcode-output-download-v2", + input.userId, + input.outputId, + String(input.expires), + ].join("\n"); const signature = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(payload)); return base64Url(new Uint8Array(signature)); } diff --git a/apps/agent-worker/src/sandbox-route-helpers.ts b/apps/agent-worker/src/sandbox-route-helpers.ts index 29fc96c4..658fdca1 100644 --- a/apps/agent-worker/src/sandbox-route-helpers.ts +++ b/apps/agent-worker/src/sandbox-route-helpers.ts @@ -57,7 +57,10 @@ export async function terminalProjectForThread( threadId: string, ): Promise<{ id: string; name: string; workspaceSlug: string } | null> { const parsedUserId = UserId(userId); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); try { return await withUserContext(db, parsedUserId, async (tx) => { const thread = await getThread(tx, { threadId: ThreadId(threadId), userId: parsedUserId }); diff --git a/apps/agent-worker/src/skill-proposal-http-routes.ts b/apps/agent-worker/src/skill-proposal-http-routes.ts new file mode 100644 index 00000000..7652dfbe --- /dev/null +++ b/apps/agent-worker/src/skill-proposal-http-routes.ts @@ -0,0 +1,327 @@ +import { + createDb, + createThreadMessage, + deleteUserSkill, + findSkillConfirmationMessage, + getThreadAgentRunMessage, + getUserSkillById, + getUserSkillByName, + lockSkillProposal, + type MessageRecord, + type UserSkillRecord, + upsertUserSkill, + withUserContext, +} from "@cheatcode/db"; +import { APIError } from "@cheatcode/observability"; +import { + AgentRunId, + CHEATCODE_DATA_SCHEMAS, + SandboxIdeSessionSchema, + SkillProposalConfirmResponseSchema, + ThreadId, + type UIMessagePart, + UserId, + UserSkillSchema, +} from "@cheatcode/types"; +import type { Context, Hono } from "hono"; +import { z } from "zod"; +import type { AgentEnv } from "./agent-env"; +import { sandboxForUser } from "./agent-routing"; +import { terminalDisplayCwd } from "./sandbox-route-helpers"; +import { parseRunRouteParam, parseThreadRouteParam, readGatewayUserId } from "./tenancy"; +import { + serializeUserSkillMarkdown, + userSkillDirectoryPath, + userSkillFilePath, + writeUserSkillMirror, +} from "./user-skill-files"; +import { + collectUserSkillPackageFromSandbox, + deleteUserSkillPackage, + persistUserSkillPackage, + readUserSkillPackage, + writeUserSkillPackageMirror, +} from "./user-skill-packages"; + +const IdSchema = z.string().uuid(); +type AgentContext = Context<{ Bindings: AgentEnv }>; +type SkillProposal = z.infer<(typeof CHEATCODE_DATA_SCHEMAS)["skill-proposed"]>; + +export function registerSkillProposalHttpRoutes(app: Hono<{ Bindings: AgentEnv }>): void { + app.post( + "/v1/threads/:threadId/skill-proposals/:runId/:proposalId/confirm", + confirmSkillProposal, + ); + app.post("/v1/skills/:skillId/open", openUserSkill); + app.delete("/v1/skills/:skillId", deleteSavedUserSkill); +} + +async function deleteSavedUserSkill(c: AgentContext): Promise { + const userId = UserId(readGatewayUserId(c.req.raw.headers)); + const skillId = parsedId(c.req.param("skillId"), "skill"); + const skill = await readSkill(c.env, userId, skillId); + if (!skill) { + throw new APIError(404, "not_found_skill", "Skill not found", { retriable: false }); + } + await removeSkillPackageFiles(c.env, userId, skill); + await deleteSkillRecord(c.env, userId, skillId); + return new Response(null, { status: 204 }); +} + +async function confirmSkillProposal(c: AgentContext): Promise { + const userId = UserId(readGatewayUserId(c.req.raw.headers)); + const threadId = ThreadId(parseThreadRouteParam(c.req.param("threadId") ?? "")); + const runId = AgentRunId(parseRunRouteParam(c.req.param("runId") ?? "")); + const proposalId = parsedId(c.req.param("proposalId"), "proposal"); + const confirmed = await persistProposal(c.env, { proposalId, runId, threadId, userId }); + await persistAndMirrorSkillPackage(c.env, userId, confirmed.skill); + return c.json( + SkillProposalConfirmResponseSchema.parse({ + message: messageResponse(confirmed.message), + skill: skillResponse(confirmed.skill), + }), + ); +} + +async function openUserSkill(c: AgentContext): Promise { + const userId = UserId(readGatewayUserId(c.req.raw.headers)); + const skillId = parsedId(c.req.param("skillId"), "skill"); + const skill = await readSkill(c.env, userId, skillId); + if (!skill) { + throw new APIError(404, "not_found_skill", "Skill not found", { retriable: false }); + } + const filePath = await mirrorSkillFile(c.env, userId, skill); + const sandbox = await sandboxForUser(c.env, userId); + const session = await sandbox.exposeCodeServer({ + initialFilePath: filePath, + workspacePath: userSkillDirectoryPath(skill.name), + }); + return c.json( + SandboxIdeSessionSchema.parse({ + ...session, + displayWorkspacePath: terminalDisplayCwd(session.workspacePath), + }), + ); +} + +async function persistProposal( + env: AgentEnv, + input: { proposalId: string; runId: AgentRunId; threadId: ThreadId; userId: UserId }, +): Promise<{ message: MessageRecord; skill: UserSkillRecord }> { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + return await withUserContext(db, input.userId, async (tx) => { + await lockSkillProposal(tx, input.proposalId); + const proposalMessage = await getThreadAgentRunMessage(tx, input); + const proposal = proposalFromMessage(proposalMessage, input.proposalId); + const existing = await findSkillConfirmationMessage(tx, input); + if (existing) { + const skill = await skillForExistingConfirmation(tx, input.userId, existing, proposal.name); + if (!skill) { + throw new APIError( + 409, + "conflict_state_invalid", + "This skill proposal was already created and later removed.", + { retriable: false }, + ); + } + return { message: existing, skill }; + } + const skill = await upsertUserSkill(tx, { + body: proposal.body, + category: proposal.category, + description: proposal.description, + name: proposal.name, + tags: proposal.tags, + userId: input.userId, + }); + const message = await createThreadMessage(tx, { + parts: confirmationParts(proposal, skill), + role: "assistant", + threadId: input.threadId, + userId: input.userId, + }); + return { message, skill }; + }); + } finally { + await close(); + } +} + +function proposalFromMessage(message: MessageRecord | null, proposalId: string): SkillProposal { + if (!message) { + throw new APIError(404, "not_found_skill", "Skill proposal not found", { + retriable: false, + }); + } + for (const part of message.parts) { + if (part.type !== "data-skill-proposed" || part.data.proposalId !== proposalId) { + continue; + } + return CHEATCODE_DATA_SCHEMAS["skill-proposed"].parse(part.data); + } + throw new APIError(404, "not_found_skill", "Skill proposal not found", { + retriable: false, + }); +} + +async function skillForExistingConfirmation( + db: Parameters[0], + userId: UserId, + message: MessageRecord, + proposalName: string, +): Promise { + const created = message.parts.find((part) => part.type === "data-skill-created"); + return created?.type === "data-skill-created" && created.data.id + ? getUserSkillById(db, userId, created.data.id) + : getUserSkillByName(db, userId, proposalName); +} + +function confirmationParts(proposal: SkillProposal, skill: UserSkillRecord): UIMessagePart[] { + const filePath = userSkillFilePath(skill.name); + return [ + { + state: "done", + text: [ + `Created and saved the new custom Cheatcode skill: **${proposal.name}**.`, + "", + "### What It Does", + proposal.description, + "", + "### Validation", + "- Confirmed the skill instructions are valid markdown.", + "- Persisted it to your custom skill registry.", + "- Mirrored it to the Cheatcode computer as `SKILL.md` for review and editing.", + ].join("\n"), + type: "text", + }, + { + data: { + v: 1, + description: proposal.description, + filePath, + id: skill.id, + name: proposal.name, + proposalId: proposal.proposalId, + slug: proposal.slug, + }, + type: "data-skill-created", + }, + ]; +} + +async function mirrorSkillFile( + env: AgentEnv, + userId: UserId, + skill: UserSkillRecord, +): Promise { + const sandbox = await sandboxForUser(env, userId); + const packageValue = await readUserSkillPackage(env.R2_OUTPUTS, userId, skill.id); + return packageValue + ? writeUserSkillPackageMirror(sandbox, skill, packageValue) + : writeUserSkillMirror(sandbox, skill); +} + +async function persistAndMirrorSkillPackage( + env: AgentEnv, + userId: UserId, + skill: UserSkillRecord, +): Promise { + const sandbox = await sandboxForUser(env, userId); + const collected = await collectUserSkillPackageFromSandbox(sandbox, skill); + const skillMarkdown = await serializeUserSkillMarkdown(skill); + const files = collected.some((file) => file.path === "SKILL.md") + ? collected.map((file) => + file.path === "SKILL.md" ? { content: skillMarkdown, path: file.path } : file, + ) + : [{ content: skillMarkdown, path: "SKILL.md" }, ...collected]; + const packageValue = await persistUserSkillPackage(env.R2_OUTPUTS, userId, skill.id, files); + return writeUserSkillPackageMirror(sandbox, skill, packageValue); +} + +async function readSkill( + env: AgentEnv, + userId: UserId, + skillId: string, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + return await withUserContext(db, userId, (tx) => getUserSkillById(tx, userId, skillId)); + } finally { + await close(); + } +} + +async function removeSkillPackageFiles( + env: AgentEnv, + userId: UserId, + skill: UserSkillRecord, +): Promise { + const sandbox = await sandboxForUser(env, userId); + if (!sandbox.deleteFile) { + throw new APIError( + 503, + "unavailable_maintenance", + "The skill workspace cannot be cleaned up right now", + { retriable: true }, + ); + } + await Promise.all([ + deleteUserSkillPackage(env.R2_OUTPUTS, userId, skill.id), + sandbox.deleteFile({ path: userSkillDirectoryPath(skill.name), recursive: true }), + ]); +} + +async function deleteSkillRecord(env: AgentEnv, userId: UserId, skillId: string): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const deleted = await withUserContext(db, userId, (tx) => deleteUserSkill(tx, userId, skillId)); + if (!deleted) { + throw new APIError(404, "not_found_skill", "Skill not found", { retriable: false }); + } + } finally { + await close(); + } +} + +function parsedId(value: string | undefined, label: string): string { + const parsed = IdSchema.safeParse(value); + if (!parsed.success) { + throw new APIError(400, "invalid_path_param", `Invalid ${label} id`, { retriable: false }); + } + return parsed.data; +} + +function skillResponse(skill: UserSkillRecord): unknown { + return UserSkillSchema.parse({ + category: skill.category, + createdAt: skill.createdAt.toISOString(), + description: skill.description, + id: skill.id, + name: skill.name, + tags: skill.tags, + updatedAt: skill.updatedAt.toISOString(), + }); +} + +function messageResponse(message: MessageRecord): unknown { + return { + agentRunId: message.agentRunId, + agentRunSegment: message.agentRunSegment, + agentRunSegmentFinal: message.agentRunSegmentFinal, + createdAt: message.createdAt.toISOString(), + id: message.id, + parts: message.parts, + role: message.role, + threadId: message.threadId, + }; +} diff --git a/apps/agent-worker/src/skill-runtime-auth.ts b/apps/agent-worker/src/skill-runtime-auth.ts new file mode 100644 index 00000000..deea0b00 --- /dev/null +++ b/apps/agent-worker/src/skill-runtime-auth.ts @@ -0,0 +1,91 @@ +import { + SkillRuntimeCapabilityError, + type SkillRuntimeScope, + type VerifiedSkillRuntimeCapability, + verifySkillRuntimeCapability, +} from "@cheatcode/auth"; +import { createDb, findAgentRunForUser, withUserContext } from "@cheatcode/db"; +import { resolveWorkerSecret } from "@cheatcode/env"; +import { APIError } from "@cheatcode/observability"; +import { AgentRunId, UserId } from "@cheatcode/types"; +import type { AgentEnv } from "./agent-env"; + +export interface SkillRuntimePrincipal extends VerifiedSkillRuntimeCapability { + userId: ReturnType; +} + +/** Verifies the sandbox capability and binds it to a still-active persisted run. */ +export async function requireSkillRuntimePrincipal( + env: AgentEnv, + headers: Headers, + requiredScope: SkillRuntimeScope, +): Promise { + const token = bearerToken(headers); + const secret = await resolveWorkerSecret(env.SKILL_RUNTIME_TOKEN_SECRET); + if (!secret) { + throw new APIError(503, "unavailable_maintenance", "Skill runtime is unavailable", { + retriable: true, + }); + } + const capability = await verifiedCapability(token, secret, requiredScope); + const userId = UserId(capability.userId); + await requireActiveRun(env, capability, userId); + return { ...capability, userId }; +} + +async function verifiedCapability( + token: string, + secret: string, + requiredScope: SkillRuntimeScope, +): Promise { + try { + return await verifySkillRuntimeCapability({ requiredScope, secret, token }); + } catch (error) { + const expired = error instanceof SkillRuntimeCapabilityError && error.reason === "expired"; + throw new APIError( + 401, + expired ? "auth_token_expired" : "auth_token_invalid", + expired ? "Skill runtime session expired" : "Invalid skill runtime session", + { retriable: expired }, + ); + } +} + +async function requireActiveRun( + env: AgentEnv, + capability: VerifiedSkillRuntimeCapability, + userId: ReturnType, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + const run = await withUserContext(db, userId, (tx) => + findAgentRunForUser(tx, { runId: AgentRunId(capability.runId), userId }), + ); + if (!run || !["pending", "running"].includes(run.status)) { + throw new APIError(409, "conflict_state_invalid", "Skill runtime run is not active", { + retriable: false, + }); + } + if ((run.projectId ?? null) !== capability.projectId) { + throw new APIError(403, "permission_denied", "Skill runtime project mismatch", { + retriable: false, + }); + } + } finally { + await close(); + } +} + +function bearerToken(headers: Headers): string { + const authorization = headers.get("Authorization") ?? ""; + const [scheme, token, ...extra] = authorization.trim().split(/\s+/u); + if (scheme !== "Bearer" || !token || extra.length > 0) { + throw new APIError(401, "auth_token_missing", "Missing skill runtime capability", { + retriable: false, + }); + } + return token; +} diff --git a/apps/agent-worker/src/skill-runtime-execution-routes.ts b/apps/agent-worker/src/skill-runtime-execution-routes.ts new file mode 100644 index 00000000..2867f1c7 --- /dev/null +++ b/apps/agent-worker/src/skill-runtime-execution-routes.ts @@ -0,0 +1,150 @@ +import { ComposioClient } from "@cheatcode/composio"; +import { createLogger, readJsonRequest } from "@cheatcode/observability"; +import { IntegrationNameSchema } from "@cheatcode/types/integrations"; +import type { Context, Hono } from "hono"; +import { z } from "zod"; +import type { AgentEnv } from "./agent-env"; +import { agentRunForRunId } from "./agent-routing"; +import { resolveComposioRuntimeCredentials } from "./durable-objects/composio-provider"; +import { requireSkillRuntimePrincipal } from "./skill-runtime-auth"; + +type AgentContext = Context<{ Bindings: AgentEnv }>; +const MAX_EXECUTION_REQUEST_BYTES = 256 * 1024; +const COMPOSIO_TIMEOUT_MS = 30_000; +const ToolRequestSchema = z + .object({ + arguments: z.record(z.string(), z.unknown()).default({}), + projectId: z.string().uuid().optional(), + toolkitSlug: IntegrationNameSchema, + toolSlug: z.string().trim().min(1).max(200), + }) + .strict(); +const ProxyRequestSchema = z + .object({ + body: z.unknown().optional(), + endpoint: z.string().trim().min(1).max(500), + method: z.enum(["GET", "POST", "PATCH", "DELETE"]).default("POST"), + projectId: z.string().uuid().optional(), + toolkitSlug: IntegrationNameSchema, + }) + .strict(); +const FrontendEventSchema = z.object({ event: z.unknown() }).passthrough(); + +export function registerSkillRuntimeExecutionRoutes(app: Hono<{ Bindings: AgentEnv }>): void { + app.post("/skill-runtime/composio/tool", executeComposioTool); + app.post("/skill-runtime/composio/proxy", rejectUnsafeComposioProxy); + app.post("/skill-runtime/skill-frontend-events", acceptFrontendEvent); + app.post("/skill-runtime/browser/live-preview", startBrowserTakeover); + app.post("/skill-runtime/browser/request-user-control", startBrowserTakeover); +} + +async function executeComposioTool(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal( + c.env, + c.req.raw.headers, + "integrations:execute", + ); + const input = ToolRequestSchema.parse( + await readJsonRequest(c.req.raw, MAX_EXECUTION_REQUEST_BYTES, "Composio tool request"), + ); + requireMatchingProject(principal.projectId, input.projectId); + const logger = createLogger({ runId: principal.runId, userId: principal.userId }); + const runtime = await resolveComposioRuntimeCredentials( + c.env, + { userId: principal.userId }, + logger, + ); + const connectionId = runtime.composioConnectedAccounts?.[input.toolkitSlug]; + if (!runtime.composioApiKey || !runtime.composioUserId || !connectionId) { + return c.json(failedTool(`Connect ${input.toolkitSlug} in Skills first.`)); + } + const version = await resolveToolVersion( + runtime.composioApiKey, + input.toolkitSlug, + input.toolSlug, + ); + const quota = await runtime.composioQuotaMeter?.consumeCall( + `skill:${principal.runId}:${crypto.randomUUID()}`, + ); + if (quota && !quota.allowed) { + return c.json(failedTool("Composio monthly call quota exhausted.")); + } + try { + const result = await new ComposioClient(runtime.composioApiKey).executeTool( + input.toolSlug, + { + arguments: input.arguments, + connectedAccountId: connectionId, + userId: runtime.composioUserId, + version, + }, + COMPOSIO_TIMEOUT_MS, + ); + return c.json(result); + } catch (error) { + logger.warn("skill_runtime_composio_execution_failed", { error, toolSlug: input.toolSlug }); + return c.json(failedTool("Composio tool execution failed.")); + } +} + +async function resolveToolVersion( + apiKey: string, + toolkitSlug: string, + toolSlug: string, +): Promise { + const page = await new ComposioClient(apiKey).listTools( + { limit: 100, search: toolSlug, toolkit: toolkitSlug }, + COMPOSIO_TIMEOUT_MS, + ); + const tool = page.items.find((item) => item.slug.toLowerCase() === toolSlug.toLowerCase()); + return tool?.version ?? "latest"; +} + +async function rejectUnsafeComposioProxy(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal( + c.env, + c.req.raw.headers, + "integrations:execute", + ); + const input = ProxyRequestSchema.parse( + await readJsonRequest(c.req.raw, MAX_EXECUTION_REQUEST_BYTES, "Composio proxy request"), + ); + requireMatchingProject(principal.projectId, input.projectId); + return c.json( + { + error: + "This provider endpoint is not in Cheatcode's validated Composio tool catalog. Use /composio/tool with a catalog tool slug.", + }, + 400, + ); +} + +async function acceptFrontendEvent(c: AgentContext): Promise { + await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "events:write"); + FrontendEventSchema.parse(await readJsonRequest(c.req.raw, 64 * 1024, "Skill frontend event")); + return c.json({ delivered: false }); +} + +async function startBrowserTakeover(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "events:write"); + return agentRunForRunId(c.env, principal.runId).fetch( + "https://agent-run.internal/browser-takeover/start", + { + headers: { "X-Cheatcode-User-Id": principal.userId }, + method: "POST", + }, + ); +} + +function failedTool(error: string) { + return { data: null, error, successful: false }; +} + +function requireMatchingProject( + capabilityProjectId: string | null, + requestedProjectId: string | undefined, +): void { + if (requestedProjectId && requestedProjectId !== capabilityProjectId) { + throw new Error("Skill runtime project mismatch"); + } +} diff --git a/apps/agent-worker/src/skill-runtime-managed-routes.ts b/apps/agent-worker/src/skill-runtime-managed-routes.ts new file mode 100644 index 00000000..ce5c262f --- /dev/null +++ b/apps/agent-worker/src/skill-runtime-managed-routes.ts @@ -0,0 +1,459 @@ +import { + createDb, + getUserSkillByName, + listUserIntegrations, + listUserSkillRecords, + setDefaultUserIntegration, + type UserIntegrationRecord, + type UserSkillRecord, + upsertUserSkill, + withUserContext, +} from "@cheatcode/db"; +import { APIError, readJsonRequest } from "@cheatcode/observability"; +import { SKILL_MANIFEST } from "@cheatcode/skills/manifest"; +import type { Context, Hono } from "hono"; +import { z } from "zod"; +import type { AgentEnv } from "./agent-env"; +import { sandboxForUser } from "./agent-routing"; +import { requireSkillRuntimePrincipal } from "./skill-runtime-auth"; +import { parsePortableSkillMarkdown, userSkillSlug } from "./user-skill-files"; +import { + persistUserSkillPackage, + readUserSkillPackage, + UserSkillPackageFileSchema, + writeUserSkillPackageMirror, +} from "./user-skill-packages"; + +type AgentContext = Context<{ Bindings: AgentEnv }>; +type RuntimePrincipal = Awaited>; + +const MAX_RUNTIME_REQUEST_BYTES = 1024 * 1024 + 32 * 1024; +const SkillSlugSchema = z + .string() + .trim() + .min(1) + .max(80) + .regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/u); +const SaveCustomSkillSchema = z + .object({ + files: z.array(UserSkillPackageFileSchema).min(1).max(20), + skillSlug: SkillSlugSchema, + }) + .strict(); +const SkillActionSchema = z + .object({ action: z.enum(["enable", "disable"]), skillSlug: SkillSlugSchema }) + .strict(); +const SkillSelectionSchema = z.object({ skillSlug: SkillSlugSchema }).strict(); +const DefaultAccountSchema = z + .object({ connectedAccountId: z.string().trim().min(1).max(256) }) + .strict(); +const CreateRequestSchema = z + .object({ + requestSummary: z.string().trim().min(1).max(500), + skillName: z.string().trim().min(1).max(80), + skillSlug: SkillSlugSchema.optional(), + }) + .strict(); + +const KNOWN_INTEGRATIONS = [ + "gmail", + "github", + "google_calendar", + "google_docs", + "google_drive", + "google_sheets", + "linear", + "notion", + "slack", +] as const; + +interface ManagedSkillItem { + alwaysEnabled: boolean; + canDisable: boolean; + canEnable: boolean; + connectedAccountCount?: number; + description: string; + editable: boolean; + enabled: boolean; + isConnected?: boolean; + name: string; + requiresConnection?: boolean; + slug: string; + source: "built_in" | "custom" | "integration"; +} + +export function registerSkillRuntimeManagedRoutes(app: Hono<{ Bindings: AgentEnv }>): void { + app.get("/skill-runtime/managed-skills", listManagedSkills); + app.post("/skill-runtime/managed-skills/custom/save", saveCustomSkill); + app.post("/skill-runtime/managed-skills/custom/prepare-create-request", prepareCreateRequest); + app.post("/skill-runtime/managed-skills/prepare-change", prepareSkillChange); + app.post("/skill-runtime/managed-skills/prepare-connect-account", prepareConnectAccount); + app.post("/skill-runtime/managed-skills/connect-link", connectLinkFallback); + app.get("/skill-runtime/managed-skills/connected-accounts", listConnectedAccounts); + app.patch("/skill-runtime/managed-skills/connected-accounts/default", switchDefaultAccount); +} + +async function listManagedSkills(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "skills:read"); + const state = await loadManagedState(c.env, principal); + return c.json({ skills: managedSkillItems(state.skills, state.integrations) }); +} + +async function saveCustomSkill(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "skills:write"); + const input = SaveCustomSkillSchema.parse( + await readJsonRequest(c.req.raw, MAX_RUNTIME_REQUEST_BYTES, "Custom skill package"), + ); + const markdown = input.files.find((file) => file.path === "SKILL.md")?.content; + if (!markdown) { + throw invalidSkillPackage("Custom skill package must include SKILL.md"); + } + const parsed = parsePortableSkillMarkdown(markdown, input.skillSlug); + if (!parsed.success) { + throw invalidSkillPackage("Custom SKILL.md frontmatter or body is invalid"); + } + return persistCustomSkill(c, principal, input, parsed.data); +} + +async function persistCustomSkill( + c: AgentContext, + principal: RuntimePrincipal, + input: z.infer, + parsed: { body: string; category: string; description: string; name: string; tags: string[] }, +): Promise { + const existing = await findSkillByName(c.env, principal, parsed.name); + const skill = await upsertRuntimeSkill(c.env, principal, parsed); + const previous = await readUserSkillPackage(c.env.R2_OUTPUTS, principal.userId, skill.id); + const packageValue = await persistUserSkillPackage( + c.env.R2_OUTPUTS, + principal.userId, + skill.id, + input.files, + ); + await writeUserSkillPackageMirror( + await sandboxForUser(c.env, principal.userId), + skill, + packageValue, + ); + return c.json({ + created: existing === null, + saved: previous?.revision !== packageValue.revision, + skill: runtimeSkillResponse(skill, input.skillSlug, packageValue.revision), + source: "custom" as const, + }); +} + +async function prepareCreateRequest(c: AgentContext): Promise { + await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "skills:write"); + const input = CreateRequestSchema.parse( + await readJsonRequest(c.req.raw, 8 * 1024, "Custom skill creation request"), + ); + const skillSlug = input.skillSlug ?? userSkillSlug(input.skillName); + return c.json({ + requiresConfirmation: true, + uiAction: { + kind: "create_skill" as const, + requestSummary: input.requestSummary, + skillName: input.skillName, + skillSlug, + }, + }); +} + +async function prepareSkillChange(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "skills:write"); + const input = SkillActionSchema.parse( + await readJsonRequest(c.req.raw, 8 * 1024, "Managed skill change"), + ); + const state = await loadManagedState(c.env, principal); + const skill = managedSkillItems(state.skills, state.integrations).find( + (item) => item.slug === input.skillSlug, + ); + if (!skill) throw skillNotFound(input.skillSlug); + return c.json(managedSkillChangeResponse(skill, input.action)); +} + +async function prepareConnectAccount(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal( + c.env, + c.req.raw.headers, + "integrations:execute", + ); + const input = SkillSelectionSchema.parse( + await readJsonRequest(c.req.raw, 8 * 1024, "Managed integration connection"), + ); + const state = await loadManagedState(c.env, principal); + const skill = managedSkillItems(state.skills, state.integrations).find( + (item) => item.slug === input.skillSlug && item.source === "integration", + ); + if (!skill) throw skillNotFound(input.skillSlug); + return c.json({ action: "connect", skill, uiAction: connectUiAction(skill) }); +} + +async function connectLinkFallback(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal( + c.env, + c.req.raw.headers, + "integrations:execute", + ); + const input = SkillSelectionSchema.passthrough().parse( + await readJsonRequest(c.req.raw, 8 * 1024, "Managed integration link"), + ); + const state = await loadManagedState(c.env, principal); + const connected = state.integrations.some( + (item) => item.integration === input.skillSlug && isActiveStatus(item.status), + ); + return c.json({ + alreadyConnected: connected, + integrationName: titleCaseSlug(input.skillSlug), + integrationSlug: input.skillSlug, + message: connected + ? `${titleCaseSlug(input.skillSlug)} is already connected.` + : `Connect ${titleCaseSlug(input.skillSlug)} from the Cheatcode Skills screen.`, + }); +} + +async function listConnectedAccounts(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal(c.env, c.req.raw.headers, "skills:read"); + const state = await loadManagedState(c.env, principal); + return c.json({ integrations: connectedIntegrationItems(state.integrations) }); +} + +async function switchDefaultAccount(c: AgentContext): Promise { + const principal = await requireSkillRuntimePrincipal( + c.env, + c.req.raw.headers, + "integrations:execute", + ); + const input = DefaultAccountSchema.parse( + await readJsonRequest(c.req.raw, 8 * 1024, "Default integration account"), + ); + const integration = await makeAccountDefault(c.env, principal, input.connectedAccountId); + const state = await loadManagedState(c.env, principal); + const integrations = connectedIntegrationItems(state.integrations); + return c.json({ + connectedAccountId: input.connectedAccountId, + integration: integrations.find((item) => item.integrationSlug === integration) ?? null, + integrationName: titleCaseSlug(integration), + integrationSlug: integration, + integrations, + success: true, + }); +} + +async function loadManagedState(env: AgentEnv, principal: RuntimePrincipal) { + return withRuntimeDb(env, principal, async (db) => ({ + integrations: await listUserIntegrations(db, principal.userId), + skills: await listUserSkillRecords(db, principal.userId), + })); +} + +async function findSkillByName( + env: AgentEnv, + principal: RuntimePrincipal, + name: string, +): Promise { + return withRuntimeDb(env, principal, (db) => getUserSkillByName(db, principal.userId, name)); +} + +async function upsertRuntimeSkill( + env: AgentEnv, + principal: RuntimePrincipal, + skill: { body: string; category: string; description: string; name: string; tags: string[] }, +): Promise { + return withRuntimeDb(env, principal, (db) => + upsertUserSkill(db, { ...skill, userId: principal.userId }), + ); +} + +async function makeAccountDefault( + env: AgentEnv, + principal: RuntimePrincipal, + connectedAccountId: string, +): Promise { + const integration = await withRuntimeDb(env, principal, async (db) => { + const accounts = await listUserIntegrations(db, principal.userId); + const account = accounts.find((item) => item.composioConnectionId === connectedAccountId); + if (!account) return null; + const updated = await setDefaultUserIntegration(db, { + composioConnectionId: connectedAccountId, + integration: account.integration, + userId: principal.userId, + }); + return updated ? account.integration : null; + }); + if (!integration) throw skillNotFound("connected account"); + return integration; +} + +async function withRuntimeDb( + env: AgentEnv, + principal: RuntimePrincipal, + operation: (db: Parameters[0]) => Promise, +): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_agent", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_AGENT, + }); + try { + return await withUserContext(db, principal.userId, operation); + } finally { + await close(); + } +} + +function managedSkillItems( + customSkills: UserSkillRecord[], + accounts: UserIntegrationRecord[], +): ManagedSkillItem[] { + return [ + ...SKILL_MANIFEST.map((skill) => builtInSkill(skill)), + ...customSkills.map(customSkill), + ...integrationSlugs(accounts).map((slug) => integrationSkill(slug, accounts)), + ].sort((left, right) => left.name.localeCompare(right.name)); +} + +function builtInSkill(skill: (typeof SKILL_MANIFEST)[number]): ManagedSkillItem { + return { + alwaysEnabled: true, + canDisable: false, + canEnable: false, + description: skill.description, + editable: false, + enabled: true, + name: titleCaseSlug(skill.name), + slug: skill.name, + source: "built_in", + }; +} + +function customSkill(skill: UserSkillRecord): ManagedSkillItem { + return { + alwaysEnabled: true, + canDisable: false, + canEnable: false, + description: skill.description, + editable: true, + enabled: true, + name: skill.name, + slug: userSkillSlug(skill.name), + source: "custom", + }; +} + +function integrationSkill(slug: string, accounts: UserIntegrationRecord[]): ManagedSkillItem { + const connected = accounts.filter( + (account) => account.integration === slug && isActiveStatus(account.status), + ); + return { + alwaysEnabled: false, + canDisable: false, + canEnable: connected.length === 0, + connectedAccountCount: connected.length, + description: `Use ${titleCaseSlug(slug)} through a connected account.`, + editable: false, + enabled: connected.length > 0, + isConnected: connected.length > 0, + name: titleCaseSlug(slug), + requiresConnection: true, + slug, + source: "integration", + }; +} + +function integrationSlugs(accounts: UserIntegrationRecord[]): string[] { + return [...new Set([...KNOWN_INTEGRATIONS, ...accounts.map((item) => item.integration)])].sort(); +} + +function managedSkillChangeResponse(skill: ManagedSkillItem, action: "enable" | "disable") { + if (skill.source === "integration" && action === "enable" && !skill.isConnected) { + return { action, requiresConfirmation: true, skill, uiAction: connectUiAction(skill) }; + } + return { + action, + message: skill.alwaysEnabled + ? `${skill.name} is always available in Cheatcode.` + : `${skill.name} follows its connected-account state.`, + requiresConfirmation: false, + skill, + }; +} + +function connectUiAction(skill: ManagedSkillItem) { + return { + integrationName: skill.name, + integrationSlug: skill.slug, + kind: "connect_account" as const, + skillName: skill.name, + skillSlug: skill.slug, + skillSource: "integration" as const, + }; +} + +function connectedIntegrationItems(accounts: UserIntegrationRecord[]) { + const groups = groupIntegrationAccounts(accounts); + return [...groups.entries()].map(([integration, items]) => ({ + connectedAccountId: + items.find((item) => item.isDefault)?.composioConnectionId ?? + items[0]?.composioConnectionId ?? + null, + connectedAccounts: items.map((item) => ({ + id: item.composioConnectionId, + isDefault: item.isDefault, + isSelected: item.isDefault, + label: item.composioConnectionId, + status: item.status, + })), + defaultConnectedAccountId: items.find((item) => item.isDefault)?.composioConnectionId ?? null, + integrationName: titleCaseSlug(integration), + integrationSlug: integration, + skillName: titleCaseSlug(integration), + skillSlug: integration, + })); +} + +function groupIntegrationAccounts( + accounts: UserIntegrationRecord[], +): Map { + const groups = new Map(); + for (const account of accounts) { + const existing = groups.get(account.integration) ?? []; + existing.push(account); + groups.set(account.integration, existing); + } + return groups; +} + +function runtimeSkillResponse(skill: UserSkillRecord, slug: string, signature: string) { + return { + archivedAt: null, + description: skill.description, + latestRevision: Math.max(1, Math.floor(skill.updatedAt.getTime() / 1000)), + name: skill.name, + signature, + slug, + updatedAt: skill.updatedAt.toISOString(), + }; +} + +function isActiveStatus(status: string): boolean { + return ["active", "authorized", "connected", "enabled"].includes(status.trim().toLowerCase()); +} + +function titleCaseSlug(slug: string): string { + return slug + .split(/[-_]+/u) + .filter(Boolean) + .map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`) + .join(" "); +} + +function invalidSkillPackage(message: string): APIError { + return new APIError(400, "tool_validation_failed", message, { retriable: false }); +} + +function skillNotFound(skill: string): APIError { + return new APIError(404, "not_found_skill", `Managed skill not found: ${skill}`, { + retriable: false, + }); +} diff --git a/apps/agent-worker/src/user-skill-files.ts b/apps/agent-worker/src/user-skill-files.ts new file mode 100644 index 00000000..b620769c --- /dev/null +++ b/apps/agent-worker/src/user-skill-files.ts @@ -0,0 +1,305 @@ +import type { UserSkillRecord } from "@cheatcode/db"; +import { createLogger } from "@cheatcode/observability"; +import type { SandboxLike } from "@cheatcode/sandbox-contracts"; +import { z } from "zod"; +import { SANDBOX_WORKSPACE_ROOT } from "./sandbox-route-helpers"; + +const USER_SKILLS_DIRECTORY = `${SANDBOX_WORKSPACE_ROOT}/.cheatcode/skills`; + +const RevisionSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const MirroredSkillSchema = z + .object({ + body: z.string().trim().min(1).max(40_000), + category: z.string().trim().min(1).max(80), + description: z.string().trim().min(1).max(400), + name: z.string().trim().min(1).max(80), + registryRevision: RevisionSchema.nullable(), + skillId: z.string().uuid(), + tags: z.array(z.string().trim().min(1).max(40)).max(12), + }) + .strict(); + +const PortableSkillSchema = z + .object({ + body: z.string().trim().min(1).max(40_000), + category: z.enum(["Builder & Apps", "Research & Docs", "Data & Media"]), + description: z.string().trim().min(1).max(400), + name: z.string().trim().min(1).max(80), + tags: z.array(z.string().trim().min(1).max(40)).max(12), + }) + .strict(); + +type MirroredUserSkill = z.infer; + +export type UserSkillMirrorResolution = + | { kind: "registry" } + | { kind: "promote"; mirror: MirroredUserSkill } + | { kind: "conflict"; reason: string }; + +export function userSkillSlug(name: string): string { + const slug = name + .normalize("NFKD") + .toLowerCase() + .replaceAll(/[^a-z0-9]+/gu, "-") + .replaceAll(/^-+|-+$/gu, "") + .slice(0, 80); + return slug || "custom-skill"; +} + +export function userSkillFilePath(name: string): string { + return `${USER_SKILLS_DIRECTORY}/${userSkillSlug(name)}/SKILL.md`; +} + +export function userSkillDirectoryPath(name: string): string { + return `${USER_SKILLS_DIRECTORY}/${userSkillSlug(name)}`; +} + +export async function writeUserSkillMirror( + sandbox: SandboxLike, + skill: UserSkillRecord, +): Promise { + if (typeof sandbox.writeFile !== "function") { + throw new Error("Sandbox does not support custom skill mirrors."); + } + const path = userSkillFilePath(skill.name); + await sandbox.writeFile({ + content: await serializeUserSkillMarkdown(skill), + encoding: "utf8", + path, + }); + return path; +} + +export async function resolveUserSkillMirror( + sandbox: SandboxLike, + skill: UserSkillRecord, +): Promise { + const currentRevision = await userSkillRevision(skill); + const markdown = await readUserSkillMirror(sandbox, skill.name); + if (markdown === null) { + await writeMirrorBestEffort(sandbox, skill, "user_skill_mirror_missing_write_failed"); + return { kind: "registry" }; + } + const parsed = parseUserSkillMarkdown(markdown); + if (!parsed.success) { + logMirrorConflict(skill, "invalid_mirror"); + return { kind: "conflict", reason: "invalid_mirror" }; + } + if (parsed.data.skillId !== skill.id || parsed.data.name !== skill.name) { + logMirrorConflict(skill, "identity_mismatch"); + return { kind: "conflict", reason: "identity_mismatch" }; + } + const mirrorRevision = await userSkillRevision({ ...skill, ...parsed.data }); + const storedRevision = parsed.data.registryRevision; + if (storedRevision === null) { + return mirrorRevision === currentRevision + ? normalizeRegistryMirror(sandbox, skill) + : { kind: "promote", mirror: parsed.data }; + } + if (mirrorRevision === storedRevision) { + return currentRevision === storedRevision + ? { kind: "registry" } + : normalizeRegistryMirror(sandbox, skill); + } + if (currentRevision === storedRevision) { + return { kind: "promote", mirror: parsed.data }; + } + logMirrorConflict(skill, "concurrent_edit"); + return { kind: "conflict", reason: "concurrent_edit" }; +} + +function parseUserSkillMarkdown( + markdown: string, +): ReturnType { + const normalized = markdown.replaceAll("\r\n", "\n").trim(); + if (!normalized.startsWith("---\n")) { + return MirroredSkillSchema.safeParse({}); + } + const end = normalized.indexOf("\n---\n", 4); + if (end === -1) { + return MirroredSkillSchema.safeParse({}); + } + const values = parseFrontmatter(normalized.slice(4, end)); + return MirroredSkillSchema.safeParse({ + body: normalized.slice(end + 5).trim(), + category: values.get("category"), + description: values.get("description"), + name: values.get("name"), + registryRevision: values.get("registry-revision") ?? null, + skillId: values.get("skill-id"), + tags: parseTags(values.get("tags")), + }); +} + +/** Parses an authored, portable SKILL.md before registry identity is assigned. */ +export function parsePortableSkillMarkdown( + markdown: string, + fallbackSlug: string, +): ReturnType { + const normalized = markdown.replaceAll("\r\n", "\n").trim(); + const parts = portableSkillParts(normalized); + const values = parsePortableFrontmatter(parts.frontmatter); + const fallbackName = titleCaseSlug(fallbackSlug); + return PortableSkillSchema.safeParse({ + body: parts.body, + category: values.get("category") ?? "Builder & Apps", + description: values.get("description") ?? `Custom Cheatcode skill: ${fallbackName}.`, + name: values.get("name") ?? fallbackName, + tags: parseTags(values.get("tags")), + }); +} + +export async function serializeUserSkillMarkdown(skill: UserSkillRecord): Promise { + const revision = await userSkillRevision(skill); + return [ + "---", + `skill-id: ${JSON.stringify(skill.id)}`, + `name: ${JSON.stringify(skill.name)}`, + `description: ${JSON.stringify(skill.description)}`, + `category: ${JSON.stringify(skill.category)}`, + `tags: ${JSON.stringify(skill.tags)}`, + `registry-revision: ${JSON.stringify(revision)}`, + "---", + "", + skill.body.trim(), + "", + ].join("\n"); +} + +async function userSkillRevision( + skill: Pick, +): Promise { + const canonical = JSON.stringify({ + body: skill.body.trim(), + category: skill.category.trim(), + description: skill.description.trim(), + id: skill.id, + name: skill.name.trim(), + tags: skill.tags, + }); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(canonical)); + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +async function readUserSkillMirror(sandbox: SandboxLike, name: string): Promise { + if (typeof sandbox.readFile !== "function") return null; + const file = await sandbox + .readFile({ encoding: "utf8", path: userSkillFilePath(name) }) + .catch(() => null); + return file?.encoding === "utf8" ? file.content : null; +} + +function parseFrontmatter(frontmatter: string): Map { + const values = new Map(); + for (const line of frontmatter.split("\n")) { + const separator = line.indexOf(":"); + if (separator === -1) continue; + const key = line.slice(0, separator).trim(); + const raw = line.slice(separator + 1).trim(); + values.set(key, parseScalar(raw)); + } + return values; +} + +function portableSkillParts(markdown: string): { body: string; frontmatter: string } { + if (!markdown.startsWith("---\n")) { + return { body: markdown, frontmatter: "" }; + } + const end = markdown.indexOf("\n---\n", 4); + return end === -1 + ? { body: markdown, frontmatter: "" } + : { body: markdown.slice(end + 5).trim(), frontmatter: markdown.slice(4, end) }; +} + +function parsePortableFrontmatter(frontmatter: string): Map { + const values = new Map(); + const lines = frontmatter.split("\n"); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const separator = line.indexOf(":"); + if (separator === -1 || /^\s/u.test(line)) continue; + const key = line.slice(0, separator).trim(); + const raw = line.slice(separator + 1).trim(); + if (raw === ">" || raw === ">-" || raw === "|") { + const folded = collectIndentedValue(lines, index + 1); + values.set(key, folded.value); + index = folded.lastIndex; + continue; + } + values.set(key, parseScalar(raw)); + } + return values; +} + +function collectIndentedValue( + lines: string[], + startIndex: number, +): { lastIndex: number; value: string } { + const content: string[] = []; + let lastIndex = startIndex - 1; + for (let index = startIndex; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + if (line && !/^\s/u.test(line)) break; + content.push(line.trim()); + lastIndex = index; + } + return { lastIndex, value: content.filter(Boolean).join(" ") }; +} + +function titleCaseSlug(slug: string): string { + return slug + .split(/[-_]+/u) + .filter(Boolean) + .map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`) + .join(" ") + .slice(0, 80); +} + +function parseScalar(raw: string): string { + if (!raw.startsWith('"')) return raw; + try { + const parsed: unknown = JSON.parse(raw); + return typeof parsed === "string" ? parsed : raw; + } catch { + return raw; + } +} + +function parseTags(raw: string | undefined): unknown { + if (raw === undefined) return []; + try { + return JSON.parse(raw) as unknown; + } catch { + return raw + .replace(/^\[|\]$/gu, "") + .split(",") + .map((tag) => tag.trim()) + .filter(Boolean); + } +} + +async function normalizeRegistryMirror( + sandbox: SandboxLike, + skill: UserSkillRecord, +): Promise { + await writeMirrorBestEffort(sandbox, skill, "user_skill_mirror_normalize_failed"); + return { kind: "registry" }; +} + +async function writeMirrorBestEffort( + sandbox: SandboxLike, + skill: UserSkillRecord, + event: string, +): Promise { + await writeUserSkillMirror(sandbox, skill).catch((error: unknown) => { + createLogger().warn(event, { error, skillId: skill.id }); + }); +} + +function logMirrorConflict(skill: UserSkillRecord, reason: string): void { + createLogger().warn("user_skill_mirror_conflict", { + reason, + skillId: skill.id, + userId: skill.userId, + }); +} diff --git a/apps/agent-worker/src/user-skill-packages.ts b/apps/agent-worker/src/user-skill-packages.ts new file mode 100644 index 00000000..678b22f7 --- /dev/null +++ b/apps/agent-worker/src/user-skill-packages.ts @@ -0,0 +1,156 @@ +import type { UserSkillRecord } from "@cheatcode/db"; +import type { SandboxLike } from "@cheatcode/sandbox-contracts"; +import type { UserId } from "@cheatcode/types"; +import { z } from "zod"; +import { + serializeUserSkillMarkdown, + userSkillDirectoryPath, + userSkillFilePath, +} from "./user-skill-files"; + +const MAX_PACKAGE_FILES = 20; +const MAX_PACKAGE_BYTES = 1024 * 1024; +const PackageFilePathSchema = z + .string() + .min(1) + .max(240) + .regex( + /^(?:SKILL\.md|[^/]+\.md|[^/]+\.ts|(?:[^/]+\/)*(?:[^/]+\.md|[^/]+\.ts)|package\.json|\.env)$/u, + ) + .refine((value) => !value.split("/").includes(".."), "Skill file paths cannot traverse."); + +export const UserSkillPackageFileSchema = z + .object({ + content: z.string().max(MAX_PACKAGE_BYTES), + path: PackageFilePathSchema, + }) + .strict(); + +const UserSkillPackageSchema = z + .object({ + files: z.array(UserSkillPackageFileSchema).min(1).max(MAX_PACKAGE_FILES), + revision: z.string().regex(/^[a-f0-9]{64}$/u), + skillId: z.string().uuid(), + v: z.literal(1), + }) + .strict() + .superRefine((value, context) => { + const paths = new Set(); + let bytes = 0; + for (const file of value.files) { + if (paths.has(file.path)) { + context.addIssue({ code: "custom", message: `Duplicate skill file: ${file.path}` }); + } + paths.add(file.path); + bytes += new TextEncoder().encode(file.content).byteLength; + } + if (!paths.has("SKILL.md")) { + context.addIssue({ code: "custom", message: "A skill package must include SKILL.md." }); + } + if (bytes > MAX_PACKAGE_BYTES) { + context.addIssue({ + code: "custom", + message: `Skill package exceeds ${MAX_PACKAGE_BYTES} bytes.`, + }); + } + }); + +export type UserSkillPackage = z.infer; +export type UserSkillPackageFile = z.infer; + +async function createUserSkillPackage( + skillId: string, + files: UserSkillPackageFile[], +): Promise { + const normalized = files + .map((file) => ({ content: file.content.replaceAll("\r\n", "\n"), path: file.path })) + .sort((left, right) => left.path.localeCompare(right.path)); + const revision = await sha256Hex(JSON.stringify(normalized)); + return UserSkillPackageSchema.parse({ files: normalized, revision, skillId, v: 1 }); +} + +export async function persistUserSkillPackage( + bucket: R2Bucket, + userId: UserId, + skillId: string, + files: UserSkillPackageFile[], +): Promise { + const packageValue = await createUserSkillPackage(skillId, files); + await bucket.put(userSkillPackageKey(userId, skillId), JSON.stringify(packageValue), { + httpMetadata: { contentType: "application/json; charset=utf-8" }, + customMetadata: { revision: packageValue.revision, skillId }, + }); + return packageValue; +} + +export async function readUserSkillPackage( + bucket: R2Bucket, + userId: UserId, + skillId: string, +): Promise { + const object = await bucket.get(userSkillPackageKey(userId, skillId)); + if (!object) return null; + const parsed = UserSkillPackageSchema.safeParse(await object.json()); + return parsed.success && parsed.data.skillId === skillId ? parsed.data : null; +} + +export async function deleteUserSkillPackage( + bucket: R2Bucket, + userId: UserId, + skillId: string, +): Promise { + await bucket.delete(userSkillPackageKey(userId, skillId)); +} + +export async function collectUserSkillPackageFromSandbox( + sandbox: SandboxLike, + skill: UserSkillRecord, +): Promise { + const fallback = [{ content: await serializeUserSkillMarkdown(skill), path: "SKILL.md" }]; + if (!sandbox.listFiles || !sandbox.readFile) { + return fallback; + } + const directory = userSkillDirectoryPath(skill.name); + const listing = await sandbox + .listFiles({ includeHidden: true, path: directory, recursive: true }) + .catch(() => null); + if (!listing) return fallback; + const candidates = listing.files + .filter((entry) => entry.type === "file") + .map((entry) => ({ absolutePath: entry.path, path: entry.relativePath })) + .filter((entry) => PackageFilePathSchema.safeParse(entry.path).success) + .sort((left, right) => left.path.localeCompare(right.path)) + .slice(0, MAX_PACKAGE_FILES); + const files: UserSkillPackageFile[] = []; + for (const candidate of candidates) { + const file = await sandbox.readFile({ encoding: "utf8", path: candidate.absolutePath }); + if (file.encoding === "utf8") files.push({ content: file.content, path: candidate.path }); + } + return files.some((file) => file.path === "SKILL.md") ? files : [...fallback, ...files]; +} + +export async function writeUserSkillPackageMirror( + sandbox: SandboxLike, + skill: Pick, + packageValue: UserSkillPackage, +): Promise { + if (!sandbox.writeFile) throw new Error("Sandbox does not support custom skill packages."); + const directory = userSkillDirectoryPath(skill.name); + for (const file of packageValue.files) { + await sandbox.writeFile({ + content: file.content, + encoding: "utf8", + path: `${directory}/${file.path}`, + }); + } + return userSkillFilePath(skill.name); +} + +function userSkillPackageKey(userId: UserId, skillId: string): string { + return `${userId}/skills/${skillId}/package.json`; +} + +async function sha256Hex(value: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} diff --git a/apps/agent-worker/wrangler.jsonc b/apps/agent-worker/wrangler.jsonc index 1a7c9439..94644cd3 100644 --- a/apps/agent-worker/wrangler.jsonc +++ b/apps/agent-worker/wrangler.jsonc @@ -4,20 +4,30 @@ "workers_dev": false, "preview_urls": false, "main": "src/index.ts", - "compatibility_date": "2026-05-20", + "compatibility_date": "2026-07-15", "compatibility_flags": ["nodejs_compat"], - "version_metadata": { "binding": "CF_VERSION_METADATA" }, + "version_metadata": { + "binding": "CF_VERSION_METADATA" + }, "vars": { "CHEATCODE_ENVIRONMENT": "production", + "CHEATCODE_RELEASE_GATE": "open", "DAYTONA_API_URL": "https://app.daytona.io/api", + "DAYTONA_ORG_ID": "300a0c8b-dc3b-4c5f-b31c-0f9e0344d00d", "DAYTONA_PREVIEW_HOST_SUFFIXES": "daytonaproxy01.net,proxy.daytona.work", "DAYTONA_TARGET": "us", "DAYTONA_SANDBOX_SNAPSHOT": "cheatcode-sandbox-viewer-bundle-3336016022d8", + "DAYTONA_WORKSPACE_VOLUME": "cheatcode-workspaces-production", "OUTPUT_DOWNLOAD_BASE_URL": "https://gateway.trycheatcode.com", "PREVIEW_HOSTNAME": "trycheatcode.com", - "R2_OUTPUTS_BUCKET_NAME": "cheatcode-outputs" + "SKILL_RUNTIME_BASE_URL": "https://gateway.trycheatcode.com/skill-runtime" }, "secrets_store_secrets": [ + { + "binding": "DATABASE_CONTEXT_SIGNING_SECRET_AGENT", + "store_id": "ba25994718db4707ab99a498e22eb5a6", + "secret_name": "database-context-signing-secret-agent" + }, { "binding": "DAYTONA_API_KEY", "store_id": "ba25994718db4707ab99a498e22eb5a6", @@ -34,9 +44,14 @@ "secret_name": "composio-api-key" }, { - "binding": "INTERNAL_MAINTENANCE_SECRET", + "binding": "WEBHOOKS_TO_AGENT_LIFECYCLE_SECRET", + "store_id": "ba25994718db4707ab99a498e22eb5a6", + "secret_name": "webhooks-to-agent-lifecycle-secret" + }, + { + "binding": "RELEASE_DATABASE_READINESS_SECRET", "store_id": "ba25994718db4707ab99a498e22eb5a6", - "secret_name": "internal-maintenance-secret" + "secret_name": "release-database-readiness-secret" }, { "binding": "DEEPSEEK_PLATFORM_API_KEY", @@ -47,6 +62,11 @@ "binding": "OUTPUT_DOWNLOAD_SIGNING_SECRET", "store_id": "ba25994718db4707ab99a498e22eb5a6", "secret_name": "output-download-signing-secret" + }, + { + "binding": "SKILL_RUNTIME_TOKEN_SECRET", + "store_id": "ba25994718db4707ab99a498e22eb5a6", + "secret_name": "skill-runtime-token-secret" } ], "durable_objects": { @@ -76,11 +96,20 @@ "new_sqlite_classes": ["ProjectSandbox"] } ], + "workflows": [ + { + "name": "cheatcode-agent-runs", + "binding": "AGENT_RUN_WORKFLOW", + "class_name": "AgentRunWorkflow", + "limits": { + "steps": 25000 + } + } + ], "hyperdrive": [ { "binding": "HYPERDRIVE", - "id": "b7cead054a6a4207a475b9544971f04a", - "localConnectionString": "postgresql://app_worker:app_worker@localhost:54322/postgres" + "id": "6fda3a31b1dc46e8ac736cbad79d5bc8" } ], "r2_buckets": [ @@ -95,8 +124,6 @@ ], "kv_namespaces": [ { - // Daytona sandbox lifecycle cache (webhooks-worker writes on sandbox.state.updated); read by - // the preview-status endpoint to avoid polling Daytona. Falls back to a live read when cold. "binding": "SANDBOX_STATE", "id": "af927eb58bcc481c97e5bf5323fc2907", "preview_id": "af927eb58bcc481c97e5bf5323fc2907" diff --git a/apps/gateway-worker/.dev.vars.example b/apps/gateway-worker/.dev.vars.example deleted file mode 100644 index 5a163aa1..00000000 --- a/apps/gateway-worker/.dev.vars.example +++ /dev/null @@ -1,8 +0,0 @@ -# Use one of these for local Clerk verification. -CLERK_SECRET_KEY= -CLERK_JWT_KEY= -CLERK_AUTHORIZED_PARTIES=http://localhost:3000,http://127.0.0.1:3000 -POLAR_ACCESS_TOKEN= -COMPOSIO_API_KEY= -COMPOSIO_AUTH_CONFIGS={"github":"ac_...","gmail":"ac_...","slack":"ac_...","notion":"ac_...","linear":"ac_..."} -INTERNAL_MAINTENANCE_SECRET= diff --git a/apps/gateway-worker/README.md b/apps/gateway-worker/README.md index 6200d431..a8f85739 100644 --- a/apps/gateway-worker/README.md +++ b/apps/gateway-worker/README.md @@ -4,16 +4,21 @@ Public Hono API entrypoint. It verifies Clerk JWTs, resolves the Clerk subject t internal `users.id` UUID, lazily syncs the user from Clerk on first authenticated request when webhooks have not run yet, rate-limits requests, and forwards agent work to `agent-worker` via Service Binding. +The bootstrap reads one canonical Clerk identity snapshot including `updated_at`; +the database compare-and-swap prevents a slower Backend API response or delayed webhook +from regressing a newer email, display name, or avatar. Provider key writes validate each supported BYOK provider through `packages/byok` before calling the Vault-backed RPC. Invalid keys are rejected before plaintext is sent to storage, and new providers are blocked when the -current entitlement tier has reached its BYOK slot limit. `/internal/users/:userId/delete-state` -is an HMAC-protected maintenance route used by webhooks to clear the user's -`QuotaTracker` durable state during the Clerk deletion lifecycle. Destructive -gateway-to-agent cleanup and webhooks-to-gateway deletion calls use the shared -`ccm1` method/path/millisecond-timestamp/body-hash signature contract and shared -deletion schemas; no legacy signature format is accepted. +current entitlement tier has reached its BYOK slot limit. Deleting a key reranks +the remaining provider catalog in the same transaction so a freed tier slot is +available immediately. Project and thread deletes enqueue an exact-generation +resource-deletion job through the webhooks Service Binding. That call uses the +isolated `ccm2` resource-deletion capability and binds gateway issuer, webhooks +audience, method, path, timestamp, nonce, and exact body hash. The webhooks +receiver pins `webhooks.internal`; no shared key or legacy signature format is +accepted. User-scoped Postgres transactions contain database work only. Secrets Store, KV, Durable Object, service-binding, weather, Polar, and Composio operations @@ -34,7 +39,9 @@ canonical live rate-limit state. Billing routes create Polar checkout/portal sessions and manage end-of-period cancellation/reactivation through `/v1/billing/state`, `/v1/billing/cancel`, and `/v1/billing/reactivate`. They update V2 entitlement state and clear the -entitlement KV cache; final product verification still happens by operating the +entitlement KV cache. Checkout accepts only an optional same-origin local path; +the gateway derives the trusted frontend origin and both Polar redirect URLs, so +callers cannot provide an external success or return URL. Final product verification still happens by operating the Settings UI directly with `agent-browser`, not a billing test script. Run creation requires the current Clerk primary email to be verified before the @@ -42,6 +49,8 @@ request is forwarded to `agent-worker`, so authenticated but unverified users do not spawn Daytona sandbox work. JWT verification also requires `azp` to match one of the exact HTTP(S) origins in `CLERK_AUTHORIZED_PARTIES`; production is pinned to `https://trycheatcode.com`. +Resolved Clerk Backend API keys fail closed unless they are `sk_live_` in +production or `sk_test_` in laptop development. Run-creation idempotency bodies are capped at 64 KiB. The Durable Object uses a five-minute in-flight claim lease and retains completed keys for 24 hours; completion @@ -56,8 +65,9 @@ one run. Reusing a key for a different body or thread fails closed. Public Clerk credentials, cookies, proxy credentials, plaintext idempotency keys, and caller-supplied `X-Cheatcode-*` headers terminate at the gateway. Normal service-binding -requests receive only gateway-minted internal identity/idempotency headers; local preview -traffic has a separate, explicit capability/cookie bridge. +requests receive only gateway-minted internal identity/idempotency headers. Artifact downloads +use that boundary to mint an owner-checked short-lived URL; only the resulting HMAC-bound +streaming URL is public. Local preview traffic has a separate, explicit capability/cookie bridge. Composio account sync follows provider cursors instead of treating the first page as complete, and fails closed if a user exceeds the 1,000-account safety @@ -86,35 +96,43 @@ and accepts authenticated `/v1/user-events` activation pings from the real web U Production releases use `CHEATCODE_RELEASE_GATE` as a fail-closed deployment barrier. The deploy operation first publishes the final gateway bundle with the gate set to `closed`; every public route, including `/health`, returns a -non-cacheable `503` while the agent converges. The closed health body reports -the gateway and service-bound agent release identities so deployment can verify -both before publishing the same gateway bundle with the gate set to `open`. -The exact HMAC-authenticated user-state deletion route remains available so a -Clerk deletion lifecycle is not stranded during the release window. - -If a barrier step fails, the deploy operation re-deploys and verifies the closed -gate before stopping. If close-gate recovery itself cannot be verified, the gate -state is reported as unconfirmed and requires immediate inspection. Once closed, -recover by rerunning the complete deployment from the same immutable commit. To -abandon it, keep gateway closed, review and roll agent back if it changed, then -roll gateway back to the matching known-good open version and verify `/health`; -never bypass convergence by flipping the gate in the dashboard. - -The HTTP barrier stops new public work but cannot terminate a request or Durable -Object execution accepted before closure. Changes to active `AgentRun` behavior -or gateway-owned Durable Object state therefore need an explicit drain/state -migration decision; the release gate alone is not an atomic Durable Object -migration mechanism. - -`IdempotencyStore` owns one exact SQLite table shape. Constructor initialization -runs behind the Durable Object input gate and transactionally rebuilds an older -deployed shape while preserving valid completed and in-flight entries; unknown -lossy shapes fail closed instead of silently discarding request outcomes. +non-cacheable `503`. Agent and webhooks are then deployed with their own gates +set to `draining`; the gateway health body proves both service-bound downstream +SHAs and gates. The release drains AgentRun and every webhook, ops, and +resource-deletion Workflow before redeploying both services `closed` and allowing +DDL. In steady state, the public 200 `/health` response also fails closed unless +both downstream services report `open` at the gateway's exact release SHA. After closed reconciliation, +contractions, and Vercel promotion, agent and webhooks reopen first and gateway +opens last. Internal lifecycle work reaches quota state through the webhooks +Worker's direct cross-Worker Durable Object binding, so gateway has no maintenance +bypass route during the closed window. + +If a barrier step fails, the deploy operation re-deploys and verifies all three +writer gates closed before stopping. If recovery cannot be verified, writer state +is reported as unconfirmed and requires immediate inspection. Once closed, +recover by rerunning the complete deployment from the same immutable commit. If +that release cannot continue, keep the gateway closed and dispatch a reviewed, +forward-compatible `stage-closed` release that explicitly names the superseded +closed SHA. Never recover a schema contraction by deploying older code or bypass +convergence by flipping the gate in the dashboard. + +The HTTP barrier stops new public work, and the draining agent/webhook gates fence +new admissions while pinned Workflow and Durable Object continuations finish. The +coordinated release drains relational AgentRun state and every retained writer +Workflow before moving those services to `closed` and running DDL. Durable Object schema changes use +explicit in-place reconciliation; the gate alone is not an atomic migration. + +`IdempotencyStore` owns one exact SQLite table shape in its stable namespace and +reconciles dormant objects to that shape when they are next activated. Run +creation is also durably idempotent in Postgres, so request-cache evolution +cannot create a duplicate run. `/v1/tools` and `/v1/agents` read the shared framework-free capability catalog from `@cheatcode/types`. The Mastra registries are statically constrained to the same exact names; workflows are exposed through tools and are not reported as -agents. +agents. Each tool summary also declares whether it uses the sandbox and whether +it produces an artifact; AgentRun stream status and deliverable routing derive +from those same traits instead of maintaining parallel tool-name lists. ## Public exports @@ -134,15 +152,20 @@ pnpm --filter @cheatcode/gateway-worker typecheck ## Env - `CHEATCODE_ENVIRONMENT` (`production` in committed Wrangler config; local generated config overrides it) -- `CHEATCODE_RELEASE_GATE` (`open` normally; generated production release config closes it during agent convergence) +- `CHEATCODE_RELEASE_GATE` (`open` normally; coordinated production releases close gateway first while agent/webhooks drain, then close all writers) - `CHEATCODE_RELEASE_SHA` (required for production deployments) - `CF_VERSION_METADATA` - `AGENT` +- `WEBHOOKS` +- `PREVIEW_PROXY` (generated local-only Service Binding; production preview + traffic reaches the preview Worker through its wildcard route) - `RATE_LIMITER` - `QUOTA_TRACKER` - `IDEMPOTENCY` - `ENTITLEMENTS_CACHE` -- `HYPERDRIVE` +- `HYPERDRIVE` (dedicated config whose database login is exactly `app_gateway`) +- `DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY` (role-specific Secrets Store binding; + must match the `app_gateway` Supabase Vault HMAC secret) - `CLERK_SECRET_KEY` or `CLERK_JWT_KEY` - `CLERK_AUTHORIZED_PARTIES` (comma-separated exact HTTP(S) origins) - `POLAR_ACCESS_TOKEN` @@ -150,5 +173,8 @@ pnpm --filter @cheatcode/gateway-worker typecheck - `POLAR_PRODUCT_ID_PRO`, `POLAR_PRODUCT_ID_PREMIUM`, `POLAR_PRODUCT_ID_ULTRA`, `POLAR_PRODUCT_ID_MAX` - `COMPOSIO_API_KEY` - `COMPOSIO_AUTH_CONFIGS` -- `INTERNAL_MAINTENANCE_SECRET` +- `GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET` (ccm2 `resource-deletion` + capability shared only with the webhooks verifier) +- `RELEASE_DATABASE_READINESS_SECRET` (ccm2 `database-readiness` capability; + the release environment receives no destructive capability key) - `USER_EVENTS`, `ERROR_EVENTS`, `PERFORMANCE_METRICS` diff --git a/apps/gateway-worker/package.json b/apps/gateway-worker/package.json index 9e8acee4..227170f0 100644 --- a/apps/gateway-worker/package.json +++ b/apps/gateway-worker/package.json @@ -7,7 +7,7 @@ "types": "./src/index.ts", "scripts": { "build": "wrangler deploy --dry-run", - "dev": "wrangler dev --var CHEATCODE_ENVIRONMENT:development", + "deploy": "wrangler deploy", "lint": "biome check .", "typecheck": "tsc -p tsconfig.json --noEmit" }, @@ -17,6 +17,7 @@ "@cheatcode/byok": "workspace:*", "@cheatcode/composio": "workspace:*", "@cheatcode/db": "workspace:*", + "@cheatcode/durable-storage": "workspace:*", "@cheatcode/env": "workspace:*", "@cheatcode/observability": "workspace:*", "@cheatcode/types": "workspace:*", diff --git a/apps/gateway-worker/src/account-http-routes.ts b/apps/gateway-worker/src/account-http-routes.ts index 1067cdfe..bf5c1fa7 100644 --- a/apps/gateway-worker/src/account-http-routes.ts +++ b/apps/gateway-worker/src/account-http-routes.ts @@ -1,5 +1,4 @@ import { createDb } from "@cheatcode/db"; -import { getMeRoute, updateMeRoute } from "./account-routes"; import { getActivityHistoryRoute } from "./activity-routes"; import { authenticate, readRequiredSecret } from "./authenticate"; import { myUsageRoute } from "./billing-routes"; @@ -11,16 +10,6 @@ import { rateLimit } from "./rate-limit"; import { listRecentThreadsRoute, searchWorkspaceRoute } from "./search-routes"; export function registerAccountHttpRoutes(app: GatewayApp): void { - app.get("/v1/me", async (c) => { - const userId = await authenticate(c.req.raw, c.env, c.executionCtx); - await rateLimit(c, userId, "GET /v1/me"); - return getMeRoute(c.env, c.executionCtx, userId); - }); - app.patch("/v1/me", async (c) => { - const userId = await authenticate(c.req.raw, c.env, c.executionCtx); - await rateLimit(c, userId, "PATCH /v1/me"); - return updateMeRoute(c.env, c.executionCtx, c.req.raw, userId); - }); app.get("/v1/me/profile", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/me/profile"); @@ -58,7 +47,10 @@ export function registerAccountHttpRoutes(app: GatewayApp): void { async function limitsRoute(c: GatewayContext): Promise { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/limits"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const snapshot = await buildLimitsSnapshot(c.env, db, userId); return c.json(snapshot); diff --git a/apps/gateway-worker/src/account-routes.ts b/apps/gateway-worker/src/account-routes.ts deleted file mode 100644 index dc177cc8..00000000 --- a/apps/gateway-worker/src/account-routes.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { createDb, getUserAccount, updateUserAccount, withUserContext } from "@cheatcode/db"; -import { APIError, readJsonRequest } from "@cheatcode/observability"; -import { MeResponseSchema, UpdateMeSchema, type UserId } from "@cheatcode/types"; -import type { WaitUntilContext } from "./wait-until-context"; - -export interface AccountRouteEnv { - HYPERDRIVE: Hyperdrive; -} - -const MAX_ACCOUNT_REQUEST_BYTES = 4 * 1024; - -function accountResponse(record: { - id: string; - email: string; - displayName: string | null; - avatarUrl: string | null; -}) { - return MeResponseSchema.parse({ - avatarUrl: record.avatarUrl, - displayName: record.displayName, - email: record.email, - id: record.id, - }); -} - -export async function getMeRoute( - env: AccountRouteEnv, - ctx: WaitUntilContext, - userId: UserId, -): Promise { - const { db, close } = createDb(env.HYPERDRIVE); - try { - const record = await withUserContext(db, userId, (tx) => getUserAccount(tx, userId)); - if (!record) { - throw new APIError(404, "not_found_user", "User not found", { retriable: false }); - } - return Response.json(accountResponse(record)); - } finally { - ctx.waitUntil(close()); - } -} - -export async function updateMeRoute( - env: AccountRouteEnv, - ctx: WaitUntilContext, - request: Request, - userId: UserId, -): Promise { - const parsed = UpdateMeSchema.safeParse( - await readJsonRequest(request, MAX_ACCOUNT_REQUEST_BYTES, "Account request"), - ); - if (!parsed.success) { - throw new APIError(400, "invalid_request_body", "Invalid account payload", { - details: { issues: parsed.error.issues.map((issue) => issue.message) }, - retriable: false, - }); - } - const { db, close } = createDb(env.HYPERDRIVE); - try { - const record = await withUserContext(db, userId, (tx) => - updateUserAccount(tx, userId, { - ...(parsed.data.displayName === undefined ? {} : { displayName: parsed.data.displayName }), - }), - ); - if (!record) { - throw new APIError(404, "not_found_user", "User not found", { retriable: false }); - } - return Response.json(accountResponse(record)); - } finally { - ctx.waitUntil(close()); - } -} diff --git a/apps/gateway-worker/src/activity-routes.ts b/apps/gateway-worker/src/activity-routes.ts index d4705c71..2b18a2d1 100644 --- a/apps/gateway-worker/src/activity-routes.ts +++ b/apps/gateway-worker/src/activity-routes.ts @@ -5,6 +5,7 @@ import { listAgentRunStartPoints, withUserContext, } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; import { APIError, readBoundedResponseJson } from "@cheatcode/observability"; import { type ActivityHistoryResponse, @@ -19,6 +20,7 @@ import { QuotaHistoryResultSchema } from "./durable-objects/quota-tracker-contra import type { WaitUntilContext } from "./wait-until-context"; export interface ActivityRouteEnv { + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; QUOTA_TRACKER: DurableObjectNamespace; } @@ -37,7 +39,10 @@ export async function getActivityHistoryRoute( userId: UserId, ): Promise { const query = parseActivityQuery(request); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const sandboxHours = await listSandboxHourHistory(env, userId, query.days); const response = await withUserContext(db, userId, (tx) => diff --git a/apps/gateway-worker/src/agent-forwarding.ts b/apps/gateway-worker/src/agent-forwarding.ts index 729d4e82..06cd4eff 100644 --- a/apps/gateway-worker/src/agent-forwarding.ts +++ b/apps/gateway-worker/src/agent-forwarding.ts @@ -34,6 +34,20 @@ export function agentServiceRequest(request: Request, userId?: string): Request return new Request(request, { headers: agentServiceHeaders(request.headers, userId) }); } +/** + * Preserves only the run-scoped capability needed by sandbox skill packages. + * Clerk cookies and caller-controlled internal headers never cross this route. + */ +export function skillRuntimeServiceRequest(request: Request): Request { + const authorization = request.headers.get("Authorization"); + const headers = new Headers(); + if (authorization) headers.set("Authorization", authorization); + headers.set("Content-Type", request.headers.get("Content-Type") ?? "application/json"); + const requestId = request.headers.get("X-Request-Id"); + if (requestId) headers.set("X-Request-Id", requestId); + return new Request(request, { headers }); +} + export async function forwardAgentRequest(c: GatewayContext, route: string): Promise { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); const headers = await rateLimit(c, userId, route); diff --git a/apps/gateway-worker/src/agent-http-routes.ts b/apps/gateway-worker/src/agent-http-routes.ts index 4013faaf..020c76ef 100644 --- a/apps/gateway-worker/src/agent-http-routes.ts +++ b/apps/gateway-worker/src/agent-http-routes.ts @@ -1,11 +1,16 @@ -import { agentServiceHeaders, forwardAgentRequest } from "./agent-forwarding"; -import { decideRunApprovalRoute, readSandboxConsoleRoute } from "./agent-proxy-routes"; +import { + agentServiceHeaders, + forwardAgentRequest, + skillRuntimeServiceRequest, +} from "./agent-forwarding"; +import { readSandboxConsoleRoute } from "./agent-proxy-routes"; import { authenticate, requireVerifiedClerkEmail } from "./authenticate"; import type { GatewayApp, GatewayContext } from "./gateway-env"; import { completeIdempotentRunRequest, prepareIdempotentRunRequest } from "./idempotency"; import { rateLimit, withRateLimitHeaders } from "./rate-limit"; export function registerAgentHttpRoutes(app: GatewayApp): void { + app.all("/skill-runtime/*", (c) => c.env.AGENT.fetch(skillRuntimeServiceRequest(c.req.raw))); app.post("/v1/threads/:threadId/runs", async (c) => createRunRoute(c)); for (const [path, route] of GET_AGENT_ROUTES) { app.get(path, (c) => forwardAgentRequest(c, route)); @@ -16,13 +21,13 @@ export function registerAgentHttpRoutes(app: GatewayApp): void { app.patch("/v1/threads/:threadId/sandbox/file", (c) => forwardAgentRequest(c, "PATCH /v1/threads/:threadId/sandbox/file"), ); - app.post("/v1/runs/:runId/approvals/:approvalId", (c) => decideRunApprovalRoute(c)); app.get("/v1/threads/:threadId/sandbox/console", (c) => readSandboxConsoleRoute(c)); } const GET_AGENT_ROUTES = [ ["/v1/threads/:threadId/runs/stream", "GET /v1/threads/:threadId/runs/stream"], ["/v1/threads/:threadId/runs/status", "GET /v1/threads/:threadId/runs/status"], + ["/v1/threads/:threadId/browser-takeover", "GET /v1/threads/:threadId/browser-takeover"], ["/v1/computer/ide", "GET /v1/computer/ide"], ["/v1/computer/terminal/context", "GET /v1/computer/terminal/context"], ["/v1/threads/:threadId/sandbox/files", "GET /v1/threads/:threadId/sandbox/files"], @@ -41,6 +46,19 @@ const GET_AGENT_ROUTES = [ const POST_AGENT_ROUTES = [ ["/v1/runs/:runId/cancel", "POST /v1/runs/:runId/cancel"], + [ + "/v1/threads/:threadId/skill-proposals/:runId/:proposalId/confirm", + "POST /v1/threads/:threadId/skill-proposals/:runId/:proposalId/confirm", + ], + ["/v1/skills/:skillId/open", "POST /v1/skills/:skillId/open"], + [ + "/v1/threads/:threadId/browser-takeover/start", + "POST /v1/threads/:threadId/browser-takeover/start", + ], + [ + "/v1/threads/:threadId/browser-takeover/resume", + "POST /v1/threads/:threadId/browser-takeover/resume", + ], ["/v1/computer/terminal", "POST /v1/computer/terminal"], ["/v1/threads/:threadId/sandbox/preview/wake", "POST /v1/threads/:threadId/sandbox/preview/wake"], ["/v1/threads/:threadId/sandbox/terminal", "POST /v1/threads/:threadId/sandbox/terminal"], diff --git a/apps/gateway-worker/src/agent-proxy-routes.ts b/apps/gateway-worker/src/agent-proxy-routes.ts index aebb9005..3c2e7c3a 100644 --- a/apps/gateway-worker/src/agent-proxy-routes.ts +++ b/apps/gateway-worker/src/agent-proxy-routes.ts @@ -1,65 +1,17 @@ -import { - APIError, - readBoundedRequestText, - readBoundedResponseJson, -} from "@cheatcode/observability"; -import { - ApprovalDecisionRequestSchema, - ApprovalDecisionResponseSchema, - SandboxConsoleQuerySchema, - SandboxConsoleSnapshotSchema, -} from "@cheatcode/types"; +import { APIError, readBoundedResponseJson } from "@cheatcode/observability"; +import { SandboxConsoleQuerySchema, SandboxConsoleSnapshotSchema } from "@cheatcode/types"; import type { Context } from "hono"; import type { z } from "zod"; -import { agentServiceHeaders, agentServiceRequest } from "./agent-forwarding"; +import { agentServiceRequest } from "./agent-forwarding"; import { authenticate } from "./authenticate"; import type { GatewayEnv } from "./gateway-env"; import { rateLimit, withRateLimitHeaders } from "./rate-limit"; type GatewayProxyContext = Context<{ Bindings: GatewayEnv }>; -const APPROVAL_DECISION_ROUTE = "POST /v1/runs/:runId/approvals/:approvalId"; -const MAX_APPROVAL_BODY_BYTES = 4 * 1024; -const MAX_APPROVAL_RESPONSE_BYTES = 16 * 1024; const MAX_CONSOLE_RESPONSE_BYTES = 2 * 1024 * 1024; const SANDBOX_CONSOLE_ROUTE = "GET /v1/threads/:threadId/sandbox/console"; -/** - * `POST /v1/runs/:runId/approvals/:approvalId` — validates the allow/deny body - * at the public boundary, forwards verbatim to the agent-worker (which routes to - * the AgentRun DO for ownership + state-machine enforcement, emitting `404` - * for a missing run or `409` when the approval cannot be resolved), then re-parses the - * `200` resolution before returning it. - */ -export async function decideRunApprovalRoute(c: GatewayProxyContext): Promise { - const userId = await authenticate(c.req.raw, c.env, c.executionCtx); - const rateLimitHeaders = await rateLimit(c, userId, APPROVAL_DECISION_ROUTE); - const rawBody = await readBoundedRequestText( - c.req.raw, - MAX_APPROVAL_BODY_BYTES, - "Approval decision", - ); - const parsed = ApprovalDecisionRequestSchema.safeParse(parseJsonRequestBody(rawBody)); - if (!parsed.success) { - throw invalidRequestBody("Invalid approval decision payload", parsed.error); - } - const forwarded = new Request(c.req.raw.url, { - body: rawBody, - headers: agentServiceHeaders(c.req.raw.headers, userId), - method: "POST", - }); - const response = await c.env.AGENT.fetch(forwarded); - return withRateLimitHeaders( - await parseForwardedJsonResponse( - response, - ApprovalDecisionResponseSchema, - MAX_APPROVAL_RESPONSE_BYTES, - "Agent approval", - ), - rateLimitHeaders, - ); -} - /** * `GET /v1/threads/:threadId/sandbox/console` — cursor-poll for dev-server logs * (`read.expensive`). Validates the cursor/lastPid query at the boundary, @@ -86,19 +38,6 @@ export async function readSandboxConsoleRoute(c: GatewayProxyContext): Promise( response: Response, schema: Schema, @@ -116,13 +55,6 @@ async function parseForwardedJsonResponse( }); } -function invalidRequestBody(message: string, error: z.ZodError): APIError { - return new APIError(400, "invalid_request_body", message, { - details: { issues: error.issues.map((issue) => issue.message) }, - retriable: false, - }); -} - function invalidQueryParam(message: string, error: z.ZodError): APIError { return new APIError(400, "invalid_query_param", message, { details: { issues: error.issues.map((issue) => issue.message) }, diff --git a/apps/gateway-worker/src/authenticate.ts b/apps/gateway-worker/src/authenticate.ts index c0fe5c2e..2fbf784e 100644 --- a/apps/gateway-worker/src/authenticate.ts +++ b/apps/gateway-worker/src/authenticate.ts @@ -1,14 +1,15 @@ import { - fetchClerkUserPrimaryEmail, + type ClerkUserSyncSnapshot, fetchClerkUserPrimaryEmailStatus, + fetchClerkUserSyncSnapshot, verifyClerkBearerToken, } from "@cheatcode/auth"; import { createDb, type Database, resolveInternalUserId, + syncClerkUser, UserDeletionBlockedError, - upsertClerkUser, } from "@cheatcode/db"; import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; import { APIError } from "@cheatcode/observability"; @@ -24,6 +25,7 @@ export interface AuthEnv { CLERK_AUTHORIZED_PARTIES?: string; CLERK_JWT_KEY?: WorkerSecret; CLERK_SECRET_KEY?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; } @@ -34,7 +36,10 @@ export async function authenticate( ): Promise { const { secretKey, verificationOptions } = await clerkVerification(env); const session = await verifyClerkBearerToken(request, verificationOptions); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { return await resolveOrSyncClerkUser(db, session.clerkUserId, secretKey); } finally { @@ -44,7 +49,7 @@ export async function authenticate( async function clerkVerification(env: AuthEnv) { const jwtKey = await readOptionalSecret(env.CLERK_JWT_KEY, "CLERK_JWT_KEY"); - const secretKey = await readOptionalSecret(env.CLERK_SECRET_KEY, "CLERK_SECRET_KEY"); + const secretKey = await readOptionalClerkSecret(env); if (!jwtKey && !secretKey) { throw new APIError(503, "unavailable_maintenance", "Clerk verification is not configured", { hint: "Set CLERK_JWT_KEY or CLERK_SECRET_KEY in the gateway Worker environment.", @@ -80,15 +85,23 @@ async function resolveOrSyncClerkUser( retriable: true, }); } - const email = await fetchClerkUserEmail(clerkUserId, secretKey); - if (!email) { + const snapshot = await fetchCanonicalClerkSnapshot(clerkUserId, secretKey); + if (!snapshot.email) { throw new APIError(404, "not_found_user", "Authenticated user is missing an email", { hint: "Add a primary email address to the Clerk user, then retry.", retriable: false, }); } try { - return (await upsertClerkUser(db, { clerkId: clerkUserId, email })).userId; + return ( + await syncClerkUser(db, { + avatarUrl: snapshot.avatarUrl, + clerkId: clerkUserId, + clerkUpdatedAtMs: snapshot.clerkUpdatedAtMs, + displayName: snapshot.displayName, + email: snapshot.email, + }) + ).userId; } catch (error) { throw mapClerkSyncError(error); } @@ -110,9 +123,12 @@ function mapClerkSyncError(error: unknown): unknown { }); } -async function fetchClerkUserEmail(clerkUserId: string, secretKey: string): Promise { +async function fetchCanonicalClerkSnapshot( + clerkUserId: string, + secretKey: string, +): Promise { try { - return await fetchClerkUserPrimaryEmail({ clerkUserId, secretKey }); + return await fetchClerkUserSyncSnapshot({ clerkUserId, secretKey }); } catch { throw new APIError(503, "unavailable_maintenance", "Unable to sync Clerk user", { hint: "Verify CLERK_SECRET_KEY and Clerk Backend API availability.", @@ -122,7 +138,7 @@ async function fetchClerkUserEmail(clerkUserId: string, secretKey: string): Prom } export async function requireVerifiedClerkEmail(request: Request, env: AuthEnv): Promise { - const secretKey = await readRequiredSecret(env.CLERK_SECRET_KEY, "CLERK_SECRET_KEY"); + const secretKey = await readRequiredClerkSecret(env); const session = await verifyClerkBearerToken(request, { authorizedParties: clerkAuthorizedParties(env), secretKey, @@ -187,7 +203,7 @@ async function fetchClerkEmailStatus(clerkUserId: string, secretKey: string) { } } -export async function readOptionalSecret( +async function readOptionalSecret( secret: WorkerSecret | undefined, name: string, ): Promise { @@ -214,3 +230,42 @@ export async function readRequiredSecret( } return value; } + +export async function readOptionalClerkSecret( + env: Pick, +): Promise { + const value = await readOptionalSecret(env.CLERK_SECRET_KEY, "CLERK_SECRET_KEY"); + return value ? assertClerkSecretKeyFamily(value, env.CHEATCODE_ENVIRONMENT) : undefined; +} + +async function readRequiredClerkSecret( + env: Pick, +): Promise { + const value = await readOptionalClerkSecret(env); + if (!value) { + throw new APIError(503, "unavailable_maintenance", "CLERK_SECRET_KEY is not configured", { + hint: "Set CLERK_SECRET_KEY in the gateway Worker environment.", + retriable: false, + }); + } + return value; +} + +function assertClerkSecretKeyFamily( + value: string, + environment: AuthEnv["CHEATCODE_ENVIRONMENT"], +): string { + const requiredPrefix = environment === "production" ? "sk_live_" : "sk_test_"; + if (!value.startsWith(requiredPrefix)) { + throw new APIError( + 503, + "unavailable_maintenance", + "Clerk secret key does not match the deployment environment", + { + hint: `Use a ${requiredPrefix} Clerk key for ${environment}.`, + retriable: false, + }, + ); + } + return value; +} diff --git a/apps/gateway-worker/src/billing-routes.ts b/apps/gateway-worker/src/billing-routes.ts index 4284350b..51f3ac90 100644 --- a/apps/gateway-worker/src/billing-routes.ts +++ b/apps/gateway-worker/src/billing-routes.ts @@ -34,6 +34,7 @@ import { type UserId, } from "@cheatcode/types"; import type { Context } from "hono"; +import { resolveCorsOrigin } from "./cors"; import type { GatewayEnv } from "./gateway-env"; import { resolveEntitlement } from "./limits"; import { rateLimit } from "./rate-limit"; @@ -48,6 +49,8 @@ const POLAR_PRODUCT_ID_ENV = { } as const satisfies Record; const BILLING_REQUEST_MAX_BYTES = 8 * 1024; +const PRODUCTION_WEB_ORIGIN = "https://trycheatcode.com"; +const LOCAL_WEB_ORIGIN = "http://localhost:3000"; type BillingContext = Context<{ Bindings: GatewayEnv }>; @@ -62,7 +65,10 @@ export async function billingStateRoute( ): Promise { const userId = await deps.authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/billing/state"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const entitlement = await withUserContext(db, userId, (tx) => findEntitlementByUserId(tx, userId), @@ -90,16 +96,20 @@ export async function billingCheckoutRoute( } const accessToken = await deps.readRequiredSecret(c.env.POLAR_ACCESS_TOKEN, "POLAR_ACCESS_TOKEN"); const productId = polarProductIdForTier(c.env, parsedInput.data.tier); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const user = await requireBillingUser(db, userId); + const redirect = checkoutRedirectUrls(c, parsedInput.data.returnPath); const url = await createCheckoutUrl({ accessToken, customerEmail: user.email, productId, - ...(parsedInput.data.returnUrl ? { returnUrl: parsedInput.data.returnUrl } : {}), + returnUrl: redirect.returnUrl, ...(c.env.POLAR_SERVER ? { server: c.env.POLAR_SERVER } : {}), - ...(parsedInput.data.successUrl ? { successUrl: parsedInput.data.successUrl } : {}), + successUrl: redirect.successUrl, userId, }); return c.json(BillingUrlResponseSchema.parse({ url })); @@ -108,13 +118,34 @@ export async function billingCheckoutRoute( } } +function checkoutRedirectUrls( + c: BillingContext, + returnPath = "/pricing", +): { returnUrl: string; successUrl: string } { + const returnUrl = new URL(returnPath, billingWebOrigin(c)); + const successUrl = new URL(returnUrl); + successUrl.searchParams.set("checkout", "success"); + return { returnUrl: returnUrl.toString(), successUrl: successUrl.toString() }; +} + +function billingWebOrigin(c: BillingContext): string { + if (c.env.CHEATCODE_ENVIRONMENT === "production") { + return PRODUCTION_WEB_ORIGIN; + } + const requestOrigin = c.req.header("Origin"); + return resolveCorsOrigin(requestOrigin, "development") ?? LOCAL_WEB_ORIGIN; +} + export async function billingCatalogRoute( c: BillingContext, deps: BillingRouteDeps, ): Promise { const userId = await deps.authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/billing/catalog"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const entitlement = await resolveEntitlement(c.env, db, userId); return c.json(BillingCatalogResponseSchema.parse(buildBillingCatalog(c.env, entitlement.tier))); @@ -126,7 +157,10 @@ export async function billingCatalogRoute( export async function myUsageRoute(c: BillingContext, deps: BillingRouteDeps): Promise { const userId = await deps.authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/me/usage"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const summary = await buildSandboxUsageSummary(c.env, db, userId); return c.json(SandboxUsageSummaryResponseSchema.parse(summary)); @@ -142,7 +176,10 @@ export async function billingPortalRoute( const userId = await deps.authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "POST /v1/billing/portal"); const accessToken = await deps.readRequiredSecret(c.env.POLAR_ACCESS_TOKEN, "POLAR_ACCESS_TOKEN"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const user = await requireBillingUser(db, userId); const customerId = @@ -194,7 +231,10 @@ export async function billingCancelRoute( }); } const accessToken = await deps.readRequiredSecret(c.env.POLAR_ACCESS_TOKEN, "POLAR_ACCESS_TOKEN"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const entitlement = await loadSubscriptionEntitlement(db, userId); const result = await cancelSubscriptionAtPeriodEnd({ @@ -220,7 +260,10 @@ export async function billingReactivateRoute( const userId = await deps.authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "POST /v1/billing/reactivate"); const accessToken = await deps.readRequiredSecret(c.env.POLAR_ACCESS_TOKEN, "POLAR_ACCESS_TOKEN"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const entitlement = await loadSubscriptionEntitlement(db, userId); const result = await reactivateSubscription({ diff --git a/apps/gateway-worker/src/core-http-routes.ts b/apps/gateway-worker/src/core-http-routes.ts index 53c8594c..791abc3c 100644 --- a/apps/gateway-worker/src/core-http-routes.ts +++ b/apps/gateway-worker/src/core-http-routes.ts @@ -1,40 +1,21 @@ -import { verifyInternalMaintenanceRequest } from "@cheatcode/auth"; -import { - APIError, - readBoundedRequestText, - readBoundedResponseJson, -} from "@cheatcode/observability"; -import { - InternalGatewayStateDeleteBodySchema, - InternalStateDeleteResponseSchema, - UserId as toUserId, - type UserId, -} from "@cheatcode/types"; -import { z } from "zod"; +import { APIError } from "@cheatcode/observability"; +import type { UserId } from "@cheatcode/types"; import { agentServiceRequest } from "./agent-forwarding"; -import { authenticate, readRequiredSecret } from "./authenticate"; +import { authenticate } from "./authenticate"; +import { registerGatewayDatabaseReadinessRoute } from "./database-readiness"; +import { registerGatewayDurableObjectStorageRoute } from "./durable-object-storage"; import type { GatewayApp, GatewayEnv } from "./gateway-env"; import { OPENAPI_DOCUMENT, openApiDocsHtml } from "./openapi"; import { rateLimit, rateLimitPublic, withRateLimitHeaders } from "./rate-limit"; -import { createUserSkillRoute, deleteUserSkillRoute, listUserSkillsRoute } from "./skills-routes"; +import { readDownstreamReleaseHealth } from "./release-health"; +import { listUserSkillsRoute } from "./skills-routes"; import { clientErrorRoute, clientUserEventRoute, vitalsRoute } from "./telemetry-routes"; import type { WaitUntilContext } from "./wait-until-context"; -const InternalMaintenanceUserIdSchema = z.string().uuid(); -const MAX_AGENT_HEALTH_RESPONSE_BYTES = 16 * 1024; -const MAX_INTERNAL_MAINTENANCE_BODY_BYTES = 1024; -const AgentHealthSchema = z - .object({ - ok: z.literal(true), - releaseSha: z.string().min(1), - versionId: z.string().min(1).nullable(), - worker: z.literal("agent"), - }) - .strict(); - export function registerCoreHttpRoutes(app: GatewayApp): void { + registerGatewayDatabaseReadinessRoute(app); + registerGatewayDurableObjectStorageRoute(app); registerHealthRoute(app); - registerMaintenanceRoute(app); registerDiscoveryRoutes(app); registerTelemetryRoutes(app); registerOutputRoute(app); @@ -45,10 +26,33 @@ function registerHealthRoute(app: GatewayApp): void { app.get("/health", async (c) => { const headers = await rateLimitPublic(c, "GET /health", "publicRead"); const releaseSha = c.env.CHEATCODE_RELEASE_SHA ?? "development"; - const agent = await readAgentHealth(c.env); - if (agent.releaseSha !== releaseSha) { + if (c.env.CHEATCODE_RELEASE_GATE !== "open") { + throw new APIError(503, "unavailable_maintenance", "Gateway release is not open", { + details: { + gatewayReleaseGate: c.env.CHEATCODE_RELEASE_GATE ?? null, + gatewayReleaseSha: releaseSha, + }, + retriable: true, + }); + } + const [{ health: agent }, { health: webhooks }] = await Promise.all([ + readDownstreamReleaseHealth(c.env, "agent"), + readDownstreamReleaseHealth(c.env, "webhooks"), + ]); + if ( + agent.releaseSha !== releaseSha || + agent.releaseGate !== "open" || + webhooks.releaseSha !== releaseSha || + webhooks.releaseGate !== "open" + ) { throw new APIError(503, "unavailable_maintenance", "Release is still converging", { - details: { agentReleaseSha: agent.releaseSha, gatewayReleaseSha: releaseSha }, + details: { + agentReleaseGate: agent.releaseGate, + agentReleaseSha: agent.releaseSha, + gatewayReleaseSha: releaseSha, + webhooksReleaseGate: webhooks.releaseGate, + webhooksReleaseSha: webhooks.releaseSha, + }, retriable: true, }); } @@ -56,44 +60,16 @@ function registerHealthRoute(app: GatewayApp): void { c.json({ agent, ok: true, + releaseGate: c.env.CHEATCODE_RELEASE_GATE, releaseSha, versionId: c.env.CF_VERSION_METADATA?.id ?? null, + webhooks, }), headers, ); }); } -function registerMaintenanceRoute(app: GatewayApp): void { - app.post("/internal/users/:userId/delete-state", async (c) => { - const rawBody = await readBoundedRequestText( - c.req.raw, - MAX_INTERNAL_MAINTENANCE_BODY_BYTES, - "Internal maintenance request", - ); - const secret = await readRequiredSecret( - c.env.INTERNAL_MAINTENANCE_SECRET, - "INTERNAL_MAINTENANCE_SECRET", - ); - await verifyInternalMaintenanceRequest({ rawBody, request: c.req.raw, secret }); - InternalGatewayStateDeleteBodySchema.parse(parseMaintenanceJson(rawBody)); - const userId = toUserId(InternalMaintenanceUserIdSchema.parse(c.req.param("userId"))); - const quotaTracker = c.env.QUOTA_TRACKER.get(c.env.QUOTA_TRACKER.idFromName(`quota:${userId}`)); - const response = await quotaTracker.fetch("https://quota.internal/delete-all", { - method: "POST", - }); - if (!response.ok) { - await response.body?.cancel().catch(() => undefined); - throw new APIError(503, "unavailable_maintenance", "Quota durable state deletion failed", { - details: { status: response.status }, - retriable: true, - }); - } - await response.body?.cancel().catch(() => undefined); - return c.json(InternalStateDeleteResponseSchema.parse({ ok: true })); - }); -} - function registerDiscoveryRoutes(app: GatewayApp): void { app.get("/openapi.json", async (c) => { const headers = await rateLimitPublic(c, "GET /openapi.json", "publicRead"); @@ -127,56 +103,33 @@ function registerTelemetryRoutes(app: GatewayApp): void { } function registerOutputRoute(app: GatewayApp): void { + app.post("/v1/outputs/:outputId/download-url", async (c) => { + const userId = await authenticate(c.req.raw, c.env, c.executionCtx); + const headers = await rateLimit(c, userId, "POST /v1/outputs/:outputId/download-url"); + return withRateLimitHeaders( + await c.env.AGENT.fetch(agentServiceRequest(c.req.raw, userId)), + headers, + ); + }); app.get("/v1/outputs/:outputId/download", async (c) => { const headers = await rateLimitPublic(c, "GET /v1/outputs/:outputId/download", "publicRead"); return withRateLimitHeaders(await c.env.AGENT.fetch(agentServiceRequest(c.req.raw)), headers); }); } -async function readAgentHealth(env: GatewayEnv): Promise> { - let response: Response; - try { - response = await env.AGENT.fetch( - new Request("https://agent.internal/health", { signal: AbortSignal.timeout(3_000) }), - ); - } catch { - throw new APIError(503, "unavailable_maintenance", "Agent service is unavailable", { - retriable: true, - }); - } - if (!response.ok) { - await response.body?.cancel().catch(() => undefined); - throw new APIError(503, "unavailable_maintenance", "Agent service is unhealthy", { - details: { status: response.status }, - retriable: true, - }); - } - try { - return AgentHealthSchema.parse( - await readBoundedResponseJson(response, MAX_AGENT_HEALTH_RESPONSE_BYTES, "Agent health"), - ); - } catch { - throw new APIError(503, "unavailable_maintenance", "Agent health response is invalid", { - retriable: true, - }); - } -} - function registerSkillRoutes(app: GatewayApp): void { app.get("/v1/skills", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/skills"); return listUserSkillsRoute(c.env, c.executionCtx, userId); }); - app.post("/v1/skills", async (c) => { - const userId = await authenticate(c.req.raw, c.env, c.executionCtx); - await rateLimit(c, userId, "POST /v1/skills"); - return createUserSkillRoute(c.env, c.executionCtx, c.req.raw, userId); - }); app.delete("/v1/skills/:skillId", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); - await rateLimit(c, userId, "DELETE /v1/skills/:skillId"); - return deleteUserSkillRoute(c.env, c.executionCtx, userId, c.req.param("skillId")); + const headers = await rateLimit(c, userId, "DELETE /v1/skills/:skillId"); + return withRateLimitHeaders( + await c.env.AGENT.fetch(agentServiceRequest(c.req.raw, userId)), + headers, + ); }); } @@ -194,13 +147,3 @@ async function optionalTelemetryUser( return "anonymous"; } } - -function parseMaintenanceJson(rawBody: string): unknown { - try { - return JSON.parse(rawBody) as unknown; - } catch { - throw new APIError(400, "invalid_request_body", "Internal maintenance body must be JSON", { - retriable: false, - }); - } -} diff --git a/apps/gateway-worker/src/database-readiness.ts b/apps/gateway-worker/src/database-readiness.ts new file mode 100644 index 00000000..f355188a --- /dev/null +++ b/apps/gateway-worker/src/database-readiness.ts @@ -0,0 +1,234 @@ +import { + assertInternalMaintenanceEnvelope, + createInternalMaintenanceHeaders, + fetchClerkInstanceIdentity, + verifyInternalMaintenanceRequest, +} from "@cheatcode/auth"; +import { assertDatabaseRuntimeReadiness, createDb } from "@cheatcode/db"; +import { resolveWorkerSecret } from "@cheatcode/env"; +import { + APIError, + readBoundedRequestText, + readBoundedResponseJson, +} from "@cheatcode/observability"; +import { + AgentDatabaseReadinessResponseSchema, + ClerkInstanceIdentitySchema, + GatewayDatabaseReadinessAggregateResponseSchema, + INTERNAL_DATABASE_READINESS_PATH, + InternalDatabaseReadinessRequestSchema, + WebhooksDatabaseReadinessResponseSchema, +} from "@cheatcode/types"; +import type { ZodType } from "zod"; +import type { GatewayApp, GatewayContext, GatewayEnv } from "./gateway-env"; +import { requireDatabaseReadinessSecret } from "./internal-maintenance"; + +const MAX_READINESS_BODY_BYTES = 4 * 1024; +const MAX_READINESS_RESPONSE_BYTES = 16 * 1024; + +export function registerGatewayDatabaseReadinessRoute(app: GatewayApp): void { + app.post(INTERNAL_DATABASE_READINESS_PATH, handleGatewayDatabaseReadiness); +} + +async function handleGatewayDatabaseReadiness(c: GatewayContext): Promise { + const { rawBody, releaseSha, secret } = await authenticateReadinessRequest(c); + const [agent, webhooks, clerk] = await Promise.all([ + readDownstreamReadiness(c.env, "agent", rawBody, secret, AgentDatabaseReadinessResponseSchema), + readDownstreamReadiness( + c.env, + "webhooks", + rawBody, + secret, + WebhooksDatabaseReadinessResponseSchema, + ), + readClerkInstanceIdentity(c.env), + assertGatewayDatabaseReady(c.env), + ]); + if (agent.releaseSha !== releaseSha || webhooks.releaseSha !== releaseSha) { + throw releaseMismatch("A downstream database-readiness release does not match the gateway"); + } + return c.json( + GatewayDatabaseReadinessAggregateResponseSchema.parse({ + agent, + clerk, + databaseRole: "app_gateway", + ok: true, + releaseSha, + versionId: c.env.CF_VERSION_METADATA?.id ?? null, + webhooks, + worker: "gateway", + }), + ); +} + +async function readClerkInstanceIdentity(env: GatewayEnv) { + const secretKey = await resolveWorkerSecret(env.CLERK_SECRET_KEY); + if (!secretKey?.trim()) { + throw new APIError(503, "unavailable_maintenance", "Clerk identity readiness failed", { + retriable: false, + }); + } + const identity = ClerkInstanceIdentitySchema.parse( + await fetchClerkInstanceIdentity({ secretKey }), + ); + if (env.CHEATCODE_ENVIRONMENT === "production" && identity.environmentType !== "production") { + throw new APIError(503, "unavailable_maintenance", "Clerk identity readiness failed", { + retriable: false, + }); + } + return identity; +} + +async function authenticateReadinessRequest(c: GatewayContext): Promise<{ + rawBody: string; + releaseSha: string; + secret: string; +}> { + assertClosedRelease(c.env); + assertGatewayReadinessHostname(c.req.raw, c.env.CHEATCODE_ENVIRONMENT); + assertInternalMaintenanceEnvelope(c.req.raw, { + audience: "gateway", + capability: "database-readiness", + issuer: "release-control", + }); + const rawBody = await readBoundedRequestText( + c.req.raw, + MAX_READINESS_BODY_BYTES, + "Database readiness request", + ); + const secret = await requireDatabaseReadinessSecret(c.env); + await verifyInternalMaintenanceRequest({ + expectedAudience: "gateway", + expectedCapability: "database-readiness", + expectedIssuer: "release-control", + expectedMethod: "POST", + expectedPathname: INTERNAL_DATABASE_READINESS_PATH, + rawBody, + request: c.req.raw, + secret, + }); + const { releaseSha } = InternalDatabaseReadinessRequestSchema.parse(parseJson(rawBody)); + assertReleaseSha(c.env, releaseSha); + return { rawBody, releaseSha, secret }; +} + +async function assertGatewayDatabaseReady(env: GatewayEnv): Promise { + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); + try { + await assertDatabaseRuntimeReadiness(db, "app_gateway"); + } catch (error) { + throw new APIError(503, "unavailable_maintenance", "Gateway database readiness failed", { + cause: error, + retriable: true, + }); + } finally { + await close(); + } +} + +async function readDownstreamReadiness( + env: GatewayEnv, + worker: "agent" | "webhooks", + rawBody: string, + secret: string, + schema: ZodType, +): Promise { + const headers = await createInternalMaintenanceHeaders({ + audience: worker, + capability: "database-readiness", + issuer: "gateway", + method: "POST", + pathname: INTERNAL_DATABASE_READINESS_PATH, + rawBody, + secret, + }); + headers.set("content-type", "application/json"); + const binding = worker === "agent" ? env.AGENT : env.WEBHOOKS; + let response: Response; + try { + response = await binding.fetch( + `https://${worker}.internal${INTERNAL_DATABASE_READINESS_PATH}`, + { + body: rawBody, + headers, + method: "POST", + signal: AbortSignal.timeout(5_000), + }, + ); + } catch (error) { + throw downstreamReadinessError(worker, error); + } + if (!response.ok) { + const status = response.status; + await response.body?.cancel().catch(() => undefined); + throw downstreamReadinessError(worker, undefined, status); + } + try { + return schema.parse( + await readBoundedResponseJson( + response, + MAX_READINESS_RESPONSE_BYTES, + `${worker} database readiness`, + ), + ); + } catch (error) { + throw downstreamReadinessError(worker, error); + } +} + +function assertClosedRelease(env: GatewayEnv): void { + if (env.CHEATCODE_RELEASE_GATE !== "closed") { + throw releaseMismatch("Database readiness requires the closed release gate"); + } +} + +function assertGatewayReadinessHostname( + request: Request, + environment: GatewayEnv["CHEATCODE_ENVIRONMENT"], +): void { + const hostname = new URL(request.url).hostname; + const isAllowed = + environment === "production" + ? hostname === "gateway.trycheatcode.com" + : hostname === "127.0.0.1" || hostname === "localhost"; + if (!isAllowed) { + throw new APIError(404, "not_found_run", "Database readiness route was not found", { + retriable: false, + }); + } +} + +function assertReleaseSha(env: GatewayEnv, releaseSha: string): void { + if (env.CHEATCODE_RELEASE_SHA !== releaseSha) { + throw releaseMismatch("Database readiness release does not match the gateway"); + } +} + +function parseJson(rawBody: string): unknown { + try { + return JSON.parse(rawBody) as unknown; + } catch { + throw new APIError(400, "invalid_request_body", "Database readiness body must be JSON", { + retriable: false, + }); + } +} + +function releaseMismatch(message: string): APIError { + return new APIError(409, "conflict_state_invalid", message, { retriable: false }); +} + +function downstreamReadinessError( + worker: "agent" | "webhooks", + cause?: unknown, + status?: number, +): APIError { + return new APIError(503, "unavailable_maintenance", `${worker} database readiness failed`, { + cause, + ...(status === undefined ? {} : { details: { status } }), + retriable: true, + }); +} diff --git a/apps/gateway-worker/src/durable-object-storage.ts b/apps/gateway-worker/src/durable-object-storage.ts new file mode 100644 index 00000000..e251a916 --- /dev/null +++ b/apps/gateway-worker/src/durable-object-storage.ts @@ -0,0 +1,181 @@ +import { + assertInternalMaintenanceEnvelope, + createInternalMaintenanceHeaders, + verifyInternalMaintenanceRequest, +} from "@cheatcode/auth"; +import { + APIError, + readBoundedRequestText, + readBoundedResponseJson, +} from "@cheatcode/observability"; +import { + INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + type InternalDurableObjectStorageRequest, + InternalDurableObjectStorageRequestSchema, + type InternalDurableObjectStorageResponse, + InternalDurableObjectStorageResponseSchema, +} from "@cheatcode/types"; +import type { GatewayApp, GatewayContext, GatewayEnv } from "./gateway-env"; +import { requireDatabaseReadinessSecret } from "./internal-maintenance"; + +const MAX_STORAGE_BODY_BYTES = 4 * 1024; +const MAX_STORAGE_RESPONSE_BYTES = 16 * 1024; +const DOWNSTREAM_STORAGE_ATTESTATION_TIMEOUT_MS = 2 * 60 * 1_000; + +export function registerGatewayDurableObjectStorageRoute(app: GatewayApp): void { + app.post(INTERNAL_DURABLE_OBJECT_STORAGE_PATH, handleDurableObjectStorage); +} + +async function handleDurableObjectStorage(c: GatewayContext): Promise { + const { input, rawBody, secret } = await authenticateRequest(c); + const result = await routeStorageRequest(c.env, input, rawBody, secret); + return c.json(assertMatchingEvidence(input, result)); +} + +async function authenticateRequest(c: GatewayContext): Promise<{ + input: InternalDurableObjectStorageRequest; + rawBody: string; + secret: string; +}> { + assertClosedRelease(c.env, c.req.raw); + assertInternalMaintenanceEnvelope(c.req.raw, { + audience: "gateway", + capability: "durable-object-schema", + issuer: "release-control", + }); + const rawBody = await readBoundedRequestText( + c.req.raw, + MAX_STORAGE_BODY_BYTES, + "Durable Object storage request", + ); + const secret = await requireDatabaseReadinessSecret(c.env); + await verifyInternalMaintenanceRequest({ + expectedAudience: "gateway", + expectedCapability: "durable-object-schema", + expectedIssuer: "release-control", + expectedMethod: "POST", + expectedPathname: INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + rawBody, + request: c.req.raw, + secret, + }); + const input = InternalDurableObjectStorageRequestSchema.parse(parseJson(rawBody)); + if (input.releaseSha !== c.env.CHEATCODE_RELEASE_SHA) { + throw mismatch("Durable Object request does not match the closed gateway release"); + } + return { input, rawBody, secret }; +} + +function assertClosedRelease(env: GatewayEnv, request: Request): void { + if (env.CHEATCODE_RELEASE_GATE !== "closed") { + throw mismatch("Durable Object reconciliation requires the closed release gate"); + } + const hostname = new URL(request.url).hostname; + const validHost = + env.CHEATCODE_ENVIRONMENT === "production" + ? hostname === "gateway.trycheatcode.com" + : hostname === "127.0.0.1" || hostname === "localhost"; + if (!validHost) { + throw new APIError(404, "not_found_run", "Durable Object route was not found", { + retriable: false, + }); + } +} + +async function routeStorageRequest( + env: GatewayEnv, + input: InternalDurableObjectStorageRequest, + rawBody: string, + secret: string, +): Promise { + if (input.className === "AgentRun" || input.className === "ProjectSandbox") { + return readDownstreamStorage(env.AGENT, "agent", input, rawBody, secret); + } + if (input.className === "WebhookIdempotencyStore") { + return readDownstreamStorage(env.WEBHOOKS, "webhooks", input, rawBody, secret); + } + if (input.className === "IdempotencyStore") { + const id = env.IDEMPOTENCY.idFromString(input.objectId); + return env.IDEMPOTENCY.get(id).reconcileStorageSchema(input); + } + if (input.className === "QuotaTracker") { + const id = env.QUOTA_TRACKER.idFromString(input.objectId); + return env.QUOTA_TRACKER.get(id).reconcileStorageSchema(input); + } + const id = env.RATE_LIMITER.idFromString(input.objectId); + return env.RATE_LIMITER.get(id).reconcileStorageSchema(input); +} + +async function readDownstreamStorage( + binding: Fetcher, + worker: "agent" | "webhooks", + input: InternalDurableObjectStorageRequest, + rawBody: string, + secret: string, +): Promise { + const headers = await createInternalMaintenanceHeaders({ + audience: worker, + capability: "durable-object-schema", + issuer: "gateway", + method: "POST", + pathname: INTERNAL_DURABLE_OBJECT_STORAGE_PATH, + rawBody, + secret, + }); + headers.set("content-type", "application/json"); + const response = await binding.fetch( + `https://${worker}.internal${INTERNAL_DURABLE_OBJECT_STORAGE_PATH}`, + { + body: rawBody, + headers, + method: "POST", + signal: AbortSignal.timeout(DOWNSTREAM_STORAGE_ATTESTATION_TIMEOUT_MS), + }, + ); + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + throw new APIError(503, "unavailable_maintenance", `${worker} storage attestation failed`, { + details: { status: response.status }, + retriable: true, + }); + } + return assertMatchingEvidence( + input, + InternalDurableObjectStorageResponseSchema.parse( + await readBoundedResponseJson( + response, + MAX_STORAGE_RESPONSE_BYTES, + `${worker} Durable Object storage attestation`, + ), + ), + ); +} + +function assertMatchingEvidence( + input: InternalDurableObjectStorageRequest, + evidence: InternalDurableObjectStorageResponse, +): InternalDurableObjectStorageResponse { + const parsed = InternalDurableObjectStorageResponseSchema.parse(evidence); + if ( + parsed.className !== input.className || + parsed.objectId !== input.objectId || + parsed.releaseSha !== input.releaseSha + ) { + throw mismatch("Durable Object storage evidence does not match the request"); + } + return parsed; +} + +function parseJson(rawBody: string): unknown { + try { + return JSON.parse(rawBody) as unknown; + } catch { + throw new APIError(400, "invalid_request_body", "Durable Object body must be JSON", { + retriable: false, + }); + } +} + +function mismatch(message: string): APIError { + return new APIError(409, "conflict_state_invalid", message, { retriable: false }); +} diff --git a/apps/gateway-worker/src/durable-objects/idempotency-storage.ts b/apps/gateway-worker/src/durable-objects/idempotency-storage.ts index 23486a51..ae332fd9 100644 --- a/apps/gateway-worker/src/durable-objects/idempotency-storage.ts +++ b/apps/gateway-worker/src/durable-objects/idempotency-storage.ts @@ -1,157 +1,131 @@ -const IDEMPOTENCY_ENTRY_COLUMNS = [ - { defaultValue: null, isNotNull: false, isPrimaryKey: true, name: "key", type: "TEXT" }, - { defaultValue: null, isNotNull: true, isPrimaryKey: false, name: "body_hash", type: "TEXT" }, - { defaultValue: null, isNotNull: false, isPrimaryKey: false, name: "claim_id", type: "TEXT" }, - { defaultValue: null, isNotNull: true, isPrimaryKey: false, name: "state", type: "TEXT" }, - { - defaultValue: null, - isNotNull: false, - isPrimaryKey: false, - name: "response_status", - type: "INTEGER", - }, - { - defaultValue: null, - isNotNull: false, - isPrimaryKey: false, - name: "response_headers_json", - type: "TEXT", - }, - { - defaultValue: null, - isNotNull: false, - isPrimaryKey: false, - name: "response_body", - type: "TEXT", - }, - { - defaultValue: null, - isNotNull: true, - isPrimaryKey: false, - name: "expires_at", - type: "INTEGER", - }, -] as const; +import { + assertExactSqliteSchema, + assertSqliteRowCountPreserved, + type ExpectedSqliteObject, + setCurrentSqliteStorageVersion, +} from "@cheatcode/durable-storage"; + +interface ExpectedColumn { + defaultValue: string | null; + isNotNull: boolean; + isPrimaryKey: boolean; + name: string; + type: string; +} -const REQUIRED_PERSISTED_COLUMNS = [ - "key", - "body_hash", - "state", - "response_status", - "response_headers_json", - "response_body", - "expires_at", +const IDEMPOTENCY_COLUMNS = [ + column("key", "TEXT", true, true), + column("body_hash", "TEXT", true), + column("claim_id", "TEXT"), + column("state", "TEXT", true), + column("response_status", "INTEGER"), + column("response_headers_json", "TEXT"), + column("response_body", "TEXT"), + column("expires_at", "INTEGER", true), ] as const; -/** Reconcile deployed Durable Object storage to the one current idempotency schema. */ +const REQUIRED_COLUMNS = IDEMPOTENCY_COLUMNS.filter(({ name }) => name !== "claim_id"); + +const IDEMPOTENCY_TABLE_SQL = `CREATE TABLE idempotency_entry ( + key TEXT PRIMARY KEY CHECK (length(key) BETWEEN 1 AND 255), + body_hash TEXT NOT NULL CHECK (length(body_hash) = 64 AND body_hash NOT GLOB '*[^a-f0-9]*'), + claim_id TEXT CHECK (claim_id IS NULL OR length(claim_id) = 36), + state TEXT NOT NULL CHECK (state IN ('in_flight', 'completed')), + response_status INTEGER CHECK (response_status IS NULL OR response_status BETWEEN 100 AND 599), + response_headers_json TEXT, + response_body TEXT CHECK (response_body IS NULL OR length(cast(response_body AS blob)) <= 65536), + expires_at INTEGER NOT NULL CHECK (expires_at >= 0) +) STRICT`; + +const IDEMPOTENCY_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [ + { + name: "idempotency_entry", + sql: IDEMPOTENCY_TABLE_SQL, + tableName: "idempotency_entry", + type: "table", + }, +]; + +/** Reconciles every dormant object to the one current persisted schema. */ export function initializeIdempotencyStorage(ctx: DurableObjectState): void { - const columns = ctx.storage.sql.exec("PRAGMA table_info(idempotency_entry)").toArray(); + normalizeIdempotencyStorage(ctx, false); + assertIdempotencyStorage(ctx); +} + +export function hasIdempotencyStorage(ctx: DurableObjectState): boolean { + return tableColumns(ctx, "idempotency_entry").length > 0; +} + +/** One-shot cutover normalizer; a later release removes this force-rebuild entrypoint. */ +export function reconcileIdempotencyStorage(ctx: DurableObjectState): void { + normalizeIdempotencyStorage(ctx, true); + assertExactSqliteSchema(ctx, IDEMPOTENCY_STORAGE_SCHEMA); +} + +export function assertIdempotencyStorage(ctx: DurableObjectState): void { + assertExactSqliteSchema(ctx, IDEMPOTENCY_STORAGE_SCHEMA); +} + +function normalizeIdempotencyStorage(ctx: DurableObjectState, forceRebuild: boolean): void { + const columns = tableColumns(ctx, "idempotency_entry"); if (columns.length === 0) { - createIdempotencyEntryTable(ctx, "idempotency_entry"); + createIdempotencyTable(ctx, "idempotency_entry"); + setCurrentSqliteStorageVersion(ctx); return; } - if (hasExactColumns(columns, IDEMPOTENCY_ENTRY_COLUMNS)) { + if (!forceRebuild && hasExactColumns(columns, IDEMPOTENCY_COLUMNS)) { + setCurrentSqliteStorageVersion(ctx); return; } - if (!REQUIRED_PERSISTED_COLUMNS.every((name) => hasColumn(columns, name))) { - throw new Error("Unsupported idempotency_entry schema; refusing a lossy reconciliation."); + if (!REQUIRED_COLUMNS.every(({ name }) => hasColumn(columns, name))) { + throw new Error("Unsupported idempotency_entry schema; refusing lossy evolution."); } - - const hasClaimId = hasColumn(columns, "claim_id"); + const claimId = hasColumn(columns, "claim_id") ? "claim_id" : "NULL"; ctx.storage.transactionSync(() => { - ctx.storage.sql.exec("DROP TABLE IF EXISTS idempotency_entry_current"); - createIdempotencyEntryTable(ctx, "idempotency_entry_current"); - copyIdempotencyEntries(ctx, hasClaimId); + ctx.storage.sql.exec("DROP TABLE IF EXISTS idempotency_entry_next"); + createIdempotencyTable(ctx, "idempotency_entry_next"); + ctx.storage.sql.exec( + `INSERT INTO idempotency_entry_next ( + key, body_hash, claim_id, state, response_status, + response_headers_json, response_body, expires_at + ) + SELECT key, body_hash, ${claimId}, state, response_status, + response_headers_json, response_body, expires_at + FROM idempotency_entry`, + ); + assertSqliteRowCountPreserved(ctx, "idempotency_entry", "idempotency_entry_next"); ctx.storage.sql.exec("DROP TABLE idempotency_entry"); - ctx.storage.sql.exec("ALTER TABLE idempotency_entry_current RENAME TO idempotency_entry"); + ctx.storage.sql.exec("ALTER TABLE idempotency_entry_next RENAME TO idempotency_entry"); }); + setCurrentSqliteStorageVersion(ctx); } -function createIdempotencyEntryTable( +function createIdempotencyTable( ctx: DurableObjectState, - table: "idempotency_entry" | "idempotency_entry_current", + table: "idempotency_entry" | "idempotency_entry_next", ): void { - if (table === "idempotency_entry") { - ctx.storage.sql.exec( - `CREATE TABLE idempotency_entry ( - key TEXT PRIMARY KEY, - body_hash TEXT NOT NULL, - claim_id TEXT, - state TEXT NOT NULL CHECK (state IN ('in_flight', 'completed')), - response_status INTEGER, - response_headers_json TEXT, - response_body TEXT, - expires_at INTEGER NOT NULL - )`, - ); - return; - } - ctx.storage.sql.exec( - `CREATE TABLE idempotency_entry_current ( - key TEXT PRIMARY KEY, - body_hash TEXT NOT NULL, - claim_id TEXT, - state TEXT NOT NULL CHECK (state IN ('in_flight', 'completed')), - response_status INTEGER, - response_headers_json TEXT, - response_body TEXT, - expires_at INTEGER NOT NULL - )`, - ); + ctx.storage.sql.exec(IDEMPOTENCY_TABLE_SQL.replace("idempotency_entry", table)); } -function copyIdempotencyEntries(ctx: DurableObjectState, hasClaimId: boolean): void { - if (hasClaimId) { - ctx.storage.sql.exec( - `INSERT OR REPLACE INTO idempotency_entry_current ( - key, body_hash, claim_id, state, response_status, - response_headers_json, response_body, expires_at - ) - SELECT key, body_hash, claim_id, state, response_status, - response_headers_json, response_body, expires_at - FROM idempotency_entry - WHERE typeof(key) = 'text' - AND typeof(body_hash) = 'text' - AND (claim_id IS NULL OR typeof(claim_id) = 'text') - AND state IN ('in_flight', 'completed') - AND (response_status IS NULL OR typeof(response_status) = 'integer') - AND (response_headers_json IS NULL OR typeof(response_headers_json) = 'text') - AND (response_body IS NULL OR typeof(response_body) = 'text') - AND typeof(expires_at) = 'integer'`, - ); - return; - } - ctx.storage.sql.exec( - `INSERT OR REPLACE INTO idempotency_entry_current ( - key, body_hash, claim_id, state, response_status, - response_headers_json, response_body, expires_at - ) - SELECT key, body_hash, NULL, state, response_status, - response_headers_json, response_body, expires_at - FROM idempotency_entry - WHERE typeof(key) = 'text' - AND typeof(body_hash) = 'text' - AND state IN ('in_flight', 'completed') - AND (response_status IS NULL OR typeof(response_status) = 'integer') - AND (response_headers_json IS NULL OR typeof(response_headers_json) = 'text') - AND (response_body IS NULL OR typeof(response_body) = 'text') - AND typeof(expires_at) = 'integer'`, - ); +function tableColumns(ctx: DurableObjectState, table: string): unknown[] { + return ctx.storage.sql.exec(`PRAGMA table_info(${table})`).toArray(); } -function hasExactColumns( - rows: unknown[], - expected: ReadonlyArray<{ - defaultValue: string | null; - isNotNull: boolean; - isPrimaryKey: boolean; - name: string; - type: string; - }>, -): boolean { +function hasExactColumns(rows: unknown[], expected: readonly ExpectedColumn[]): boolean { return ( rows.length === expected.length && - expected.every((column, index) => isColumn(rows[index], index, column)) + expected.every((value, index) => { + const row = rows[index]; + return ( + isRecord(row) && + row["cid"] === index && + row["name"] === value.name && + row["type"] === value.type && + row["notnull"] === Number(value.isNotNull) && + row["pk"] === Number(value.isPrimaryKey) && + row["dflt_value"] === value.defaultValue + ); + }) ); } @@ -159,28 +133,14 @@ function hasColumn(rows: unknown[], name: string): boolean { return rows.some((row) => isRecord(row) && row["name"] === name); } -function isColumn( - value: unknown, - index: number, - expected: { - defaultValue: string | null; - isNotNull: boolean; - isPrimaryKey: boolean; - name: string; - type: string; - }, -): boolean { - if (!isRecord(value)) { - return false; - } - return ( - value["cid"] === index && - value["name"] === expected.name && - value["type"] === expected.type && - value["notnull"] === Number(expected.isNotNull) && - value["dflt_value"] === expected.defaultValue && - value["pk"] === Number(expected.isPrimaryKey) - ); +function column( + name: string, + type: string, + isNotNull = false, + isPrimaryKey = false, + defaultValue: string | null = null, +): ExpectedColumn { + return { defaultValue, isNotNull, isPrimaryKey, name, type }; } function isRecord(value: unknown): value is Record { diff --git a/apps/gateway-worker/src/durable-objects/idempotency.ts b/apps/gateway-worker/src/durable-objects/idempotency.ts index 3815cca5..b416ded1 100644 --- a/apps/gateway-worker/src/durable-objects/idempotency.ts +++ b/apps/gateway-worker/src/durable-objects/idempotency.ts @@ -1,5 +1,14 @@ import { DurableObject } from "cloudflare:workers"; +import { + assertStorageReconciliationRequest, + reconcileExactSqliteStorage, + storageSchemaEvidence, +} from "@cheatcode/durable-storage"; import { readJsonRequest } from "@cheatcode/observability"; +import type { + InternalDurableObjectStorageRequest, + InternalDurableObjectStorageResponse, +} from "@cheatcode/types"; import { z } from "zod"; import { IdempotencyBeginBodySchema, @@ -7,7 +16,16 @@ import { IdempotencyBeginResultSchema, IdempotencyCompleteBodySchema, } from "./idempotency-contract"; -import { initializeIdempotencyStorage } from "./idempotency-storage"; +import { + assertIdempotencyStorage, + hasIdempotencyStorage, + initializeIdempotencyStorage, + reconcileIdempotencyStorage, +} from "./idempotency-storage"; +import { + gatewayDurableObjectClosedResponse, + rearmClosedGatewayDurableObjectAlarm, +} from "./release-gate"; interface IdempotencyRow { body_hash: string; @@ -19,11 +37,32 @@ interface IdempotencyRow { state: "completed" | "in_flight"; } -type IdempotencyEnv = Record; +interface IdempotencyEnv { + CHEATCODE_RELEASE_GATE: "closed" | "open"; + CHEATCODE_RELEASE_SHA?: string; +} const MAX_IDEMPOTENCY_REQUEST_BYTES = 1024 * 1024; export class IdempotencyStore extends DurableObject { + private isStorageInitialized = false; + + public reconcileStorageSchema( + value: InternalDurableObjectStorageRequest, + ): InternalDurableObjectStorageResponse { + const input = assertStorageReconciliationRequest(this.ctx, this.env, value, "IdempotencyStore"); + reconcileExactSqliteStorage( + input.mode, + () => assertIdempotencyStorage(this.ctx), + () => reconcileIdempotencyStorage(this.ctx), + ); + this.isStorageInitialized = true; + return storageSchemaEvidence(input); + } + public override async fetch(request: Request): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return gatewayDurableObjectClosedResponse(); + } if (request.method !== "POST") { return new Response("Method not allowed", { status: 405 }); } @@ -45,19 +84,22 @@ export class IdempotencyStore extends DurableObject { } public override async alarm(): Promise { + if (!hasIdempotencyStorage(this.ctx)) { + await this.ctx.storage.deleteAlarm(); + return; + } + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + await rearmClosedGatewayDurableObjectAlarm(this.ctx); + return; + } + this.isStorageInitialized = true; this.deleteExpired(Date.now()); await this.scheduleNextAlarm(); } - public constructor(ctx: DurableObjectState, env: IdempotencyEnv) { - super(ctx, env); - this.ctx.blockConcurrencyWhile(async () => { - initializeIdempotencyStorage(this.ctx); - }); - } - private async begin(value: unknown): Promise { const input = IdempotencyBeginBodySchema.parse(value); + this.ensureStorage(); this.deleteExpired(input.now); const row = this.readRow(input.key); if (!row) { @@ -97,6 +139,7 @@ export class IdempotencyStore extends DurableObject { private async complete(value: unknown): Promise { const input = IdempotencyCompleteBodySchema.parse(value); + this.ensureStorage(); this.ctx.storage.sql.exec( `UPDATE idempotency_entry SET state = 'completed', @@ -135,7 +178,20 @@ export class IdempotencyStore extends DurableObject { await this.ctx.storage.setAlarm(expiresAt); return; } - await this.ctx.storage.deleteAlarm(); + await this.ctx.storage.deleteAll(); + this.isStorageInitialized = false; + } + + private ensureStorage(): void { + if (this.isStorageInitialized) { + return; + } + if (hasIdempotencyStorage(this.ctx)) { + assertIdempotencyStorage(this.ctx); + } else { + initializeIdempotencyStorage(this.ctx); + } + this.isStorageInitialized = true; } } diff --git a/apps/gateway-worker/src/durable-objects/quota-tracker-storage.ts b/apps/gateway-worker/src/durable-objects/quota-tracker-storage.ts new file mode 100644 index 00000000..b3dd7392 --- /dev/null +++ b/apps/gateway-worker/src/durable-objects/quota-tracker-storage.ts @@ -0,0 +1,232 @@ +import { + assertExactSqliteSchema, + assertSqliteRowCountPreserved, + type ExpectedSqliteObject, + setCurrentSqliteStorageVersion, +} from "@cheatcode/durable-storage"; +import { QUOTA_FEATURES } from "@cheatcode/types/quota"; + +const MAX_FINITE_REAL = "1.7976931348623157e308"; +const FEATURE_CHECK = "feature IN ('composio_calls', 'sandbox_hours')"; +const PERIOD_KEY_CHECK = + "length(period_key) = 7 AND period_key GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]'"; + +const COUNTER_SQL = `CREATE TABLE counter ( + feature TEXT NOT NULL CHECK (${FEATURE_CHECK}), + period_key TEXT NOT NULL CHECK (${PERIOD_KEY_CHECK}), + used REAL NOT NULL DEFAULT 0 CHECK (used >= 0 AND abs(used) <= ${MAX_FINITE_REAL}), + updated_at INTEGER NOT NULL CHECK (updated_at >= 0), + PRIMARY KEY (feature, period_key) +) STRICT`; +const LIMIT_OVERRIDE_SQL = `CREATE TABLE limit_override ( + feature TEXT PRIMARY KEY CHECK (${FEATURE_CHECK}), + limit_val REAL NOT NULL CHECK (limit_val >= 0 AND abs(limit_val) <= ${MAX_FINITE_REAL}), + entitlement_version INTEGER NOT NULL CHECK (entitlement_version >= 0) +) STRICT`; +const USAGE_EVENT_SQL = `CREATE TABLE usage_event ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feature TEXT NOT NULL CHECK (${FEATURE_CHECK}), + amount REAL NOT NULL CHECK (amount > 0 AND abs(amount) <= ${MAX_FINITE_REAL}), + recorded_at INTEGER NOT NULL CHECK (recorded_at >= 0) +) STRICT`; +const QUOTA_OPERATION_SQL = `CREATE TABLE quota_operation ( + event_id TEXT PRIMARY KEY CHECK (length(event_id) BETWEEN 1 AND 200), + operation TEXT NOT NULL CHECK (operation IN ('record', 'try-consume')), + feature TEXT NOT NULL CHECK (${FEATURE_CHECK}), + period_key TEXT NOT NULL CHECK (${PERIOD_KEY_CHECK}), + amount REAL NOT NULL CHECK (amount > 0 AND abs(amount) <= ${MAX_FINITE_REAL}), + allowed INTEGER NOT NULL CHECK (allowed IN (0, 1)), + limit_val REAL NOT NULL CHECK (limit_val >= 0 AND abs(limit_val) <= ${MAX_FINITE_REAL}), + remaining REAL NOT NULL CHECK (remaining >= 0 AND abs(remaining) <= ${MAX_FINITE_REAL}), + used REAL NOT NULL CHECK (used >= 0 AND abs(used) <= ${MAX_FINITE_REAL}), + recorded_at INTEGER NOT NULL CHECK (recorded_at >= 0) +) STRICT`; +const USAGE_EVENT_INDEX_SQL = + "CREATE INDEX usage_event_feature_time_idx ON usage_event(feature, recorded_at)"; +const QUOTA_OPERATION_INDEX_SQL = + "CREATE INDEX quota_operation_feature_time_idx ON quota_operation(feature, recorded_at)"; + +const QUOTA_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [ + { name: "counter", sql: COUNTER_SQL, tableName: "counter", type: "table" }, + { + name: "limit_override", + sql: LIMIT_OVERRIDE_SQL, + tableName: "limit_override", + type: "table", + }, + { + name: "quota_operation", + sql: QUOTA_OPERATION_SQL, + tableName: "quota_operation", + type: "table", + }, + { name: "usage_event", sql: USAGE_EVENT_SQL, tableName: "usage_event", type: "table" }, + { + name: "quota_operation_feature_time_idx", + sql: QUOTA_OPERATION_INDEX_SQL, + tableName: "quota_operation", + type: "index", + }, + { + name: "usage_event_feature_time_idx", + sql: USAGE_EVENT_INDEX_SQL, + tableName: "usage_event", + type: "index", + }, +]; + +/** Force-normalizes all quota tables after the release barrier has drained every caller. */ +export function reconcileQuotaTrackerStorage(ctx: DurableObjectState): void { + ensureSourceTables(ctx); + const limitColumns = ctx.storage.sql.exec("PRAGMA table_info(limit_override)").toArray(); + if (!hasColumn(limitColumns, "feature") || !hasColumn(limitColumns, "limit_val")) { + throw new Error("Unsupported quota limit schema; refusing lossy evolution."); + } + const hasEntitlementVersion = hasColumn(limitColumns, "entitlement_version"); + assertQuotaSourceRows(ctx, hasEntitlementVersion); + const entitlementVersion = hasEntitlementVersion ? "entitlement_version" : "0"; + ctx.storage.transactionSync(() => rebuildQuotaTables(ctx, entitlementVersion)); + setCurrentSqliteStorageVersion(ctx); + assertQuotaTrackerStorage(ctx); +} + +export function assertQuotaTrackerStorage(ctx: DurableObjectState): void { + assertExactSqliteSchema(ctx, QUOTA_STORAGE_SCHEMA); +} + +export function initializeQuotaTrackerStorage(ctx: DurableObjectState): void { + ensureSourceTables(ctx); + ctx.storage.sql.exec(USAGE_EVENT_INDEX_SQL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS")); + ctx.storage.sql.exec( + QUOTA_OPERATION_INDEX_SQL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS"), + ); + setCurrentSqliteStorageVersion(ctx); + assertQuotaTrackerStorage(ctx); +} + +export function hasQuotaTrackerStorage(ctx: DurableObjectState): boolean { + return ( + ctx.storage.sql + .exec( + "SELECT 1 AS present FROM sqlite_schema WHERE type = 'table' AND name = 'counter' LIMIT 1", + ) + .toArray().length > 0 + ); +} + +function ensureSourceTables(ctx: DurableObjectState): void { + for (const sql of [COUNTER_SQL, LIMIT_OVERRIDE_SQL, USAGE_EVENT_SQL, QUOTA_OPERATION_SQL]) { + ctx.storage.sql.exec(sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS")); + } +} + +function rebuildQuotaTables(ctx: DurableObjectState, entitlementVersion: string): void { + ctx.storage.sql.exec("DROP INDEX IF EXISTS usage_event_feature_time_idx"); + ctx.storage.sql.exec("DROP INDEX IF EXISTS quota_operation_feature_time_idx"); + for (const table of ["counter", "limit_override", "usage_event", "quota_operation"] as const) { + ctx.storage.sql.exec(`ALTER TABLE ${table} RENAME TO ${table}_reconcile_source`); + } + for (const sql of [COUNTER_SQL, LIMIT_OVERRIDE_SQL, USAGE_EVENT_SQL, QUOTA_OPERATION_SQL]) { + ctx.storage.sql.exec(sql); + } + copyQuotaRows(ctx, entitlementVersion); + for (const table of ["counter", "limit_override", "usage_event", "quota_operation"] as const) { + assertSqliteRowCountPreserved(ctx, `${table}_reconcile_source`, table); + ctx.storage.sql.exec(`DROP TABLE ${table}_reconcile_source`); + } + ctx.storage.sql.exec(USAGE_EVENT_INDEX_SQL); + ctx.storage.sql.exec(QUOTA_OPERATION_INDEX_SQL); +} + +function copyQuotaRows(ctx: DurableObjectState, entitlementVersion: string): void { + ctx.storage.sql.exec( + `INSERT INTO counter (feature, period_key, used, updated_at) + SELECT feature, period_key, used, updated_at FROM counter_reconcile_source`, + ); + ctx.storage.sql.exec( + `INSERT INTO limit_override (feature, limit_val, entitlement_version) + SELECT feature, limit_val, ${entitlementVersion} FROM limit_override_reconcile_source`, + ); + ctx.storage.sql.exec( + `INSERT INTO usage_event (id, feature, amount, recorded_at) + SELECT id, feature, amount, recorded_at FROM usage_event_reconcile_source`, + ); + ctx.storage.sql.exec( + `INSERT INTO quota_operation + (event_id, operation, feature, period_key, amount, allowed, + limit_val, remaining, used, recorded_at) + SELECT event_id, operation, feature, period_key, amount, allowed, + limit_val, remaining, used, recorded_at + FROM quota_operation_reconcile_source`, + ); +} + +function assertQuotaSourceRows(ctx: DurableObjectState, hasEntitlementVersion: boolean): void { + const features = [QUOTA_FEATURES.composioCalls, QUOTA_FEATURES.sandboxHours] as const; + assertNoInvalidRows( + ctx, + `SELECT 1 FROM counter WHERE + typeof(feature) <> 'text' OR feature NOT IN (?, ?) OR + typeof(period_key) <> 'text' OR length(period_key) <> 7 OR + period_key NOT GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]' OR + typeof(used) NOT IN ('integer', 'real') OR used < 0 OR abs(used) > ${MAX_FINITE_REAL} OR + typeof(updated_at) <> 'integer' OR updated_at < 0 LIMIT 1`, + features, + ); + const entitlementPredicate = hasEntitlementVersion + ? " OR typeof(entitlement_version) <> 'integer' OR entitlement_version < 0" + : ""; + assertNoInvalidRows( + ctx, + `SELECT 1 FROM limit_override WHERE + typeof(feature) <> 'text' OR feature NOT IN (?, ?) OR + typeof(limit_val) NOT IN ('integer', 'real') OR limit_val < 0 OR + abs(limit_val) > ${MAX_FINITE_REAL}${entitlementPredicate} LIMIT 1`, + features, + ); + assertNoInvalidRows( + ctx, + `SELECT 1 FROM usage_event WHERE + typeof(id) <> 'integer' OR typeof(feature) <> 'text' OR feature NOT IN (?, ?) OR + typeof(amount) NOT IN ('integer', 'real') OR amount <= 0 OR + abs(amount) > ${MAX_FINITE_REAL} OR typeof(recorded_at) <> 'integer' OR + recorded_at < 0 LIMIT 1`, + features, + ); + assertNoInvalidRows( + ctx, + `SELECT 1 FROM quota_operation WHERE + typeof(event_id) <> 'text' OR length(event_id) NOT BETWEEN 1 AND 200 OR + typeof(operation) <> 'text' OR operation NOT IN ('record', 'try-consume') OR + typeof(feature) <> 'text' OR feature NOT IN (?, ?) OR + typeof(period_key) <> 'text' OR length(period_key) <> 7 OR + period_key NOT GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]' OR + typeof(amount) NOT IN ('integer', 'real') OR amount <= 0 OR + abs(amount) > ${MAX_FINITE_REAL} OR typeof(allowed) <> 'integer' OR + allowed NOT IN (0, 1) OR typeof(limit_val) NOT IN ('integer', 'real') OR + limit_val < 0 OR abs(limit_val) > ${MAX_FINITE_REAL} OR + typeof(remaining) NOT IN ('integer', 'real') OR remaining < 0 OR + abs(remaining) > ${MAX_FINITE_REAL} OR typeof(used) NOT IN ('integer', 'real') OR + used < 0 OR abs(used) > ${MAX_FINITE_REAL} OR + typeof(recorded_at) <> 'integer' OR recorded_at < 0 LIMIT 1`, + features, + ); +} + +function assertNoInvalidRows( + ctx: DurableObjectState, + sql: string, + features: readonly [string, string], +): void { + if (ctx.storage.sql.exec(sql, ...features).toArray().length > 0) { + throw new Error("Quota storage contains invalid or retired data; refusing lossy evolution."); + } +} + +function hasColumn(rows: unknown[], name: string): boolean { + return rows.some((row) => isRecord(row) && row["name"] === name); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/apps/gateway-worker/src/durable-objects/quota-tracker.ts b/apps/gateway-worker/src/durable-objects/quota-tracker.ts index 78d3173a..ec95cc65 100644 --- a/apps/gateway-worker/src/durable-objects/quota-tracker.ts +++ b/apps/gateway-worker/src/durable-objects/quota-tracker.ts @@ -1,7 +1,15 @@ import { DurableObject } from "cloudflare:workers"; +import { + assertStorageReconciliationRequest, + reconcileExactSqliteStorage, + storageSchemaEvidence, +} from "@cheatcode/durable-storage"; import { readJsonRequest } from "@cheatcode/observability"; +import type { + InternalDurableObjectStorageRequest, + InternalDurableObjectStorageResponse, +} from "@cheatcode/types"; import { - QUOTA_FEATURES, QUOTA_TRACKER_MAX_REQUEST_BYTES, type QuotaFeature, QuotaFeatureSchema, @@ -23,6 +31,17 @@ import { type QuotaSnapshotResult, QuotaSnapshotResultSchema, } from "./quota-tracker-contract"; +import { + assertQuotaTrackerStorage, + hasQuotaTrackerStorage, + initializeQuotaTrackerStorage, + reconcileQuotaTrackerStorage, +} from "./quota-tracker-storage"; +import { + assertGatewayDurableObjectOpen, + gatewayDurableObjectClosedResponse, + rearmClosedGatewayDurableObjectAlarm, +} from "./release-gate"; import { nextGatewayDurableObjectAlarm, QUOTA_TRACKER_RETENTION_MS } from "./retention"; interface CounterRow { @@ -34,8 +53,6 @@ interface LimitRow { limit_val: number; } -const LIMIT_OVERRIDE_COLUMNS = ["feature", "limit_val", "entitlement_version"] as const; - interface HistoryRow { amount: number; recorded_at: number; @@ -61,7 +78,10 @@ interface QuotaOperationInput { periodKey: string; } -type QuotaTrackerEnv = Record; +interface QuotaTrackerEnv { + CHEATCODE_RELEASE_GATE: "closed" | "open"; + CHEATCODE_RELEASE_SHA?: string; +} function isCounterRow(value: unknown): value is CounterRow { return isRecord(value) && typeof value["used"] === "number"; @@ -105,12 +125,28 @@ function isRecord(value: unknown): value is Record { } export class QuotaTracker extends DurableObject { + private isStorageInitialized = false; + + public reconcileStorageSchema( + value: InternalDurableObjectStorageRequest, + ): InternalDurableObjectStorageResponse { + const input = assertStorageReconciliationRequest(this.ctx, this.env, value, "QuotaTracker"); + reconcileExactSqliteStorage( + input.mode, + () => assertQuotaTrackerStorage(this.ctx), + () => reconcileQuotaTrackerStorage(this.ctx), + ); + this.isStorageInitialized = true; + return storageSchemaEvidence(input); + } + public async tryConsume( feature: QuotaFeature, amount: number, periodEnd: Date, eventId: string, ): Promise { + this.ensureStorage(); const periodKey = periodKeyFromDate(periodEnd); const input: QuotaOperationInput = { amount, @@ -125,6 +161,7 @@ export class QuotaTracker extends DurableObject { } public async peek(feature: QuotaFeature, periodEnd: Date): Promise { + this.ensureStorage(); const limit = this.readLimit(feature); const used = this.readUsed(feature, periodKeyFromDate(periodEnd)); return QuotaUsageResponseSchema.parse({ @@ -141,6 +178,7 @@ export class QuotaTracker extends DurableObject { eventId: string, recordedAt: Date, ): Promise { + this.ensureStorage(); const periodKey = periodKeyFromDate(periodEnd); const input: QuotaOperationInput = { amount, @@ -157,6 +195,7 @@ export class QuotaTracker extends DurableObject { } public async history(feature: QuotaFeature, from: Date): Promise { + this.ensureStorage(); const events = this.ctx.storage.sql .exec( `SELECT SUM(amount) AS amount, @@ -177,6 +216,7 @@ export class QuotaTracker extends DurableObject { limit: number, entitlementVersion: number, ): Promise { + this.ensureStorage(); this.ctx.storage.sql.exec( `INSERT INTO limit_override (feature, limit_val, entitlement_version) VALUES (?, ?, ?) @@ -191,14 +231,13 @@ export class QuotaTracker extends DurableObject { } public async deleteAllState(): Promise { - this.ctx.storage.sql.exec("DELETE FROM counter"); - this.ctx.storage.sql.exec("DELETE FROM limit_override"); - this.ctx.storage.sql.exec("DELETE FROM quota_operation"); - this.ctx.storage.sql.exec("DELETE FROM usage_event"); - await this.ctx.storage.deleteAlarm(); + assertGatewayDurableObjectOpen(this.env); + await this.ctx.storage.deleteAll(); + this.isStorageInitialized = false; } public async snapshot(periodEnd: Date): Promise { + this.ensureStorage(); const periodKey = periodKeyFromDate(periodEnd); const rawRows = this.ctx.storage.sql .exec("SELECT feature, limit_val FROM limit_override ORDER BY feature") @@ -220,6 +259,9 @@ export class QuotaTracker extends DurableObject { } public override async fetch(request: Request): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return gatewayDurableObjectClosedResponse(); + } if (request.method !== "POST") { return new Response("Method not allowed", { status: 405 }); } @@ -276,6 +318,15 @@ export class QuotaTracker extends DurableObject { } public override async alarm(): Promise { + if (!hasQuotaTrackerStorage(this.ctx)) { + await this.ctx.storage.deleteAlarm(); + return; + } + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + await rearmClosedGatewayDurableObjectAlarm(this.ctx); + return; + } + this.isStorageInitialized = true; this.ctx.storage.sql.exec( "DELETE FROM counter WHERE updated_at < ?", Date.now() - QUOTA_TRACKER_RETENTION_MS, @@ -291,49 +342,17 @@ export class QuotaTracker extends DurableObject { await this.refreshCleanupAlarm(); } - public constructor(ctx: DurableObjectState, env: QuotaTrackerEnv) { - super(ctx, env); - this.ctx.blockConcurrencyWhile(async () => { - this.ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS counter ( - feature TEXT NOT NULL, - period_key TEXT NOT NULL, - used REAL NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL, - PRIMARY KEY (feature, period_key) - )`, - ); - this.ensureLimitOverrideSchema(); - this.ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS usage_event ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - feature TEXT NOT NULL, - amount REAL NOT NULL, - recorded_at INTEGER NOT NULL - )`, - ); - this.ctx.storage.sql.exec( - "CREATE INDEX IF NOT EXISTS usage_event_feature_time_idx ON usage_event(feature, recorded_at)", - ); - this.ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS quota_operation ( - event_id TEXT PRIMARY KEY, - operation TEXT NOT NULL, - feature TEXT NOT NULL, - period_key TEXT NOT NULL, - amount REAL NOT NULL, - allowed INTEGER NOT NULL, - limit_val REAL NOT NULL, - remaining REAL NOT NULL, - used REAL NOT NULL, - recorded_at INTEGER NOT NULL - )`, - ); - this.ctx.storage.sql.exec( - "CREATE INDEX IF NOT EXISTS quota_operation_feature_time_idx ON quota_operation(feature, recorded_at)", - ); - this.deleteUnsupportedFeatures(); - }); + private ensureStorage(): void { + assertGatewayDurableObjectOpen(this.env); + if (this.isStorageInitialized) { + return; + } + if (hasQuotaTrackerStorage(this.ctx)) { + assertQuotaTrackerStorage(this.ctx); + } else { + initializeQuotaTrackerStorage(this.ctx); + } + this.isStorageInitialized = true; } private consumeOnce(input: QuotaOperationInput): QuotaTryConsumeResponse { @@ -432,54 +451,6 @@ export class QuotaTracker extends DurableObject { return false; } - private deleteUnsupportedFeatures(): void { - for (const table of ["counter", "limit_override", "quota_operation", "usage_event"] as const) { - this.ctx.storage.sql.exec( - `DELETE FROM ${table} WHERE feature NOT IN (?, ?)`, - QUOTA_FEATURES.composioCalls, - QUOTA_FEATURES.sandboxHours, - ); - } - } - - private ensureLimitOverrideSchema(): void { - this.ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS limit_override ( - feature TEXT PRIMARY KEY, - limit_val REAL NOT NULL, - entitlement_version INTEGER NOT NULL - )`, - ); - const columns = this.ctx.storage.sql.exec("PRAGMA table_info(limit_override)").toArray(); - if (hasExactLimitOverrideColumns(columns)) { - return; - } - const canPreserveLimits = - hasNamedColumn(columns, "feature") && hasNamedColumn(columns, "limit_val"); - this.ctx.storage.transactionSync(() => { - this.ctx.storage.sql.exec("DROP TABLE IF EXISTS limit_override_next"); - this.ctx.storage.sql.exec( - `CREATE TABLE limit_override_next ( - feature TEXT PRIMARY KEY, - limit_val REAL NOT NULL, - entitlement_version INTEGER NOT NULL - )`, - ); - if (canPreserveLimits) { - this.ctx.storage.sql.exec( - `INSERT OR REPLACE INTO limit_override_next (feature, limit_val, entitlement_version) - SELECT feature, limit_val, 0 - FROM limit_override - WHERE typeof(feature) = 'text' - AND typeof(limit_val) IN ('integer', 'real') - AND limit_val >= 0`, - ); - } - this.ctx.storage.sql.exec("DROP TABLE limit_override"); - this.ctx.storage.sql.exec("ALTER TABLE limit_override_next RENAME TO limit_override"); - }); - } - private readLimit(feature: QuotaFeature): number { const [rawRow] = this.ctx.storage.sql .exec("SELECT limit_val FROM limit_override WHERE feature = ?", feature) @@ -547,20 +518,6 @@ function historyResult(rows: unknown[]): QuotaHistoryResult { ); } -function hasExactLimitOverrideColumns(rows: unknown[]): boolean { - if (rows.length !== LIMIT_OVERRIDE_COLUMNS.length) { - return false; - } - return LIMIT_OVERRIDE_COLUMNS.every((name, index) => { - const row = rows[index]; - return isRecord(row) && row["name"] === name; - }); -} - -function hasNamedColumn(rows: unknown[], name: string): boolean { - return rows.some((row) => isRecord(row) && row["name"] === name); -} - function periodKeyFromDate(date: Date): string { const year = date.getUTCFullYear(); const month = String(date.getUTCMonth() + 1).padStart(2, "0"); diff --git a/apps/gateway-worker/src/durable-objects/rate-limit-contract.ts b/apps/gateway-worker/src/durable-objects/rate-limit-contract.ts index 16c623fc..4fe003c9 100644 --- a/apps/gateway-worker/src/durable-objects/rate-limit-contract.ts +++ b/apps/gateway-worker/src/durable-objects/rate-limit-contract.ts @@ -3,13 +3,13 @@ import { z } from "zod"; const RateLimitConfigSchema = z .object({ capacity: z.number().int().positive(), - refillPerSec: z.number().positive(), + refillPerSec: z.number().finite().positive(), }) .strict(); export const RateLimitConsumeBodySchema = z .object({ - key: z.string().min(1), + key: z.string().min(1).max(256), cost: z.number().int().positive(), config: RateLimitConfigSchema, }) diff --git a/apps/gateway-worker/src/durable-objects/rate-limiter-storage.ts b/apps/gateway-worker/src/durable-objects/rate-limiter-storage.ts new file mode 100644 index 00000000..d13f1969 --- /dev/null +++ b/apps/gateway-worker/src/durable-objects/rate-limiter-storage.ts @@ -0,0 +1,74 @@ +import { + assertExactSqliteSchema, + assertSqliteRowCountPreserved, + type ExpectedSqliteObject, + setCurrentSqliteStorageVersion, +} from "@cheatcode/durable-storage"; + +const MAX_FINITE_REAL = "1.7976931348623157e308"; +const BUCKET_TABLE_SQL = `CREATE TABLE bucket ( + key TEXT PRIMARY KEY CHECK (length(key) BETWEEN 1 AND 256), + tokens REAL NOT NULL CHECK (tokens >= 0 AND abs(tokens) <= ${MAX_FINITE_REAL}), + last_refill_ms INTEGER NOT NULL CHECK (last_refill_ms >= 0), + capacity INTEGER NOT NULL CHECK (capacity > 0), + refill_per_sec REAL NOT NULL CHECK (refill_per_sec > 0 AND abs(refill_per_sec) <= ${MAX_FINITE_REAL}), + CHECK (tokens <= capacity) +) STRICT`; + +const RATE_LIMITER_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [ + { name: "bucket", sql: BUCKET_TABLE_SQL, tableName: "bucket", type: "table" }, +]; + +export function initializeRateLimiterStorage(ctx: DurableObjectState): void { + ensureRateLimiterTable(ctx); + setCurrentSqliteStorageVersion(ctx); + assertRateLimiterStorage(ctx); +} + +export function hasRateLimiterStorage(ctx: DurableObjectState): boolean { + return ctx.storage.sql.exec("PRAGMA table_info(bucket)").toArray().length > 0; +} + +/** Rebuilds the one live table so same-column legacy constraints cannot survive the cutover. */ +export function reconcileRateLimiterStorage(ctx: DurableObjectState): void { + ensureRateLimiterTable(ctx); + assertRateLimiterSourceRows(ctx); + ctx.storage.transactionSync(() => { + ctx.storage.sql.exec("ALTER TABLE bucket RENAME TO bucket_reconcile_source"); + ctx.storage.sql.exec(BUCKET_TABLE_SQL); + ctx.storage.sql.exec( + `INSERT INTO bucket (key, tokens, last_refill_ms, capacity, refill_per_sec) + SELECT key, tokens, last_refill_ms, capacity, refill_per_sec + FROM bucket_reconcile_source`, + ); + assertSqliteRowCountPreserved(ctx, "bucket_reconcile_source", "bucket"); + ctx.storage.sql.exec("DROP TABLE bucket_reconcile_source"); + }); + setCurrentSqliteStorageVersion(ctx); + assertRateLimiterStorage(ctx); +} + +function ensureRateLimiterTable(ctx: DurableObjectState): void { + ctx.storage.sql.exec(BUCKET_TABLE_SQL.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS")); +} + +export function assertRateLimiterStorage(ctx: DurableObjectState): void { + assertExactSqliteSchema(ctx, RATE_LIMITER_STORAGE_SCHEMA); +} + +function assertRateLimiterSourceRows(ctx: DurableObjectState): void { + const invalid = ctx.storage.sql + .exec( + `SELECT 1 FROM bucket WHERE + typeof(key) <> 'text' OR length(key) NOT BETWEEN 1 AND 256 OR + typeof(tokens) NOT IN ('integer', 'real') OR tokens < 0 OR + abs(tokens) > ${MAX_FINITE_REAL} OR typeof(last_refill_ms) <> 'integer' OR + last_refill_ms < 0 OR typeof(capacity) <> 'integer' OR capacity <= 0 OR + tokens > capacity OR typeof(refill_per_sec) NOT IN ('integer', 'real') OR + refill_per_sec <= 0 OR abs(refill_per_sec) > ${MAX_FINITE_REAL} LIMIT 1`, + ) + .toArray(); + if (invalid.length > 0) { + throw new Error("Rate limiter contains invalid data; refusing lossy evolution."); + } +} diff --git a/apps/gateway-worker/src/durable-objects/rate-limiter.ts b/apps/gateway-worker/src/durable-objects/rate-limiter.ts index 7dcc6777..35769131 100644 --- a/apps/gateway-worker/src/durable-objects/rate-limiter.ts +++ b/apps/gateway-worker/src/durable-objects/rate-limiter.ts @@ -1,10 +1,30 @@ import { DurableObject } from "cloudflare:workers"; +import { + assertStorageReconciliationRequest, + reconcileExactSqliteStorage, + storageSchemaEvidence, +} from "@cheatcode/durable-storage"; import { readJsonRequest } from "@cheatcode/observability"; +import type { + InternalDurableObjectStorageRequest, + InternalDurableObjectStorageResponse, +} from "@cheatcode/types"; import { type RateLimitConfig, RateLimitConsumeBodySchema, type RateLimitResult, } from "./rate-limit-contract"; +import { + assertRateLimiterStorage, + hasRateLimiterStorage, + initializeRateLimiterStorage, + reconcileRateLimiterStorage, +} from "./rate-limiter-storage"; +import { + assertGatewayDurableObjectOpen, + gatewayDurableObjectClosedResponse, + rearmClosedGatewayDurableObjectAlarm, +} from "./release-gate"; import { nextGatewayDurableObjectAlarm, RATE_LIMITER_RETENTION_MS } from "./retention"; interface BucketRow { @@ -12,7 +32,10 @@ interface BucketRow { last_refill_ms: number; } -type RateLimiterEnv = Record; +interface RateLimiterEnv { + CHEATCODE_RELEASE_GATE: "closed" | "open"; + CHEATCODE_RELEASE_SHA?: string; +} const MAX_RATE_LIMIT_REQUEST_BYTES = 16 * 1024; function isBucketRow(value: unknown): value is BucketRow { @@ -24,11 +47,28 @@ function isBucketRow(value: unknown): value is BucketRow { } export class RateLimiter extends DurableObject { + private isStorageInitialized = false; + + public reconcileStorageSchema( + value: InternalDurableObjectStorageRequest, + ): InternalDurableObjectStorageResponse { + const input = assertStorageReconciliationRequest(this.ctx, this.env, value, "RateLimiter"); + reconcileExactSqliteStorage( + input.mode, + () => assertRateLimiterStorage(this.ctx), + () => reconcileRateLimiterStorage(this.ctx), + ); + this.isStorageInitialized = true; + return storageSchemaEvidence(input); + } + public async consume( key: string, cost: number, config: RateLimitConfig, ): Promise { + assertGatewayDurableObjectOpen(this.env); + this.ensureStorage(); const now = Date.now(); const [rawRow] = this.ctx.storage.sql .exec("SELECT tokens, last_refill_ms FROM bucket WHERE key = ?", key) @@ -65,6 +105,9 @@ export class RateLimiter extends DurableObject { } public override async fetch(request: Request): Promise { + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + return gatewayDurableObjectClosedResponse(); + } if (request.method !== "POST") { return new Response("Method not allowed", { status: 405 }); } @@ -75,6 +118,15 @@ export class RateLimiter extends DurableObject { } public override async alarm(): Promise { + if (!hasRateLimiterStorage(this.ctx)) { + await this.ctx.storage.deleteAlarm(); + return; + } + if (this.env.CHEATCODE_RELEASE_GATE === "closed") { + await rearmClosedGatewayDurableObjectAlarm(this.ctx); + return; + } + this.isStorageInitialized = true; this.ctx.storage.sql.exec( "DELETE FROM bucket WHERE last_refill_ms < ?", Date.now() - RATE_LIMITER_RETENTION_MS, @@ -82,23 +134,21 @@ export class RateLimiter extends DurableObject { if (this.hasBuckets()) { await this.ensureCleanupAlarm(); } else { - await this.ctx.storage.deleteAlarm(); + await this.ctx.storage.deleteAll(); + this.isStorageInitialized = false; } } - public constructor(ctx: DurableObjectState, env: RateLimiterEnv) { - super(ctx, env); - this.ctx.blockConcurrencyWhile(async () => { - this.ctx.storage.sql.exec( - `CREATE TABLE IF NOT EXISTS bucket ( - key TEXT PRIMARY KEY, - tokens REAL NOT NULL, - last_refill_ms INTEGER NOT NULL, - capacity INTEGER NOT NULL, - refill_per_sec REAL NOT NULL - )`, - ); - }); + private ensureStorage(): void { + if (this.isStorageInitialized) { + return; + } + if (hasRateLimiterStorage(this.ctx)) { + assertRateLimiterStorage(this.ctx); + } else { + initializeRateLimiterStorage(this.ctx); + } + this.isStorageInitialized = true; } private hasBuckets(): boolean { diff --git a/apps/gateway-worker/src/durable-objects/release-gate.ts b/apps/gateway-worker/src/durable-objects/release-gate.ts new file mode 100644 index 00000000..b2c72192 --- /dev/null +++ b/apps/gateway-worker/src/durable-objects/release-gate.ts @@ -0,0 +1,28 @@ +import { APIError } from "@cheatcode/observability"; + +const CLOSED_GATE_ALARM_RECHECK_MS = 5 * 60 * 1_000; + +interface GatewayDurableObjectEnv { + CHEATCODE_RELEASE_GATE: "closed" | "open"; +} + +export function assertGatewayDurableObjectOpen(env: GatewayDurableObjectEnv): void { + if (env.CHEATCODE_RELEASE_GATE === "closed") { + throw new Error("Gateway Durable Object is fenced by the closed release gate."); + } +} + +export function gatewayDurableObjectClosedResponse(): Response { + const response = new APIError(503, "unavailable_maintenance", "Release is in progress", { + details: { releaseGate: "closed", worker: "gateway" }, + retriable: true, + }).toResponse(`req_${crypto.randomUUID().replaceAll("-", "")}`); + response.headers.set("Cache-Control", "no-store"); + response.headers.set("Retry-After", "5"); + return response; +} + +/** Keep cleanup work pending without touching application tables during reconciliation. */ +export function rearmClosedGatewayDurableObjectAlarm(ctx: DurableObjectState): Promise { + return ctx.storage.setAlarm(Date.now() + CLOSED_GATE_ALARM_RECHECK_MS); +} diff --git a/apps/gateway-worker/src/gateway-env.ts b/apps/gateway-worker/src/gateway-env.ts index 2f92805b..3e1d4587 100644 --- a/apps/gateway-worker/src/gateway-env.ts +++ b/apps/gateway-worker/src/gateway-env.ts @@ -10,25 +10,29 @@ export interface GatewayEnv extends AnalyticsBindings, IdempotencyBindings { AGENT: Fetcher; CF_VERSION_METADATA?: CloudflareVersionMetadata; CHEATCODE_ENVIRONMENT: "development" | "production"; - CHEATCODE_RELEASE_GATE?: "open" | "closed"; + CHEATCODE_RELEASE_GATE: "open" | "closed"; CHEATCODE_RELEASE_SHA?: string; CLERK_AUTHORIZED_PARTIES?: string; CLERK_JWT_KEY?: WorkerSecret; CLERK_SECRET_KEY?: WorkerSecret; COMPOSIO_API_KEY?: WorkerSecret; COMPOSIO_AUTH_CONFIGS?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; ENTITLEMENTS_CACHE: KVNamespace; + GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET: WorkerSecret; HYPERDRIVE: Hyperdrive; IDEMPOTENCY: DurableObjectNamespace; - INTERNAL_MAINTENANCE_SECRET?: WorkerSecret; POLAR_ACCESS_TOKEN?: WorkerSecret; POLAR_PRODUCT_ID_MAX?: string; POLAR_PRODUCT_ID_PREMIUM?: string; POLAR_PRODUCT_ID_PRO?: string; POLAR_PRODUCT_ID_ULTRA?: string; POLAR_SERVER?: "production" | "sandbox"; + PREVIEW_PROXY?: Fetcher; QUOTA_TRACKER: DurableObjectNamespace; RATE_LIMITER: DurableObjectNamespace; + RELEASE_DATABASE_READINESS_SECRET: WorkerSecret; + WEBHOOKS: Fetcher; } export type GatewayApp = Hono<{ Bindings: GatewayEnv }>; diff --git a/apps/gateway-worker/src/greeting-routes.ts b/apps/gateway-worker/src/greeting-routes.ts index 7792f3e2..277810dc 100644 --- a/apps/gateway-worker/src/greeting-routes.ts +++ b/apps/gateway-worker/src/greeting-routes.ts @@ -1,4 +1,5 @@ import { createDb, sumWorkedMinutesToday, withUserContext } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; import { createLogger, readBoundedResponseJson, @@ -9,6 +10,7 @@ import { z } from "zod"; import type { WaitUntilContext } from "./wait-until-context"; export interface GreetingRouteEnv { + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; } @@ -82,7 +84,10 @@ async function resolveWorkedMinutesToday( userId: UserId, timezone: string | null, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { return await withUserContext(db, userId, (tx) => sumWorkedMinutesToday(tx, userId, timezone ?? "UTC"), diff --git a/apps/gateway-worker/src/index.ts b/apps/gateway-worker/src/index.ts index 8c31541d..a56390fa 100644 --- a/apps/gateway-worker/src/index.ts +++ b/apps/gateway-worker/src/index.ts @@ -4,16 +4,18 @@ import { createLogger, emitErrorEvent, emitPerformanceMetric, - readBoundedResponseJson, safeErrorTelemetry, toAPIError, withErrorHandler, } from "@cheatcode/observability"; +import { + INTERNAL_DATABASE_READINESS_PATH, + INTERNAL_DURABLE_OBJECT_STORAGE_PATH, +} from "@cheatcode/types"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { routePath } from "hono/route"; import { secureHeaders } from "hono/secure-headers"; -import { z } from "zod"; import { registerAccountHttpRoutes } from "./account-http-routes"; import { registerAgentHttpRoutes } from "./agent-http-routes"; import { registerBillingHttpRoutes } from "./billing-http-routes"; @@ -25,13 +27,7 @@ import { RateLimiter } from "./durable-objects/rate-limiter"; import { formatGatewayRouteError } from "./error-handling"; import type { GatewayContext, GatewayEnv } from "./gateway-env"; import { registerIntegrationHttpRoutes } from "./integration-http-routes"; -import { - localPreviewOriginRequest, - resolveLocalPreviewProxyRequest, - resolveLocalSandboxPreviewHost, - rewriteLocalPreviewRequest, - withLocalPreviewCookie, -} from "./local-preview-proxy"; +import { resolveLocalPreviewRoute } from "./local-preview-routing"; import { assertOpenApiRouteParity, gatewayOperationIdForRegisteredRoute, @@ -41,38 +37,10 @@ import { import { registerProjectHttpRoutes } from "./project-http-routes"; import { registerProviderHttpRoutes } from "./provider-http-routes"; import { withRateLimitErrorHeaders } from "./rate-limit"; +import { type DownstreamWorker, readDownstreamReleaseHealth } from "./release-health"; export { IdempotencyStore, QuotaTracker, RateLimiter }; -const LocalPreviewOriginResponseSchema = z.object({ - originalHost: z.string().min(1), - signed: z.boolean(), - token: z.string(), - url: z.string().url(), -}); -type LocalPreviewOriginResponse = z.infer; - -interface LocalPreviewOriginRequestInput { - clientHost: string; - cookie?: string; - host: string; - origin?: string; - url: string; -} -const ReleaseGateAgentHealthSchema = z - .object({ - ok: z.literal(true), - releaseSha: z.string().min(1), - versionId: z.string().min(1).nullable(), - worker: z.literal("agent"), - }) - .strict(); -const DAYTONA_TOKEN_HEADER = "x-daytona-preview-token"; -const DAYTONA_SKIP_WARNING_HEADER = "X-Daytona-Skip-Preview-Warning"; -const FORWARDED_HOST_HEADER = "X-Forwarded-Host"; -const INTERNAL_USER_DELETE_PATH = /^\/internal\/users\/[^/]+\/delete-state$/u; -const MAX_AGENT_HEALTH_RESPONSE_BYTES = 16 * 1024; -const MAX_LOCAL_PREVIEW_ORIGIN_RESPONSE_BYTES = 32 * 1024; const CORS_EXPOSED_HEADERS = [ "Content-Disposition", "Location", @@ -242,54 +210,42 @@ async function routeGatewayRequest( }); return releaseGate; } - const originalLocalPreviewHost = resolveLocalSandboxPreviewHost(request); const requestWithId = isWebSocketUpgrade(request) ? request : new Request(request); if (!isWebSocketUpgrade(requestWithId)) { requestWithId.headers.set("X-Request-Id", id); } - const websocketResponse = await localPreviewWebSocketResponse(request, env); - if (websocketResponse) { - return websocketResponse; + const localPreview = + env.CHEATCODE_ENVIRONMENT === "development" ? resolveLocalPreviewRoute(requestWithId) : null; + if (localPreview?.kind === "redirect") { + return withRequestId(localPreview.response, id); } - const localPreviewProxy = resolveLocalPreviewProxyRequest(requestWithId); - if (localPreviewProxy) { - return withLocalPreviewCookie( - await env.AGENT.fetch(localPreviewProxy.request), - id, - localPreviewProxy.encodedHost, - ); - } - const localPreviewHost = - originalLocalPreviewHost ?? resolveLocalSandboxPreviewHost(requestWithId); - if (localPreviewHost) { - return withRequestId( - await env.AGENT.fetch(rewriteLocalPreviewRequest(requestWithId, localPreviewHost)), - id, - ); + if (localPreview?.kind === "proxy") { + if (!env.PREVIEW_PROXY) { + throw new APIError(503, "unavailable_maintenance", "Local preview proxy is not configured", { + retriable: false, + }); + } + return withRequestId(await env.PREVIEW_PROXY.fetch(localPreview.request), id); } return withRequestId(await gatewayApp.fetch(requestWithId, env, ctx), id); } -async function localPreviewWebSocketResponse( - request: Request, - env: GatewayEnv, -): Promise { - if (!isWebSocketUpgrade(request)) { - return undefined; - } - const originRequest = localPreviewOriginRequest(request); - return originRequest ? proxyLocalPreviewWebSocket(request, env, originRequest) : undefined; -} - async function releaseGateResponse( request: Request, env: GatewayEnv, requestIdValue: string, ): Promise { - if (env.CHEATCODE_RELEASE_GATE !== "closed" || isInternalLifecycleRequest(request)) { + if (env.CHEATCODE_RELEASE_GATE !== "closed") { return undefined; } const url = new URL(request.url); + if ( + request.method === "POST" && + (url.pathname === INTERNAL_DATABASE_READINESS_PATH || + url.pathname === INTERNAL_DURABLE_OBJECT_STORAGE_PATH) + ) { + return undefined; + } const details: Record = { releaseGate: "closed", releaseSha: env.CHEATCODE_RELEASE_SHA ?? null, @@ -297,7 +253,12 @@ async function releaseGateResponse( worker: "gateway", }; if (request.method === "GET" && url.pathname === "/health") { - details["agent"] = await readReleaseGateAgentHealth(env); + const [agent, webhooks] = await Promise.all([ + readReleaseGateDownstreamHealth(env, "agent"), + readReleaseGateDownstreamHealth(env, "webhooks"), + ]); + details["agent"] = agent; + details["webhooks"] = webhooks; } const response = new APIError(503, "unavailable_maintenance", "Release is in progress", { details, @@ -307,35 +268,20 @@ async function releaseGateResponse( return response; } -function isInternalLifecycleRequest(request: Request): boolean { - // HMAC-authenticated deletion must remain available while public traffic is drained. - return request.method === "POST" && INTERNAL_USER_DELETE_PATH.test(new URL(request.url).pathname); -} - -async function readReleaseGateAgentHealth(env: GatewayEnv): Promise> { +async function readReleaseGateDownstreamHealth( + env: GatewayEnv, + worker: DownstreamWorker, +): Promise> { try { - const response = await env.AGENT.fetch( - new Request("https://agent.internal/health", { signal: AbortSignal.timeout(3_000) }), - ); - if (!response.ok) { - await response.body?.cancel().catch(() => undefined); - return unavailableAgentHealth(response.status); - } - const health = ReleaseGateAgentHealthSchema.parse( - await readBoundedResponseJson( - response, - MAX_AGENT_HEALTH_RESPONSE_BYTES, - "Agent release health", - ), - ); - return { ...health, status: response.status }; + const { health, status } = await readDownstreamReleaseHealth(env, worker); + return { ...health, status }; } catch { - return unavailableAgentHealth(null); + return unavailableDownstreamHealth(worker); } } -function unavailableAgentHealth(status: number | null): Record { - return { ok: false, releaseSha: null, status, versionId: null, worker: "agent" }; +function unavailableDownstreamHealth(worker: DownstreamWorker): Record { + return { ok: false, releaseSha: null, status: null, versionId: null, worker }; } function applyReleaseGateHeaders( @@ -355,82 +301,6 @@ function applyReleaseGateHeaders( } } -async function proxyLocalPreviewWebSocket( - request: Request, - env: GatewayEnv, - originRequest: LocalPreviewOriginRequestInput, -): Promise { - const originResponse = await env.AGENT.fetch( - new Request("http://agent.internal/__internal/local-preview-origin", { - headers: localPreviewOriginHeaders(originRequest), - }), - ); - if (!originResponse.ok) { - return originResponse; - } - const origin = LocalPreviewOriginResponseSchema.parse( - await readBoundedResponseJson( - originResponse, - MAX_LOCAL_PREVIEW_ORIGIN_RESPONSE_BYTES, - "Agent local preview origin", - ), - ); - const websocketRequest = buildLocalPreviewWebSocketRequest(request, originRequest, origin); - const response = await fetch(websocketRequest); - if (response.webSocket) { - return new Response(null, { status: 101, webSocket: response.webSocket }); - } - return response; -} - -function localPreviewOriginHeaders(originRequest: LocalPreviewOriginRequestInput): Headers { - const headers = new Headers({ - "X-Cheatcode-Local-Preview-Client-Host": originRequest.clientHost, - "X-Cheatcode-Local-Preview-Host": originRequest.host, - "X-Cheatcode-Local-Preview-Url": originRequest.url, - }); - if (originRequest.cookie) headers.set("X-Cheatcode-Local-Preview-Cookie", originRequest.cookie); - if (originRequest.origin) headers.set("Origin", originRequest.origin); - return headers; -} - -function buildLocalPreviewWebSocketRequest( - request: Request, - originRequest: LocalPreviewOriginRequestInput, - origin: LocalPreviewOriginResponse, -): Request { - const localUrl = new URL(originRequest.url); - const upstreamUrl = localPreviewUpstreamUrl(origin.url, localUrl); - const websocketRequest = new Request(upstreamUrl.toString(), request); - websocketRequest.headers.delete("Host"); - websocketRequest.headers.delete("Cookie"); - if (!origin.signed) { - websocketRequest.headers.set(DAYTONA_TOKEN_HEADER, origin.token); - } - websocketRequest.headers.set(DAYTONA_SKIP_WARNING_HEADER, "true"); - websocketRequest.headers.set(FORWARDED_HOST_HEADER, origin.originalHost); - const browserOrigin = - request.headers.get("Origin") ?? - `${new URL(originRequest.url).protocol}//${origin.originalHost}`; - const browserProtocol = new URL(browserOrigin).protocol.replace(":", ""); - websocketRequest.headers.set("Origin", browserOrigin); - websocketRequest.headers.set("Forwarded", `host=${origin.originalHost};proto=${browserProtocol}`); - websocketRequest.headers.set("X-Forwarded-Proto", browserProtocol); - return websocketRequest; -} - -function localPreviewUpstreamUrl(originUrl: string, requestUrl: URL): URL { - const upstreamUrl = new URL(originUrl); - const requestParams = new URLSearchParams(requestUrl.search); - upstreamUrl.pathname = requestUrl.pathname; - for (const [key, value] of requestParams) { - upstreamUrl.searchParams.append(key, value); - } - upstreamUrl.searchParams.delete("__cc_pt"); - upstreamUrl.searchParams.delete("cc_preview_reload"); - return upstreamUrl; -} - function isWebSocketUpgrade(request: Request): boolean { return (request.headers.get("Upgrade") ?? "").toLowerCase() === "websocket"; } diff --git a/apps/gateway-worker/src/integration-http-routes.ts b/apps/gateway-worker/src/integration-http-routes.ts index c6eada09..50f3e1c9 100644 --- a/apps/gateway-worker/src/integration-http-routes.ts +++ b/apps/gateway-worker/src/integration-http-routes.ts @@ -16,7 +16,10 @@ export function registerIntegrationHttpRoutes(app: GatewayApp): void { app.get("/v1/integrations", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/integrations"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const integrations = await listIntegrationSummaries(db, c.env, userId); return c.json(integrations); @@ -27,7 +30,10 @@ export function registerIntegrationHttpRoutes(app: GatewayApp): void { app.get("/v1/integrations/catalog", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/integrations/catalog"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const catalog = await getIntegrationCatalog(db, c.env, userId); return c.json(catalog); @@ -44,7 +50,10 @@ export function registerIntegrationHttpRoutes(app: GatewayApp): void { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "POST /v1/integrations/:name/connect"); const integration = parseIntegrationName(c.req.param("name")); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { return await connectIntegration({ db, env: c.env, integration, request: c.req.raw, userId }); } finally { @@ -59,7 +68,10 @@ function registerIntegrationAccountRoutes(app: GatewayApp): void { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "POST /v1/integrations/:name/accounts/:connectionId/default"); const integration = parseIntegrationName(c.req.param("name")); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { await makeIntegrationAccountDefault({ composioConnectionId: parseComposioConnectionId(c.req.param("connectionId")), @@ -76,7 +88,10 @@ function registerIntegrationAccountRoutes(app: GatewayApp): void { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "DELETE /v1/integrations/:name/accounts/:connectionId"); const integration = parseIntegrationName(c.req.param("name")); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { await deleteIntegrationAccount({ composioConnectionId: parseComposioConnectionId(c.req.param("connectionId")), diff --git a/apps/gateway-worker/src/internal-maintenance.ts b/apps/gateway-worker/src/internal-maintenance.ts new file mode 100644 index 00000000..93efc8ad --- /dev/null +++ b/apps/gateway-worker/src/internal-maintenance.ts @@ -0,0 +1,52 @@ +import { assertDistinctHmacSecrets } from "@cheatcode/auth"; +import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; +import { APIError } from "@cheatcode/observability"; + +export interface GatewayMaintenanceSecretBindings { + GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET: WorkerSecret; + RELEASE_DATABASE_READINESS_SECRET: WorkerSecret; +} + +export async function requireResourceDeletionSecret( + env: GatewayMaintenanceSecretBindings, +): Promise { + return (await requireGatewayMaintenanceSecrets(env)).resourceDeletion; +} + +export function requireDatabaseReadinessSecret( + env: GatewayMaintenanceSecretBindings, +): Promise { + return requireGatewayMaintenanceSecrets(env).then((secrets) => secrets.databaseReadiness); +} + +async function requireGatewayMaintenanceSecrets(env: GatewayMaintenanceSecretBindings): Promise<{ + databaseReadiness: string; + resourceDeletion: string; +}> { + try { + const [databaseReadiness, resourceDeletion] = await Promise.all([ + resolveRequiredSecret(env.RELEASE_DATABASE_READINESS_SECRET), + resolveRequiredSecret(env.GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET), + ]); + assertDistinctHmacSecrets([databaseReadiness, resourceDeletion]); + return { databaseReadiness, resourceDeletion }; + } catch { + // Secret-store failures share one bounded internal-maintenance error contract. + } + throw new APIError( + 503, + "unavailable_maintenance", + "Gateway maintenance secrets are unavailable", + { + retriable: false, + }, + ); +} + +async function resolveRequiredSecret(binding: WorkerSecret): Promise { + const secret = await resolveWorkerSecret(binding); + if (!secret?.trim()) { + throw new Error("Maintenance secret is missing"); + } + return secret; +} diff --git a/apps/gateway-worker/src/limits.ts b/apps/gateway-worker/src/limits.ts index 75136542..1f8c7898 100644 --- a/apps/gateway-worker/src/limits.ts +++ b/apps/gateway-worker/src/limits.ts @@ -2,7 +2,6 @@ import { type EntitlementCache, EntitlementCacheSchema, entitlementCacheFromValues, - tierLimits, } from "@cheatcode/billing"; import { countActiveProjects, @@ -13,12 +12,7 @@ import { withUserContext, } from "@cheatcode/db"; import { APIError, createLogger, readBoundedResponseJson } from "@cheatcode/observability"; -import { - type LimitsSnapshot, - LimitsSnapshotSchema, - type Provider, - type UserId, -} from "@cheatcode/types"; +import { type LimitsSnapshot, LimitsSnapshotSchema, type UserId } from "@cheatcode/types"; import { QUOTA_FEATURES, QUOTA_TRACKER_MAX_RESPONSE_BYTES, @@ -87,32 +81,6 @@ export async function enforceActiveProjectLimit(db: Database, userId: UserId): P }); } -export function enforceByokProviderSlotLimit( - entitlement: EntitlementCache, - provider: Provider, - existingKeys: readonly { disabledAt?: null | string; provider: Provider }[], -): void { - const activeKeys = existingKeys.filter( - (key) => key.disabledAt === null || key.disabledAt === undefined, - ); - if (activeKeys.some((key) => key.provider === provider)) { - return; - } - const limit = tierLimits(entitlement.tier).byokProviderSlots; - if (limit === null || activeKeys.length < limit) { - return; - } - throw new APIError(403, "permission_plan_required", "BYOK provider slot limit reached", { - details: { - limit, - tier: entitlement.tier, - used: activeKeys.length, - }, - hint: "Upgrade your plan or remove an existing provider key before adding another one.", - retriable: false, - }); -} - export async function resolveEntitlement( env: LimitBindings, db: Database, @@ -130,10 +98,7 @@ export async function resolveEntitlement( } /** DB-only authoritative entitlement read for mutation transactions. */ -export async function resolveDatabaseEntitlement( - db: Database, - userId: UserId, -): Promise { +async function resolveDatabaseEntitlement(db: Database, userId: UserId): Promise { const row = await findEntitlementByUserId(db, userId); return entitlementCacheFromValues(row ?? { tier: "free" }); } diff --git a/apps/gateway-worker/src/local-preview-proxy.ts b/apps/gateway-worker/src/local-preview-proxy.ts deleted file mode 100644 index df84d4bf..00000000 --- a/apps/gateway-worker/src/local-preview-proxy.ts +++ /dev/null @@ -1,215 +0,0 @@ -const LOCAL_PREVIEW_COOKIE_NAME = "__cheatcode_preview_host"; -const LOCAL_PREVIEW_CLIENT_HOST_HEADER = "X-Cheatcode-Local-Preview-Client-Host"; -const LOCAL_PREVIEW_ENCODED_HOST_PATTERN = /^[A-Za-z0-9_-]{1,512}$/; -const LOCAL_PREVIEW_HOST_PATTERN = - /^(?:\d{4,5}-[a-z0-9-]+-[a-z0-9_]+|[a-z0-9-]+--\d{1,5})\.localhost$/; -const LOCAL_PREVIEW_PATH_PREFIX = "/__sandbox/"; - -export interface LocalPreviewProxyRequest { - encodedHost?: string; - request: Request; -} - -export interface LocalPreviewOriginRequest { - clientHost: string; - cookie?: string; - host: string; - origin?: string; - url: string; -} - -export function resolveLocalPreviewProxyRequest(request: Request): LocalPreviewProxyRequest | null { - const pathProxy = rewriteLocalPreviewPathRequest(request); - if (pathProxy) { - return pathProxy; - } - const cookieHost = resolveLocalPreviewCookieHost(request); - if (!cookieHost) { - return null; - } - return { request: rewriteLocalPreviewRequest(request, cookieHost) }; -} - -export function withLocalPreviewCookie( - response: Response, - id: string, - encodedHost?: string, -): Response { - if (response.status === 101 || response.webSocket) { - return response; - } - const wrapped = new Response(response.body, response); - wrapped.headers.set("X-Request-Id", id); - if (encodedHost) { - wrapped.headers.append( - "Set-Cookie", - `${LOCAL_PREVIEW_COOKIE_NAME}=${encodedHost}; Path=/; Max-Age=3600; SameSite=Lax`, - ); - } - return wrapped; -} - -export function localPreviewOriginRequest(request: Request): LocalPreviewOriginRequest | null { - const proxy = resolveLocalPreviewProxyRequest(request); - if (proxy) { - return originRequestFromRewrittenRequest(proxy.request); - } - const localPreviewHost = resolveLocalSandboxPreviewHost(request); - if (!localPreviewHost) { - return null; - } - return originRequestFromRewrittenRequest(rewriteLocalPreviewRequest(request, localPreviewHost)); -} - -export function resolveLocalSandboxPreviewHost(request: Request): string | null { - const url = new URL(request.url); - if (isLocalPreviewHost(url.host)) { - return url.host; - } - return ( - resolveLocalPreviewHostHeader(request, "Host") ?? - resolveLocalPreviewHostHeader(request, "MF-Original-Hostname") - ); -} - -export function rewriteLocalPreviewRequest( - request: Request, - host: string, - pathname?: string, -): Request { - const url = new URL(request.url); - const { hostname, port } = splitHost(host); - url.hostname = hostname; - url.port = port ?? url.port; - if (pathname) { - url.pathname = pathname; - } - const headers = new Headers(request.headers); - const clientHost = headers.get("Host") ?? url.host; - headers.set(LOCAL_PREVIEW_CLIENT_HOST_HEADER, clientHost); - headers.set("Host", host); - if (isWebSocketUpgrade(request)) { - const websocketRequest = new Request(url.toString(), request); - websocketRequest.headers.set(LOCAL_PREVIEW_CLIENT_HOST_HEADER, clientHost); - websocketRequest.headers.set("Host", host); - return websocketRequest; - } - const init: RequestInit = { - headers, - method: request.method, - redirect: "manual", - }; - if (request.method !== "GET" && request.method !== "HEAD") { - init.body = request.body; - } - return new Request(url.toString(), init); -} - -function originRequestFromRewrittenRequest(request: Request): LocalPreviewOriginRequest { - const clientHost = - request.headers.get(LOCAL_PREVIEW_CLIENT_HOST_HEADER) ?? - request.headers.get("Host") ?? - new URL(request.url).host; - const cookie = request.headers.get("Cookie"); - const origin = request.headers.get("Origin"); - return { - clientHost, - ...(cookie ? { cookie } : {}), - host: request.headers.get("Host") ?? new URL(request.url).host, - ...(origin ? { origin } : {}), - url: request.url, - }; -} - -function rewriteLocalPreviewPathRequest(request: Request): LocalPreviewProxyRequest | null { - const url = new URL(request.url); - if (!url.pathname.startsWith(LOCAL_PREVIEW_PATH_PREFIX)) { - return null; - } - - const previewPath = url.pathname.slice(LOCAL_PREVIEW_PATH_PREFIX.length); - const slashIndex = previewPath.indexOf("/"); - const encodedHost = slashIndex === -1 ? previewPath : previewPath.slice(0, slashIndex); - const host = decodePreviewHost(encodedHost); - if (!host) { - return null; - } - - const proxiedPath = slashIndex === -1 ? "/" : previewPath.slice(slashIndex); - return { - encodedHost, - request: rewriteLocalPreviewRequest(request, host, proxiedPath), - }; -} - -function resolveLocalPreviewCookieHost(request: Request): string | null { - if (!shouldProxyLocalPreviewCookieRequest(request)) { - return null; - } - const encodedHost = readCookie(request.headers.get("Cookie"), LOCAL_PREVIEW_COOKIE_NAME); - return encodedHost ? decodePreviewHost(encodedHost) : null; -} - -function shouldProxyLocalPreviewCookieRequest(request: Request): boolean { - if (request.method !== "GET" && request.method !== "HEAD") { - return false; - } - const url = new URL(request.url); - if (url.pathname.startsWith("/v1/") || url.pathname.startsWith(LOCAL_PREVIEW_PATH_PREFIX)) { - return false; - } - return url.pathname !== "/health"; -} - -function decodePreviewHost(encodedHost: string): string | null { - if (!LOCAL_PREVIEW_ENCODED_HOST_PATTERN.test(encodedHost)) { - return null; - } - try { - const normalized = encodedHost.replaceAll("-", "+").replaceAll("_", "/"); - const padding = "=".repeat((4 - (normalized.length % 4)) % 4); - const host = atob(`${normalized}${padding}`); - return isLocalPreviewHost(host) ? host : null; - } catch { - return null; - } -} - -function readCookie(cookieHeader: string | null, name: string): string | null { - if (!cookieHeader) { - return null; - } - for (const cookie of cookieHeader.split(";")) { - const trimmed = cookie.trim(); - const separatorIndex = trimmed.indexOf("="); - if (separatorIndex === -1) { - continue; - } - if (trimmed.slice(0, separatorIndex) === name) { - return trimmed.slice(separatorIndex + 1); - } - } - return null; -} - -function resolveLocalPreviewHostHeader(request: Request, headerName: string): string | null { - const host = request.headers.get(headerName); - if (!host) { - return null; - } - return isLocalPreviewHost(host) ? host : null; -} - -function isLocalPreviewHost(host: string): boolean { - const { hostname } = splitHost(host); - return LOCAL_PREVIEW_HOST_PATTERN.test(hostname); -} - -function isWebSocketUpgrade(request: Request): boolean { - return (request.headers.get("Upgrade") ?? "").toLowerCase() === "websocket"; -} - -function splitHost(host: string): { hostname: string; port?: string } { - const [hostname = "", port] = host.split(":"); - return port ? { hostname, port } : { hostname }; -} diff --git a/apps/gateway-worker/src/local-preview-routing.ts b/apps/gateway-worker/src/local-preview-routing.ts new file mode 100644 index 00000000..fa51d7c1 --- /dev/null +++ b/apps/gateway-worker/src/local-preview-routing.ts @@ -0,0 +1,145 @@ +import { readCookieValue } from "@cheatcode/auth"; + +const LOCAL_GATEWAY_PORT = "8787"; +const LOCAL_PREVIEW_ENCODED_HOST_PATTERN = /^[A-Za-z0-9_-]{1,512}$/u; +const LOCAL_PREVIEW_HOST_PATTERN = /^([a-z0-9]+(?:-[a-z0-9]+)*)--(\d{1,5})\.localhost$/u; +const LOCAL_PREVIEW_PATH_PREFIX = "/__sandbox/"; +const LOCAL_PREVIEW_SESSION_COOKIE = "cc_pt"; +const LOCAL_PREVIEW_TOKEN_QUERY = "__cc_pt"; +const MAX_LOCAL_PREVIEW_TOKEN_LENGTH = 2_048; + +export type LocalPreviewRoute = + | { kind: "proxy"; request: Request } + | { kind: "redirect"; response: Response }; + +/** + * Keep the gateway as the sole local listener while routing preview traffic to + * the real preview-proxy Worker. Path-form URLs are only a browser handoff; + * every authenticated request runs on the canonical `*.localhost` origin. + */ +export function resolveLocalPreviewRoute(request: Request): LocalPreviewRoute | null { + const redirect = localPreviewPathRedirect(request); + if (redirect) { + return { kind: "redirect", response: redirect }; + } + const host = resolveLocalPreviewHost(request); + if (!host) { + return null; + } + return { kind: "proxy", request: requestForHost(request, host) }; +} + +function localPreviewPathRedirect(request: Request): Response | null { + const url = new URL(request.url); + if (!url.pathname.startsWith(LOCAL_PREVIEW_PATH_PREFIX)) { + return null; + } + const previewPath = url.pathname.slice(LOCAL_PREVIEW_PATH_PREFIX.length); + const slashIndex = previewPath.indexOf("/"); + const encodedHost = slashIndex === -1 ? previewPath : previewPath.slice(0, slashIndex); + const host = decodePreviewHost(encodedHost); + if (!host) { + return null; + } + url.host = host; + url.pathname = slashIndex === -1 ? "/" : previewPath.slice(slashIndex); + return new Response(null, { + headers: { + "Cache-Control": "private, no-store", + Location: url.toString(), + // The preview proxy requires the exact app-origin referrer after it + // exchanges the query handoff for a cookie. `origin` discloses neither + // the local path-form handoff nor its signed query credential. + "Referrer-Policy": "origin", + }, + status: request.method === "GET" || request.method === "HEAD" ? 302 : 307, + }); +} + +function resolveLocalPreviewHost(request: Request): string | null { + const url = new URL(request.url); + const capabilityToken = + url.searchParams.get(LOCAL_PREVIEW_TOKEN_QUERY) ?? + readCookieValue(request.headers.get("Cookie"), LOCAL_PREVIEW_SESSION_COOKIE); + for (const candidate of [ + url.host, + request.headers.get("Host"), + request.headers.get("MF-Original-Hostname"), + localPreviewAudience(capabilityToken), + ]) { + if (candidate && isLocalPreviewHost(candidate)) { + return candidate; + } + } + return null; +} + +function localPreviewAudience(token: string | null): string | null { + if (!token || token.length > MAX_LOCAL_PREVIEW_TOKEN_LENGTH) { + return null; + } + const [prefix, encodedPayload, signature, ...extra] = token.split("."); + if (prefix !== "ccp1" || !encodedPayload || !signature || extra.length > 0) { + return null; + } + try { + const normalized = encodedPayload.replaceAll("-", "+").replaceAll("_", "/"); + const padding = "=".repeat((4 - (normalized.length % 4)) % 4); + const payload: unknown = JSON.parse(atob(`${normalized}${padding}`)); + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + return null; + } + const audience = Reflect.get(payload, "aud"); + return typeof audience === "string" && isLocalPreviewHost(audience) ? audience : null; + } catch { + return null; + } +} + +function decodePreviewHost(encodedHost: string): string | null { + if (!LOCAL_PREVIEW_ENCODED_HOST_PATTERN.test(encodedHost)) { + return null; + } + try { + const normalized = encodedHost.replaceAll("-", "+").replaceAll("_", "/"); + const padding = "=".repeat((4 - (normalized.length % 4)) % 4); + const host = atob(`${normalized}${padding}`); + return isLocalPreviewHost(host) ? host : null; + } catch { + return null; + } +} + +function isLocalPreviewHost(host: string): boolean { + const url = safeLocalUrl(host); + if (!url || url.port !== LOCAL_GATEWAY_PORT) { + return false; + } + const match = LOCAL_PREVIEW_HOST_PATTERN.exec(url.hostname); + const port = Number(match?.[2]); + return Boolean(match?.[1]) && Number.isInteger(port) && port >= 1 && port <= 65_535; +} + +function safeLocalUrl(host: string): URL | null { + try { + const url = new URL(`http://${host}`); + return url.host !== host || + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ? null + : url; + } catch { + return null; + } +} + +function requestForHost(request: Request, host: string): Request { + const url = new URL(request.url); + url.host = host; + const rewritten = new Request(url, request); + rewritten.headers.set("Host", host); + return rewritten; +} diff --git a/apps/gateway-worker/src/openapi-account-routes.ts b/apps/gateway-worker/src/openapi-account-routes.ts index 5c7440b9..92c80bf6 100644 --- a/apps/gateway-worker/src/openapi-account-routes.ts +++ b/apps/gateway-worker/src/openapi-account-routes.ts @@ -1,8 +1,6 @@ import { ActivityHistoryResponseSchema, LimitsSnapshotSchema, - MeResponseSchema, - UpdateMeSchema, UpdateUserProfileSchema, UserProfileSchema, } from "@cheatcode/types"; @@ -18,10 +16,6 @@ import { withJsonSchemaConstraints, zodJsonSchema } from "./openapi-zod"; export const accountSchemas: Record = { ActivityHistory: zodJsonSchema(ActivityHistoryResponseSchema), LimitsSnapshot: zodJsonSchema(LimitsSnapshotSchema), - MeResponse: zodJsonSchema(MeResponseSchema), - UpdateMe: withJsonSchemaConstraints(zodJsonSchema(UpdateMeSchema, "input"), { - minProperties: 1, - }), UpdateUserProfile: withJsonSchemaConstraints(zodJsonSchema(UpdateUserProfileSchema, "input"), { minProperties: 1, }), @@ -31,29 +25,10 @@ export const accountSchemas: Record = { const activityDaysParameter: JsonValue = { in: "query", name: "days", - schema: { default: 30, maximum: 90, minimum: 1, type: "integer" }, + schema: { default: 30, maximum: 366, minimum: 1, type: "integer" }, }; export const accountRoutes: OpenApiRoute[] = [ - { - method: "get", - operationId: "getMe", - path: "/v1/me", - responses: { "200": jsonResponse("Current user", schemaRef("MeResponse")) }, - security: [{ bearerAuth: [] }], - summary: "Get current user", - tags: ["account"], - }, - { - method: "patch", - operationId: "updateMe", - path: "/v1/me", - requestBody: jsonBody(schemaRef("UpdateMe")), - responses: { "200": jsonResponse("Updated user", schemaRef("MeResponse")) }, - security: [{ bearerAuth: [] }], - summary: "Update the current user", - tags: ["account"], - }, { method: "get", operationId: "getMyProfile", diff --git a/apps/gateway-worker/src/openapi-builder.ts b/apps/gateway-worker/src/openapi-builder.ts index 19a1a068..f954d466 100644 --- a/apps/gateway-worker/src/openapi-builder.ts +++ b/apps/gateway-worker/src/openapi-builder.ts @@ -244,29 +244,13 @@ function pathParameterSchema(name: string): JsonValue { } if (name === "provider") { return { - enum: [ - "anthropic", - "openai", - "google", - "openrouter", - "deepseek", - "exa", - "firecrawl", - "llamaparse", - ], + enum: ["anthropic", "openai", "google", "openrouter", "deepseek", "exa", "firecrawl"], type: "string", }; } return { type: "string" }; } -const UUID_PATH_PARAMETERS = new Set([ - "approvalId", - "outputId", - "projectId", - "runId", - "skillId", - "threadId", -]); +const UUID_PATH_PARAMETERS = new Set(["outputId", "projectId", "runId", "skillId", "threadId"]); const COMPONENT_SCHEMA_PREFIX = "#/components/schemas/"; diff --git a/apps/gateway-worker/src/openapi-project-routes.ts b/apps/gateway-worker/src/openapi-project-routes.ts index d75a788e..324d06a5 100644 --- a/apps/gateway-worker/src/openapi-project-routes.ts +++ b/apps/gateway-worker/src/openapi-project-routes.ts @@ -80,9 +80,9 @@ export const projectRoutes: OpenApiRoute[] = [ method: "delete", operationId: "deleteProject", path: "/v1/projects/{projectId}", - responses: { "204": emptyResponse("Project archived") }, + responses: { "202": emptyResponse("Project deletion accepted") }, security: [{ bearerAuth: [] }], - summary: "Archive a project", + summary: "Delete a project", tags: ["projects"], }, { @@ -144,9 +144,9 @@ export const projectRoutes: OpenApiRoute[] = [ method: "delete", operationId: "deleteThread", path: "/v1/threads/{threadId}", - responses: { "204": emptyResponse("Thread archived") }, + responses: { "202": emptyResponse("Thread deletion accepted") }, security: [{ bearerAuth: [] }], - summary: "Archive a thread", + summary: "Delete a thread", tags: ["threads"], }, { diff --git a/apps/gateway-worker/src/openapi-run-control-routes.ts b/apps/gateway-worker/src/openapi-run-control-routes.ts deleted file mode 100644 index 2bc7f08f..00000000 --- a/apps/gateway-worker/src/openapi-run-control-routes.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { ApprovalDecisionRequestSchema, ApprovalDecisionResponseSchema } from "@cheatcode/types"; -import { - type JsonValue, - jsonBody, - jsonResponse, - type OpenApiRoute, - schemaRef, -} from "./openapi-builder"; -import { zodJsonSchema } from "./openapi-zod"; - -export const runControlSchemas: Record = { - ApprovalDecisionRequest: zodJsonSchema(ApprovalDecisionRequestSchema, "input"), - ApprovalDecisionResponse: zodJsonSchema(ApprovalDecisionResponseSchema), -}; - -export const runControlRoutes: OpenApiRoute[] = [ - { - method: "post", - operationId: "decideRunApproval", - path: "/v1/runs/{runId}/approvals/{approvalId}", - requestBody: jsonBody(schemaRef("ApprovalDecisionRequest")), - responses: { - "200": jsonResponse("Approval resolution", schemaRef("ApprovalDecisionResponse")), - "404": jsonResponse("Run not found", schemaRef("Error")), - "409": jsonResponse("Approval unavailable or decision conflict", schemaRef("Error")), - }, - security: [{ bearerAuth: [] }], - summary: "Resolve a pending tool-approval or model-fallback request", - tags: ["runs"], - }, -]; diff --git a/apps/gateway-worker/src/openapi-sandbox-routes.ts b/apps/gateway-worker/src/openapi-sandbox-routes.ts index 335fa3bf..6a763466 100644 --- a/apps/gateway-worker/src/openapi-sandbox-routes.ts +++ b/apps/gateway-worker/src/openapi-sandbox-routes.ts @@ -1,4 +1,8 @@ import { + BrowserTakeoverResumeResultSchema, + BrowserTakeoverResumeSchema, + BrowserTakeoverSessionSchema, + BrowserTakeoverStatusSchema, SandboxConsoleSnapshotSchema, SandboxFileListSchema, SandboxFilePreviewSchema, @@ -22,6 +26,10 @@ import { import { zodJsonSchema } from "./openapi-zod"; export const sandboxSchemas: Record = { + BrowserTakeoverResume: zodJsonSchema(BrowserTakeoverResumeSchema, "input"), + BrowserTakeoverResumeResult: zodJsonSchema(BrowserTakeoverResumeResultSchema), + BrowserTakeoverSession: zodJsonSchema(BrowserTakeoverSessionSchema), + BrowserTakeoverStatus: zodJsonSchema(BrowserTakeoverStatusSchema), OpenSandboxTerminal: zodJsonSchema(SandboxTerminalCommandSchema, "input"), SandboxConsoleSnapshot: zodJsonSchema(SandboxConsoleSnapshotSchema), SandboxFile: zodJsonSchema(SandboxFileSchema), @@ -69,6 +77,40 @@ const sandboxFileListParameters: JsonValue[] = [ ]; export const sandboxRoutes: OpenApiRoute[] = [ + { + method: "get", + operationId: "getBrowserTakeoverStatus", + path: "/v1/threads/{threadId}/browser-takeover", + responses: { + "200": jsonResponse("Browser takeover status", schemaRef("BrowserTakeoverStatus")), + }, + security: [{ bearerAuth: [] }], + summary: "Read the active browser takeover state", + tags: ["sandbox"], + }, + { + method: "post", + operationId: "startBrowserTakeover", + path: "/v1/threads/{threadId}/browser-takeover/start", + responses: { + "200": jsonResponse("Browser takeover session", schemaRef("BrowserTakeoverSession")), + }, + security: [{ bearerAuth: [] }], + summary: "Pause automation and take over its live browser", + tags: ["sandbox"], + }, + { + method: "post", + operationId: "resumeBrowserAutomation", + path: "/v1/threads/{threadId}/browser-takeover/resume", + requestBody: jsonBody(schemaRef("BrowserTakeoverResume")), + responses: { + "200": jsonResponse("Browser automation resumed", schemaRef("BrowserTakeoverResumeResult")), + }, + security: [{ bearerAuth: [] }], + summary: "Close human control and resume browser automation", + tags: ["sandbox"], + }, { method: "get", operationId: "openComputerIde", diff --git a/apps/gateway-worker/src/openapi-skill-routes.ts b/apps/gateway-worker/src/openapi-skill-routes.ts index 489eb991..2fc17881 100644 --- a/apps/gateway-worker/src/openapi-skill-routes.ts +++ b/apps/gateway-worker/src/openapi-skill-routes.ts @@ -1,8 +1,11 @@ -import { CreateUserSkillSchema, UserSkillSchema, UserSkillsResponseSchema } from "@cheatcode/types"; +import { + SkillProposalConfirmResponseSchema, + UserSkillSchema, + UserSkillsResponseSchema, +} from "@cheatcode/types"; import { emptyResponse, type JsonValue, - jsonBody, jsonResponse, type OpenApiRoute, schemaRef, @@ -10,7 +13,7 @@ import { import { zodJsonSchema } from "./openapi-zod"; export const skillSchemas: Record = { - CreateUserSkill: zodJsonSchema(CreateUserSkillSchema, "input"), + SkillProposalConfirmResponse: zodJsonSchema(SkillProposalConfirmResponseSchema), UserSkill: zodJsonSchema(UserSkillSchema), UserSkillsResponse: zodJsonSchema(UserSkillsResponseSchema), }; @@ -27,12 +30,24 @@ export const skillRoutes: OpenApiRoute[] = [ }, { method: "post", - operationId: "createUserSkill", - path: "/v1/skills", - requestBody: jsonBody(schemaRef("CreateUserSkill")), - responses: { "201": jsonResponse("Created user skill", schemaRef("UserSkill")) }, + operationId: "confirmSkillProposal", + path: "/v1/threads/{threadId}/skill-proposals/{runId}/{proposalId}/confirm", + responses: { + "200": jsonResponse("Confirmed skill proposal", schemaRef("SkillProposalConfirmResponse")), + }, + security: [{ bearerAuth: [] }], + summary: "Create a skill from a persisted agent proposal", + tags: ["skills"], + }, + { + method: "post", + operationId: "openUserSkill", + path: "/v1/skills/{skillId}/open", + responses: { + "200": jsonResponse("Skill file IDE session", schemaRef("SandboxIdeSession")), + }, security: [{ bearerAuth: [] }], - summary: "Create or update a user skill by name", + summary: "Open a custom skill file in the Computer", tags: ["skills"], }, { diff --git a/apps/gateway-worker/src/openapi.ts b/apps/gateway-worker/src/openapi.ts index 311de2f0..050b36f4 100644 --- a/apps/gateway-worker/src/openapi.ts +++ b/apps/gateway-worker/src/openapi.ts @@ -4,6 +4,7 @@ import { ClientUserEventBodySchema, CreateRunSchema, ErrorResponseSchema, + OutputDownloadUrlResponseSchema, RunStatusSnapshotSchema, ToolDomainSchema, ToolSummarySchema, @@ -25,7 +26,6 @@ import { import { discoveryRoutes, discoverySchemas } from "./openapi-discovery-routes"; import { integrationSchemas } from "./openapi-integration-schemas"; import { projectRoutes, projectSchemas } from "./openapi-project-routes"; -import { runControlRoutes, runControlSchemas } from "./openapi-run-control-routes"; import { sandboxRoutes, sandboxSchemas } from "./openapi-sandbox-routes"; import { arraySchemaFor, idempotencyKeyParameter, objectSchemaFor } from "./openapi-schema-utils"; import { skillRoutes, skillSchemas } from "./openapi-skill-routes"; @@ -71,6 +71,7 @@ const COMPONENT_SCHEMAS: Record = { required: ["openapi"], type: "object", }, + OutputDownloadUrl: zodJsonSchema(OutputDownloadUrlResponseSchema), RunStatus: zodJsonSchema(RunStatusSnapshotSchema), Tool: zodJsonSchema(ToolSummarySchema), WebVitals: zodJsonSchema(WebVitalsBodySchema, "input"), @@ -102,6 +103,12 @@ const outputDownloadParameters: JsonValue[] = [ required: true, schema: { maxLength: 256, minLength: 32, type: "string" }, }, + { + in: "query", + name: "userId", + required: true, + schema: { format: "uuid", type: "string" }, + }, ]; const routes: OpenApiRoute[] = [ @@ -166,7 +173,22 @@ const routes: OpenApiRoute[] = [ summary: "Cancel an agent run", tags: ["runs"], }, - ...runControlRoutes, + { + method: "post", + operationId: "createOutputDownloadUrl", + path: "/v1/outputs/{outputId}/download-url", + rateLimited: true, + responses: { + "200": jsonResponse("Short-lived output download URL", schemaRef("OutputDownloadUrl")), + "400": jsonResponse("Invalid output id", schemaRef("Error")), + "404": jsonResponse("Not found", schemaRef("Error")), + "410": jsonResponse("Expired", schemaRef("Error")), + "503": jsonResponse("Signing service unavailable", schemaRef("Error")), + }, + security: [{ bearerAuth: [] }], + summary: "Create a short-lived generated output download URL", + tags: ["outputs"], + }, { method: "get", operationId: "downloadOutput", @@ -372,7 +394,6 @@ export const OPENAPI_DOCUMENT = buildOpenApiDocument({ ...billingSchemas, ...discoverySchemas, ...projectSchemas, - ...runControlSchemas, ...sandboxSchemas, ...skillSchemas, }, diff --git a/apps/gateway-worker/src/profile-routes.ts b/apps/gateway-worker/src/profile-routes.ts index 113c8710..4c0608b8 100644 --- a/apps/gateway-worker/src/profile-routes.ts +++ b/apps/gateway-worker/src/profile-routes.ts @@ -21,13 +21,14 @@ import { UserProfileSchema, } from "@cheatcode/types"; import type { z } from "zod"; -import { clerkAuthorizedParties, readOptionalSecret } from "./authenticate"; +import { clerkAuthorizedParties, readOptionalClerkSecret } from "./authenticate"; import type { WaitUntilContext } from "./wait-until-context"; export interface ProfileRouteEnv { CHEATCODE_ENVIRONMENT: "development" | "production"; CLERK_AUTHORIZED_PARTIES?: string; CLERK_SECRET_KEY?: WorkerSecret; + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; } @@ -38,7 +39,10 @@ export async function getMyProfileRoute( ctx: WaitUntilContext, userId: UserId, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const record = await withUserContext(db, userId, (tx) => getUserProfile(tx, userId)); return Response.json(UserProfileSchema.parse(profileResponse(record))); @@ -60,7 +64,10 @@ export async function updateMyProfileRoute( throw invalidRequestBody("Invalid profile payload", parsed.error); } const body = parsed.data; - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); let result: UserProfileRecord; try { result = await withUserContext(db, userId, (tx) => @@ -106,7 +113,7 @@ async function mirrorOnboardingClaim( ): Promise { const logger = createLogger({ userId }); try { - const secretKey = await readOptionalSecret(env.CLERK_SECRET_KEY, "CLERK_SECRET_KEY"); + const secretKey = await readOptionalClerkSecret(env); if (!secretKey) { logger.warn("onboarding_claim_mirror_skipped"); return; diff --git a/apps/gateway-worker/src/project-routes.ts b/apps/gateway-worker/src/project-routes.ts index a4125db9..56ee968a 100644 --- a/apps/gateway-worker/src/project-routes.ts +++ b/apps/gateway-worker/src/project-routes.ts @@ -1,7 +1,6 @@ -import { createInternalMaintenanceHeaders } from "@cheatcode/auth"; import { beginProjectDeletion, - completeProjectWorkspaceCleanup, + beginThreadDeletion, createDb, createProject, createThread, @@ -14,20 +13,17 @@ import { lockUserProjectMutations, type MessageRecord, type ProjectSummaryRecord, - softDeleteThread, type ThreadRecord, updateProject, updateThread, withUserContext, } from "@cheatcode/db"; -import { resolveWorkerSecret, type WorkerSecret } from "@cheatcode/env"; +import type { WorkerSecret } from "@cheatcode/env"; import { APIError, readJsonRequest } from "@cheatcode/observability"; import { CreateProjectSchema, type CreateThread, CreateThreadSchema, - InternalAgentStateDeleteBodySchema, - internalUserStateDeletePath, Paginated, PaginationQuerySchema, ProjectId, @@ -43,26 +39,40 @@ import { } from "@cheatcode/types"; import { z } from "zod"; import { enforceActiveProjectLimit, type LimitBindings } from "./limits"; +import { + enqueueResourceDeletion, + type ResourceDeletionEnqueueEnv, +} from "./resource-deletion-enqueue"; import type { WaitUntilContext } from "./wait-until-context"; const MAX_PROJECT_REQUEST_BYTES = 64 * 1024; -export interface ProjectRouteEnv extends LimitBindings { - AGENT: Fetcher; +export interface ProjectRouteEnv extends LimitBindings, ResourceDeletionEnqueueEnv { + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; - INTERNAL_MAINTENANCE_SECRET?: WorkerSecret; } const IdParamSchema = z.string().uuid(); -const PageCursorSchema = z - .object({ - at: z.string().datetime(), - id: z.string().uuid(), - kind: z.enum(["messages", "projects", "threads"]), +const CursorIdentitySchema = z.object({ + at: z.string().datetime(), + id: z.string().uuid(), +}); +const PageCursorSchema = z.discriminatedUnion("kind", [ + CursorIdentitySchema.extend({ + kind: z.literal("messages"), + segment: z.number().int().nonnegative(), + v: z.literal(2), + }).strict(), + CursorIdentitySchema.extend({ + kind: z.literal("projects"), + v: z.literal(1), + }).strict(), + CursorIdentitySchema.extend({ + kind: z.literal("threads"), v: z.literal(1), - }) - .strict(); -type PageCursorKind = z.infer["kind"]; + }).strict(), +]); +type PageCursorKind = "messages" | "projects" | "threads"; export async function listProjectsRoute( env: ProjectRouteEnv, @@ -71,7 +81,10 @@ export async function listProjectsRoute( userId: UserId, ): Promise { const pagination = parsePagination(request, "projects"); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const projects = await withUserContext(db, userId, (tx) => listProjects(tx, { @@ -104,7 +117,10 @@ export async function createProjectRoute( if (!parsedInput.success) { throw invalidRequestBody("Invalid project payload", parsedInput.error); } - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const project = await withUserContext(db, userId, async (tx) => { await enforceActiveProjectLimit(tx, userId); @@ -117,9 +133,6 @@ export async function createProjectRoute( ...(parsedInput.data.defaultModel === undefined ? {} : { defaultModel: parsedInput.data.defaultModel }), - ...(parsedInput.data.masterInstructions - ? { masterInstructions: parsedInput.data.masterInstructions } - : {}), userId, }); }); @@ -135,7 +148,10 @@ export async function getProjectRoute( projectId: ProjectIdType, userId: UserId, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const project = await withUserContext(db, userId, (tx) => getProject(tx, { projectId, userId }), @@ -162,16 +178,16 @@ export async function updateProjectRoute( if (!parsedInput.success) { throw invalidRequestBody("Invalid project update payload", parsedInput.error); } - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const input = parsedInput.data; const project = await withUserContext(db, userId, (tx) => updateWritableProject(tx, projectId, userId, { ...(input.importRepoUrl === undefined ? {} : { importRepoUrl: input.importRepoUrl }), ...(input.defaultModel === undefined ? {} : { defaultModel: input.defaultModel }), - ...(input.masterInstructions === undefined - ? {} - : { masterInstructions: input.masterInstructions }), ...(input.name === undefined ? {} : { name: input.name }), }), ); @@ -190,8 +206,10 @@ export async function deleteProjectRoute( projectId: ProjectIdType, userId: UserId, ): Promise { - const maintenanceSecret = await readMaintenanceSecret(env); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const deletion = await withUserContext(db, userId, (tx) => beginProjectDeletion(tx, { projectId, userId }), @@ -205,34 +223,14 @@ export async function deleteProjectRoute( retriable: true, }); } - if (deletion.type === "cleanup-completed") { - return new Response(null, { status: 204 }); - } - const cleanupBody = JSON.stringify( - InternalAgentStateDeleteBodySchema.parse({ - scope: "project", - workspaceSlug: deletion.workspaceSlug, - }), - ); - const pathname = internalUserStateDeletePath(userId); - const cleanupHeaders = await createInternalMaintenanceHeaders({ - method: "POST", - pathname, - rawBody: cleanupBody, - secret: maintenanceSecret, + await enqueueResourceDeletion(env, { + deletedAt: deletion.deletedAt.toISOString(), + kind: "project-deletion", + projectId, + userId, + workspaceSlug: deletion.workspaceSlug, }); - cleanupHeaders.set("content-type", "application/json"); - await deleteProjectAgentState(env, pathname, cleanupBody, cleanupHeaders); - const completed = await withUserContext(db, userId, (tx) => - completeProjectWorkspaceCleanup(tx, { projectId, userId }), - ); - if (!completed) { - throw new APIError(503, "unavailable_maintenance", "Project cleanup state was not saved", { - hint: "Retry deletion. Workspace cleanup is idempotent.", - retriable: true, - }); - } - return new Response(null, { status: 204 }); + return new Response(null, { status: 202 }); } finally { ctx.waitUntil(close()); } @@ -246,7 +244,10 @@ export async function listProjectThreadsRoute( userId: UserId, ): Promise { const pagination = parsePagination(request, "threads"); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const threadRows = await withUserContext(db, userId, async (tx) => { await requireProject(tx, projectId, userId); @@ -269,8 +270,8 @@ export async function listProjectThreadsRoute( /** * `POST /v1/threads` — create a chat (chat-first). With no `projectId` the chat is * project-less; its `mode`/`importRepoUrl`/`defaultModel` ride the thread as launch - * intent until the first run lazily materializes the project. With `projectId` set - * it's the deliberate "add a chat to an existing project" grouping. + * intent until a workspace-backed tool lazily materializes the project. With + * `projectId` set it's the deliberate "add a chat to an existing project" grouping. */ export async function createChatRoute( env: ProjectRouteEnv, @@ -285,7 +286,10 @@ export async function createChatRoute( throw invalidRequestBody("Invalid thread payload", parsedInput.error); } const input = parsedInput.data; - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const thread = await withUserContext(db, userId, (tx) => createThreadForRequest(tx, input, userId), @@ -341,7 +345,10 @@ export async function getThreadRoute( threadId: ThreadIdType, userId: UserId, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const thread = await withUserContext(db, userId, (tx) => getThread(tx, { threadId, userId })); if (!thread) { @@ -366,7 +373,10 @@ export async function updateThreadRoute( if (!parsedInput.success) { throw invalidRequestBody("Invalid thread update payload", parsedInput.error); } - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const thread = await withUserContext(db, userId, (tx) => updateThread(tx, { threadId, title: parsedInput.data.title, userId }), @@ -386,21 +396,31 @@ export async function deleteThreadRoute( threadId: ThreadIdType, userId: UserId, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const deleted = await withUserContext(db, userId, (tx) => - softDeleteThread(tx, { threadId, userId }), + beginThreadDeletion(tx, { threadId, userId }), ); - if (deleted === "not-found") { + if (deleted.type === "not-found") { throw threadNotFound(); } - if (deleted === "active-run") { + if (deleted.type === "active-run") { throw new APIError(409, "conflict_run_already_active", "Thread has an active agent run", { hint: "Cancel or wait for the run to finish, then retry deletion.", retriable: true, }); } - return new Response(null, { status: 204 }); + await enqueueResourceDeletion(env, { + deletedAt: deleted.deletedAt.toISOString(), + kind: "thread-deletion", + projectId: deleted.projectId, + threadId, + userId, + }); + return new Response(null, { status: 202 }); } finally { ctx.waitUntil(close()); } @@ -414,7 +434,10 @@ export async function listThreadMessagesRoute( userId: UserId, ): Promise { const pagination = parsePagination(request, "messages"); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const rows = await withUserContext(db, userId, async (tx) => { await requireThread(tx, threadId, userId); @@ -515,12 +538,10 @@ async function requireThread( function projectResponse(project: ProjectSummaryRecord) { return { archiveAfter: project.archiveAfter?.toISOString() ?? null, - archivedPendingAction: project.archivedPendingAction, createdAt: project.createdAt.toISOString(), defaultModel: project.defaultModel, id: project.id, importRepoUrl: project.importRepoUrl ?? null, - masterInstructions: project.masterInstructions, mode: project.mode, name: project.name, overQuota: project.overQuota, @@ -534,6 +555,7 @@ function threadResponse(thread: ThreadRecord) { activeRunId: thread.activeRunId, createdAt: thread.createdAt.toISOString(), id: thread.id, + latestModelId: thread.latestModelId, pendingInitialPrompt: thread.projectId === null && thread.activeRunId === null ? (thread.launchIntent?.initialPrompt ?? null) @@ -547,6 +569,8 @@ function threadResponse(thread: ThreadRecord) { function messageResponse(message: MessageRecord) { return { agentRunId: message.agentRunId, + agentRunSegment: message.agentRunSegment, + agentRunSegmentFinal: message.agentRunSegmentFinal, createdAt: message.createdAt.toISOString(), id: message.id, parts: message.parts, @@ -556,7 +580,7 @@ function messageResponse(message: MessageRecord) { } interface RoutePagination { - cursor?: { at: string; id: string }; + cursor?: { at: string; id: string; segment?: number }; limit: number; } @@ -573,10 +597,17 @@ function parsePagination(request: Request, kind: PageCursorKind): RoutePaginatio return { limit: parsed.data.limit }; } const cursor = decodePageCursor(parsed.data.cursor, kind); - return { cursor: { at: cursor.at, id: cursor.id }, limit: parsed.data.limit }; + return { + cursor: { + at: cursor.at, + id: cursor.id, + ...(cursor.kind === "messages" ? { segment: cursor.segment } : {}), + }, + limit: parsed.data.limit, + }; } -function paginateRows( +function paginateRows( rows: T[], pagination: RoutePagination, kind: PageCursorKind, @@ -587,11 +618,24 @@ function paginateRows( return { data, has_more: hasMore, - next_cursor: - hasMore && last ? encodePageCursor({ at: last.pageCursorAt, id: last.id, kind, v: 1 }) : null, + next_cursor: hasMore && last ? encodePageCursor(pageCursorFromRow(last, kind)) : null, }; } +function pageCursorFromRow( + row: { agentRunSegment?: number; id: string; pageCursorAt: string }, + kind: PageCursorKind, +): z.infer { + const identity = { at: row.pageCursorAt, id: row.id }; + if (kind === "messages") { + if (!Number.isSafeInteger(row.agentRunSegment) || Number(row.agentRunSegment) < 0) { + throw new TypeError("Message page row is missing its transcript segment."); + } + return { ...identity, kind, segment: Number(row.agentRunSegment), v: 2 }; + } + return { ...identity, kind, v: 1 }; +} + function decodePageCursor(value: string, expectedKind: PageCursorKind) { try { if (!/^[A-Za-z0-9_-]+$/u.test(value)) { @@ -626,46 +670,6 @@ function invalidRequestBody(message: string, error: z.ZodError): APIError { }); } -async function deleteProjectAgentState( - env: ProjectRouteEnv, - pathname: string, - body: string, - headers: Headers, -): Promise { - const response = await env.AGENT.fetch(`https://agent.internal${pathname}`, { - body, - headers, - method: "POST", - }); - if (!response.ok) { - await response.body?.cancel().catch(() => undefined); - throw new APIError(503, "unavailable_maintenance", "Project sandbox cleanup failed", { - details: { status: response.status }, - retriable: true, - }); - } - await response.body?.cancel().catch(() => undefined); -} - -async function readMaintenanceSecret(env: ProjectRouteEnv): Promise { - let value: string | undefined; - try { - value = await resolveWorkerSecret(env.INTERNAL_MAINTENANCE_SECRET); - } catch { - throw new APIError(503, "unavailable_maintenance", "Maintenance secret is unavailable", { - hint: "Verify INTERNAL_MAINTENANCE_SECRET on the gateway Worker.", - retriable: false, - }); - } - if (!value) { - throw new APIError(503, "unavailable_maintenance", "Maintenance secret is not configured", { - hint: "Set INTERNAL_MAINTENANCE_SECRET on the gateway Worker.", - retriable: false, - }); - } - return value; -} - function invalidPathParam(message: string, error: z.ZodError): APIError { return new APIError(400, "invalid_path_param", message, { details: { issues: error.issues.map((issue) => issue.message) }, diff --git a/apps/gateway-worker/src/provider-http-routes.ts b/apps/gateway-worker/src/provider-http-routes.ts index 953d836f..3bfdffa0 100644 --- a/apps/gateway-worker/src/provider-http-routes.ts +++ b/apps/gateway-worker/src/provider-http-routes.ts @@ -4,17 +4,11 @@ import { setProviderKey, validateProviderKey, } from "@cheatcode/byok"; -import { - createDb, - lockUserEntitlementMutations, - lockUserProviderKeyMutations, - withUserContext, -} from "@cheatcode/db"; +import { createDb, lockUserProviderKeyMutations, withUserContext } from "@cheatcode/db"; import { APIError, emitUserEvent, readJsonRequest } from "@cheatcode/observability"; import { ProviderSchema, ToolDomainSchema, UpsertProviderKeySchema } from "@cheatcode/types"; import { authenticate } from "./authenticate"; import type { GatewayApp, GatewayContext } from "./gateway-env"; -import { enforceByokProviderSlotLimit, resolveDatabaseEntitlement } from "./limits"; import { listAgentsRoute, listToolsRoute } from "./metadata-routes"; import { rateLimit } from "./rate-limit"; @@ -41,7 +35,10 @@ export function registerProviderHttpRoutes(app: GatewayApp): void { app.get("/v1/provider-keys", async (c) => { const userId = await authenticate(c.req.raw, c.env, c.executionCtx); await rateLimit(c, userId, "GET /v1/provider-keys"); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { return c.json(await withUserContext(db, userId, (tx) => listProviderKeys(tx))); } finally { @@ -66,14 +63,14 @@ async function upsertProviderKey(c: GatewayContext): Promise { } const input = parsedInput.data; await validateProviderKey(input.provider, input.key); - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const result = await withUserContext(db, userId, async (tx) => { - await lockUserEntitlementMutations(tx, userId); await lockUserProviderKeyMutations(tx, userId); - const entitlement = await resolveDatabaseEntitlement(tx, userId); const existingKeys = await listProviderKeys(tx); - enforceByokProviderSlotLimit(entitlement, input.provider, existingKeys); await setProviderKey(tx, input.provider, input.key); const keys = await listProviderKeys(tx); const summary = @@ -103,7 +100,10 @@ async function deleteProviderKeyRoute(c: GatewayContext): Promise { retriable: false, }); } - const { db, close } = createDb(c.env.HYPERDRIVE); + const { db, close } = createDb(c.env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: c.env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { await withUserContext(db, userId, async (tx) => { await lockUserProviderKeyMutations(tx, userId); diff --git a/apps/gateway-worker/src/release-health.ts b/apps/gateway-worker/src/release-health.ts new file mode 100644 index 00000000..47dd249f --- /dev/null +++ b/apps/gateway-worker/src/release-health.ts @@ -0,0 +1,87 @@ +import { APIError, readBoundedResponseJson } from "@cheatcode/observability"; +import { z } from "zod"; +import type { GatewayEnv } from "./gateway-env"; + +const MAX_RELEASE_HEALTH_RESPONSE_BYTES = 16 * 1024; +const DownstreamReleaseHealthSchema = z + .object({ + ok: z.literal(true), + releaseGate: z.enum(["closed", "draining", "open"]), + releaseSha: z.string().min(1), + versionId: z.string().min(1).nullable(), + worker: z.enum(["agent", "webhooks"]), + }) + .strict(); + +export type DownstreamWorker = z.infer["worker"]; +type DownstreamReleaseHealth = z.infer; + +export interface DownstreamReleaseHealthResult { + health: DownstreamReleaseHealth; + status: number; +} + +export async function readDownstreamReleaseHealth( + env: Pick, + worker: DownstreamWorker, +): Promise { + const response = await fetchHealth(env, worker); + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + throw unhealthyService(worker, response.status); + } + try { + const health = DownstreamReleaseHealthSchema.parse( + await readBoundedResponseJson( + response, + MAX_RELEASE_HEALTH_RESPONSE_BYTES, + `${serviceLabel(worker)} health`, + ), + ); + if (health.worker !== worker) { + throw new Error("Downstream health identified the wrong Worker"); + } + return { health, status: response.status }; + } catch { + throw new APIError( + 503, + "unavailable_maintenance", + `${serviceLabel(worker)} health response is invalid`, + { retriable: true }, + ); + } +} + +async function fetchHealth( + env: Pick, + worker: DownstreamWorker, +): Promise { + try { + const binding = worker === "agent" ? env.AGENT : env.WEBHOOKS; + return await binding.fetch( + new Request(`https://${worker}.internal/health`, { + signal: AbortSignal.timeout(3_000), + }), + ); + } catch { + throw new APIError( + 503, + "unavailable_maintenance", + `${serviceLabel(worker)} service is unavailable`, + { retriable: true }, + ); + } +} + +function unhealthyService(worker: DownstreamWorker, status: number): APIError { + return new APIError( + 503, + "unavailable_maintenance", + `${serviceLabel(worker)} service is unhealthy`, + { details: { status }, retriable: true }, + ); +} + +function serviceLabel(worker: DownstreamWorker): string { + return worker === "agent" ? "Agent" : "Webhooks"; +} diff --git a/apps/gateway-worker/src/resource-deletion-enqueue.ts b/apps/gateway-worker/src/resource-deletion-enqueue.ts new file mode 100644 index 00000000..c0c01ba7 --- /dev/null +++ b/apps/gateway-worker/src/resource-deletion-enqueue.ts @@ -0,0 +1,54 @@ +import { createInternalMaintenanceHeaders } from "@cheatcode/auth"; +import { APIError, readBoundedResponseJson } from "@cheatcode/observability"; +import { + INTERNAL_RESOURCE_DELETION_PATH, + type InternalResourceDeletionRequest, + InternalResourceDeletionRequestSchema, +} from "@cheatcode/types"; +import { z } from "zod"; +import { + type GatewayMaintenanceSecretBindings, + requireResourceDeletionSecret, +} from "./internal-maintenance"; + +const MAX_ENQUEUE_RESPONSE_BYTES = 16 * 1024; +const EnqueueResponseSchema = z + .object({ jobId: z.string().uuid().nullable(), ok: z.literal(true) }) + .strict(); + +export interface ResourceDeletionEnqueueEnv extends GatewayMaintenanceSecretBindings { + WEBHOOKS: Fetcher; +} + +export async function enqueueResourceDeletion( + env: ResourceDeletionEnqueueEnv, + input: InternalResourceDeletionRequest, +): Promise { + const rawBody = JSON.stringify(InternalResourceDeletionRequestSchema.parse(input)); + const headers = await createInternalMaintenanceHeaders({ + audience: "webhooks", + capability: "resource-deletion", + issuer: "gateway", + method: "POST", + pathname: INTERNAL_RESOURCE_DELETION_PATH, + rawBody, + secret: await requireResourceDeletionSecret(env), + }); + headers.set("content-type", "application/json"); + const response = await env.WEBHOOKS.fetch( + `https://webhooks.internal${INTERNAL_RESOURCE_DELETION_PATH}`, + { body: rawBody, headers, method: "POST" }, + ); + if (!response.ok) { + const status = response.status; + await response.body?.cancel().catch(() => undefined); + throw new APIError(503, "unavailable_maintenance", "Resource deletion enqueue failed", { + details: { status }, + retriable: true, + }); + } + const result = EnqueueResponseSchema.parse( + await readBoundedResponseJson(response, MAX_ENQUEUE_RESPONSE_BYTES, "Webhooks Worker"), + ); + return result.jobId; +} diff --git a/apps/gateway-worker/src/search-routes.ts b/apps/gateway-worker/src/search-routes.ts index cd8fe14e..e02201fe 100644 --- a/apps/gateway-worker/src/search-routes.ts +++ b/apps/gateway-worker/src/search-routes.ts @@ -6,6 +6,7 @@ import { type WorkspaceThreadSearchRecord, withUserContext, } from "@cheatcode/db"; +import type { WorkerSecret } from "@cheatcode/env"; import { APIError, createLogger } from "@cheatcode/observability"; import { RecentThreadsQuerySchema, @@ -19,6 +20,7 @@ import type { z } from "zod"; import type { WaitUntilContext } from "./wait-until-context"; export interface SearchRouteEnv { + DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY: WorkerSecret; HYPERDRIVE: Hyperdrive; } @@ -30,7 +32,10 @@ export async function searchWorkspaceRoute( ): Promise { const query = parseSearchQuery(request); const startedAt = performance.now(); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const records = await withUserContext(db, userId, (tx) => searchWorkspace(tx, userId, { limit: query.limit, q: query.q }), @@ -52,7 +57,10 @@ export async function listRecentThreadsRoute( userId: UserId, ): Promise { const limit = parseRecentThreadsLimit(request); - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const records = await withUserContext(db, userId, (tx) => listRecentThreads(tx, userId, limit)); const response = RecentThreadsResponseSchema.parse({ diff --git a/apps/gateway-worker/src/skills-routes.ts b/apps/gateway-worker/src/skills-routes.ts index 621c79b4..af0e4472 100644 --- a/apps/gateway-worker/src/skills-routes.ts +++ b/apps/gateway-worker/src/skills-routes.ts @@ -1,25 +1,13 @@ import { createDb, - deleteUserSkill, listUserSkillSummaries, - UserSkillLimitExceededError, type UserSkillSummaryRecord, - upsertUserSkill, withUserContext, } from "@cheatcode/db"; -import { APIError, readJsonRequest } from "@cheatcode/observability"; -import { - CreateUserSkillSchema, - type UserId, - UserSkillSchema, - UserSkillsResponseSchema, -} from "@cheatcode/types"; -import { z } from "zod"; +import { type UserId, UserSkillSchema, UserSkillsResponseSchema } from "@cheatcode/types"; import type { GatewayEnv } from "./gateway-env"; import type { WaitUntilContext } from "./wait-until-context"; -const IdParamSchema = z.string().uuid(); -const MAX_SKILL_REQUEST_BYTES = 64 * 1024; function skillSummary(record: UserSkillSummaryRecord): unknown { return UserSkillSchema.parse({ category: record.category, @@ -38,7 +26,10 @@ export async function listUserSkillsRoute( ctx: WaitUntilContext, userId: UserId, ): Promise { - const { db, close } = createDb(env.HYPERDRIVE); + const { db, close } = createDb(env.HYPERDRIVE, { + audience: "app_gateway", + signingSecret: env.DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY, + }); try { const rows = await withUserContext(db, userId, (tx) => listUserSkillSummaries(tx, userId)); return Response.json(UserSkillsResponseSchema.parse({ skills: rows.map(skillSummary) })); @@ -46,67 +37,3 @@ export async function listUserSkillsRoute( ctx.waitUntil(close()); } } - -/** - * `POST /v1/skills` — create or update (by name) a custom skill. Used by the agent's - * `skill_create` tool path and the manual creation form. - */ -export async function createUserSkillRoute( - env: GatewayEnv, - ctx: WaitUntilContext, - request: Request, - userId: UserId, -): Promise { - const parsed = CreateUserSkillSchema.safeParse( - await readJsonRequest(request, MAX_SKILL_REQUEST_BYTES, "Skill request"), - ); - if (!parsed.success) { - throw new APIError(400, "invalid_request_body", "Invalid skill payload", { - details: { issues: parsed.error.issues.map((issue) => issue.message) }, - retriable: false, - }); - } - const input = parsed.data; - const { db, close } = createDb(env.HYPERDRIVE); - try { - const record = await withUserContext(db, userId, (tx) => - upsertUserSkill(tx, { ...input, userId }), - ); - return Response.json(skillSummary(record), { status: 201 }); - } catch (error) { - if (error instanceof UserSkillLimitExceededError) { - throw new APIError(409, "conflict_state_invalid", error.message, { - hint: "Delete an existing custom skill before creating another.", - retriable: false, - }); - } - throw error; - } finally { - ctx.waitUntil(close()); - } -} - -/** `DELETE /v1/skills/:id` — soft-delete a custom skill the caller owns. */ -export async function deleteUserSkillRoute( - env: GatewayEnv, - ctx: WaitUntilContext, - userId: UserId, - skillId: string, -): Promise { - const parsed = IdParamSchema.safeParse(skillId); - if (!parsed.success) { - throw new APIError(400, "invalid_path_param", "Invalid skill id", { retriable: false }); - } - const { db, close } = createDb(env.HYPERDRIVE); - try { - const deleted = await withUserContext(db, userId, (tx) => - deleteUserSkill(tx, userId, parsed.data), - ); - if (!deleted) { - throw new APIError(404, "not_found_skill", "Skill not found", { retriable: false }); - } - return new Response(null, { status: 204 }); - } finally { - ctx.waitUntil(close()); - } -} diff --git a/apps/gateway-worker/wrangler.jsonc b/apps/gateway-worker/wrangler.jsonc index e30f22c7..de8fc933 100644 --- a/apps/gateway-worker/wrangler.jsonc +++ b/apps/gateway-worker/wrangler.jsonc @@ -4,12 +4,11 @@ "workers_dev": false, "preview_urls": false, "main": "src/index.ts", - "compatibility_date": "2026-05-20", + "compatibility_date": "2026-07-15", "compatibility_flags": ["nodejs_compat"], - "version_metadata": { "binding": "CF_VERSION_METADATA" }, - // Polar product ids per paid tier — non-secret config (they appear in checkout URLs), - // bound as plain vars. Ultra/Max have no Polar product yet, so those tiers 503 until - // the owner creates the products and adds their ids here. + "version_metadata": { + "binding": "CF_VERSION_METADATA" + }, "vars": { "CHEATCODE_ENVIRONMENT": "production", "CHEATCODE_RELEASE_GATE": "open", @@ -27,9 +26,18 @@ { "binding": "AGENT", "service": "cheatcode-agent" + }, + { + "binding": "WEBHOOKS", + "service": "cheatcode-webhooks" } ], "secrets_store_secrets": [ + { + "binding": "DATABASE_CONTEXT_SIGNING_SECRET_GATEWAY", + "store_id": "ba25994718db4707ab99a498e22eb5a6", + "secret_name": "database-context-signing-secret-gateway" + }, { "binding": "CLERK_SECRET_KEY", "store_id": "ba25994718db4707ab99a498e22eb5a6", @@ -51,9 +59,14 @@ "secret_name": "composio-auth-configs" }, { - "binding": "INTERNAL_MAINTENANCE_SECRET", + "binding": "GATEWAY_TO_WEBHOOKS_RESOURCE_DELETION_SECRET", + "store_id": "ba25994718db4707ab99a498e22eb5a6", + "secret_name": "gateway-to-webhooks-resource-deletion-secret" + }, + { + "binding": "RELEASE_DATABASE_READINESS_SECRET", "store_id": "ba25994718db4707ab99a498e22eb5a6", - "secret_name": "internal-maintenance-secret" + "secret_name": "release-database-readiness-secret" } ], "durable_objects": { @@ -82,8 +95,7 @@ "hyperdrive": [ { "binding": "HYPERDRIVE", - "id": "b7cead054a6a4207a475b9544971f04a", - "localConnectionString": "postgresql://app_worker:app_worker@localhost:54322/postgres" + "id": "67d13808cb3548ea942aec0f4d569400" } ], "migrations": [ diff --git a/apps/preview-proxy/.dev.vars.example b/apps/preview-proxy/.dev.vars.example deleted file mode 100644 index dba1c9bf..00000000 --- a/apps/preview-proxy/.dev.vars.example +++ /dev/null @@ -1,5 +0,0 @@ -# Daytona control-plane API key (resolves the per-sandbox preview origin + token). -DAYTONA_API_KEY= -# Shared HMAC secret for the preview access-token contract (must match the -# value the agent-worker uses to MINT cc_pt tokens). -PREVIEW_TOKEN_SECRET= diff --git a/apps/preview-proxy/README.md b/apps/preview-proxy/README.md index a259cd3a..7dd5890c 100644 --- a/apps/preview-proxy/README.md +++ b/apps/preview-proxy/README.md @@ -48,7 +48,12 @@ additionally requires: Any failure returns `401` (no redirect). A `handoff` capability is accepted only from `__cc_pt`, expires after at most 60 seconds, and is accepted only on GET/HEAD navigation/session-exchange requests. A `session` capability is accepted only -from the `__Host-cc_pt` cookie and expires after at most 10 minutes. +from the environment's host-only cookie and expires after at most 10 minutes. +Production uses `__Host-cc_pt`; local HTTP uses the dev-only `cc_pt` name because +Chrome does not accept the `__Host-` prefix over local HTTP. Both cookies are +`Secure; SameSite=None; Partitioned`: the documented local app origin is +`127.0.0.1`, while isolated preview hosts are beneath `localhost`, so the local +iframe intentionally exercises the same cross-site cookie boundary as production. ### Cookie hand-off @@ -104,7 +109,7 @@ depend on the code-execution tool package. - Forwards method, body (streamed), and headers to `{originUrl}{path+search}`. - On the dedicated code-server port only, buffers at most 4 MiB of workbench HTML and injects the shared parent-frame bridge. The bridge accepts messages - only from `https://trycheatcode.com` and posts state only to that exact origin; + only from `CHEATCODE_APP_ORIGIN` and posts state only to that exact origin; arbitrary generated-app HTML remains streamed and unmodified. - Injects: - `x-daytona-preview-token: ` @@ -125,7 +130,7 @@ depend on the code-execution tool package. - Rejects browser requests that rely on the session cookie when `Origin` or Fetch Metadata identifies a different preview origin. Cookie-authenticated iframe navigations additionally require a referrer from the exact preview - origin or `https://trycheatcode.com`; this blocks sibling-preview navigation + origin or `CHEATCODE_APP_ORIGIN`; this blocks sibling-preview navigation attacks before sandbox code handles a state-changing GET. The Vercel iframes use `referrerpolicy="origin"`, so the query-to-cookie redirect retains that trusted signal without disclosing an app path or query. This check runs before @@ -137,7 +142,7 @@ depend on the code-execution tool package. - Marks authenticated HTML `private, no-store`, makes other responses private while preserving browser-cache directives, and varies responses by the cookie/origin/Fetch-Metadata inputs used at the boundary. -- Adds `frame-ancestors 'self' https://trycheatcode.com`, +- Adds `frame-ancestors 'self' `, `Origin-Agent-Cluster: ?1`, and `X-Robots-Tag: noindex, nofollow` to non-WS responses. OAC prevents `document.domain` relaxation when the browser honors it, but it is a browser hint rather than a complete security boundary. @@ -169,6 +174,7 @@ released together. | Binding | Type | Source | | --------------------- | --------------- | --------------------------------------- | +| `CHEATCODE_APP_ORIGIN` | var | exact trusted Vercel/local app origin | | `CHEATCODE_ENVIRONMENT` | var | `wrangler.jsonc` (`production`) | | `CHEATCODE_RELEASE_SHA` | release var | guarded deploy command | | `CF_VERSION_METADATA` | version metadata | Cloudflare runtime | @@ -179,16 +185,39 @@ released together. | `PREVIEW_TOKEN_SECRET`| Secrets Store | `preview-token-secret` | Secrets bind from store `ba25994718db4707ab99a498e22eb5a6` (shared with -`agent-worker`). Local dev: copy `.dev.vars.example` to `.dev.vars`. Secrets are -resolved request-scoped via `resolveWorkerSecret`; the token and API key are -never logged. +`agent-worker`). Local development reads the same bindings from the root, +git-ignored `.env.local`; there is no per-Worker credential file. Secrets are +resolved request-scoped via `resolveWorkerSecret`; the token and API key are never +logged. + +Root `pnpm dev` runs this Worker as the fourth member of the chained Wrangler +process. The gateway's generated local-only Service Binding routes +`*.localhost:8787` HTTP and WebSocket traffic here; a path-form handoff is first +redirected to that canonical local origin so the session cookie is scoped +correctly. The redirect preserves only an origin referrer, which is the trusted +navigation signal required by the cookie-authenticated follow-up and cannot +disclose the handoff path or token. No preview domain or cloud development +deployment is required. Optional Analytics Engine bindings `ERROR_EVENTS` and `PERFORMANCE_METRICS` feed the shared `@cheatcode/observability` emitters. ## DNS / route setup -- Route: `*.${PREVIEW_HOSTNAME}/*` on the preview domain's Cloudflare zone. +- Worker route: `*.${PREVIEW_HOSTNAME}/*` points only to + `cheatcode-preview-proxy`. Cloudflare's most-specific-route rule lets the exact + gateway and webhooks routes override it. +- Exact no-script routes for `clerk.trycheatcode.com/*`, + `docs.trycheatcode.com/*`, and `www.trycheatcode.com/*` negate the wildcard + for Clerk, documentation, and the Vercel frontend hostname. + `preview.trycheatcode.com` deliberately has no exact route and inherits the + preview wildcard for release health checks. +- [`infra/cloudflare/production-route-contract.json`](../../infra/cloudflare/production-route-contract.json) + is the production contract. `stage-closed` creates only a missing exact + no-script route through the Workers Routes API before deploying the wildcard; + it never updates or deletes an existing route. Conflicts, semantic duplicates, + and overlapping wildcard routes stop the release. The contract is checked + again after preview deployment and before the gateway can reopen. - DNS: a **proxied** wildcard record `*` -> the zone (orange-cloud) so Cloudflare terminates TLS and runs this Worker for every sub-subdomain. - TLS: the wildcard is one label deep beyond the configured apex; Cloudflare @@ -196,6 +225,11 @@ the shared `@cheatcode/observability` emitters. no extra certificate is required. (If preview hosts ever gain another label, Advanced Certificate Manager / a custom cert would be needed.) +The production Cloudflare token needs Zone Read plus Workers Routes Read and +Write. This follows Cloudflare's documented +[route matching and no-script negation](https://developers.cloudflare.com/workers/configuration/routing/routes/#matching-behavior) +and the [Workers Routes API](https://developers.cloudflare.com/api/resources/workers/subresources/routes/). + ## Code checks ```bash diff --git a/apps/preview-proxy/package.json b/apps/preview-proxy/package.json index 7dfa4654..60513585 100644 --- a/apps/preview-proxy/package.json +++ b/apps/preview-proxy/package.json @@ -7,7 +7,7 @@ "types": "./src/index.ts", "scripts": { "build": "wrangler deploy --dry-run", - "dev": "wrangler dev --var CHEATCODE_ENVIRONMENT:development", + "deploy": "wrangler deploy", "lint": "biome check .", "typecheck": "tsc -p tsconfig.json --noEmit" }, diff --git a/apps/preview-proxy/src/env.ts b/apps/preview-proxy/src/env.ts index dd8adab1..8814d145 100644 --- a/apps/preview-proxy/src/env.ts +++ b/apps/preview-proxy/src/env.ts @@ -16,6 +16,7 @@ import { z } from "zod"; export const PreviewProxyEnvSchema = z .object({ ...WorkerReleaseBindingsSchema, + CHEATCODE_APP_ORIGIN: z.string().url(), DAYTONA_API_KEY: WorkerSecretSchema, DAYTONA_API_URL: z.string().url(), DAYTONA_PREVIEW_HOST_SUFFIXES: z.string().min(1).max(1_024).optional(), @@ -30,6 +31,23 @@ export const PreviewProxyEnvSchema = z path: ["CHEATCODE_RELEASE_SHA"], }); } + if ( + env.CHEATCODE_ENVIRONMENT === "production" && + env.CHEATCODE_APP_ORIGIN !== "https://trycheatcode.com" + ) { + context.addIssue({ + code: "custom", + message: "Production previews require the canonical Vercel application origin", + path: ["CHEATCODE_APP_ORIGIN"], + }); + } + if (!isExactAppOrigin(env.CHEATCODE_APP_ORIGIN, env.CHEATCODE_ENVIRONMENT)) { + context.addIssue({ + code: "custom", + message: "Preview application origin must be an exact trusted HTTP(S) origin", + path: ["CHEATCODE_APP_ORIGIN"], + }); + } if (env.CHEATCODE_ENVIRONMENT === "production" && env.PREVIEW_HOSTNAME.includes(":")) { context.addIssue({ code: "custom", @@ -48,6 +66,7 @@ export const PreviewProxyEnvSchema = z export interface PreviewProxyEnv extends AnalyticsBindings { CF_VERSION_METADATA?: CloudflareVersionMetadata; + CHEATCODE_APP_ORIGIN: string; CHEATCODE_ENVIRONMENT: "development" | "production"; CHEATCODE_RELEASE_SHA?: string; DAYTONA_API_KEY: WorkerSecret; @@ -56,3 +75,27 @@ export interface PreviewProxyEnv extends AnalyticsBindings { PREVIEW_HOSTNAME: string; PREVIEW_TOKEN_SECRET: WorkerSecret; } + +function isExactAppOrigin(value: string, environment: "development" | "production"): boolean { + try { + const url = new URL(value); + if ( + url.origin !== value || + url.pathname !== "/" || + url.search || + url.hash || + url.username || + url.password + ) { + return false; + } + if (environment === "production") { + return url.protocol === "https:"; + } + return ( + url.protocol === "http:" && (url.hostname === "localhost" || url.hostname === "127.0.0.1") + ); + } catch { + return false; + } +} diff --git a/apps/preview-proxy/src/index.ts b/apps/preview-proxy/src/index.ts index ac013911..b15284f6 100644 --- a/apps/preview-proxy/src/index.ts +++ b/apps/preview-proxy/src/index.ts @@ -11,14 +11,17 @@ import { import { type PreviewProxyEnv, PreviewProxyEnvSchema } from "./env"; import { type PreviewTarget, parsePreviewHost } from "./host"; import { - CHEATCODE_APP_ORIGIN, - PREVIEW_SESSION_COOKIE, PREVIEW_SESSION_PATH, PREVIEW_TOKEN_QUERY, + previewSessionCookieName, } from "./preview-session"; import { proxyPreviewRequest } from "./proxy"; import { assertPreviewRequestContext } from "./request-context"; -import { authorizePreviewRequest, mintPreviewSessionToken } from "./token"; +import { + authorizePreviewRequest, + type MintedPreviewSession, + mintPreviewSessionToken, +} from "./token"; const WORKER_NAME = "preview-proxy"; const SECURITY_VARY_HEADERS = [ @@ -30,7 +33,6 @@ const SECURITY_VARY_HEADERS = [ ]; async function handlePreviewRequest(request: Request, env: PreviewProxyEnv): Promise { - PreviewProxyEnvSchema.parse(env); const url = new URL(request.url); const originalHost = url.host; if (request.method === "GET" && url.pathname === "/health") { @@ -42,16 +44,25 @@ async function handlePreviewRequest(request: Request, env: PreviewProxyEnv): Pro audience: originalHost, request, secret, + sessionCookieName: previewSessionCookieName(env.CHEATCODE_ENVIRONMENT), target, url, }); - assertPreviewRequestContext({ fromQuery: authorized.fromQuery, request, url }); + assertPreviewRequestContext({ + fromQuery: authorized.fromQuery, + request, + trustedAppOrigin: env.CHEATCODE_APP_ORIGIN, + trustedPreviewOrigin: `${url.protocol}//${authorized.verified.audience}`, + url, + }); assertNavigationHandoff(authorized.fromQuery, request.method); // Exchange a query token for a host-only cookie before sandbox content runs so the // credential leaves the visible URL and can never reach the origin. url.searchParams.delete(PREVIEW_TOKEN_QUERY); const session = await mintHandoffSession(authorized.fromQuery, originalHost, secret, target); - const setCookie = session ? buildSessionCookie(session.token, session.expiresAt) : undefined; + const setCookie = session + ? buildSessionCookie(session.token, session.expiresAt, env.CHEATCODE_ENVIRONMENT) + : undefined; if (url.pathname === PREVIEW_SESSION_PATH) { if (!setCookie) { throw new APIError(400, "invalid_request_body", "Preview session refresh requires a token", { @@ -80,8 +91,11 @@ async function mintHandoffSession( audience: string, secret: string, target: PreviewTarget, -) { - return isFromQuery ? mintPreviewSessionToken({ audience, secret, target }) : undefined; +): Promise { + if (!isFromQuery) { + return undefined; + } + return await mintPreviewSessionToken({ audience, secret, target }); } function assertNavigationHandoff(isFromQuery: boolean, method: string): void { @@ -146,11 +160,17 @@ function previewSessionRedirect(url: URL, setCookie: string): Response { }); } -function buildSessionCookie(token: string, exp: number): string { +function buildSessionCookie( + token: string, + exp: number, + environment: PreviewProxyEnv["CHEATCODE_ENVIRONMENT"], +): string { const maxAgeSeconds = Math.max(0, Math.floor((exp - Date.now()) / 1000)); - // Partition the third-party iframe credential by the top-level app site. The - // __Host- prefix and Secure/Path attributes keep it host-only. - return `${PREVIEW_SESSION_COOKIE}=${token}; HttpOnly; Secure; SameSite=None; Partitioned; Path=/; Max-Age=${maxAgeSeconds}`; + const name = previewSessionCookieName(environment); + // Partition the iframe credential by the top-level app site. Development uses + // a non-prefixed name because Chrome rejects __Host- over local HTTP; both + // environments retain the same host-only, cross-site isolation attributes. + return `${name}=${token}; HttpOnly; Secure; SameSite=None; Partitioned; Path=/; Max-Age=${maxAgeSeconds}`; } function requestId(): string { @@ -167,14 +187,14 @@ function withRequestId(response: Response, id: string): Response { return wrapped; } -function withPreviewSecurityHeaders(response: Response): Response { +function withPreviewSecurityHeaders(response: Response, appOrigin?: string): Response { if (response.status === 101 || response.webSocket) { return response; } const wrapped = new Response(response.body, response); wrapped.headers.append( "Content-Security-Policy", - `frame-ancestors 'self' ${CHEATCODE_APP_ORIGIN}`, + appOrigin ? `frame-ancestors 'self' ${appOrigin}` : "frame-ancestors 'none'", ); wrapped.headers.set("Origin-Agent-Cluster", "?1"); wrapped.headers.set("X-Robots-Tag", "noindex, nofollow"); @@ -205,6 +225,28 @@ function routeName(request: Request): string { return `${request.method} ${new URL(request.url).pathname}`; } +function requestContextTelemetry(request: Request) { + return { + fetchDestination: request.headers.get("Sec-Fetch-Dest") ?? "missing", + fetchMode: request.headers.get("Sec-Fetch-Mode") ?? "missing", + fetchSite: request.headers.get("Sec-Fetch-Site") ?? "missing", + origin: safeHeaderOrigin(request.headers.get("Origin")), + referrerOrigin: safeHeaderOrigin(request.headers.get("Referer")), + requestOrigin: new URL(request.url).origin, + }; +} + +function safeHeaderOrigin(value: string | null): string { + if (!value) { + return "missing"; + } + try { + return new URL(value).origin; + } catch { + return "invalid"; + } +} + function statusClass(status: number): string { if (status >= 500) { return "5xx"; @@ -223,9 +265,13 @@ const previewProxyHandler = { const id = requestId(); const startedAt = performance.now(); let status = 500; + let appOrigin: string | undefined; try { + PreviewProxyEnvSchema.parse(env); + appOrigin = env.CHEATCODE_APP_ORIGIN; const response = withPreviewSecurityHeaders( withRequestId(await handlePreviewRequest(request, env), id), + appOrigin, ); status = response.status; return response; @@ -243,9 +289,10 @@ const previewProxyHandler = { createLogger({ requestId: id }).error("preview_proxy_request_failed", { apiCode: apiError.code, httpStatus: apiError.status, + ...(apiError.code === "permission_denied" ? requestContextTelemetry(request) : {}), ...safeErrorTelemetry(error), }); - return withPreviewSecurityHeaders(apiError.toResponse(id)); + return withPreviewSecurityHeaders(apiError.toResponse(id), appOrigin); } finally { emitPerformanceMetric(env, { route: WORKER_NAME, diff --git a/apps/preview-proxy/src/preview-session.ts b/apps/preview-proxy/src/preview-session.ts index 4965c356..89c47a01 100644 --- a/apps/preview-proxy/src/preview-session.ts +++ b/apps/preview-proxy/src/preview-session.ts @@ -1,4 +1,19 @@ -export const CHEATCODE_APP_ORIGIN = "https://trycheatcode.com"; -export const PREVIEW_SESSION_COOKIE = "__Host-cc_pt"; +const DEVELOPMENT_PREVIEW_SESSION_COOKIE = "cc_pt"; +const PRODUCTION_PREVIEW_SESSION_COOKIE = "__Host-cc_pt"; + export const PREVIEW_SESSION_PATH = "/.well-known/cheatcode-preview-session"; export const PREVIEW_TOKEN_QUERY = "__cc_pt"; + +export function previewSessionCookieName(environment: "development" | "production"): string { + return environment === "production" + ? PRODUCTION_PREVIEW_SESSION_COOKIE + : DEVELOPMENT_PREVIEW_SESSION_COOKIE; +} + +export function isReservedPreviewCookieName(value: string): boolean { + const normalized = value.toLowerCase(); + return ( + normalized === DEVELOPMENT_PREVIEW_SESSION_COOKIE.toLowerCase() || + normalized === PRODUCTION_PREVIEW_SESSION_COOKIE.toLowerCase() + ); +} diff --git a/apps/preview-proxy/src/proxy.ts b/apps/preview-proxy/src/proxy.ts index 78616304..65bf4e59 100644 --- a/apps/preview-proxy/src/proxy.ts +++ b/apps/preview-proxy/src/proxy.ts @@ -12,11 +12,7 @@ import { refreshPreviewOriginAfterAuthFailure, resolvePreviewOrigin, } from "./origin"; -import { - CHEATCODE_APP_ORIGIN, - PREVIEW_SESSION_COOKIE, - PREVIEW_TOKEN_QUERY, -} from "./preview-session"; +import { isReservedPreviewCookieName, PREVIEW_TOKEN_QUERY } from "./preview-session"; import { relayPreviewWebSocket } from "./websocket-relay"; /** @@ -177,7 +173,7 @@ async function transformCodeServerResponse( "Code-server HTML", ); const body = isCodeServerWorkbenchHtml(html) - ? injectCodeServerParentBridge(html, CHEATCODE_APP_ORIGIN) + ? injectCodeServerParentBridge(html, input.env.CHEATCODE_APP_ORIGIN) : html; const headers = new Headers(response.headers); headers.delete("Content-Encoding"); @@ -198,7 +194,7 @@ async function forwardWebSocket(input: ProxyInput, origin: PreviewOrigin): Promi wsRequest.headers.set(DAYTONA_TOKEN_HEADER, origin.token); wsRequest.headers.set(DAYTONA_SKIP_WARNING_HEADER, "true"); setCanonicalForwardingHeaders(wsRequest.headers, input.originalHost, input.url.protocol); - wsRequest.headers.set("Origin", new URL(origin.url).origin); + wsRequest.headers.set("Origin", websocketOrigin(input, origin)); const response = await fetch(wsRequest); if (response.webSocket) { return relayPreviewWebSocket( @@ -210,6 +206,16 @@ async function forwardWebSocket(input: ProxyInput, origin: PreviewOrigin): Promi return buildClientResponse(response, input, origin); } +function websocketOrigin(input: ProxyInput, origin: PreviewOrigin): string { + if (!isCodeServerRequest(input)) { + return new URL(origin.url).origin; + } + // Code Server validates WebSocket origins against the public preview host + // configured when the sandbox starts. Replacing that origin with Daytona's + // private upstream host makes the otherwise valid workbench socket fail 403. + return `${input.url.protocol}//${input.originalHost}`; +} + function buildForwardHeaders( source: Headers, origin: PreviewOrigin, @@ -326,7 +332,7 @@ function sanitizeOriginCookie(cookie: string): string | null { const first = parts.shift()?.trim(); const separator = first?.indexOf("=") ?? -1; const name = separator > 0 && first ? first.slice(0, separator).trim() : ""; - if (!validOriginCookieName(name) || name.toLowerCase() === PREVIEW_SESSION_COOKIE.toLowerCase()) { + if (!validOriginCookieName(name) || isReservedPreviewCookieName(name)) { return null; } const attributes = parts.filter((part) => !/^\s*domain\s*=/iu.test(part)); diff --git a/apps/preview-proxy/src/request-context.ts b/apps/preview-proxy/src/request-context.ts index 3f6c9ca1..7c06913e 100644 --- a/apps/preview-proxy/src/request-context.ts +++ b/apps/preview-proxy/src/request-context.ts @@ -1,5 +1,4 @@ import { APIError } from "@cheatcode/observability"; -import { CHEATCODE_APP_ORIGIN } from "./preview-session"; const NAVIGATION_DESTINATIONS = new Set(["document", "frame", "iframe"]); @@ -13,17 +12,33 @@ const NAVIGATION_DESTINATIONS = new Set(["document", "frame", "iframe"]); export function assertPreviewRequestContext(input: { fromQuery: boolean; request: Request; + trustedAppOrigin: string; + trustedPreviewOrigin: string; url: URL; }): void { if (input.fromQuery) { return; } + const fetchSite = input.request.headers.get("Sec-Fetch-Site")?.toLowerCase(); + // Fetch Metadata is browser-controlled. Checking it before Origin avoids false + // denials when a local service binding rewrites standard origin headers while + // preserving the browser's same-origin classification. + if (fetchSite === "same-origin" || fetchSite === "none") { + return; + } const origin = input.request.headers.get("Origin"); - if (origin && origin !== input.url.origin) { + if (origin && !isTrustedPreviewOrigin(origin, input)) { throw crossOriginDenied(); } - const referrerOrigin = readTrustedReferrerOrigin(input.request, input.url); - const fetchSite = input.request.headers.get("Sec-Fetch-Site")?.toLowerCase(); + if (origin) { + return; + } + const referrerOrigin = readTrustedReferrerOrigin( + input.request, + input.url, + input.trustedAppOrigin, + input.trustedPreviewOrigin, + ); if (!fetchSite) { // Older/non-browser clients do not always send Fetch Metadata. Fail closed // unless another browser-controlled same-origin/trusted-app signal exists. @@ -32,31 +47,44 @@ export function assertPreviewRequestContext(input: { } throw crossOriginDenied(); } - if (fetchSite === "same-origin" || fetchSite === "none" || origin) { + if (isTrustedNavigation(input.request, fetchSite, referrerOrigin)) { return; } - const mode = input.request.headers.get("Sec-Fetch-Mode")?.toLowerCase(); - const destination = input.request.headers.get("Sec-Fetch-Dest")?.toLowerCase() ?? ""; - if ( + throw crossOriginDenied(); +} + +function isTrustedNavigation( + request: Request, + fetchSite: string, + referrerOrigin: string | null, +): boolean { + const mode = request.headers.get("Sec-Fetch-Mode")?.toLowerCase(); + const destination = request.headers.get("Sec-Fetch-Dest")?.toLowerCase() ?? ""; + return ( + Boolean(referrerOrigin) && (fetchSite === "same-site" || fetchSite === "cross-site") && mode === "navigate" && NAVIGATION_DESTINATIONS.has(destination) - ) { - if (referrerOrigin) { - return; - } - } - throw crossOriginDenied(); + ); } -function readTrustedReferrerOrigin(request: Request, url: URL): string | null { +function readTrustedReferrerOrigin( + request: Request, + url: URL, + trustedAppOrigin: string, + trustedPreviewOrigin: string, +): string | null { const referer = request.headers.get("Referer"); if (!referer) { return null; } try { const referrerOrigin = new URL(referer).origin; - if (referrerOrigin === url.origin || referrerOrigin === CHEATCODE_APP_ORIGIN) { + if ( + referrerOrigin === url.origin || + referrerOrigin === trustedPreviewOrigin || + referrerOrigin === trustedAppOrigin + ) { return referrerOrigin; } } catch { @@ -65,6 +93,13 @@ function readTrustedReferrerOrigin(request: Request, url: URL): string | null { throw crossOriginDenied(); } +function isTrustedPreviewOrigin( + origin: string, + input: Pick[0], "trustedPreviewOrigin" | "url">, +): boolean { + return origin === input.url.origin || origin === input.trustedPreviewOrigin; +} + function crossOriginDenied(): APIError { return new APIError(403, "permission_denied", "Cross-origin preview request denied", { retriable: false, diff --git a/apps/preview-proxy/src/token.ts b/apps/preview-proxy/src/token.ts index c6673c77..202973b3 100644 --- a/apps/preview-proxy/src/token.ts +++ b/apps/preview-proxy/src/token.ts @@ -2,12 +2,13 @@ import { mintPreviewCapability, PreviewCapabilityError, type PreviewCapabilityKind, + readCookieValue, type VerifiedPreviewCapability, verifyPreviewCapability, } from "@cheatcode/auth"; import { APIError } from "@cheatcode/observability"; import type { PreviewTarget } from "./host"; -import { PREVIEW_SESSION_COOKIE, PREVIEW_TOKEN_QUERY } from "./preview-session"; +import { PREVIEW_TOKEN_QUERY } from "./preview-session"; interface PreviewTokenSource { readonly kind: PreviewCapabilityKind; @@ -25,12 +26,16 @@ export interface MintedPreviewSession { } /** Query credentials are handoffs; host-only cookie credentials are sessions. */ -function readPreviewToken(request: Request, url: URL): PreviewTokenSource | null { +function readPreviewToken( + request: Request, + url: URL, + sessionCookieName: string, +): PreviewTokenSource | null { const queryToken = url.searchParams.get(PREVIEW_TOKEN_QUERY); if (queryToken) { return { kind: "handoff", token: queryToken }; } - const cookieToken = readCookie(request.headers.get("Cookie"), PREVIEW_SESSION_COOKIE); + const cookieToken = readCookieValue(request.headers.get("Cookie"), sessionCookieName); if (cookieToken) { return { kind: "session", token: cookieToken }; } @@ -42,10 +47,11 @@ export async function authorizePreviewRequest(input: { audience: string; request: Request; secret: string; + sessionCookieName: string; target: PreviewTarget; url: URL; }): Promise { - const source = readPreviewToken(input.request, input.url); + const source = readPreviewToken(input.request, input.url, input.sessionCookieName); if (!source) { throw new APIError(401, "auth_token_missing", "Missing preview access token", { retriable: false, @@ -115,20 +121,3 @@ function invalidToken(): APIError { retriable: false, }); } - -function readCookie(cookieHeader: string | null, name: string): string | null { - if (!cookieHeader) { - return null; - } - for (const cookie of cookieHeader.split(";")) { - const trimmed = cookie.trim(); - const separatorIndex = trimmed.indexOf("="); - if (separatorIndex === -1) { - continue; - } - if (trimmed.slice(0, separatorIndex) === name) { - return trimmed.slice(separatorIndex + 1); - } - } - return null; -} diff --git a/apps/preview-proxy/wrangler.jsonc b/apps/preview-proxy/wrangler.jsonc index 94cdd767..b492d072 100644 --- a/apps/preview-proxy/wrangler.jsonc +++ b/apps/preview-proxy/wrangler.jsonc @@ -4,7 +4,7 @@ "workers_dev": false, "preview_urls": false, "main": "src/index.ts", - "compatibility_date": "2026-05-20", + "compatibility_date": "2026-07-15", "compatibility_flags": ["nodejs_compat"], "version_metadata": { "binding": "CF_VERSION_METADATA" }, "routes": [ @@ -14,6 +14,7 @@ } ], "vars": { + "CHEATCODE_APP_ORIGIN": "https://trycheatcode.com", "CHEATCODE_ENVIRONMENT": "production", "DAYTONA_API_URL": "https://app.daytona.io/api", "DAYTONA_PREVIEW_HOST_SUFFIXES": "daytonaproxy01.net,proxy.daytona.work", diff --git a/apps/web/README.md b/apps/web/README.md index 004c0610..f9c68032 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -4,6 +4,15 @@ Next.js 16 app shell with Clerk auth and AI SDK chat streaming. Production runs The Settings Billing panel consumes gateway billing state directly and exposes checkout, portal, cancel-at-period-end, and reactivation controls. +Persisted assistant runs may cross message pages, but each API row stays bounded. The history +query actively follows cursors until every segment through the final marker is loaded, then +losslessly reconstructs structured fragments and merges the run under its stable run ID. A +partial or corrupt transcript is never rendered as a duplicate assistant message. + +Deliverable parts contain durable output identity and presentation metadata, never an expiring +URL. A download click calls the authenticated gateway mint endpoint, validates its bounded response, +and follows the resulting short-lived capability directly to the streaming response. + ## Public exports Framework app only. @@ -21,17 +30,31 @@ or run browser-flow scripts for web acceptance testing. ## Env - `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` -- `NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA` (provided automatically by Vercel's enabled system variables) +- `NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA` (embedded from the exact release SHA by the prebuilt workflow) - `NEXT_PUBLIC_GATEWAY_URL` - `NEXT_PUBLIC_PREVIEW_HOSTNAME` (must match both preview Workers and the Cloudflare wildcard route) - `CLERK_SECRET_KEY` +- `VERCEL_ENV` (actual Vercel runtime environment) +- `VERCEL_TARGET_ENV` (Vercel build/deployment target) +- `VERCEL_URL` (actual immutable deployment hostname) -Local development and Vercel Preview require Clerk `pk_test_`/`sk_test_` keys. -Vercel Production fails its build unless both Clerk keys use the `pk_live_`/`sk_live_` -prefixes. Middleware also restricts Clerk session-token authorized parties to the exact +Local development requires Clerk `pk_test_`/`sk_test_` keys. Every Vercel +deployment requires the production `pk_live_`/`sk_live_` keys; development keys +exist only in root `.env.local` on the laptop. Middleware also restricts Clerk session-token authorized parties to the exact loopback, Vercel deployment, or production request origin for that environment. Preview deployments are matched to their exact system-provided `VERCEL_URL`; no wildcard Vercel origin is trusted. +The prebuilt production build explicitly sets `VERCEL_TARGET_ENV=production`, +which selects the live Clerk, canonical gateway, preview-hostname, and exact-SHA +validation branch before a deployment URL exists. Only an actual Vercel +`production` or `preview` runtime requires `VERCEL_URL`; Vercel supplies +`VERCEL_ENV` and `VERCEL_URL` after the prebuilt artifact is deployed. +`next.config.ts` and the runtime env accessor share the pure validators exported +by `@cheatcode/env/web-config`; all four public build values are explicit and +missing values have no local or production fallback. +The config loads the repository-root `.env.local` through `@next/env` for local +builds and strips all loaded Worker-only values before Next evaluates the app; +no second env file under `apps/web` is used. The production CSP admits the exact validated `NEXT_PUBLIC_GATEWAY_URL`; a real Vercel Production build pins that value to `https://gateway.trycheatcode.com`, while optimized local QA can use its loopback Wrangler origin. @@ -43,6 +66,12 @@ guarded release workflow can verify `/api/health` before promoting the deploymen `apps/web/vercel.json` disables automatic Git deployments for every branch. The guarded `Production Release` workflow builds, stages, and verifies one immutable exact-SHA Vercel production deployment without assigning production domains. It -then releases and verifies the Cloudflare backend, promotes that already-verified -deployment, and waits for `trycheatcode.com` to report the same release SHA before -post-deploy database contractions can run. +applies expand-only migrations before closing and draining every database-writing +Worker. A successful stage persists the exact deployment ID, immutable URL, SHA, +control ref, and stage run identity as a GitHub artifact; OPEN accepts the stage +run ID, not an operator-copied URL. The separate reconciliation phase stays closed. +OPEN validates that handoff and reconciliation evidence, applies contractions, +promotes the exact deployment ID, proves `trycheatcode.com` resolves to it, and +then invokes backend OPEN. Backend OPEN redeploys all writers CLOSED on their +dedicated database roles, proves signed three-role readiness, and reopens agent and +webhooks before opening gateway last, with one final canonical alias/SHA check. diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 7cdcd230..594bed10 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -1,13 +1,37 @@ +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { parseWebBuildEnvironment, WEB_APPLICATION_ENV_KEYS } from "@cheatcode/env/web-config"; import type { NextConfig } from "next"; -const IS_VERCEL_PRODUCTION = process.env["VERCEL_ENV"] === "production"; -const GATEWAY_ORIGIN = readGatewayOrigin(process.env["NEXT_PUBLIC_GATEWAY_URL"]); -const PREVIEW_HOSTNAME = readPreviewHostname(process.env["NEXT_PUBLIC_PREVIEW_HOSTNAME"]); +const REPOSITORY_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const { loadEnvConfig } = createRequire(import.meta.url)("@next/env") as typeof import("@next/env"); +// Next preloads env relative to apps/web; reload from the monorepo root so the +// single laptop env works, then prevent Worker-only secrets reaching Next. +const loadedRootEnvironment = loadEnvConfig( + REPOSITORY_ROOT, + process.env.NODE_ENV !== "production", + undefined, + true, +); +for (const key of Object.keys(loadedRootEnvironment.parsedEnv ?? {})) { + if (!WEB_APPLICATION_ENV_KEYS.has(key)) { + delete process.env[key]; + } +} + +const WEB_BUILD_ENVIRONMENT = parseWebBuildEnvironment({ + NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"], + NEXT_PUBLIC_GATEWAY_URL: process.env["NEXT_PUBLIC_GATEWAY_URL"], + NEXT_PUBLIC_PREVIEW_HOSTNAME: process.env["NEXT_PUBLIC_PREVIEW_HOSTNAME"], + NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA: process.env["NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA"], + VERCEL_ENV: process.env["VERCEL_ENV"], + VERCEL_TARGET_ENV: process.env["VERCEL_TARGET_ENV"], +}); +const GATEWAY_ORIGIN = WEB_BUILD_ENVIRONMENT.gatewayOrigin; +const PREVIEW_HOSTNAME = WEB_BUILD_ENVIRONMENT.previewHostname; const PREVIEW_HTTPS_ORIGIN = `https://*.${PREVIEW_HOSTNAME}`; const PREVIEW_WSS_ORIGIN = `wss://*.${PREVIEW_HOSTNAME}`; -const CLERK_FRONTEND_HOSTNAME = readClerkFrontendHostname( - process.env["NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY"], -); +const CLERK_FRONTEND_HOSTNAME = WEB_BUILD_ENVIRONMENT.clerkFrontendHostname; const CLERK_FRONTEND_ORIGIN = `https://${CLERK_FRONTEND_HOSTNAME}`; const CLERK_WEBSOCKET_ORIGIN = `wss://${CLERK_FRONTEND_HOSTNAME}`; @@ -29,6 +53,7 @@ const CONTENT_SECURITY_POLICY = [ ].join("; "); const nextConfig = { + allowedDevOrigins: ["127.0.0.1", "localhost"], cacheComponents: true, devIndicators: false, async headers() { @@ -59,87 +84,3 @@ const nextConfig = { } satisfies NextConfig; export default nextConfig; - -function readGatewayOrigin(value: string | undefined): string { - const configured = (value ?? "http://localhost:8787").trim(); - let parsed: URL; - try { - parsed = new URL(configured); - } catch { - throw new TypeError("NEXT_PUBLIC_GATEWAY_URL must be a valid URL"); - } - if (configured !== parsed.origin) { - throw new TypeError( - "NEXT_PUBLIC_GATEWAY_URL must be an origin without credentials, path, query, or fragment", - ); - } - const isLoopback = - parsed.hostname === "localhost" || - parsed.hostname === "127.0.0.1" || - parsed.hostname === "[::1]"; - if (parsed.protocol !== "https:" && !(isLoopback && parsed.protocol === "http:")) { - throw new TypeError( - "NEXT_PUBLIC_GATEWAY_URL must use HTTPS except for a loopback development origin", - ); - } - if (IS_VERCEL_PRODUCTION && parsed.origin !== "https://gateway.trycheatcode.com") { - throw new TypeError( - "Vercel Production requires https://gateway.trycheatcode.com as its gateway origin", - ); - } - return parsed.origin; -} - -function readPreviewHostname(value: string | undefined): string { - const hostname = (value ?? "trycheatcode.com").trim().toLowerCase().replace(/\.$/u, ""); - if (!isValidHostname(hostname)) { - throw new TypeError("NEXT_PUBLIC_PREVIEW_HOSTNAME must be a valid hostname"); - } - if (IS_VERCEL_PRODUCTION && hostname !== "trycheatcode.com") { - throw new TypeError( - "Vercel Production previews require the owned trycheatcode.com wildcard route", - ); - } - return hostname; -} - -function readClerkFrontendHostname(value: string | undefined): string { - const expectedEnvironment = IS_VERCEL_PRODUCTION ? "live" : "test"; - const prefix = `pk_${expectedEnvironment}_`; - if (!value?.startsWith(prefix)) { - throw new TypeError( - `${IS_VERCEL_PRODUCTION ? "Vercel Production" : "Development and Vercel Preview"} requires a Clerk ${prefix} publishable key`, - ); - } - const encodedHostname = value.slice(prefix.length); - if (!/^[A-Za-z0-9_-]+$/u.test(encodedHostname)) { - throw new TypeError("Clerk publishable key payload must be base64url encoded"); - } - const decoded = Buffer.from(encodedHostname, "base64url").toString("utf8"); - if (!decoded.endsWith("$") || decoded.slice(0, -1).includes("$")) { - throw new TypeError("Clerk publishable key payload is malformed"); - } - const hostname = decoded.slice(0, -1).toLowerCase(); - if (!isValidHostname(hostname)) { - throw new TypeError("Clerk publishable key contains an invalid Frontend API hostname"); - } - if (IS_VERCEL_PRODUCTION && hostname !== "clerk.trycheatcode.com") { - throw new TypeError("Vercel Production requires the clerk.trycheatcode.com Clerk instance"); - } - if (!IS_VERCEL_PRODUCTION && !hostname.endsWith(".clerk.accounts.dev")) { - throw new TypeError("Development and Vercel Preview require a Clerk development instance"); - } - return hostname; -} - -function isValidHostname(hostname: string): boolean { - const labels = hostname.split("."); - return ( - hostname.length <= 253 && - labels.length >= 2 && - labels.every( - (label) => - label.length >= 1 && label.length <= 63 && /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(label), - ) - ); -} diff --git a/apps/web/package.json b/apps/web/package.json index 1d9a7728..e31bd002 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -3,9 +3,9 @@ "private": true, "version": "0.0.0", "type": "module", - "packageManager": "pnpm@10.34.5", + "packageManager": "pnpm@11.8.0", "engines": { - "node": "22.x" + "node": "22.22.2" }, "scripts": { "build": "next build", @@ -20,6 +20,7 @@ "@cheatcode/types": "workspace:*", "@cheatcode/ui": "workspace:*", "@clerk/nextjs": "catalog:", + "@next/env": "catalog:", "@clerk/ui": "catalog:", "@hookform/resolvers": "catalog:", "@tanstack/react-query": "catalog:", diff --git a/apps/web/src/app/(app)/billing/page.tsx b/apps/web/src/app/(app)/billing/page.tsx index 0340eb1d..04aa714b 100644 --- a/apps/web/src/app/(app)/billing/page.tsx +++ b/apps/web/src/app/(app)/billing/page.tsx @@ -1,11 +1,5 @@ -import { BillingPanel } from "@/components/settings/billing-panel"; +import { redirect } from "next/navigation"; export default function BillingPage() { - return ( -
-
- -
-
- ); + redirect("/usage"); } diff --git a/apps/web/src/app/(app)/error.tsx b/apps/web/src/app/(app)/error.tsx index 7fd22d8b..09d90210 100644 --- a/apps/web/src/app/(app)/error.tsx +++ b/apps/web/src/app/(app)/error.tsx @@ -1,7 +1,7 @@ "use client"; +import { SquareAsterisk } from "@cheatcode/ui"; import { useEffect } from "react"; -import { SquareAsterisk } from "@/components/ui/icons"; import { RecoveryCard } from "@/components/ui/recovery-card"; import { reportClientError } from "@/lib/error-reporter"; diff --git a/apps/web/src/app/(app)/usage/page.tsx b/apps/web/src/app/(app)/usage/page.tsx new file mode 100644 index 00000000..9b1d9549 --- /dev/null +++ b/apps/web/src/app/(app)/usage/page.tsx @@ -0,0 +1,11 @@ +import { UsagePanel } from "@/components/settings/usage-panel"; + +export default function UsagePage() { + return ( +
+
+ +
+
+ ); +} diff --git a/apps/web/src/app/global-error.tsx b/apps/web/src/app/global-error.tsx index e8ebcf18..e6884b72 100644 --- a/apps/web/src/app/global-error.tsx +++ b/apps/web/src/app/global-error.tsx @@ -1,7 +1,7 @@ "use client"; +import { SquareAsterisk } from "@cheatcode/ui"; import { useEffect } from "react"; -import { SquareAsterisk } from "@/components/ui/icons"; import { RecoveryCard } from "@/components/ui/recovery-card"; import { reportClientError } from "@/lib/error-reporter"; diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 8548c472..a9c634b4 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -708,7 +708,8 @@ } } -/* Unlayered focus rules outrank Tailwind's layered `outline-none` utility. */ +/* Cheatcode uses component-level focus fills and shadows instead of browser outlines. */ +body :where( button, a, @@ -718,35 +719,6 @@ summary, [contenteditable="true"], [tabindex]:not([tabindex="-1"]) -):focus:not(:focus-visible) { - outline: none; -} - -body - :where( - button, - a, - input, - textarea, - select, - summary, - [contenteditable="true"], - [tabindex]:not([tabindex="-1"]) - ):focus-visible { - outline: 2px solid var(--focus-ring); - outline-offset: 2px; -} - -.cheatcode-auth-dialog - :where( - button, - a, - input, - textarea, - select, - summary, - [contenteditable="true"], - [tabindex]:not([tabindex="-1"]) - ):focus-visible { - outline-color: #fff; +):is(:focus, :focus-visible) { + outline: 0; } diff --git a/apps/web/src/components/auth/auth-modal.tsx b/apps/web/src/components/auth/auth-modal.tsx index b3c673d0..bda30fb1 100644 --- a/apps/web/src/components/auth/auth-modal.tsx +++ b/apps/web/src/components/auth/auth-modal.tsx @@ -1,9 +1,8 @@ "use client"; -import { ModalShell } from "@cheatcode/ui"; +import { ModalShell, X } from "@cheatcode/ui"; import { SignIn, SignUp } from "@clerk/nextjs"; import { useSyncExternalStore } from "react"; -import { X } from "@/components/ui/icons"; import { clerkAuthAppearance } from "./clerk-auth-appearance"; export type AuthMode = "sign-in" | "sign-up"; diff --git a/apps/web/src/components/auth/auth-route-page.tsx b/apps/web/src/components/auth/auth-route-page.tsx index 1934f90d..1f2e87e9 100644 --- a/apps/web/src/components/auth/auth-route-page.tsx +++ b/apps/web/src/components/auth/auth-route-page.tsx @@ -1,11 +1,11 @@ "use client"; +import { Monitor } from "@cheatcode/ui"; import { useAuth } from "@clerk/nextjs"; import { useRouter } from "next/navigation"; import { useSyncExternalStore } from "react"; import { AuthModal, type AuthMode } from "@/components/auth/auth-modal"; import { CheatcodeMark } from "@/components/ui/cheatcode-mark"; -import { Monitor } from "@/components/ui/icons"; import { safeLocalRedirect } from "@/lib/navigation/safe-local-redirect"; export function AuthRoutePage({ mode }: { mode: AuthMode }) { @@ -54,7 +54,6 @@ function subscribeToLocationChanges(callback: () => void): () => void { function readRedirectPath(): string { const params = new URLSearchParams(window.location.search); - const candidate = - params.get("redirect_url") ?? params.get("redirectUrl") ?? params.get("redirect") ?? "/"; + const candidate = params.get("redirect_url") ?? "/"; return safeLocalRedirect(candidate, window.location.origin) ?? "/"; } diff --git a/apps/web/src/components/billing/manage-subscription-dialog.tsx b/apps/web/src/components/billing/manage-subscription-dialog.tsx index b797141e..4722eb02 100644 --- a/apps/web/src/components/billing/manage-subscription-dialog.tsx +++ b/apps/web/src/components/billing/manage-subscription-dialog.tsx @@ -6,14 +6,17 @@ import type { BillingStateResponse, BillingSubscriptionActionResponse, } from "@cheatcode/types"; -import { ModalShell } from "@cheatcode/ui"; +import { ChevronDown, CreditCard, Loader2, ModalShell } from "@cheatcode/ui"; import { type QueryClient, useMutation, useQueryClient } from "@tanstack/react-query"; import { useState } from "react"; import { toast } from "sonner"; import { CheatcodeLoader } from "@/components/ui/cheatcode-loader"; -import { ChevronDown, CreditCard, Loader2 } from "@/components/ui/icons"; import { RecoveryCard } from "@/components/ui/recovery-card"; -import { requestBillingCancellation, requestBillingReactivation } from "@/lib/api/billing"; +import { + requestBillingCancellation, + requestBillingPortal, + requestBillingReactivation, +} from "@/lib/api/billing"; import { BILLING_STATE_QUERY_KEY, useBillingStateQuery } from "@/lib/hooks/use-billing"; const CANCELLATION_REASON_LABELS: Record = { @@ -73,8 +76,10 @@ function useManageSubscriptionController({ const [reason, setReason] = useState(""); const [comment, setComment] = useState(""); const cancelMutation = useCancellationMutation(getToken, queryClient, () => setStep("overview")); + const portalMutation = useBillingPortalMutation(getToken); const reactivateMutation = useReactivationMutation(getToken, queryClient, planDisplayName); - const isBusy = cancelMutation.isPending || reactivateMutation.isPending; + const isBusy = + cancelMutation.isPending || portalMutation.isPending || reactivateMutation.isPending; function closeDialog() { if (isBusy) return; @@ -97,6 +102,7 @@ function useManageSubscriptionController({ comment, confirmCancellation, isBusy, + openBillingPortal: () => portalMutation.mutate(), reactivate: () => reactivateMutation.mutate(), reason, setComment, @@ -140,6 +146,15 @@ function useReactivationMutation( }); } +function useBillingPortalMutation(getToken: () => Promise) { + return useMutation({ + mutationFn: () => requestBillingPortal(getToken), + onError: (error) => + toast.error(error instanceof Error ? error.message : "Billing portal couldn't open"), + onSuccess: (url) => window.location.assign(url), + }); +} + function ManageDialogFrame({ controller, planDisplayName, @@ -199,6 +214,7 @@ function ManageDialogBody({ isBusy={controller.isBusy} onCancel={() => controller.setStep("cancel")} onClose={controller.closeDialog} + onOpenBillingPortal={controller.openBillingPortal} onReactivate={controller.reactivate} planDisplayName={planDisplayName} sandboxHoursTotal={sandboxHoursTotal} @@ -230,6 +246,7 @@ function PlanOverview({ isBusy, onCancel, onClose, + onOpenBillingPortal, onReactivate, planDisplayName, sandboxHoursTotal, @@ -238,6 +255,7 @@ function PlanOverview({ isBusy: boolean; onCancel: () => void; onClose: () => void; + onOpenBillingPortal: () => void; onReactivate: () => void; planDisplayName: string; sandboxHoursTotal: number; @@ -255,6 +273,7 @@ function PlanOverview({ isBusy={isBusy} onCancel={onCancel} onClose={onClose} + onOpenBillingPortal={onOpenBillingPortal} onReactivate={onReactivate} planDisplayName={planDisplayName} state={state} @@ -293,6 +312,7 @@ function PlanOverviewActions({ isBusy, onCancel, onClose, + onOpenBillingPortal, onReactivate, planDisplayName, state, @@ -300,6 +320,7 @@ function PlanOverviewActions({ isBusy: boolean; onCancel: () => void; onClose: () => void; + onOpenBillingPortal: () => void; onReactivate: () => void; planDisplayName: string; state: BillingStateResponse; @@ -310,6 +331,14 @@ function PlanOverviewActions({ {state.canCancel ? : null}
+ - -
- ); -} - -export function ApprovalDecisionBlock({ data }: { data: ApprovalDecisionData }) { - return ( -
-
decision
-
{decisionLabel(data)}
-
- ); -} - -/** Informational model-transition notice (the interactive pause is an approval-request). */ -export function ModelFallbackBlock({ data }: { data: ModelFallbackData }) { - return ( -
-
model fallback
-
- Switched from {data.fromModel} to {data.toModel} -
-
- Reason: {fallbackReasonLabel(data.reason)} -
- - Open Models & Keys - -
- ); -} - -function decisionLabel(data: ApprovalDecisionData): string { - const verb = data.decision === "allow" ? "allowed" : "denied"; - if (data.decidedBy === "user") { - return `${verb} by user`; - } - return `${verb} (${data.decidedBy})`; -} - -function fallbackReasonLabel(reason: ModelFallbackData["reason"]): string { - if (reason === "rate_limit") { - return "provider rate limit"; - } - if (reason === "provider_balance") { - return "provider balance or quota exhausted"; - } - return "provider error"; -} diff --git a/apps/web/src/components/chat/chat-context-row.tsx b/apps/web/src/components/chat/chat-context-row.tsx index d49c9c13..6b8ec11a 100644 --- a/apps/web/src/components/chat/chat-context-row.tsx +++ b/apps/web/src/components/chat/chat-context-row.tsx @@ -5,14 +5,23 @@ import { useChatContextController } from "@/components/chat/chat-context-control import { ChatContextView } from "@/components/chat/chat-context-view"; export function ChatContextRow({ + isRunning, project, threadId, title, }: { + isRunning: boolean; project: ProjectSummary | null; threadId: string; title: null | string | undefined; }) { const controller = useChatContextController({ project, threadId, title }); - return ; + return ( + + ); } diff --git a/apps/web/src/components/chat/chat-context-view.tsx b/apps/web/src/components/chat/chat-context-view.tsx index 63a23c5a..69d33e7f 100644 --- a/apps/web/src/components/chat/chat-context-view.tsx +++ b/apps/web/src/components/chat/chat-context-view.tsx @@ -1,27 +1,33 @@ "use client"; import type { ProjectSummary } from "@cheatcode/types"; +import { Clock3, Loader2, Plus, X } from "@cheatcode/ui"; import type { ChatContextController } from "@/components/chat/chat-context-controller"; import { FolderChatsSearch } from "@/components/chat/folder-chats-search"; import { CheatcodeTooltip } from "@/components/ui/cheatcode-tooltip"; -import { Clock3, Plus, X } from "@/components/ui/icons"; import type { ChatWorkspaceTab } from "@/lib/store/chat-tabs-store"; import { cn } from "@/lib/ui/cn"; export function ChatContextView({ controller, + isRunning, project, threadId, }: { controller: ChatContextController; + isRunning: boolean; project: ProjectSummary | null; threadId: string; }) { return (
- +
- +
{controller.state.folderChatsOpen && project ? ( @@ -37,15 +43,20 @@ export function ChatContextView({ function MobileChatTitle({ activeThreadId, + isRunning, tabs, }: { activeThreadId: string; + isRunning: boolean; tabs: readonly ChatWorkspaceTab[]; }) { const activeTitle = tabs.find((tab) => tab.id === activeThreadId)?.title ?? "New chat"; return (
-

{activeTitle}

+

+ {activeTitle} + {isRunning ? : null} +

); } @@ -105,9 +116,11 @@ function FolderChatsButton({ function ChatTabStrip({ activeThreadId, controller, + isRunning, }: { activeThreadId: string; controller: ChatContextController; + isRunning: boolean; }) { return (
@@ -115,6 +128,7 @@ function ChatTabStrip({ {controller.state.tabs.map((tab) => ( controller.actions.selectTab(tab)} @@ -129,19 +143,21 @@ function ChatTabStrip({ function ChatTabPill({ isActive, + isRunning, onClose, onSelect, showClose, tab, }: { isActive: boolean; + isRunning: boolean; onClose: () => void; onSelect: () => void; showClose: boolean; tab: ChatWorkspaceTab; }) { return ( -
+
- {isActive && showClose ? ( + {isRunning ? ( + + ) : isActive && showClose ? ( - {open && canExpand ? ( -
- {data.text} + {open ? ( +
+
) : null}
); } -function thinkingLabel(data: ThinkingData): string { - if (data.delta) { - return "Thinking…"; - } - if (typeof data.durationMs === "number") { - return `Thought for ${formatThinkingDuration(data.durationMs)}`; - } - return "Thought"; -} - -function formatThinkingDuration(ms: number): string { - const seconds = ms / 1000; - if (seconds < 10) { - return `${seconds.toFixed(seconds === 0 ? 0 : 1)}s`; - } - if (seconds < 60) { - return `${Math.round(seconds)}s`; - } - const minutes = Math.floor(seconds / 60); - const remainder = Math.round(seconds % 60); - return remainder === 0 ? `${minutes}m` : `${minutes}m ${remainder}s`; -} - -export function ToolGroup({ parts }: { parts: MessagePart[] }) { +export function ToolGroup({ parts }: { parts: ToolPart[] }) { const rows = collapseToolRuns(parts); return (
@@ -238,8 +221,8 @@ export function ToolGroup({ parts }: { parts: MessagePart[] }) { ); } -function collapseToolRuns(parts: MessagePart[]): { key: string; parts: MessagePart[] }[] { - const rows: { key: string; parts: MessagePart[] }[] = []; +function collapseToolRuns(parts: ToolPart[]): { key: string; parts: ToolPart[] }[] { + const rows: { key: string; parts: ToolPart[] }[] = []; let index = 0; while (index < parts.length) { const type = parts[index]?.type; @@ -253,7 +236,7 @@ function collapseToolRuns(parts: MessagePart[]): { key: string; parts: MessagePa return rows; } -function ToolRow({ parts }: { parts: MessagePart[] }) { +function ToolRow({ parts }: { parts: ToolPart[] }) { const [open, setOpen] = useState(false); const first = parts[0]; if (!first) { @@ -288,7 +271,7 @@ function toolRowLabel(description: { arg: string | null; verb: string }, extra: return extra > 0 ? `${primary} (+${extra} more)` : primary; } -function ToolDetails({ parts }: { parts: MessagePart[] }) { +function ToolDetails({ parts }: { parts: ToolPart[] }) { const sections = buildToolDetailSections(parts); return (
@@ -306,7 +289,7 @@ function ToolDetails({ parts }: { parts: MessagePart[] }) { ); } -function buildToolDetailSections(parts: MessagePart[]): ToolDetailSection[] { +function buildToolDetailSections(parts: ToolPart[]): ToolDetailSection[] { const occurrences = new Map(); return parts.flatMap((part) => { const partIdentity = toolPartIdentity(part); @@ -379,52 +362,21 @@ function TimelineConnector({ continued }: { continued: boolean }) { ); } -function toolDetailSections(part: MessagePart): Array> { - const record = asRecord(part); +function toolDetailSections(part: ToolPart): Array> { const { name, input } = toolNameAndInput(part); - const output = record["output"] ?? record["result"]; const isCommand = isCommandTool(name); - const sections = [ + return [ { label: isCommand ? "Command" : "Input", isCommand, scroll: false, value: isCommand ? commandValue(input) : formatUnknown(summarizeToolValue(input, 0)), }, - ]; - if (output !== undefined) { - sections.push({ - label: "Output", - isCommand: false, - scroll: true, - value: formatUnknown(summarizeToolValue(output, 0)), - }); - } - return sections.filter((section) => section.value.length > 0); -} - -function toolPartIdentity(part: MessagePart): string { - const record = asRecord(part); - const data = part.type === "data-tool" ? asRecord(record["data"]) : record; - const callId = - stringRecordField(data, "toolCallId") || - stringRecordField(record, "toolCallId") || - stringRecordField(record, "id"); - if (callId) { - return `${part.type}:${callId}`; - } - const { input, name } = toolNameAndInput(part); - const output = record["output"] ?? record["result"]; - return `${part.type}:${name}:${toolValueFingerprint([input, output])}`; + ].filter((section) => section.value.length > 0); } -function toolValueFingerprint(value: unknown): string { - const serialized = formatUnknown(summarizeToolValue(value, 0)); - let hash = 2_166_136_261; - for (let index = 0; index < serialized.length; index += 1) { - hash = Math.imul(hash ^ serialized.charCodeAt(index), 16_777_619); - } - return (hash >>> 0).toString(36); +function toolPartIdentity(part: ToolPart): string { + return `${part.type}:${part.data.toolCallId}`; } function ShellCommand({ value }: { value: string }) { @@ -462,7 +414,7 @@ function isCommandTool(name: string): boolean { return name === "runCode" || name.startsWith("shell_") || name === "start_dev_server"; } -function describeTool(part: MessagePart): { verb: string; arg: string | null } { +function describeTool(part: ToolPart): { verb: string; arg: string | null } { const { name, input } = toolNameAndInput(part); const spec = TOOL_VERBS[name]; const verb = spec?.verb ?? humanizeToolName(name); @@ -475,21 +427,10 @@ function describeTool(part: MessagePart): { verb: string; arg: string | null } { return { verb, arg: null }; } -function toolNameAndInput(part: MessagePart): { input: Record; name: string } { - const record = asRecord(part); - if (part.type === "data-tool") { - const data = asRecord(record["data"]); - return { input: asRecord(data["input"]), name: stringRecordField(data, "toolName") }; - } - if (part.type === "dynamic-tool") { - return { - input: asRecord(record["input"] ?? record["args"]), - name: stringRecordField(record, "toolName"), - }; - } +function toolNameAndInput(part: ToolPart): { input: Record; name: string } { return { - input: asRecord(record["input"] ?? record["args"]), - name: part.type.replace("tool-", ""), + input: part.data.input ?? {}, + name: part.data.toolName, }; } @@ -508,19 +449,14 @@ function shortenArg(value: string): string { : `${collapsed.slice(0, MAX_TOOL_ARG_LENGTH - 1)}…`; } -export function isToolPart(part: MessagePart): boolean { - return part.type.startsWith("tool-") || part.type === "dynamic-tool" || part.type === "data-tool"; +export function isToolPart(part: MessagePart): part is ToolPart { + return part.type === "data-tool"; } function asRecord(value: unknown): Record { return value && typeof value === "object" ? (value as Record) : {}; } -function stringRecordField(record: Record, key: string): string { - const value = record[key]; - return typeof value === "string" ? value : ""; -} - function formatUnknown(value: unknown): string { if (typeof value === "string") { return value; diff --git a/apps/web/src/components/chat/message-deliverable-model.ts b/apps/web/src/components/chat/message-deliverable-model.ts index 32d5f814..21ab893f 100644 --- a/apps/web/src/components/chat/message-deliverable-model.ts +++ b/apps/web/src/components/chat/message-deliverable-model.ts @@ -8,21 +8,6 @@ export function collectDeliverables(parts: readonly MessagePart[]): ArtifactData return deliverables; } -export function artifactFallbackName(data: ArtifactData): string { - const extension = artifactExtension(data.kind, data.mimeType); - return extension ? `${data.kind}-${data.outputId.slice(0, 8)}.${extension}` : data.kind; -} - -function artifactExtension(kind: ArtifactData["kind"], mimeType: string): string | null { - if (kind === "slide") return "pptx"; - if (kind === "xlsx" || kind === "docx" || kind === "pdf") return kind; - if (kind === "folder" || kind === "link") return null; - if (mimeType === "image/svg+xml") return "svg"; - if (!mimeType.includes("/")) return null; - const extension = mimeType.split("/").at(1)?.split(";").at(0); - return extension?.replace(/[^a-z0-9]+/gi, "").toLowerCase() || null; -} - export function formatBytes(bytes: number): string { if (bytes < 1024) return `${bytes} B`; const units = ["KB", "MB", "GB"]; diff --git a/apps/web/src/components/chat/message-deliverables.tsx b/apps/web/src/components/chat/message-deliverables.tsx index e8ba9e8d..537859bb 100644 --- a/apps/web/src/components/chat/message-deliverables.tsx +++ b/apps/web/src/components/chat/message-deliverables.tsx @@ -1,18 +1,25 @@ -import { artifactFallbackName, formatBytes } from "@/components/chat/message-deliverable-model"; -import type { ArtifactData } from "@/components/chat/message-parts.types"; +"use client"; + import { Code, Download, - ExternalLink, FileSpreadsheet, FileText, - Folder, Image as ImageIcon, - Link as LinkIcon, Presentation, -} from "@/components/ui/icons"; + Video, +} from "@cheatcode/ui"; +import { useAuth } from "@clerk/nextjs"; +import { useState } from "react"; +import { toast } from "sonner"; +import { formatBytes } from "@/components/chat/message-deliverable-model"; +import type { ArtifactData } from "@/components/chat/message-parts.types"; +import { createOutputDownloadUrl } from "@/lib/api/outputs"; + +type GetToken = () => Promise; export function DeliverablesBlock({ items }: { items: readonly ArtifactData[] }) { + const { getToken } = useAuth(); return (
{items.map((item) => ( - + ))}
); } -function DeliverableChip({ data }: { data: ArtifactData }) { +function DeliverableChip({ data, getToken }: { data: ArtifactData; getToken: GetToken }) { + const [isPreparing, setIsPreparing] = useState(false); const Icon = deliverableIcon(data.kind, data.mimeType); - const label = data.filename ?? artifactFallbackName(data); - const isLink = data.kind === "link"; - const sizeLabel = typeof data.sizeBytes === "number" ? formatBytes(data.sizeBytes) : null; - const meta = [data.kind, sizeLabel].filter(Boolean).join(" · "); + const label = data.filename; + const meta = `${data.kind} · ${formatBytes(data.sizeBytes)}`; + const download = async (): Promise => { + if (isPreparing) { + return; + } + setIsPreparing(true); + try { + const capability = await createOutputDownloadUrl(getToken, data.outputId); + window.location.assign(capability.downloadUrl); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Download could not be prepared"); + } finally { + setIsPreparing(false); + } + }; return (
- void download()} + type="button" > - - {isLink ? "open" : "download"} - +
); } -function DeliverableActionIcon({ isLink }: { isLink: boolean }) { - return isLink ? ( -