diff --git a/docs/screenshot-input.md b/docs/screenshot-input.md new file mode 100644 index 0000000..0afb66f --- /dev/null +++ b/docs/screenshot-input.md @@ -0,0 +1,689 @@ +# File and screenshot input + +Status: implemented in draft PR + +Date: 2026-08-06 + +## 1. Summary + +Let the operator attach local files to an agent prompt by: + +- pasting files or screenshots from the browser clipboard; +- dragging files onto a prompt or terminal pane; or +- choosing files with a small attachment button in live views. + +The browser uploads the bytes to Agent Manager. Agent Manager stores them +outside the user's repository, then gives the target CLI a server-local path. +Raster images additionally use native image interfaces where the harness has a +reliable one. The prompt remains text at the PTY boundary; no binary data is +sent through xterm or terminal escape sequences. + +The first version covers local agent sessions. Remote agents need an additional +download protocol because they do not share the Space's filesystem and are a +separate phase. + +## 2. Why this shape + +Three constraints decide the architecture. + +### 2.1 The browser and the CLI have different clipboards + +The browser receives a pasted screenshot as a `ClipboardEvent`/`File`. The agent +CLI runs in a headless container and can only inspect that container's OS +clipboard. Asking Claude, Codex, Hermes, or another TUI to read its clipboard +therefore cannot see what the operator copied on their laptop or phone. + +Agent Manager's existing mobile paste code already documents the same boundary: +direct browser clipboard access may also be unavailable inside the Hugging Face +cross-origin iframe, while a user-driven DOM `paste` event remains readable. + +Hermes Agent's own browser dashboard independently uses the intended bridge: +extract image files from the browser transfer, upload them to the gateway, then +drive the server-side TUI with the resulting local path. This is useful prior +art, not an integration dependency. + +### 2.2 Agent Manager's terminal transport is text + +Overview replies call `POST /api/sessions/:id/input`, `deliver()` starts the +session when needed, and `runner.sendInput()` writes a bracketed text paste plus +a separate Return key to the PTY. A live browser pane similarly sends xterm input +as WebSocket `{t:'i', d}` frames. + +That transport should stay text. Sending an image through a PTY would require a +terminal graphics/clipboard protocol and corresponding support in every TUI. +Those protocols do not solve the browser/container clipboard boundary and would +couple Agent Manager to terminal-specific behavior. + +### 2.3 Coding CLIs already understand local file paths + +The exact native image affordance differs by harness, but a server-local path is +the common denominator for images, PDFs, Office documents, archives, source +files, and other regular files: + +| Harness | Observed path as of this design | First implementation | +|---|---|---| +| Claude Code | accepts an image path in a prompt; native terminals also support image paste/drop | explicit absolute path in the prompt | +| Codex | `-i/--image` supports initial images; in-session image paste reads the server clipboard | native `-i` on the first turn plus an explicit path fallback | +| Gemini CLI | `@` injects supported images as multimodal context | `@` | +| opencode | TUI image drop and `opencode run -f/--file` are supported | explicit/drop-style absolute path | +| Hermes | the current TUI supports `/image ` and detects a pasted standalone image path | `/image ` adapter, with explicit-path fallback | +| OpenClaw | no stable image-attachment CLI contract was verified | explicit path, best effort | + +Every adapter must retain the explicit-path fallback. Native image flags and +TUI commands change faster than Agent Manager, while a file that exists and a +prompt that names it remain inspectable by an agent with filesystem and vision +tools. + +## 3. Goals + +1. Paste a file or screenshot into any local agent input without first saving it + by hand. +2. Drag one or more regular files onto the input or terminal pane. +3. Work in the Hugging Face iframe and on phone-sized layouts. +4. Make upload progress, success, and failure visible. +5. Never submit a partially uploaded or missing attachment. +6. Avoid modifying or dirtying the user's repository. +7. Keep the attachment available for the lifetime of the session so resumed + agents can still inspect it. +8. Preserve ordinary text paste and Agent Manager's existing pane drag/drop. +9. Use bounded streaming uploads so files do not block the process that pumps + every PTY. +10. Preserve native image behavior while making non-image files available by + explicit local path. + +## 4. Non-goals + +- Images in agent-to-agent prompts. Agents inside the Space can already write a + file and reference its path. +- Synchronizing the operator's filesystem with a remote agent's filesystem. +- Making xterm render an attachment editor inside a third-party TUI. +- Automatically publishing screenshots with shared session traces. +- OCR, resizing, transcoding, or other server-side image processing. +- Replacing the PTY integration with each harness's structured SDK/server API. + +## 5. User experience + +### 5.1 Composers + +The Sidebar quickstart and Overview reply textarea share the same paste/drop +behavior: + +- Pasting a file adds a chip and does not insert binary or fake text + into the textarea. +- Dropping files over the composer shows a restrained dashed highlight and + adds the same chips. +- Overview and live terminal views expose a file button with an unrestricted + ``; quick creation intentionally stays prompt-first + and accepts paste/drop without another picker. +- Image chips show thumbnails; other files show a compact extension badge. +- The prompt may contain text plus files or files alone. +- A files-only submission uses `Please inspect the attached file.` (or `files` + for several) as its text. Image-only server delivery retains the more specific + `screenshot` wording. +- At most five files may be attached to one prompt. +- The send button is disabled while an upload is active. +- A failed upload leaves the draft and pending files intact and names the + failure next to the affected chip. + +Pending files remain browser `File` objects until the operator submits. This +means abandoning or editing a draft does not create server-side orphan files. +`URL.createObjectURL()` supplies local previews and is revoked when a chip is +removed or the component unmounts. + +### 5.2 Live terminal panes + +xterm owns the visible composer, so Agent Manager cannot reliably place its own +persistent attachment chips inside it. File paste/drop therefore behaves as a +short transaction: + +1. Show `uploading file…` over the bottom of the pane. +2. Upload the file without sending a prompt. +3. On success, attach it through the harness adapter or paste a formatted path + reference into the CLI composer. +4. Return focus to xterm. The operator can keep typing and presses Enter when + ready. + +An upload must never auto-submit the agent's prompt. Inserting the attachment and +submitting are separate actions, matching native TUI image paste behavior. + +Only file-bearing drag events are claimed. Session/pane drag data continues to +reach the existing group layout handlers. The terminal drop target sets +`dropEffect='copy'`; pane reordering keeps `move`. + +### 5.3 Mobile paste + +The existing key-bar Paste action becomes file-aware: + +1. Try `navigator.clipboard.read()` inside the button's user gesture and extract + one preferred representation per file item (native raster first). Rich text + with a `text/plain` representation remains text rather than becoming a file. +2. If no file exists, try `navigator.clipboard.readText()` and retain current + text behavior. +3. If either API is missing or denied, open the visible fallback textarea. +4. Its `onPaste` handler checks `clipboardData.items` for files before reading + `text/plain`. + +This keeps the cross-origin iframe fallback as the load-bearing path rather than +assuming async Clipboard API permission. + +### 5.4 Remote panes + +The attachment affordance is disabled for remote sessions in phase one, with: + +> Files are not available for remote agents yet — that agent cannot read +> files stored on this Space. + +Silently inserting a Space path would be worse: it looks attached in the UI but +cannot exist on the remote machine. + +## 6. Data model and storage + +### 6.1 Location + +Managed files live at: + +```text +${STATE_DIR}/attachments//- +``` + +This is deliberately outside `WORKSPACES_DIR`: + +- attachments do not appear as untracked repository files; +- Agent Manager owns their lifecycle; +- several sessions sharing one working directory do not share an attachment + namespace; +- a later remote download endpoint can reuse the same store; and +- the mounted `/data` bucket keeps them across browser disconnects, Space sleep, + and process restarts. + +Attachment ids are generated by the server. The client filename is decoded, +reduced to a basename, stripped of control/path characters, and bounded before +it is appended to that id. Keeping the safe name preserves extensions required +by tools that inspect PDF, DOCX, archive, source, and data files. + +### 6.2 Attachment shape + +```ts +interface Attachment { + id: string; // att_, scoped to the session + kind: 'image' | 'file'; + name: string; // sanitized original name; image extension is corrected + mime: string; + bytes: number; + path: string; // absolute server path; used by the CLI adapter + previewUrl: string; + insertText: string; // safe fallback for direct terminal insertion +} +``` + +The attachment id is untrusted input everywhere it returns from the browser. +Accept only `^att_[a-f0-9]{24}$`, resolve the extension by listing/looking up the +session's own directory, and never join an arbitrary browser-supplied filename. + +### 6.3 Lifecycle + +- Composer files are uploaded only on submit. +- Terminal files are uploaded immediately because xterm needs a server path to + insert. +- Successful attachments remain while the session exists, including while it + is stopped. +- Deleting a session removes its managed attachment directory. It does not + affect files in the session workspace, preserving the existing session-delete + contract. +- Startup may prune attachment directories whose session no longer exists and + whose newest file is older than seven days. The grace period covers a crash + between session persistence and upload completion. +- Failed and aborted uploads delete their temporary file immediately. +- Each file is capped at 100 MiB; each session is capped at 200 stored files and + 500 MiB. Uploads for one session are serialized so concurrent requests cannot + race the quota check. +- Pruning and session deletion use asynchronous filesystem operations so a + large attachment store cannot block terminal I/O. + +No separate database is required. The session-scoped directory, validated id, +extension, and `stat` provide the metadata needed in v1. + +## 7. HTTP API + +### 7.1 Upload + +```http +POST /api/sessions/:id/attachments +Content-Type: application/pdf +X-File-Name: brief.pdf + + +``` + +Successful response: + +```json +{ + "id": "att_74e0f69dc9ed772cb685999e", + "kind": "file", + "name": "brief.pdf", + "mime": "application/pdf", + "bytes": 184332, + "path": "/data/state/attachments/session-id/att_74e0f69dc9ed772cb685999e-brief.pdf", + "previewUrl": "/api/sessions/session-id/attachments/att_74e0f69dc9ed772cb685999e/raw", + "insertText": "File: /data/state/attachments/session-id/att_74e0f69dc9ed772cb685999e-brief.pdf " +} +``` + +The request body is streamed to a temporary file. While streaming, count bytes +and abort with `413` above 100 MiB. Once complete: + +1. Read the first small header from the temporary file. +2. Detect PNG, JPEG, GIF, or WebP by magic bytes. Correct misleading image + metadata from those bytes and validate the complete image envelope. +3. If a supported raster image is claimed but its bytes are invalid, reject it + with `415`. Otherwise preserve the regular file as an opaque attachment. +4. Sanitize the basename and rename atomically under the generated id. +5. Return `201`. + +Do not base64-wrap attachment data in JSON. Base64 adds roughly one third to +transfer size and forces both browser and server to hold large strings in +memory. The existing Files upload route is a good streaming pattern, but it is +not reused directly because it writes into the user workspace and has different +overwrite semantics. + +Errors: + +| Status | Meaning | +|---|---| +| `400` | session cannot accept files or malformed request | +| `404` | unknown session/attachment | +| `413` | empty, larger than 100 MiB, or over the session quota | +| `415` | a claimed PNG/JPEG/GIF/WebP is malformed | +| `429` | per-session upload backstop exceeded | +| `500` | durable write failed | + +Use a small backstop such as 20 uploads/minute/session. This is not the security +boundary—the private Space is—but protects the terminal process from accidental +paste/drop loops. + +### 7.2 Preview + +```http +GET /api/sessions/:id/attachments/:attachmentId/raw +``` + +Return the detected MIME with: + +```text +X-Content-Type-Options: nosniff +Content-Security-Policy: sandbox +Cache-Control: no-store +``` + +Validated raster images use `Content-Disposition: inline` for chip previews. +Every other file uses `Content-Disposition: attachment`; combined with `sandbox` +and `nosniff`, HTML, SVG, Office, PDF, and other browser-capable formats cannot +become active content in the Agent Manager origin. + +Attachments may contain sensitive material, so use `Cache-Control: no-store` in +the first version. If bucket reads become measurable, a short private cache can +be evaluated later without making year-long browser retention the default. + +### 7.3 Send a structured prompt + +Extend the existing route without breaking text-only callers: + +```http +POST /api/sessions/:id/input +Content-Type: application/json + +{ + "text": "Match this layout", + "attachmentIds": ["att_74e0f69dc9ed772cb685999e"] +} +``` + +The server resolves every id inside that session's attachment directory before +starting or typing into the CLI. Unknown ids reject the whole request; never send +a prompt with only a subset of its files. + +`attachmentIds` defaults to `[]`, preserving every existing UI and API caller. +The agent-to-agent `text/plain` route remains unchanged. + +## 8. Prompt delivery + +### 8.1 One normalized input + +Refactor delivery conceptually to: + +```js +deliver(session, { text, attachments }, from) +``` + +The HTTP route validates and resolves attachment ids, then the harness adapter +turns `{text, absolutePaths}` into one of: + +- a textual prompt containing explicit file paths; +- a path-injection syntax such as Gemini's `@path`; or +- a short TUI attachment command followed by the textual prompt. + +The normalized prompt recorded in traces should remain legible even when the +harness receives a native attachment: + +```text +Match this layout + +Attached files: +- /data/state/attachments//-brief.pdf +``` + +This is also the universal fallback. Do not include base64 data in the prompt or +trace. + +### 8.2 Adapter interface + +Keep version-sensitive behavior in one module rather than scattered across React +components and `runner.js`. The implementation exposes the universal formatted +prompt and any native prelude commands separately: + +```ts +formatAttachmentDelivery(cli, text, attachments): string +formatAttachmentPrelude(cli, attachments): string[] +``` + +Initial adapters: + +- `gemini`: append `@` tokens to the prompt. +- `hermes`: send `/image ` + Return for each image, then send the + prompt. If the command is unavailable, use the universal prompt. +- all others and all non-image files: use the universal explicit-path prompt. + +opencode `--file` and other native launch flags are optional follow-ups. Codex +uses its installed `--image` support on a first turn; every adapter retains the +explicit path so a missing or changed native flag is not the only route to the +file. + +### 8.3 First prompt without a boot race + +The quickstart path currently places an initial prompt on the CLI launch command +because typing into a booting TUI could lose it. Attachment quickstart needs a +two-step browser flow—create the session, then upload to its attachment scope—so +`deliver()` must preserve that property: + +1. `POST /api/sessions` without a prompt creates a stopped session. +2. Upload all pending files to the returned session id. +3. `POST /api/sessions/:id/input` with text and attachment ids. +4. If the session has never started and its CLI has `withPrompt`, store the + fully formatted prompt as `pendingPrompt`; for Codex, also retain the + validated image paths as `pendingImagePaths`; then call `ensureRunning()`. +5. `commandFor()` consumes those fields on the first launch, adding one Codex + `-i` flag per image, and clears them once the PTY starts. + +Only resumed or already-started sessions use the existing boot-then-type path. +If attachment upload fails after session creation, keep the stopped session and +the browser draft. Automatically deleting it would make recovery surprising. + +### 8.4 Terminal insertion + +Live terminal paste/drop does not call `/input`, because `/input` submits a turn. +After upload, it asks the server to insert the resolved attachments without a +Return key: + +```http +POST /api/sessions/:id/attachments/insert +Content-Type: application/json + +{"attachmentIds":["att_74e0f69dc9ed772cb685999e"]} +``` + +The server writes `insertText` to the running PTY and acknowledges only after +the write is accepted. It never sends Return. If the process stops after upload, +the browser says the file was saved but not inserted and retains its attachment +id behind a Retry action. This avoids treating a browser-local xterm paste as +proof that a disconnected or non-controlling pane reached the CLI. Completed +terminal insertions are idempotent by attachment id, so retrying after a lost +HTTP response cannot paste the same path twice. + +Hermes is the one useful native exception: the server may send `/image ` +and Return, briefly wait for the command to settle, and then restore focus without +submitting the actual prompt. This logic should still live behind the same +adapter and fall back to `insertText`. + +## 9. Frontend structure + +Add a small module, `web/src/lib/attachments.ts`, containing: + +- image-preview MIME hints and a client-side 100 MiB check; +- `filesFromTransfer(DataTransfer)`; +- `transferMayContainFile(DataTransfer)`; +- duplicate suppression across `items` and `files`; +- local preview creation/revocation; and +- sequential or bounded-concurrency upload helpers. + +Use sequential uploads initially. Five files is the maximum, bucket writes are +the bottleneck, and simpler ordering makes chip status deterministic. + +Add a reusable `Attachments` chip row used by Sidebar and Overview. The +terminal imports only the transfer/upload helpers. + +Expected file changes: + +| File | Change | +|---|---| +| `web/src/api.ts` | attachment types, upload, preview URL, `sendInput(..., attachmentIds)` | +| `web/src/lib/attachments.ts` | clipboard/drop extraction and pending-file lifecycle | +| `web/src/components/Attachments.tsx` | image previews, file badges, picker, progress/error states | +| `web/src/components/Sidebar.tsx` | quickstart paste/drop and two-step submit | +| `web/src/components/Overview.tsx` | reply attachments and file-only send | +| `web/src/components/TerminalPane.tsx` | capture-phase file paste/drop and mobile file paste | +| `web/src/styles.css` | chips, drop highlight, terminal upload overlay | +| `server/src/attachments.js` | storage, validation, lookup, preview, cleanup | +| `server/src/index.js` | routes and structured delivery | +| `server/src/runner.js` | adapter prelude sequencing/first-prompt support if needed | +| `server/src/config.js` | optional per-harness attachment-directory access flags | + +## 10. Security and privacy + +### 10.1 Treat browser metadata as untrusted + +- Detect native/inline raster images from bytes, not `Content-Type`, extension, + or `File.name`. +- Treat every non-raster format—including SVG and HTML—as an inert file and + force download; never render it as active content in the app origin. +- Generate the attachment id server-side and reduce the client name to a + bounded, control-free basename. +- Resolve only under a fixed session attachment root. +- Refuse symlinks and non-regular files during lookup/preview. +- Cap bytes while streaming and remove partial files on abort/error. +- Use `wx`/exclusive temporary creation and atomic rename; never overwrite. + +### 10.2 Keep attachments private by default + +Attachments routinely contain tokens, customer data, internal documents, and +source code. They remain on the private Agent Manager storage and are not +uploaded to the Hub by this feature. + +Session sharing should keep its current safe behavior: a trace may contain the +local attachment path, but the attachment bytes are not bundled. A future +"include attachments" option requires its own explicit consent, redaction story, +and dataset visibility review. + +### 10.3 Avoid prompt ambiguity + +Paths are data, not instructions. Format the fallback in a clearly delimited +block generated by Agent Manager, while keeping the operator's text unchanged. +The server does not derive prompt text from an original filename. + +## 11. Remote-agent phase + +Remote agents currently exchange text messages and explicitly have no shared +filesystem or file upload. Supporting attachments requires protocol work, not +just enabling the button. + +Extend a remote user message with attachments: + +```json +{ + "seq": 42, + "role": "user", + "from": "operator", + "text": "fix this alignment", + "attachments": [ + { + "id": "att_74e0f69dc9ed772cb685999e", + "name": "screenshot.png", + "mime": "image/png", + "bytes": 184332, + "url": "/api/remote/my-agent/attachments/att_74e0f69dc9ed772cb685999e" + } + ] +} +``` + +The copied remote prompt must then: + +1. Download each attachment with the same authenticated HTTP client/HF token + used for polling. +2. Store it in a fresh local temporary directory. +3. Include the resulting local path when handing the message to the remote CLI. +4. Remove or retain it according to the remote session's lifecycle. + +The endpoint must be scoped to the remote name and attachment id. Base64 in the +message log is rejected: it exceeds the current 32 KiB message contract, bloats +Markdown logs, and needlessly injects binary material into traces. + +## 12. Failure behavior + +| Failure | User-visible result | Prompt sent? | +|---|---|---| +| clipboard contains no file | ordinary text paste continues, or no-op | no | +| file larger than limit | chip says `too large (100 MB max)` | no | +| network/write failure | chip or terminal overlay says upload failed | no | +| one of several uploads fails | successful files remain, all chips stay for retry | no | +| attachment id missing at send | `attachment no longer exists`; retain draft | no | +| terminal socket closes after upload | say file was saved but not inserted; offer retry | no | +| remote target | explain unsupported phase | no | +| CLI cannot inspect path | agent sees explicit path and can report the limitation | yes | + +For multi-file composer sends, atomicity applies to delivery, not storage: files +may upload one by one, but `/input` runs only after all have succeeded. A retry +may reuse already uploaded ids while uploading only failed files. + +## 13. Testing + +### 13.1 Server + +- PNG, JPEG, GIF, and WebP magic bytes produce the right extension/MIME. +- MIME/extension lies do not affect detection. +- PDF, DOCX, text, archives, and unknown extensions are stored as inert files. +- Empty input and malformed claimed raster images are rejected. +- The byte limit trips during streaming and leaves no partial file. +- Aborted requests leave no partial file. +- Concurrent uploads produce unique files. +- Concurrent uploads cannot race the per-session byte/count quota. +- Attachment ids cannot cross sessions or traverse paths. +- Preview headers include CSP and `nosniff`. +- A structured prompt rejects if any referenced attachment is absent. +- File-only and screenshot-only prompts receive appropriate default text. +- Deleting a session removes only its managed attachment directory. +- First-turn attachment delivery uses `pendingPrompt` rather than the boot delay. + +### 13.2 Browser/Playwright + +- A synthetic clipboard PNG on the quickstart textarea creates one chip and no + text. +- A pasted DOCX creates a generic extension chip and uploads beside an image. +- A drop containing the same file through both `items` and `files` creates one + chip. +- Plain-text paste is unchanged. +- Removing a chip revokes its preview and excludes it from upload. +- A multi-file submission waits for every upload before `/input`. +- Every chip mutation remains disabled for the complete multi-file send. +- Terminal paste receives a server acknowledgement and inserts without Return. +- A terminal stopped after upload reports saved-but-not-inserted, disables new + attachments, and can retry the stored attachment after restart. +- Remote live composers explain that Space-local files are unavailable. +- Terminal file drop does not trigger pane movement. +- Pane movement data does not trigger the file drop UI. +- The mobile fallback textarea handles both file and text paste. +- Clipboard API denial reaches the fallback sheet. +- Upload failure preserves the draft and shows the server's reason. + +`server/terminal-ui.test.mjs` already supplies a Chromium/xterm harness and +clipboard permissions; extend it for the live-terminal cases. Add a focused +server attachment test for storage and validation. + +### 13.3 Manual harness matrix + +For each installed CLI, verify a first turn and a later turn with one PNG, then +repeat with a PDF or DOCX to exercise explicit-path delivery: + +- the prompt arrives once; +- the CLI/model actually inspects the file (and its visual content for images); +- the terminal remains editable before submission; +- resuming the session can still inspect the same path; and +- a path containing a configurable `DATA_DIR` with spaces is quoted correctly. + +Test Chrome and Safari desktop, Chrome Android or Safari iOS, direct Space URL, +and the Hugging Face iframe. Clipboard permission behavior differs enough that +the DOM paste fallback must be exercised explicitly. + +## 14. Rollout + +1. Land storage/API plus Overview attachments. This proves durable upload and + structured delivery without touching xterm. +2. Add Sidebar quickstart and the first-prompt `pendingPrompt` path. +3. Add terminal paste/drop and the mobile fallback. +4. Add/verify native harness adapters one at a time; retain fallback paths. +5. Observe attachment sizes/counts in logs, then tune limits if real screenshots + routinely exceed them. +6. Design and land remote download transport separately. + +No migration is required. The attachment directory is created lazily on the +first successful upload. Old web clients continue sending `{text}` and old +sessions have no attachment state. + +## 15. Alternatives considered + +### Reuse `/api/files/:id/upload` + +Rejected as the public contract. It is intentionally a general workspace file +operation, accepts arbitrary names and bytes, overwrites an existing target, and +has no size cap. Its streaming/error cleanup pattern should be reused in the new +managed route. + +### Save screenshots inside the working repository + +Rejected as the default. It makes `git status` noisy, can accidentally enter a +commit, and forces cleanup policy into user source trees. A user who wants the +image as a project asset can separately save/copy it there. + +### Put base64 images in the prompt or remote Markdown log + +Rejected. It expands bytes, consumes memory and tokens, pollutes traces, exceeds +remote message limits, and makes accidental sharing more dangerous. + +### Emulate native clipboard/image terminal protocols + +Rejected. The browser still cannot place bytes on the container's clipboard, +and support differs across terminal emulators and CLIs. Uploading produces the +local path those native mechanisms eventually need anyway. + +### Integrate every CLI's structured API + +Deferred. Codex app-server, Gemini ACP, opencode serve, Hermes gateway, and other +interfaces could carry native image blocks, but adopting all of them would +replace Agent Manager's core PTY/session architecture. A path bridge is small, +observable, and works with the terminal sessions already being managed. + +## 16. References + +- Agent Manager input delivery: `server/src/index.js`, `server/src/runner.js` +- Existing browser paste behavior: `web/src/components/TerminalPane.tsx` +- Existing streaming file uploads: `server/src/index.js`, + `web/src/components/FilesPane.tsx` +- Remote filesystem scope: `docs/remote-agents.md` +- Claude Code image workflow: + +- Gemini CLI `@` commands: + +- opencode image drag/drop: + +- OpenAI Codex CLI overview (multimodal inputs): + diff --git a/server/package.json b/server/package.json index a2d6f9b..e302320 100644 --- a/server/package.json +++ b/server/package.json @@ -13,8 +13,9 @@ "scripts": { "start": "node src/index.js", "dev": "node --watch src/index.js", - "test:ui": "node terminal-ui.test.mjs", - "test": "node test/repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node migration.test.mjs && node resize.test.mjs" + "test:ui": "node terminal-ui.test.mjs && node screenshot-input.test.mjs", + "test:screenshots": "node screenshot-input.test.mjs", + "test": "node test/attachments.test.mjs && node test/repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node migration.test.mjs && node resize.test.mjs" }, "engines": { "node": ">=20.19" diff --git a/server/screenshot-input.test.mjs b/server/screenshot-input.test.mjs new file mode 100644 index 0000000..64c4cba --- /dev/null +++ b/server/screenshot-input.test.mjs @@ -0,0 +1,303 @@ +#!/usr/bin/env node +/** + * Attachment-input integration checks in a real browser: + * - stored bytes, response MIME, and preview headers come from detection; + * - a stopped terminal never reports a false insertion success; + * - an uploaded-but-uninserted screenshot can be retried without reupload; + * - attachment chips cannot mutate an in-flight send; + * - one clipboard image stays one chip across browser DataTransfer views; + * - document files stay inert downloads and can be sent beside images; + * - the creation dialog has no redundant file picker. + * + * Set SCREENSHOT_PUBLIC_DIR to a prebuilt web/dist to skip the build. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { chromium } from 'playwright'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.dirname(HERE); +const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'am-screenshot-ui-')); +const PUBLIC_DIR = process.env.SCREENSHOT_PUBLIC_DIR || path.join(DATA_DIR, 'public'); +const API = 'http://127.0.0.1:7896'; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const png = Buffer.alloc(45); +Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(png); +png.writeUInt32BE(13, 8); Buffer.from('IHDR').copy(png, 12); +png.writeUInt32BE(1, 16); png.writeUInt32BE(1, 20); +Buffer.from('IEND').copy(png, 37); +const pdf = Buffer.from('%PDF-1.7\nopaque pdf data'); +const docx = Buffer.from('PK\x03\x04opaque office data'); + +let failures = 0; +const check = (name, ok, detail = '') => { + console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` ${detail}` : ''}`); + if (!ok) failures += 1; +}; +const waitFor = async (fn, timeout = 15_000) => { + const until = Date.now() + timeout; + while (Date.now() < until) { + try { if (await fn()) return true; } catch {} + await sleep(100); + } + return false; +}; +const apiJson = async (url, init) => { + const response = await fetch(`${API}${url}`, init); + const body = await response.json().catch(() => ({})); + return { response, body }; +}; + +if (!process.env.SCREENSHOT_PUBLIC_DIR) { + const build = spawnSync('npm', ['run', 'build', '--', '--outDir', PUBLIC_DIR], { + cwd: path.join(ROOT, 'web'), encoding: 'utf8', + }); + if (build.status !== 0) throw new Error(`web build failed:\n${build.stdout}\n${build.stderr}`); +} + +const backend = spawn('node', ['src/index.js'], { + cwd: HERE, + env: { + ...process.env, + PORT: '7896', DATA_DIR, PUBLIC_DIR, AM_BASHRC: '/nonexistent', SPACE_HOST: '', + AM_TEST_REPAINT_CMD: 'bash --noprofile --norc', + }, + stdio: ['ignore', 'pipe', 'pipe'], +}); +let logs = ''; +backend.stdout.on('data', (data) => { logs += data; }); +backend.stderr.on('data', (data) => { logs += data; }); + +let browser; +try { + if (!await waitFor(() => fetch(`${API}/api/health`).then((response) => response.ok).catch(() => false), 60_000)) { + throw new Error(`server did not start:\n${logs.slice(-2000)}`); + } + await fetch(`${API}/api/welcome/seen`, { method: 'POST' }); + + const created = await apiJson('/api/sessions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cli: 'test-repaint', name: 'screenshot-e2e', path: '.' }), + }); + const id = created.body.id; + if (!id) throw new Error(`session creation failed: ${JSON.stringify(created.body)}`); + + // The browser's declared type is deliberately non-canonical. The response + // and raw preview must reflect the PNG bytes, not that metadata. + const upload = await fetch(`${API}/api/sessions/${id}/attachments`, { + method: 'POST', headers: { 'content-type': 'application/json' }, body: png, + }); + const stored = await upload.json(); + check('upload MIME is detected from bytes', upload.status === 201 && stored.mime === 'image/png', + JSON.stringify({ status: upload.status, mime: stored.mime })); + const preview = await fetch(`${API}${stored.previewUrl}`); + check('preview uses private, sandboxed response headers', + preview.headers.get('content-type') === 'image/png' + && preview.headers.get('cache-control') === 'no-store' + && preview.headers.get('content-disposition')?.startsWith('inline;') + && preview.headers.get('x-content-type-options') === 'nosniff' + && preview.headers.get('content-security-policy') === 'sandbox'); + + const documentUpload = await fetch(`${API}/api/sessions/${id}/attachments`, { + method: 'POST', + headers: { 'content-type': 'application/pdf', 'x-file-name': encodeURIComponent('brief.pdf') }, + body: pdf, + }); + const document = await documentUpload.json(); + const documentDownload = await fetch(`${API}${document.previewUrl}`); + check('PDF uploads are preserved as inert downloadable files', + documentUpload.status === 201 + && document.kind === 'file' + && document.name === 'brief.pdf' + && document.mime === 'application/pdf' + && documentDownload.headers.get('content-disposition')?.startsWith('attachment;') + && documentDownload.headers.get('x-content-type-options') === 'nosniff' + && documentDownload.headers.get('content-security-policy') === 'sandbox', + JSON.stringify({ status: documentUpload.status, kind: document.kind, name: document.name })); + + const svgUpload = await fetch(`${API}/api/sessions/${id}/attachments`, { + method: 'POST', + headers: { 'content-type': 'image/svg+xml', 'x-file-name': encodeURIComponent('diagram.svg') }, + body: Buffer.from(''), + }); + const svg = await svgUpload.json(); + const svgDownload = await fetch(`${API}${svg.previewUrl}`); + check('browser-active SVG is accepted only as a forced-download file', + svgUpload.status === 201 + && svg.kind === 'file' + && svgDownload.headers.get('content-type') === 'image/svg+xml' + && svgDownload.headers.get('content-disposition')?.startsWith('attachment;') + && svgDownload.headers.get('content-security-policy') === 'sandbox'); + + const missingId = `att_${'0'.repeat(24)}`; + const mixedSend = await apiJson(`/api/sessions/${id}/input`, { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ text: 'must not send partially', attachmentIds: [stored.id, missingId] }), + }); + const sessionsAfterReject = await (await fetch(`${API}/api/sessions`)).json(); + check('structured send rejects atomically before starting the agent', + mixedSend.response.status === 404 + && sessionsAfterReject.find((session) => session.id === id)?.running === false, + JSON.stringify({ status: mixedSend.response.status, error: mixedSend.body.error })); + + const remoteCreated = await apiJson('/api/sessions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ cli: 'remote', name: 'remote-screenshot-e2e', path: '.' }), + }); + const remoteUpload = await fetch(`${API}/api/sessions/${remoteCreated.body.id}/attachments`, { + method: 'POST', headers: { 'content-type': 'image/png' }, body: png, + }); + const remoteUploadBody = await remoteUpload.json(); + check('remote uploads are rejected with an actionable reason', + remoteUpload.status === 400 && remoteUploadBody.error.includes('cannot read files stored on this Space')); + + const bakedChromium = '/opt/pw-browsers/chromium-1208/chrome-linux64/chrome'; + const chromiumExecutable = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH + || (fs.existsSync(bakedChromium) ? bakedChromium : undefined); + browser = await chromium.launch({ + headless: true, + ...(chromiumExecutable ? { executablePath: chromiumExecutable } : {}), + }); + const page = await browser.newPage({ viewport: { width: 1280, height: 800 } }); + await page.goto(API, { waitUntil: 'domcontentloaded' }); + await page.locator('.sidebar .row[title^="screenshot-e2e"]').first().click(); + await page.locator('.tile-terminal:not(.tile-cached) .xterm-screen').waitFor({ state: 'visible' }); + await page.locator('.pane-head .ph-image').waitFor({ state: 'visible' }); + await waitFor(() => page.locator('.pane-head .ph-image').isEnabled()); + check('terminal file picker accepts documents and other regular files', + await page.locator('.pane-head .ph-image').getAttribute('aria-label') === 'Attach files' + && await page.locator('.pane-head .image-file-input').getAttribute('accept') === null); + + // Let the upload finish, stop the process, and only then expose the response + // to the UI. The following insert request must fail authoritatively. + let disconnectedAttachmentId; + await page.route(`**/api/sessions/${id}/attachments`, async (route) => { + const response = await route.fetch(); + disconnectedAttachmentId = (await response.json()).id; + await fetch(`${API}/api/sessions/${id}/stop`, { method: 'POST' }); + await route.fulfill({ response }); + }, { times: 1 }); + await page.locator('.pane-head .image-file-input').setInputFiles({ + name: 'disconnect.png', mimeType: 'image/png', buffer: png, + }); + const retryStatus = page.locator('.term-image-status.has-action'); + await retryStatus.waitFor({ state: 'visible' }); + const failedText = await retryStatus.textContent(); + check('a stopped terminal reports saved-but-not-inserted, never success', + !!failedText?.includes('saved but not inserted') && !failedText.includes('press Enter'), + JSON.stringify({ failedText })); + await page.locator('.term-exit').waitFor({ state: 'visible' }); + check('terminal attachment picker is unavailable while stopped', + await page.locator('.pane-head .ph-image').isDisabled()); + + await page.locator('.term-exit .tx-btn').click(); + await waitFor(() => page.locator('.pane-head .ph-image').isEnabled(), 20_000); + const retry = retryStatus.locator('button'); + await retry.click(); + await page.locator('.term-image-status.success').waitFor({ state: 'visible' }); + const successText = await page.locator('.term-image-status.success').textContent(); + check('retry inserts the already-uploaded file after restart', + !!successText?.includes('inserted') && successText.includes('press Enter'), + JSON.stringify({ successText })); + const terminalTail = await (await fetch(`${API}/api/agents/${id}/tail?lines=80`)).json(); + check('terminal retry reaches the PTY without submitting the prompt', + terminalTail.text.includes('Screenshot:') && !terminalTail.text.includes('command not found')); + const repeatedInsert = await apiJson(`/api/sessions/${id}/attachments/insert`, { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ attachmentIds: [disconnectedAttachmentId] }), + }); + const tailAfterRepeat = await (await fetch(`${API}/api/agents/${id}/tail?lines=80`)).json(); + const insertedCount = (value) => (value.match(/Screenshot:/g) || []).length; + check('retry is idempotent if the successful HTTP response was lost', + repeatedInsert.body.repeated === true + && insertedCount(tailAfterRepeat.text) === insertedCount(terminalTail.text)); + + const watcher = await browser.newPage({ viewport: { width: 1000, height: 700 } }); + await watcher.goto(API, { waitUntil: 'domcontentloaded' }); + await watcher.locator('.sidebar .row[title^="screenshot-e2e"]').first().click(); + await watcher.locator('.ph-role', { hasText: 'watching' }).waitFor({ state: 'visible' }); + check('a shared-terminal watcher cannot inject into the controller composer', + await watcher.locator('.pane-head .ph-image').isDisabled() + && (await watcher.locator('.pane-head .ph-image').getAttribute('title'))?.includes('take control')); + await watcher.close(); + + // Keep the creation dialog focused on the prompt. Pasting still works, and + // pasted chips remain removable, but choosing files belongs in live views. + await page.locator('.bolt-btn').click(); + check('creation dialog omits the redundant file picker', + await page.locator('.quick .image-pick').count() === 0 + && await page.locator('.quick .image-file-input').count() === 0); + await page.locator('.quick-cli[title="Repaint fixture"]').click(); + + // Hold the second upload. Once the first + // chip says uploaded, every attachment mutation must remain disabled until + // the single logical send transaction finishes. + await page.locator('.quick-prompt').fill('inspect both files'); + await page.locator('.quick-prompt').evaluate((element, bytes) => { + const raw = atob(bytes); + const data = Uint8Array.from(raw, (character) => character.charCodeAt(0)); + // WebKit can expose one clipboard image as distinct File instances through + // items and files, including different timestamps. `files` must win rather + // than merging both browser views. + const itemFile = new File([data], 'first.jpg', { type: 'image/jpg', lastModified: 1 }); + const listedFile = new File([data], 'first.jpg', { type: 'image/jpg', lastModified: 2 }); + const event = new Event('paste', { bubbles: true, cancelable: true }); + Object.defineProperty(event, 'clipboardData', { value: { + items: [{ kind: 'file', type: 'image/jpg', getAsFile: () => itemFile }], + files: [listedFile], + } }); + element.dispatchEvent(event); + }, png.toString('base64')); + check('clipboard image paste uses the canonical file list once and preserves prompt text', + await page.locator('.quick .image-chip').count() === 1 + && await page.locator('.quick-prompt').inputValue() === 'inspect both files'); + await page.locator('.quick-prompt').evaluate((element, bytes) => { + const raw = atob(bytes); + const data = Uint8Array.from(raw, (character) => character.charCodeAt(0)); + const transfer = new DataTransfer(); + transfer.items.add(new File([data], 'requirements.docx', { + type: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + })); + element.dispatchEvent(new ClipboardEvent('paste', { + bubbles: true, cancelable: true, clipboardData: transfer, + })); + }, docx.toString('base64')); + check('DOCX paste creates a generic file chip beside the image', + await page.locator('.quick .image-chip').count() === 2 + && await page.locator('.quick .image-chip-placeholder', { hasText: 'DOCX' }).count() === 1); + let releaseSecond; + let sawSecond; + const secondReached = new Promise((resolve) => { sawSecond = resolve; }); + const secondHold = new Promise((resolve) => { releaseSecond = resolve; }); + let uploadCount = 0; + await page.route('**/api/sessions/*/attachments', async (route) => { + uploadCount += 1; + if (uploadCount === 2) { + sawSecond(); + await secondHold; + } + await route.continue(); + }); + await page.locator('.quick-prompt').press('Enter'); + await Promise.race([ + secondReached, + sleep(20_000).then(() => { throw new Error('second upload did not start'); }), + ]); + const removeButtons = page.locator('.quick .image-chip > button'); + check('attachment removal stays locked for the full send transaction', + await removeButtons.nth(0).isDisabled() + && await removeButtons.nth(1).isDisabled()); + releaseSecond(); + await page.locator('.controls').waitFor({ state: 'hidden', timeout: 30_000 }); +} finally { + try { await browser?.close(); } catch {} + backend.kill('SIGKILL'); + fs.rmSync(DATA_DIR, { recursive: true, force: true }); +} + +console.log(failures ? `\n${failures} FAILURE(S)` : '\nall checks passed'); +process.exit(failures ? 1 : 0); diff --git a/server/src/attachments.js b/server/src/attachments.js new file mode 100644 index 0000000..70cdbf1 --- /dev/null +++ b/server/src/attachments.js @@ -0,0 +1,387 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { STATE_DIR } from './config.js'; + +export const ATTACHMENT_LIMIT = 100 * 1024 * 1024; +// A single prompt is capped separately at five files. This lifetime cap keeps +// a forgotten session from growing without bound while still leaving room for +// many ordinary attachment turns. +export const SESSION_ATTACHMENT_LIMIT = 500 * 1024 * 1024; +export const SESSION_ATTACHMENT_COUNT_LIMIT = 200; +export const ATTACHMENT_ID = /^att_[a-f0-9]{24}$/; +export const IMAGE_MIMES = Object.freeze([ + 'image/png', + 'image/jpeg', + 'image/webp', + 'image/gif', +]); + +const ATTACHMENTS_DIR = path.join(STATE_DIR, 'attachments'); +const IMAGE_EXTENSIONS = Object.freeze({ + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/webp': 'webp', + 'image/gif': 'gif', +}); +const MIME_BY_EXTENSION = Object.freeze({ + ...Object.fromEntries(Object.entries(IMAGE_EXTENSIONS).map(([mime, extension]) => [extension, mime])), + jpeg: 'image/jpeg', + pdf: 'application/pdf', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + xls: 'application/vnd.ms-excel', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ppt: 'application/vnd.ms-powerpoint', + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + odt: 'application/vnd.oasis.opendocument.text', + ods: 'application/vnd.oasis.opendocument.spreadsheet', + odp: 'application/vnd.oasis.opendocument.presentation', + txt: 'text/plain', + md: 'text/markdown', + csv: 'text/csv', + html: 'text/html', + htm: 'text/html', + css: 'text/css', + js: 'text/javascript', + mjs: 'text/javascript', + svg: 'image/svg+xml', + json: 'application/json', + yaml: 'application/yaml', + yml: 'application/yaml', + xml: 'application/xml', + rtf: 'application/rtf', + zip: 'application/zip', + gz: 'application/gzip', + tar: 'application/x-tar', + '7z': 'application/x-7z-compressed', + epub: 'application/epub+zip', +}); +const ATTACHMENT_FILE = /^att_[a-f0-9]{24}(?:[-.]|$)/; +const uploadWindows = new Map(); +const uploadLocks = new Map(); + +function httpError(statusCode, message) { + const error = new Error(message); + error.statusCode = statusCode; + return error; +} + +function sessionDir(sessionId) { + // Session ids are server-generated slugs. Keep this check here as a second + // boundary: a future caller must not turn an attachment lookup into a path + // join with an arbitrary browser value. + if (!/^[a-z0-9][a-z0-9-]{0,100}$/.test(String(sessionId))) { + throw httpError(400, 'session cannot accept files'); + } + return path.join(ATTACHMENTS_DIR, sessionId); +} + +function checkUploadRate(sessionId) { + const now = Date.now(); + const recent = (uploadWindows.get(sessionId) || []).filter((at) => now - at < 60_000); + if (recent.length >= 20) throw httpError(429, 'too many file uploads — try again in a minute'); + recent.push(now); + uploadWindows.set(sessionId, recent); +} + +// Serialize writes within one session so concurrent uploads cannot each pass a +// stale quota check. Different sessions still stream in parallel. +async function withUploadLock(sessionId, task) { + const previous = uploadLocks.get(sessionId) || Promise.resolve(); + let release; + const hold = new Promise((resolve) => { release = resolve; }); + const tail = previous.catch(() => {}).then(() => hold); + uploadLocks.set(sessionId, tail); + await previous.catch(() => {}); + try { + return await task(); + } finally { + release(); + if (uploadLocks.get(sessionId) === tail) uploadLocks.delete(sessionId); + } +} + +async function attachmentUsage(dir) { + let entries = []; + try { entries = await fs.promises.readdir(dir); } catch { return { bytes: 0, count: 0 }; } + let bytes = 0; + let count = 0; + for (const name of entries) { + try { + const stat = await fs.promises.lstat(path.join(dir, name)); + if (!stat.isFile() || stat.isSymbolicLink()) continue; + bytes += stat.size; + if (ATTACHMENT_FILE.test(name)) count += 1; + } catch {} + } + return { bytes, count }; +} + +export function detectImageMime(bytes) { + if (bytes.length >= 8 + && bytes[0] === 0x89 && bytes.subarray(1, 4).toString('ascii') === 'PNG' + && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) { + return 'image/png'; + } + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg'; + } + if (bytes.length >= 6 && ['GIF87a', 'GIF89a'].includes(bytes.subarray(0, 6).toString('ascii'))) { + return 'image/gif'; + } + if (bytes.length >= 12 + && bytes.subarray(0, 4).toString('ascii') === 'RIFF' + && bytes.subarray(8, 12).toString('ascii') === 'WEBP') { + return 'image/webp'; + } + return null; +} + +function imageEnvelopeIsValid(mime, header, tail, totalBytes) { + if (mime === 'image/png') { + return totalBytes >= 45 && header.length >= 24 + && header.readUInt32BE(8) === 13 + && header.subarray(12, 16).toString('ascii') === 'IHDR' + && header.readUInt32BE(16) > 0 && header.readUInt32BE(20) > 0 + && tail.length >= 12 + && tail.readUInt32BE(tail.length - 12) === 0 + && tail.subarray(tail.length - 8, tail.length - 4).toString('ascii') === 'IEND'; + } + if (mime === 'image/jpeg') { + return totalBytes >= 6 && tail.length >= 2 + && tail[tail.length - 2] === 0xff && tail[tail.length - 1] === 0xd9; + } + if (mime === 'image/gif') { + return totalBytes >= 14 && header.length >= 10 + && header.readUInt16LE(6) > 0 && header.readUInt16LE(8) > 0 + && tail[tail.length - 1] === 0x3b; + } + if (mime === 'image/webp') { + const chunk = header.subarray(12, 16).toString('ascii'); + return totalBytes >= 20 && header.length >= 16 + && ['VP8 ', 'VP8L', 'VP8X'].includes(chunk) + && header.readUInt32LE(4) + 8 <= totalBytes; + } + return false; +} + +const normalizedMime = (value) => { + const mime = String(value || '').split(';', 1)[0].trim().toLowerCase(); + const canonical = mime === 'image/jpg' || mime === 'image/pjpeg' ? 'image/jpeg' : mime; + return /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/.test(canonical) + ? canonical : ''; +}; + +const extensionOf = (name) => path.extname(name).slice(1).toLowerCase(); +const mimeForName = (name) => { + const known = MIME_BY_EXTENSION[extensionOf(name)]; + return (typeof known === 'string' ? known : '') || 'application/octet-stream'; +}; + +function truncateUtf8(value, maxBytes) { + let result = value; + while (Buffer.byteLength(result) > maxBytes) result = result.slice(0, -1); + return result; +} + +function safeFileName(value) { + let decoded = String(Array.isArray(value) ? value[0] : value || 'attachment'); + try { decoded = decodeURIComponent(decoded); } catch {} + let name = path.basename(decoded.replace(/\\/g, '/')) + .normalize('NFKC') + .replace(/[\u0000-\u001f\u007f/\\]/g, '_') + .trim(); + if (!name || name === '.' || name === '..') name = 'attachment'; + if (Buffer.byteLength(name) > 180) { + const candidate = path.extname(name); + const extension = Buffer.byteLength(candidate) <= 24 ? candidate : ''; + const stem = extension ? name.slice(0, -extension.length) : name; + name = `${truncateUtf8(stem, 180 - Buffer.byteLength(extension))}${extension}`; + } + return name || 'attachment'; +} + +function canonicalImageName(name, mime) { + const extension = IMAGE_EXTENSIONS[mime]; + const current = path.extname(name); + const stem = current ? name.slice(0, -current.length) : name; + return `${stem || 'Screenshot'}.${extension}`; +} + +const isImageMime = (mime) => IMAGE_MIMES.includes(mime); +const claimsNativeImage = (name, declared) => + isImageMime(normalizedMime(declared)) + || Object.values(IMAGE_EXTENSIONS).includes(extensionOf(name)) + || extensionOf(name) === 'jpeg'; + +const responseShape = (sessionId, id, mime, bytes, filePath, name = path.basename(filePath)) => { + const kind = isImageMime(mime) ? 'image' : 'file'; + return { + id, + kind, + name, + mime, + bytes, + path: filePath, + previewUrl: `/api/sessions/${encodeURIComponent(sessionId)}/attachments/${id}/raw`, + insertText: `${kind === 'image' ? 'Screenshot' : 'File'}: ${filePath} `, + }; +}; + +/** Stream one browser file into the session-owned attachment store. */ +export async function receiveAttachment(readable, sessionId, { contentType = '', fileName = '' } = {}) { + // Image metadata is advisory. Byte detection remains the security boundary + // for formats rendered inline or passed to a CLI's native image interface. + // Other formats are inert files: they are never executed, and the raw route + // forces them to download rather than rendering browser-active content. + const originalName = safeFileName(fileName); + checkUploadRate(sessionId); + return withUploadLock(sessionId, async () => { + const dir = sessionDir(sessionId); + await fs.promises.mkdir(dir, { recursive: true }); + const usage = await attachmentUsage(dir); + if (usage.count >= SESSION_ATTACHMENT_COUNT_LIMIT) { + throw httpError(413, `this session already has ${SESSION_ATTACHMENT_COUNT_LIMIT} files`); + } + const id = `att_${crypto.randomBytes(12).toString('hex')}`; + const temporary = path.join(dir, `.${id}.${crypto.randomBytes(4).toString('hex')}.part`); + let bytes = 0; + const limiter = new Transform({ + transform(chunk, _encoding, callback) { + bytes += chunk.length; + if (bytes > ATTACHMENT_LIMIT) return callback(httpError(413, 'file is larger than 100 MB')); + if (usage.bytes + bytes > SESSION_ATTACHMENT_LIMIT) { + return callback(httpError(413, 'this session has reached its 500 MB attachment limit')); + } + callback(null, chunk); + }, + }); + + try { + await pipeline(readable, limiter, fs.createWriteStream(temporary, { flags: 'wx' })); + if (bytes === 0) throw httpError(413, 'file is empty'); + + const handle = await fs.promises.open(temporary, 'r'); + const header = Buffer.alloc(32); + const tail = Buffer.alloc(Math.min(16, bytes)); + let bytesRead = 0; + let tailBytesRead = 0; + try { + ({ bytesRead } = await handle.read(header, 0, header.length, 0)); + ({ bytesRead: tailBytesRead } = await handle.read(tail, 0, tail.length, Math.max(0, bytes - tail.length))); + } finally { + await handle.close(); + } + const headerBytes = header.subarray(0, bytesRead); + const tailBytes = tail.subarray(0, tailBytesRead); + const detected = detectImageMime(headerBytes); + if (detected && !imageEnvelopeIsValid(detected, headerBytes, tailBytes, bytes)) { + throw httpError(415, 'image is truncated or malformed'); + } + if (!detected && claimsNativeImage(originalName, contentType)) { + throw httpError(415, 'file does not contain a valid PNG, JPEG, GIF, or WebP image'); + } + + const storedName = detected ? canonicalImageName(originalName, detected) : originalName; + const mime = detected || mimeForName(storedName); + const finalPath = path.join(dir, `${id}-${storedName}`); + await fs.promises.rename(temporary, finalPath); + return responseShape(sessionId, id, mime, bytes, finalPath, storedName); + } catch (error) { + await fs.promises.unlink(temporary).catch(() => {}); + throw error; + } + }); +} + +/** Resolve an untrusted attachment id within exactly one session. */ +export function resolveAttachment(sessionId, attachmentId) { + if (!ATTACHMENT_ID.test(String(attachmentId))) throw httpError(404, 'attachment not found'); + const dir = sessionDir(sessionId); + let names = []; + try { names = fs.readdirSync(dir); } catch {} + for (const name of names) { + const isCurrent = name.startsWith(`${attachmentId}-`); + const legacyExtension = name.startsWith(`${attachmentId}.`) ? extensionOf(name) : ''; + if (!isCurrent && !Object.values(IMAGE_EXTENSIONS).includes(legacyExtension)) continue; + const filePath = path.join(dir, name); + try { + const stat = fs.lstatSync(filePath); + if (!stat.isFile() || stat.isSymbolicLink()) continue; + const displayName = isCurrent ? name.slice(attachmentId.length + 1) : name; + return responseShape(sessionId, attachmentId, mimeForName(displayName), stat.size, filePath, displayName); + } catch {} + } + throw httpError(404, 'attachment not found'); +} + +export function resolveAttachments(sessionId, attachmentIds) { + if (!Array.isArray(attachmentIds)) throw httpError(400, 'attachmentIds must be an array'); + if (attachmentIds.length > 5) throw httpError(400, 'at most five files may be attached'); + if (new Set(attachmentIds).size !== attachmentIds.length) throw httpError(400, 'duplicate attachment id'); + return attachmentIds.map((id) => resolveAttachment(sessionId, id)); +} + +export async function removeSessionAttachments(sessionId) { + uploadWindows.delete(sessionId); + await withUploadLock(sessionId, () => fs.promises.rm(sessionDir(sessionId), { recursive: true, force: true })); +} + +/** Remove only old orphan stores; recent crash leftovers keep a seven-day grace. */ +export async function pruneAttachmentDirs(sessionIds, now = Date.now()) { + const live = new Set(sessionIds); + const cutoff = now - 7 * 24 * 60 * 60 * 1000; + let entries = []; + try { entries = await fs.promises.readdir(ATTACHMENTS_DIR, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const dir = path.join(ATTACHMENTS_DIR, entry.name); + // A process death can happen between exclusive temp creation and rename. + // Normal request failures remove these immediately; this is the crash + // backstop, with the same grace period as orphan session directories. + try { + for (const name of await fs.promises.readdir(dir)) { + const part = path.join(dir, name); + const stat = await fs.promises.lstat(part); + if (name.endsWith('.part') && stat.isFile() && !stat.isSymbolicLink() && stat.mtimeMs < cutoff) { + await fs.promises.unlink(part); + } + } + } catch {} + if (live.has(entry.name)) continue; + let newest = 0; + try { + const names = await fs.promises.readdir(dir); + const stats = await Promise.all(names.map((name) => fs.promises.lstat(path.join(dir, name)))); + newest = Math.max((await fs.promises.stat(dir)).mtimeMs, ...stats.map((stat) => stat.mtimeMs)); + } catch { continue; } + if (newest < cutoff) await fs.promises.rm(dir, { recursive: true, force: true }); + } +} + +const quotePath = (filePath) => JSON.stringify(filePath); + +/** TUI commands that attach native image context before the textual prompt. */ +export function formatAttachmentPrelude(cli, attachments) { + if (cli !== 'hermes') return []; + return attachments + .filter((attachment) => attachment.kind === 'image') + .map((image) => `/image ${quotePath(image.path)}`); +} + +/** Keep CLI-version-specific formatting out of request handlers and React. */ +export function formatAttachmentDelivery(cli, text, attachments) { + const onlyImages = attachments.length > 0 && attachments.every((attachment) => attachment.kind === 'image'); + const prompt = String(text || '').trim() || (onlyImages + ? `Please inspect the attached screenshot${attachments.length === 1 ? '' : 's'}.` + : `Please inspect the attached file${attachments.length === 1 ? '' : 's'}.`); + if (!attachments.length) return prompt; + const paths = attachments.map((attachment) => attachment.path); + if (cli === 'gemini') { + return `${prompt}\n\n${paths.map((filePath) => `@${quotePath(filePath)}`).join('\n')}`; + } + return `${prompt}\n\nAttached files:\n${paths.map((filePath) => `- ${quotePath(filePath)}`).join('\n')}`; +} diff --git a/server/src/config.js b/server/src/config.js index 32b3ba5..8459c1e 100644 --- a/server/src/config.js +++ b/server/src/config.js @@ -84,7 +84,9 @@ export const CLIS = [ withPrompt: (q) => `claude ${q}`, setup: setupHint('ANTHROPIC_API_KEY') }, { id: 'codex', label: 'Codex', bin: 'codex', color: '#5eb6a6', run: 'codex', cont: 'codex resume --last', resizeMode: 'repaint', - withPrompt: (q) => `codex ${q}`, + // `q` and image paths arrive shell-quoted from runner.commandFor(). Repeat + // -i because Codex's variadic flag would otherwise consume the prompt. + withPrompt: (q, images = []) => `codex${images.length ? ` ${images.map((image) => `-i ${image}`).join(' ')}` : ''} ${q}`, setup: setupHint('OPENAI_API_KEY') }, { id: 'gemini', label: 'Gemini CLI', bin: 'gemini', color: '#4796e3', run: 'gemini', cont: null, resizeMode: 'repaint', withPrompt: (q) => `gemini -i ${q}`, // -i = interactive session seeded with the prompt diff --git a/server/src/index.js b/server/src/index.js index f5a2224..6a90961 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -16,7 +16,7 @@ import * as store from './sessions.js'; import * as groups from './groups.js'; import * as order from './order.js'; import * as demo from './demo.js'; -import { attach, agentInfo, deriveState, stop, stopAll, ensureRunning, sendInput, isRunning, capturePane, ghosttyReady, ghosttyError, installClaudeRepinHook } from './runner.js'; +import { attach, agentInfo, deriveState, stop, stopAll, ensureRunning, sendInput, pasteInput, isRunning, capturePane, ghosttyReady, ghosttyError, installClaudeRepinHook } from './runner.js'; // Control frames ride the terminal socket behind a leading NUL pair, which real // PTY output never begins with. Same sentinel the old copy-mode hint used, so the @@ -31,10 +31,16 @@ import { startWatchdog } from './watchdog.js'; import { shareSession, shareNamespace, findTrace, shareAccess, grantAccess, revokeAccess, importBundle, listBundles, SHAREABLE_CLIS } from './share.js'; import * as backup from './backup.js'; +import { + formatAttachmentDelivery, formatAttachmentPrelude, pruneAttachmentDirs, receiveAttachment, removeSessionAttachments, + resolveAttachment, resolveAttachments, +} from './attachments.js'; ensureDirs(); refreshVersions(); store.init(); +pruneAttachmentDirs(store.list().map((session) => session.id)) + .catch((e) => console.error('[attachments.prune]', e && e.message)); groups.init(); order.init(); demo.init(); @@ -150,7 +156,14 @@ process.on('unhandledRejection', (e) => console.error('[unhandledRejection]', e) process.on('uncaughtException', (e) => console.error('[uncaughtException]', e)); const app = express(); -app.use(express.json()); +const jsonBody = express.json(); +app.use((req, res, next) => { + // Attachments are raw streaming bodies. Skip JSON parsing even when browser + // metadata claims application/json, or the global parser would consume a + // JSON file (and reject mislabeled image bytes) before the upload route. + if (req.method === 'POST' && /^\/api\/sessions\/[^/]+\/attachments$/.test(req.path)) return next(); + return jsonBody(req, res, next); +}); // Safety lock: with no authentication, only serve the terminal backend when the // Space is private. If it's public, block every working API (and /ws below) and @@ -247,8 +260,9 @@ const operatorName = () => process.env.SPACE_AUTHOR_NAME || process.env.AM_USER * agent-to-agent API) go through here, which is what makes remote agents * reachable from everywhere the local ones are without duplicating either path. */ -async function deliver(session, text, from) { +async function deliver(session, { text, attachments = [] }, from) { if (isRemote(session.cli)) { + if (attachments.length) throw Object.assign(new Error('files are not available for remote agents yet'), { statusCode: 400 }); const name = session.remote?.name; if (!name) throw new Error('this remote pane has no name recorded'); // Delivery does NOT un-pause: a disconnected agent isn't listening, so the @@ -257,9 +271,28 @@ async function deliver(session, text, from) { remote.append(name, { role: 'user', from: from || operatorName(), text }); return false; } + const prompt = formatAttachmentDelivery(session.cli, text, attachments); + const prelude = formatAttachmentPrelude(session.cli, attachments); + // A session created before its first prompt can still use the CLI's launch + // argument. This is especially important for quickstart attachments: upload + // needs a session id first, but typing into a half-booted TUI loses turns. + const cli = cliById(session.cli); + if (!session.everStarted && cli?.withPrompt && prelude.length === 0) { + store.update(session.id, { + pendingPrompt: prompt, + pendingImagePaths: session.cli === 'codex' + ? attachments.filter((attachment) => attachment.kind === 'image').map((image) => image.path) + : undefined, + }); + return ensureRunning(store.get(session.id) || session); + } const started = ensureRunning(session); if (started) await sleep(3500); // let the CLI boot before the keystrokes land - await sendInput(session.id, text); + for (const command of prelude) { + await sendInput(session.id, command); + await sleep(500); + } + await sendInput(session.id, prompt); return started; } @@ -271,12 +304,104 @@ app.post('/api/sessions/:id/input', async (req, res) => { if (!s) return res.status(404).json({ error: 'not found' }); if (PASSIVE_CLIS.includes(s.cli)) return res.status(400).json({ error: `${s.cli} pane takes no input` }); const text = typeof (req.body || {}).text === 'string' ? req.body.text.trim() : ''; - if (!text) return res.status(400).json({ error: 'empty' }); + const attachmentIds = (req.body || {}).attachmentIds ?? []; + if (!text && (!Array.isArray(attachmentIds) || attachmentIds.length === 0)) return res.status(400).json({ error: 'empty' }); try { - const started = await deliver(s, text); + const attachments = resolveAttachments(s.id, attachmentIds); + const started = await deliver(s, { text, attachments }); res.json({ ok: true, started }); } catch (e) { - res.status(409).json({ error: String(e.message || e) }); + res.status(e.statusCode || 409).json({ error: String(e.message || e) }); + } +}); + +const canAttachFiles = (session) => session.cli !== 'shell' + && !PASSIVE_CLIS.includes(session.cli) && !isRemote(session.cli); +// A lost HTTP response must not make the Retry action paste the same path a +// second time. Attachment ids are unique and the terminal UI never intentionally +// inserts one twice, so this bounded per-session set is a natural idempotency key. +const terminalAttachmentInsertions = new Map(); + +// Managed attachments live under STATE_DIR, never in the user's repository. +// The raw body is streamed and capped in attachments.js; express.json ignores +// this exact route, so no middleware buffers uploads first. +app.post('/api/sessions/:id/attachments', async (req, res) => { + const s = store.get(req.params.id); + if (!s) return res.status(404).json({ error: 'not found' }); + if (!canAttachFiles(s)) { + const error = isRemote(s.cli) + ? 'files are not available for remote agents yet — that agent cannot read files stored on this Space' + : `${s.cli} pane cannot accept files`; + return res.status(400).json({ error }); + } + try { + const attachment = await receiveAttachment(req, s.id, { + contentType: req.headers['content-type'], + fileName: req.headers['x-file-name'], + }); + if (!res.destroyed) res.status(201).json(attachment); + } catch (e) { + if (!res.headersSent && !res.destroyed) res.status(e.statusCode || 500).json({ error: String(e.message || e) }); + } +}); + +// Insert terminal attachments without pressing Return on the operator's +// prompt. This server acknowledgement is the source of truth for the terminal +// overlay; a browser-local xterm paste can be dropped after a disconnect or +// when another viewer owns the input lease. +app.post('/api/sessions/:id/attachments/insert', async (req, res) => { + const s = store.get(req.params.id); + if (!s) return res.status(404).json({ error: 'not found' }); + if (!canAttachFiles(s)) return res.status(400).json({ error: `${s.cli} pane cannot accept files` }); + try { + const attachments = resolveAttachments(s.id, (req.body || {}).attachmentIds ?? []); + if (!attachments.length) return res.status(400).json({ error: 'no files to insert' }); + const inserted = terminalAttachmentInsertions.get(s.id) || new Set(); + const pending = attachments.filter((attachment) => !inserted.has(attachment.id)); + const mode = s.cli === 'hermes' && attachments.some((attachment) => attachment.kind === 'image') + ? 'attached' : 'inserted'; + if (!pending.length) return res.json({ ok: true, mode, repeated: true }); + terminalAttachmentInsertions.set(s.id, inserted); + const nativeImages = s.cli === 'hermes' + ? pending.filter((attachment) => attachment.kind === 'image') : []; + const prelude = formatAttachmentPrelude(s.cli, nativeImages); + if (prelude.length) { + for (let index = 0; index < prelude.length; index += 1) { + await sendInput(s.id, prelude[index]); + inserted.add(nativeImages[index].id); + await sleep(500); + } + } + const inline = pending.filter((attachment) => !inserted.has(attachment.id)); + if (inline.length) pasteInput(s.id, inline.map((attachment) => attachment.insertText).join('')); + for (const attachment of inline) inserted.add(attachment.id); + return res.json({ ok: true, mode }); + } catch (e) { + return res.status(e.statusCode || 409).json({ error: String(e.message || e) }); + } +}); + +app.get('/api/sessions/:id/attachments/:attachmentId/raw', (req, res) => { + const s = store.get(req.params.id); + if (!s) return res.status(404).json({ error: 'not found' }); + try { + const attachment = resolveAttachment(s.id, req.params.attachmentId); + const encodedName = encodeURIComponent(attachment.name).replace(/[!'()*]/g, (character) => + `%${character.charCodeAt(0).toString(16).toUpperCase()}`); + const fallbackName = attachment.name.replace(/[^\x20-\x7e]|["\\]/g, '_'); + res.set({ + 'Content-Type': attachment.mime, + 'Content-Length': String(attachment.bytes), + 'Content-Disposition': `${attachment.kind === 'image' ? 'inline' : 'attachment'}; filename="${fallbackName}"; filename*=UTF-8''${encodedName}`, + 'X-Content-Type-Options': 'nosniff', + 'Content-Security-Policy': 'sandbox', + 'Cache-Control': 'no-store', + }); + const stream = fs.createReadStream(attachment.path); + stream.on('error', () => { if (!res.headersSent) res.status(404).end(); else res.destroy(); }); + stream.pipe(res); + } catch (e) { + res.status(e.statusCode || 404).json({ error: String(e.message || e) }); } }); @@ -466,7 +591,7 @@ app.post('/api/agents/:id/prompt', promptBody, async (req, res) => { // the operator's laptop, and the [message from x:] prefix plus `from:` in // the message's frontmatter is how it can tell a peer's request from the // operator's. - const started = await deliver(s, `[message from ${from.session.name}:] ${text}`, from.session.name); + const started = await deliver(s, { text: `[message from ${from.session.name}:] ${text}` }, from.session.name); res.json({ ok: true, id: s.id, name: s.name, started }); } catch (e) { res.status(409).json({ error: String(e.message || e) }); @@ -2005,7 +2130,7 @@ app.put('/api/trace/:id/source', (req, res) => { res.json({ ok: true, traceSource: { kind, ref } }); }); -app.delete('/api/sessions/:id', (req, res) => { +app.delete('/api/sessions/:id', async (req, res) => { const s = store.get(req.params.id); if (!s) return res.status(404).json({ error: 'not found' }); stop(s.id); @@ -2016,6 +2141,8 @@ app.delete('/api/sessions/:id', (req, res) => { groups.detachSession(s.id); order.drop(`s:${s.id}`); store.remove(s.id); + terminalAttachmentInsertions.delete(s.id); + try { await removeSessionAttachments(s.id); } catch (e) { console.error('[attachments.remove]', e && e.message); } res.json({ ok: true }); }); diff --git a/server/src/runner.js b/server/src/runner.js index b5aa781..494ecae 100644 --- a/server/src/runner.js +++ b/server/src/runner.js @@ -104,6 +104,7 @@ const MAX_COLS = 1000; const MAX_ROWS = 500; const hosts = new Map(); // session id -> host +const stopping = new Set(); function djb2(s) { let h = 5381; @@ -112,7 +113,7 @@ function djb2(s) { } export function isRunning(id) { - return hosts.has(id); + return hosts.has(id) && !stopping.has(id); } export function ghosttyReady() { @@ -1168,6 +1169,9 @@ export function commandFor(session) { // (claude 'p', codex 'p', gemini -i 'p', opencode --prompt 'p') — the CLI // starts already working on it, no typing race against a booting TUI. const q0 = !session.everStarted && session.pendingPrompt ? shq(session.pendingPrompt) : ''; + const q0Images = !session.everStarted && Array.isArray(session.pendingImagePaths) + ? session.pendingImagePaths.map((image) => shq(String(image))) : []; + const firstCommand = () => cli.withPrompt(q0, q0Images); // Claude keys conversations by working directory, so grouped sessions sharing // a folder would all `--continue` onto the SAME most-recent conversation. Pin @@ -1238,7 +1242,7 @@ export function commandFor(session) { const shared = list().some((o) => o.id !== session.id && o.cli === 'opencode' && (o.path ?? o.id) === folder); const base = session.everStarted && cli.cont && !shared ? `${cli.cont} || exec ${cli.run}` - : `exec ${q0 && cli.withPrompt ? cli.withPrompt(q0) : cli.run}`; + : `exec ${q0 && cli.withPrompt ? firstCommand() : cli.run}`; return `${guard}${base}`; } @@ -1256,7 +1260,7 @@ export function commandFor(session) { // the agent is the PTY's foreground process; when it exits the session ends — // a clear "done" signal — and the fallback preserves that. if (session.everStarted && cli.cont) return `${cli.cont} || exec ${cli.run}`; - if (q0 && cli.withPrompt) return `exec ${cli.withPrompt(q0)}`; + if (q0 && cli.withPrompt) return `exec ${firstCommand()}`; return `exec ${cli.run}`; } @@ -1397,6 +1401,7 @@ export function ensureRunning(session, cols = 120, rows = 34) { term.onExit(() => { hosts.delete(session.id); + stopping.delete(session.id); if (host.gridTimer) { clearTimeout(host.gridTimer); host.gridTimer = null; } if (host.traceHistoryTimer) { clearTimeout(host.traceHistoryTimer); host.traceHistoryTimer = null; } if (host.resizeCapture) { @@ -1411,8 +1416,11 @@ export function ensureRunning(session, cols = 120, rows = 34) { }); hosts.set(session.id, host); + stopping.delete(session.id); if (!persistedHistory && captureResize) hydrateTraceHistory(session, host); - if (!session.everStarted) update(session.id, { everStarted: true, pendingPrompt: undefined }); + if (!session.everStarted) update(session.id, { + everStarted: true, pendingPrompt: undefined, pendingImagePaths: undefined, + }); if (session.cli === 'codex') scheduleCodexCapture(session, workdir); if (session.cli === 'opencode') scheduleOpencodeCapture(session, workdir); if (session.cli === 'claude') scheduleClaudeCapture(session, workdir); @@ -1496,7 +1504,7 @@ export function attach(session, cols, rows) { /** Type a line into the session's terminal (works with no browser attached). */ export async function sendInput(id, text) { const host = hosts.get(id); - if (!host) throw new Error('session is not running'); + if (!host || stopping.has(id)) throw new Error('session is not running'); // Multi-line prompts go in as a bracketed paste so the CLI's composer treats // the inner newlines as soft line breaks instead of submitting early. const payload = text.includes('\n') ? `\x1b[200~${text}\x1b[201~` : text; @@ -1508,6 +1516,16 @@ export async function sendInput(id, text) { host.pty.write('\r'); } +/** Insert text into a running terminal's composer without submitting it. */ +export function pasteInput(id, text) { + const host = hosts.get(id); + if (!host || stopping.has(id)) throw new Error('session is not running'); + const value = String(text || ''); + if (!value) return; + const payload = value.includes('\n') ? `\x1b[200~${value}\x1b[201~` : value; + host.pty.write(payload); +} + /** * The session's rendered screen plus `lines` of scrollback above it — what a * human would see in the pane. Used by the agent API so one agent can watch @@ -1534,6 +1552,7 @@ export function capturePane(id, lines = 80) { export function stop(id) { const host = hosts.get(id); if (!host) return; + stopping.add(id); try { host.pty.kill(); } catch {} } @@ -1543,6 +1562,7 @@ export function stop(id) { */ export function stopAll() { for (const host of hosts.values()) { + stopping.add(host.id); try { host.pty.kill(); } catch {} } } diff --git a/server/terminal-ui.test.mjs b/server/terminal-ui.test.mjs index a83290e..97111a3 100644 --- a/server/terminal-ui.test.mjs +++ b/server/terminal-ui.test.mjs @@ -102,7 +102,13 @@ try { })); await sleep(600); - browser = await chromium.launch({ headless: true }); + const bakedChromium = '/opt/pw-browsers/chromium-1208/chrome-linux64/chrome'; + const chromiumExecutable = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH + || (fs.existsSync(bakedChromium) ? bakedChromium : undefined); + browser = await chromium.launch({ + headless: true, + ...(chromiumExecutable ? { executablePath: chromiumExecutable } : {}), + }); const context = await browser.newContext({ viewport: { width: 1280, height: 800 }, permissions: ['clipboard-read', 'clipboard-write'], diff --git a/server/test/attachments.test.mjs b/server/test/attachments.test.mjs new file mode 100644 index 0000000..2d2284c --- /dev/null +++ b/server/test/attachments.test.mjs @@ -0,0 +1,189 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'am-attachments-')); +process.env.DATA_DIR = root; + +const { + ATTACHMENT_LIMIT, SESSION_ATTACHMENT_LIMIT, detectImageMime, formatAttachmentDelivery, + formatAttachmentPrelude, pruneAttachmentDirs, receiveAttachment, removeSessionAttachments, + resolveAttachment, resolveAttachments, +} = await import('../src/attachments.js'); +const { cliById } = await import('../src/config.js'); +const { commandFor } = await import('../src/runner.js'); + +const png = Buffer.alloc(45); +Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(png); +png.writeUInt32BE(13, 8); Buffer.from('IHDR').copy(png, 12); +png.writeUInt32BE(1, 16); png.writeUInt32BE(1, 20); +Buffer.from('IEND').copy(png, 37); +const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0xff, 0xd9]); +const gif = Buffer.alloc(14); +Buffer.from('GIF89a', 'ascii').copy(gif); gif.writeUInt16LE(1, 6); gif.writeUInt16LE(1, 8); gif[13] = 0x3b; +const webp = Buffer.alloc(20); +Buffer.from('RIFF', 'ascii').copy(webp); webp.writeUInt32LE(12, 4); +Buffer.from('WEBPVP8 ', 'ascii').copy(webp, 8); +const receive = (bytes, sessionId, contentType = 'application/octet-stream', fileName = 'attachment') => + receiveAttachment(Readable.from([bytes]), sessionId, { contentType, fileName }); + +try { + assert.equal(detectImageMime(png), 'image/png'); + assert.equal(detectImageMime(jpeg), 'image/jpeg'); + assert.equal(detectImageMime(gif), 'image/gif'); + assert.equal(detectImageMime(webp), 'image/webp'); + assert.equal(detectImageMime(Buffer.from('')), null); + + const stored = await receive(png, 'codex-123abc', 'image/png', 'screen.png'); + assert.match(stored.id, /^att_[a-f0-9]{24}$/); + assert.equal(stored.mime, 'image/png'); + assert.equal(stored.bytes, png.length); + assert.equal(fs.readFileSync(stored.path).compare(png), 0); + assert.deepEqual(resolveAttachment('codex-123abc', stored.id), stored); + assert.deepEqual(resolveAttachments('codex-123abc', [stored.id]), [stored]); + + // Attachments created by the image-only implementation remain resolvable + // after the generalized filename format ships. + const legacyId = `att_${'1'.repeat(24)}`; + const legacyPath = path.join(path.dirname(stored.path), `${legacyId}.png`); + fs.writeFileSync(legacyPath, png); + const legacy = resolveAttachment('codex-123abc', legacyId); + assert.equal(legacy.kind, 'image'); + assert.equal(legacy.mime, 'image/png'); + assert.equal(legacy.path, legacyPath); + + assert.throws(() => resolveAttachment('other-123abc', stored.id), /not found/); + assert.throws(() => resolveAttachment('codex-123abc', '../sessions.json'), /not found/); + assert.throws(() => resolveAttachments('codex-123abc', Array(6).fill(stored.id)), /at most five/); + + const mislabeled = await receive(png, 'codex-123abc', 'image/jpeg', 'wrong.jpg'); + assert.equal(mislabeled.mime, 'image/png'); + assert.equal(mislabeled.name, 'wrong.png'); + const untyped = await receive(png, 'codex-123abc', '', 'Screenshot'); + assert.equal(untyped.mime, 'image/png'); + assert.equal(untyped.name, 'Screenshot.png'); + + const docx = await receive(Buffer.from('PK\u0003\u0004opaque office data'), 'codex-123abc', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + encodeURIComponent('../../Quarterly Report.docx')); + assert.equal(docx.kind, 'file'); + assert.equal(docx.name, 'Quarterly Report.docx'); + assert.equal(docx.mime, 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'); + assert.match(path.basename(docx.path), new RegExp(`^${docx.id}-Quarterly Report\\.docx$`)); + assert.match(docx.insertText, /^File: /); + assert.deepEqual(resolveAttachment('codex-123abc', docx.id), docx); + + const pdf = await receive(Buffer.from('%PDF-1.7\nopaque pdf data'), 'codex-123abc', + 'application/pdf', 'notes.pdf'); + assert.equal(pdf.kind, 'file'); + assert.equal(pdf.mime, 'application/pdf'); + const svg = await receive(Buffer.from(''), 'codex-123abc', + 'image/svg+xml', 'diagram.svg'); + assert.equal(svg.kind, 'file'); + assert.equal(svg.mime, 'image/svg+xml'); + const opaque = await receive(Buffer.from('unknown-format'), 'codex-123abc', + 'application/x-custom-format', 'model.blend'); + assert.equal(opaque.kind, 'file'); + assert.equal(opaque.mime, 'application/octet-stream'); + await assert.rejects( + receive(png.subarray(0, 24), 'codex-123abc', 'image/png', 'broken.png'), + (error) => error.statusCode === 415 && /malformed/.test(error.message), + ); + await assert.rejects( + receive(Buffer.alloc(0), 'codex-123abc', 'application/octet-stream', 'empty.bin'), + (error) => error.statusCode === 413, + ); + const tooLarge = Readable.from((function* chunks() { + let remaining = ATTACHMENT_LIMIT + 1; + while (remaining > 0) { + const size = Math.min(1024 * 1024, remaining); + remaining -= size; + yield Buffer.alloc(size); + } + }())); + await assert.rejects( + receiveAttachment(tooLarge, 'large-123abc', { contentType: 'application/octet-stream', fileName: 'large.bin' }), + (error) => error.statusCode === 413 && /100 MB/.test(error.message), + ); + assert.equal(fs.readdirSync(path.join(root, 'state', 'attachments', 'large-123abc')).some((name) => name.endsWith('.part')), false); + + const aborted = new Readable({ + read() { + this.push(png.subarray(0, 20)); + this.destroy(new Error('request aborted')); + }, + }); + await assert.rejects(receiveAttachment(aborted, 'aborted-123abc', { + contentType: 'image/png', fileName: 'aborted.png', + }), /request aborted/); + assert.equal(fs.readdirSync(path.join(root, 'state', 'attachments', 'aborted-123abc')).some((name) => name.endsWith('.part')), false); + + const quotaDir = path.join(root, 'state', 'attachments', 'quota-123abc'); + fs.mkdirSync(quotaDir, { recursive: true }); + const quotaFile = path.join(quotaDir, 'existing.bin'); + fs.writeFileSync(quotaFile, ''); + fs.truncateSync(quotaFile, SESSION_ATTACHMENT_LIMIT); + await assert.rejects( + receive(png, 'quota-123abc', 'image/png', 'quota.png'), + (error) => error.statusCode === 413 && /500 MB/.test(error.message), + ); + + const raceQuotaDir = path.join(root, 'state', 'attachments', 'race-quota-123abc'); + fs.mkdirSync(raceQuotaDir, { recursive: true }); + const raceQuotaFile = path.join(raceQuotaDir, 'existing.bin'); + fs.writeFileSync(raceQuotaFile, ''); + fs.truncateSync(raceQuotaFile, SESSION_ATTACHMENT_LIMIT - png.length); + const raced = await Promise.allSettled([ + receive(png, 'race-quota-123abc', 'image/png', 'first.png'), + receive(png, 'race-quota-123abc', 'image/png', 'second.png'), + ]); + assert.equal(raced.filter((result) => result.status === 'fulfilled').length, 1); + assert.equal(raced.filter((result) => result.status === 'rejected' + && result.reason.statusCode === 413).length, 1); + + const concurrent = await Promise.all(Array.from({ length: 4 }, () => + receive(png, 'parallel-123abc', 'application/octet-stream', 'parallel.png'))); + assert.equal(new Set(concurrent.map((image) => image.id)).size, concurrent.length); + + const formatted = formatAttachmentDelivery('codex', 'Compare this', [stored]); + assert.match(formatted, /Compare this/); + assert.match(formatted, /Attached files:/); + assert.match(formatted, new RegExp(stored.id)); + assert.match(formatAttachmentDelivery('gemini', '', [stored]), /Please inspect the attached screenshot\./); + assert.match(formatAttachmentDelivery('codex', '', [docx]), /Please inspect the attached file\./); + assert.deepEqual(formatAttachmentPrelude('hermes', [stored]), [`/image ${JSON.stringify(stored.path)}`]); + assert.deepEqual(formatAttachmentPrelude('hermes', [docx, stored]), [`/image ${JSON.stringify(stored.path)}`]); + assert.deepEqual(formatAttachmentPrelude('codex', [stored]), []); + assert.equal( + cliById('codex').withPrompt("'compare both'", ["'/tmp/first image.png'", "'/tmp/second.png'"]), + "codex -i '/tmp/first image.png' -i '/tmp/second.png' 'compare both'", + ); + assert.equal( + commandFor({ + id: 'codex-first-image', cli: 'codex', everStarted: false, + pendingPrompt: 'compare both', pendingImagePaths: ['/tmp/first image.png'], + }), + "exec codex -i '/tmp/first image.png' 'compare both'", + ); + + const old = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + const oldPart = path.join(path.dirname(stored.path), '.crashed.part'); + fs.writeFileSync(oldPart, 'partial'); + fs.utimesSync(oldPart, old, old); + const orphan = path.join(root, 'state', 'attachments', 'orphan-123abc'); + fs.mkdirSync(orphan, { recursive: true }); + fs.writeFileSync(path.join(orphan, 'old'), 'old'); + fs.utimesSync(path.join(orphan, 'old'), old, old); + fs.utimesSync(orphan, old, old); + await pruneAttachmentDirs(['codex-123abc']); + assert.equal(fs.existsSync(oldPart), false); + assert.equal(fs.existsSync(orphan), false); + + await removeSessionAttachments('codex-123abc'); + assert.equal(fs.existsSync(path.dirname(stored.path)), false); + console.log('attachment tests passed'); +} finally { + fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 94f7d74..4138a4b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import Sidebar from './components/Sidebar'; +import type { QuickStartAttachmentOptions } from './components/Sidebar'; import TerminalPane from './components/TerminalPane'; import FilesPane from './components/FilesPane'; import TracePane from './components/TracePane'; @@ -17,6 +18,7 @@ import * as api from './api'; import type { Cli, GridSpec, MoveTarget, OverviewFilter, Session, Tree } from './types'; import { isPassive, isRemote } from './types'; import { GridGlyph, ListGlyph } from './components/icons'; +import { uploadPendingAttachments } from './lib/attachments'; // Phone-sized viewport: the app becomes two full-screen views (list ⇄ pane). function useIsMobile() { @@ -489,13 +491,43 @@ export default function App() { }; // Quickstart: server boots the agent and types the prompt; we jump straight // to the new pane so you watch it happen. - const quickStart = async (cli: string, prompt: string, name = '', path = '.') => { + const quickStart = async (cli: string, prompt: string, name = '', path = '.', attachmentOptions?: QuickStartAttachmentOptions) => { try { - const s = await api.quickStart(cli, prompt, name, path); - rememberPath(s.path); + let sessionId: string; + let sessionPath: string | null = path; + if (attachmentOptions?.attachments.length) { + if (attachmentOptions.sessionId) { + sessionId = attachmentOptions.sessionId; + } else { + // Attachments are session-scoped, so create the stopped session first, + // then upload. If an upload fails the session remains visible and the + // sidebar retains its id for a retry. + const created = await api.createSession(name, cli, undefined, path); + sessionId = created.id; + sessionPath = created.path; + attachmentOptions.onSessionCreated(created.id); + rememberPath(created.path); + await refresh(); + } + const attachments = await uploadPendingAttachments( + sessionId, attachmentOptions.attachments, attachmentOptions.onAttachmentUpdate, + ); + await api.sendInput(sessionId, prompt, attachments.map((image) => image.id)); + } else { + const created = await api.quickStart(cli, prompt, name, path); + sessionId = created.id; + sessionPath = created.path; + } + rememberPath(sessionPath); await refresh(); - setActiveRef(`s:${s.id}`); - } catch (e) { showErr('Couldn’t quickstart the agent')(e); } + setActiveRef(`s:${sessionId}`); + } catch (e) { + // Quickstart owns a persistent inline recovery state (including the + // stopped session created before an upload), so a second generic toast + // obscures the useful error and makes one failure look like two. + console.error('Couldn’t quickstart the agent', e); + throw e; + } }; // Creations land in an explicitly targeted group (the group's + button), // else the group you're currently looking at; loose otherwise. diff --git a/web/src/api.ts b/web/src/api.ts index 1e4bd1a..57d88c1 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -178,8 +178,40 @@ export const getRemotePrompt = (name: string): Promise => export const setRemotePaused = (id: string, paused: boolean): Promise => fetch(`/api/sessions/${id}/remote/paused`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ paused }) }).then(json); -export const sendInput = (id: string, text: string): Promise<{ ok: boolean; started?: boolean }> => - fetch(`/api/sessions/${id}/input`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ text }) }).then(json); +export interface Attachment { + id: string; + kind: 'image' | 'file'; + name: string; + mime: string; + bytes: number; + path: string; + previewUrl: string; + insertText: string; +} + +export const uploadAttachment = async (id: string, file: File): Promise => { + const headers: Record = { + 'x-file-name': encodeURIComponent(file.name || 'Attachment'), + }; + if (file.type) headers['content-type'] = file.type; + const response = await fetch(`/api/sessions/${id}/attachments`, { + method: 'POST', + headers, + body: file, + }); + return jsonOrError(response); +}; + +export const insertAttachments = ( + id: string, + attachmentIds: string[], +): Promise<{ ok: boolean; mode: 'inserted' | 'attached'; repeated?: boolean }> => + fetch(`/api/sessions/${id}/attachments/insert`, { + method: 'POST', headers: HEADERS, body: JSON.stringify({ attachmentIds }), + }).then(jsonOrError); + +export const sendInput = (id: string, text: string, attachmentIds: string[] = []): Promise<{ ok: boolean; started?: boolean }> => + fetch(`/api/sessions/${id}/input`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ text, attachmentIds }) }).then(jsonOrError); // ---- push notifications ---- export const getPushKey = (): Promise<{ publicKey: string; devices: number }> => diff --git a/web/src/components/Attachments.tsx b/web/src/components/Attachments.tsx new file mode 100644 index 0000000..b7e2243 --- /dev/null +++ b/web/src/components/Attachments.tsx @@ -0,0 +1,77 @@ +import { useId, useRef } from 'react'; +import type { PendingAttachment } from '../lib/attachments'; + +const fmtBytes = (bytes: number) => bytes < 1024 * 1024 + ? `${Math.max(1, Math.round(bytes / 1024))} KB` + : `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + +const fileLabel = (name: string) => { + const extension = name.split('.').pop(); + return extension && extension !== name ? extension.slice(0, 4).toUpperCase() : 'FILE'; +}; + +export default function Attachments({ attachments, disabled, disabledReason, showPicker = true, onFiles, onRemove }: { + attachments: PendingAttachment[]; + disabled?: boolean; + disabledReason?: string; + showPicker?: boolean; + onFiles: (files: File[]) => void; + onRemove: (key: string) => void; +}) { + const picker = useRef(null); + const reasonId = useId(); + const showReason = showPicker && !!disabled && !!disabledReason; + if (!showPicker && attachments.length === 0) return null; + return ( +
+ {showPicker && ( + <> + + { + onFiles(Array.from(event.currentTarget.files || [])); + event.currentTarget.value = ''; + }} + /> + + )} + {showReason && {disabledReason}} + {attachments.map((attachment) => ( +
+ {attachment.previewUrl ? ( + + + + ) : ( + + )} + + {attachment.file.name || 'Attachment'} + + {attachment.status === 'uploading' ? 'uploading…' + : attachment.error || (attachment.status === 'uploaded' ? 'uploaded' : fmtBytes(attachment.file.size))} + + + +
+ ))} +
+ ); +} diff --git a/web/src/components/Overview.tsx b/web/src/components/Overview.tsx index ea8cc14..8aa4424 100644 --- a/web/src/components/Overview.tsx +++ b/web/src/components/Overview.tsx @@ -3,8 +3,14 @@ import type { CSSProperties, ReactNode } from 'react'; import * as api from '../api'; import type { MetaSession } from '../api'; import type { Cli, OverviewFilter, Session, SessionState, Tree } from '../types'; -import { isPassive } from '../types'; +import { isPassive, isRemote } from '../types'; import { renderMarkdown } from '../lib/markdown'; +import { + defaultAttachmentPrompt, filesFromTransfer, pendingAttachmentsFromFiles, revokePendingAttachments, + transferMayContainFile, uploadPendingAttachments, +} from '../lib/attachments'; +import type { PendingAttachment } from '../lib/attachments'; +import Attachments from './Attachments'; import Logo from './Logo'; const fmtAgo = (ts: number) => { @@ -36,14 +42,49 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { }) { const d = s.digest; const [draft, setDraft] = useState(''); + const [images, setImages] = useState([]); + const imagesRef = useRef([]); + const [imageError, setImageError] = useState(null); + const [dropActive, setDropActive] = useState(false); const [sending, setSending] = useState(false); - const [failed, setFailed] = useState(false); + const [failed, setFailed] = useState(null); // Optimistic echo: the sent text becomes the prompt line the moment the // send succeeds — the digest round-trip (CLI writes transcript → rebuild → // poll) can take seconds, and a frozen card reads as "did that get lost?". const [sent, setSent] = useState<{ text: string; at: number } | null>(null); const [histIdx, setHistIdx] = useState(0); // 0 = live view, n = n-th exchange back const inputRef = useRef(null); + const allowAttachments = !isRemote(s.cli); + + useEffect(() => { imagesRef.current = images; }, [images]); + useEffect(() => () => revokePendingAttachments(imagesRef.current), []); + + const addImages = (files: File[]) => { + if (!allowAttachments || sending || !files.length) return; + const next = pendingAttachmentsFromFiles(files, imagesRef.current.length); + const merged = [...imagesRef.current, ...next.attachments]; + imagesRef.current = merged; + setImages(merged); + setImageError(next.error); + }; + const removeImage = (key: string) => { + if (sending) return; + setImages((current) => { + const removed = current.find((image) => image.key === key); + if (removed?.previewUrl) URL.revokeObjectURL(removed.previewUrl); + const next = current.filter((image) => image.key !== key); + imagesRef.current = next; + return next; + }); + setImageError(null); + }; + const updateImage = (key: string, patch: Partial) => { + setImages((current) => { + const next = current.map((image) => image.key === key ? { ...image, ...patch } : image); + imagesRef.current = next; + return next; + }); + }; // After you send (or when the transcript shows a prompt newer than the last // answer), the old answer is stale — a spinner takes its place. @@ -60,18 +101,25 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { const send = async () => { const text = draft.trim(); - if (!text || sending) return; + const batch = imagesRef.current; + if ((!text && !batch.length) || sending) return; setSending(true); - setFailed(false); + setFailed(null); try { - await api.sendInput(s.id, text); + const attachments = await uploadPendingAttachments(s.id, batch, updateImage); + await api.sendInput(s.id, text, attachments.map((image) => image.id)); + const optimisticText = text || defaultAttachmentPrompt(batch.length); setDraft(''); - setSent({ text, at: Date.now() }); + revokePendingAttachments(batch); + imagesRef.current = []; + setImages([]); + setImageError(null); + setSent({ text: optimisticText, at: Date.now() }); setHistIdx(0); if (inputRef.current) { inputRef.current.style.height = 'auto'; inputRef.current.blur(); } - } catch { - setFailed(true); - setTimeout(() => setFailed(false), 4000); + } catch (error) { + setFailed(error instanceof Error ? error.message : 'failed to reach the agent'); + setTimeout(() => setFailed(null), 5000); } setSending(false); }; @@ -147,33 +195,57 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { {showLiveProgress &&
running
} -
- ❯ -