From 7e49345b4950662dafdf43db60a1594425ecbfdd Mon Sep 17 00:00:00 2001 From: Agent Manager Date: Wed, 5 Aug 2026 23:03:27 +0000 Subject: [PATCH 1/4] Add screenshot input for local agents --- docs/screenshot-input.md | 651 ++++++++++++++++++++++++ server/package.json | 2 +- server/src/attachments.js | 232 +++++++++ server/src/index.js | 73 ++- server/test/attachments.test.mjs | 83 +++ web/src/App.tsx | 37 +- web/src/api.ts | 27 +- web/src/components/ImageAttachments.tsx | 60 +++ web/src/components/Overview.tsx | 129 +++-- web/src/components/Sidebar.tsx | 152 +++++- web/src/components/TerminalPane.tsx | 141 ++++- web/src/lib/imageAttachments.ts | 101 ++++ web/src/styles.css | 34 +- 13 files changed, 1644 insertions(+), 78 deletions(-) create mode 100644 docs/screenshot-input.md create mode 100644 server/src/attachments.js create mode 100644 server/test/attachments.test.mjs create mode 100644 web/src/components/ImageAttachments.tsx create mode 100644 web/src/lib/imageAttachments.ts diff --git a/docs/screenshot-input.md b/docs/screenshot-input.md new file mode 100644 index 0000000..a04a7e2 --- /dev/null +++ b/docs/screenshot-input.md @@ -0,0 +1,651 @@ +# Screenshot input + +Status: proposed + +Date: 2026-08-05 + +## 1. Summary + +Let the operator attach screenshots to an agent prompt by: + +- pasting an image from the browser clipboard; +- dragging image files onto a prompt or terminal pane; or +- choosing images with a small attachment button. + +The browser uploads the image bytes to Agent Manager. Agent Manager stores them +outside the user's repository, then gives the target CLI a server-local image +path. The prompt remains text at the PTY boundary; no binary data is sent through +xterm or terminal escape sequences. + +The first version covers local agent sessions. Remote agents need an additional +download protocol because they do not share the Space's filesystem and are a +separate phase. + +## 2. Why this shape + +Three constraints decide the architecture. + +### 2.1 The browser and the CLI have different clipboards + +The browser receives a pasted screenshot as a `ClipboardEvent`/`File`. The agent +CLI runs in a headless container and can only inspect that container's OS +clipboard. Asking Claude, Codex, Hermes, or another TUI to read its clipboard +therefore cannot see what the operator copied on their laptop or phone. + +Agent Manager's existing mobile paste code already documents the same boundary: +direct browser clipboard access may also be unavailable inside the Hugging Face +cross-origin iframe, while a user-driven DOM `paste` event remains readable. + +Hermes Agent's own browser dashboard independently uses the intended bridge: +extract image files from the browser transfer, upload them to the gateway, then +drive the server-side TUI with the resulting local path. This is useful prior +art, not an integration dependency. + +### 2.2 Agent Manager's terminal transport is text + +Overview replies call `POST /api/sessions/:id/input`, `deliver()` starts the +session when needed, and `runner.sendInput()` writes a bracketed text paste plus +a separate Return key to the PTY. A live browser pane similarly sends xterm input +as WebSocket `{t:'i', d}` frames. + +That transport should stay text. Sending an image through a PTY would require a +terminal graphics/clipboard protocol and corresponding support in every TUI. +Those protocols do not solve the browser/container clipboard boundary and would +couple Agent Manager to terminal-specific behavior. + +### 2.3 Coding CLIs already understand image paths + +The exact affordance differs by harness, but a server-local path is the common +denominator: + +| Harness | Observed path as of this design | First implementation | +|---|---|---| +| Claude Code | accepts an image path in a prompt; native terminals also support image paste/drop | explicit absolute path in the prompt | +| Codex | `-i/--image` supports initial images; in-session image paste reads the server clipboard | explicit path, allowing `view_image`; native first-turn flag later | +| Gemini CLI | `@` injects supported images as multimodal context | `@` | +| opencode | TUI image drop and `opencode run -f/--file` are supported | explicit/drop-style absolute path | +| Hermes | `--image` and the interactive `/image ` command are supported | `/image ` adapter, with explicit-path fallback | +| OpenClaw | no stable image-attachment CLI contract was verified | explicit path, best effort | + +Every adapter must retain the explicit-path fallback. CLI flags and TUI commands +change faster than Agent Manager, while a file that exists and a prompt that +names it remain inspectable by an agent with filesystem and vision tools. + +## 3. Goals + +1. Paste a screenshot into any local agent input without first saving it by + hand. +2. Drag one or more raster images onto the input or terminal pane. +3. Work in the Hugging Face iframe and on phone-sized layouts. +4. Make upload progress, success, and failure visible. +5. Never submit a partially uploaded or missing attachment. +6. Avoid modifying or dirtying the user's repository. +7. Keep the attachment available for the lifetime of the session so resumed + agents can still inspect it. +8. Preserve ordinary text paste and Agent Manager's existing pane drag/drop. +9. Use bounded streaming uploads so images do not block the process that pumps + every PTY. + +## 4. Non-goals + +- A general document upload system. The first version accepts raster images + only. +- Images in agent-to-agent prompts. Agents inside the Space can already write a + file and reference its path. +- Synchronizing the operator's filesystem with a remote agent's filesystem. +- Making xterm render an attachment editor inside a third-party TUI. +- Automatically publishing screenshots with shared session traces. +- OCR, resizing, transcoding, or other server-side image processing. +- Replacing the PTY integration with each harness's structured SDK/server API. + +## 5. User experience + +### 5.1 Composers + +The Sidebar quickstart and Overview reply textarea get the same attachment +behavior: + +- Pasting an image adds a thumbnail chip and does not insert binary or fake text + into the textarea. +- Dropping images over the composer shows a restrained dashed highlight and + adds the same chips. +- An image button opens ``. +- Each chip shows a thumbnail, filename or `Screenshot`, size, and remove button. +- The prompt may contain text plus images or images alone. +- An images-only submission uses `Please inspect the attached screenshot.` (or + `screenshots` for several) as its text. +- At most five images may be attached to one prompt. +- The send button is disabled while an upload is active. +- A failed upload leaves the draft and pending images intact and names the + failure next to the affected chip. + +Pending images remain browser `File` objects until the operator submits. This +means abandoning or editing a draft does not create server-side orphan files. +`URL.createObjectURL()` supplies local previews and is revoked when a chip is +removed or the component unmounts. + +### 5.2 Live terminal panes + +xterm owns the visible composer, so Agent Manager cannot reliably place its own +persistent attachment chips inside it. Image paste/drop therefore behaves as a +short transaction: + +1. Show `uploading screenshot…` over the bottom of the pane. +2. Upload the image without sending a prompt. +3. On success, attach it through the harness adapter or paste a formatted path + reference into the CLI composer. +4. Return focus to xterm. The operator can keep typing and presses Enter when + ready. + +An upload must never auto-submit the agent's prompt. Inserting the attachment and +submitting are separate actions, matching native TUI image paste behavior. + +Only image-bearing drag events are claimed. Session/pane drag data continues to +reach the existing group layout handlers. The terminal drop target sets +`dropEffect='copy'`; pane reordering keeps `move`. + +### 5.3 Mobile paste + +The existing key-bar Paste action becomes image-aware: + +1. Try `navigator.clipboard.read()` inside the button's user gesture and extract + every `image/*` item. +2. If no image exists, try `navigator.clipboard.readText()` and retain current + text behavior. +3. If either API is missing or denied, open the visible fallback textarea. +4. Its `onPaste` handler checks `clipboardData.items` for images before reading + `text/plain`. + +This keeps the cross-origin iframe fallback as the load-bearing path rather than +assuming async Clipboard API permission. + +### 5.4 Remote panes + +The attachment affordance is disabled for remote sessions in phase one, with: + +> Screenshots are not available for remote agents yet — that agent cannot read +> files stored on this Space. + +Silently inserting a Space path would be worse: it looks attached in the UI but +cannot exist on the remote machine. + +## 6. Data model and storage + +### 6.1 Location + +Managed images live at: + +```text +${STATE_DIR}/attachments//. +``` + +This is deliberately outside `WORKSPACES_DIR`: + +- screenshots do not appear as untracked repository files; +- Agent Manager owns their lifecycle; +- several sessions sharing one working directory do not share an attachment + namespace; +- a later remote download endpoint can reuse the same store; and +- the mounted `/data` bucket keeps them across browser disconnects, Space sleep, + and process restarts. + +Filenames are generated by the server, contain no spaces, and never use a +client-supplied path. The original filename is display-only and need not be +persisted in v1. + +### 6.2 Attachment shape + +```ts +interface ImageAttachment { + id: string; // att_, scoped to the session + kind: 'image'; + name: string; // generated basename + mime: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif'; + bytes: number; + path: string; // absolute server path; used by the CLI adapter + previewUrl: string; + insertText: string; // safe fallback for direct terminal insertion +} +``` + +The attachment id is untrusted input everywhere it returns from the browser. +Accept only `^att_[a-f0-9]{24}$`, resolve the extension by listing/looking up the +session's own directory, and never join an arbitrary browser-supplied filename. + +### 6.3 Lifecycle + +- Composer images are uploaded only on submit. +- Terminal images are uploaded immediately because xterm needs a server path to + insert. +- Successful attachments remain while the session exists, including while it + is stopped. +- Deleting a session removes its managed attachment directory. It does not + affect files in the session workspace, preserving the existing session-delete + contract. +- Startup may prune attachment directories whose session no longer exists and + whose newest file is older than seven days. The grace period covers a crash + between session persistence and upload completion. +- Failed and aborted uploads delete their temporary file immediately. + +No separate database is required. The session-scoped directory, validated id, +extension, and `stat` provide the metadata needed in v1. + +## 7. HTTP API + +### 7.1 Upload + +```http +POST /api/sessions/:id/attachments +Content-Type: image/png +X-File-Name: screenshot.png + + +``` + +Successful response: + +```json +{ + "id": "att_74e0f69dc9ed772cb685999e", + "kind": "image", + "name": "att_74e0f69dc9ed772cb685999e.png", + "mime": "image/png", + "bytes": 184332, + "path": "/data/state/attachments/session-id/att_74e0f69dc9ed772cb685999e.png", + "previewUrl": "/api/sessions/session-id/attachments/att_74e0f69dc9ed772cb685999e/raw", + "insertText": "Screenshot: /data/state/attachments/session-id/att_74e0f69dc9ed772cb685999e.png " +} +``` + +The request body is streamed to a temporary file. While streaming, count bytes +and abort with `413` above 25 MiB. Once complete: + +1. Read the first small header from the temporary file. +2. Detect PNG, JPEG, GIF, or WebP by magic bytes. +3. Reject an unsupported or mismatched body with `415`. +4. Rename to the detected extension atomically. +5. Return `201`. + +Do not base64-wrap image data in JSON. Base64 adds roughly one third to transfer +size and forces both browser and server to hold large strings in memory. The +existing Files upload route is a good streaming pattern, but it is not reused +directly because it accepts arbitrary names, overwrites, and has no size cap. + +Errors: + +| Status | Meaning | +|---|---| +| `400` | session cannot accept images or malformed request | +| `404` | unknown session/attachment | +| `413` | empty or larger than 25 MiB | +| `415` | bytes are not a supported raster image | +| `429` | per-session upload backstop exceeded | +| `500` | durable write failed | + +Use a small backstop such as 20 uploads/minute/session. This is not the security +boundary—the private Space is—but protects the terminal process from accidental +paste/drop loops. + +### 7.2 Preview + +```http +GET /api/sessions/:id/attachments/:attachmentId/raw +``` + +Return the detected MIME with: + +```text +X-Content-Type-Options: nosniff +Content-Security-Policy: sandbox +Cache-Control: no-store +``` + +Screenshots may contain sensitive material, so use `Cache-Control: no-store` in +the first version. If bucket reads become measurable, a short private cache can +be evaluated later without making year-long browser retention the default. + +### 7.3 Send a structured prompt + +Extend the existing route without breaking text-only callers: + +```http +POST /api/sessions/:id/input +Content-Type: application/json + +{ + "text": "Match this layout", + "attachmentIds": ["att_74e0f69dc9ed772cb685999e"] +} +``` + +The server resolves every id inside that session's attachment directory before +starting or typing into the CLI. Unknown ids reject the whole request; never send +a prompt with only a subset of its images. + +`attachmentIds` defaults to `[]`, preserving every existing UI and API caller. +The agent-to-agent `text/plain` route remains unchanged. + +## 8. Prompt delivery + +### 8.1 One normalized input + +Refactor delivery conceptually to: + +```js +deliver(session, { text, attachments }, from) +``` + +The HTTP route validates and resolves attachment ids, then the harness adapter +turns `{text, absolutePaths}` into one of: + +- a textual prompt containing explicit image paths; +- a path-injection syntax such as Gemini's `@path`; or +- a short TUI attachment command followed by the textual prompt. + +The normalized prompt recorded in traces should remain legible even when the +harness receives a native attachment: + +```text +Match this layout + +Attached screenshots: +- /data/state/attachments//.png +``` + +This is also the universal fallback. Do not include base64 data in the prompt or +trace. + +### 8.2 Adapter interface + +Keep version-sensitive behavior in one module rather than scattered across React +components and `runner.js`: + +```ts +interface AttachmentDelivery { + prompt: string; + prelude?: Array<{ text: string; submit: boolean; settleMs?: number }>; +} + +formatAttachmentDelivery(cli, text, paths): AttachmentDelivery +``` + +Initial adapters: + +- `gemini`: append `@` tokens to the prompt. +- `hermes`: send `/image ` + Return for each image, then send the + prompt. If the command is unavailable, use the universal prompt. +- all others: use the universal prompt. + +Codex `--image`, opencode `--file`, and other native launch flags are optional +follow-ups. They should be added only with version probes/tests and must not be +required for the feature to function. + +### 8.3 First prompt without a boot race + +The quickstart path currently places an initial prompt on the CLI launch command +because typing into a booting TUI could lose it. Screenshot quickstart needs a +two-step browser flow—create the session, then upload to its attachment scope—so +`deliver()` must preserve that property: + +1. `POST /api/sessions` without a prompt creates a stopped session. +2. Upload all pending images to the returned session id. +3. `POST /api/sessions/:id/input` with text and attachment ids. +4. If the session has never started and its CLI has `withPrompt`, store the + fully formatted prompt as `pendingPrompt` and call `ensureRunning()`. +5. `commandFor()` consumes `pendingPrompt` on the first launch as it does today. + +Only resumed or already-started sessions use the existing boot-then-type path. +If attachment upload fails after session creation, keep the stopped session and +the browser draft. Automatically deleting it would make recovery surprising. + +### 8.4 Terminal insertion + +Live terminal paste/drop does not call `/input`, because `/input` submits a turn. +After upload, the browser uses `insertText` returned by the attachment endpoint: + +```ts +term.paste(uploaded.insertText) +``` + +`paste()` retains xterm's bracketed-paste handling. The browser claims terminal +control before insertion exactly as it does for ordinary paste. A watcher that +cannot claim control reports `Image uploaded, but this pane is watching—interact +and try again` rather than pretending the path reached the CLI. + +Hermes is the one useful native exception: the browser may send `/image ` +and Return, wait for the command to settle, and then restore focus without +submitting the actual prompt. This logic should still live behind the same +adapter and fall back to `insertText`. + +## 9. Frontend structure + +Add a small module, for example `web/src/lib/imageAttachments.ts`, containing: + +- accepted MIME types and client-side 25 MiB check; +- `imageFilesFromTransfer(DataTransfer)`; +- `transferMayContainImage(DataTransfer)`; +- duplicate suppression across `items` and `files`; +- local preview creation/revocation; and +- sequential or bounded-concurrency upload helpers. + +Use sequential uploads initially. Five files is the maximum, bucket writes are +the bottleneck, and simpler ordering makes chip status deterministic. + +Add a reusable `ImageAttachments` chip row used by Sidebar and Overview. The +terminal imports only the transfer/upload helpers. + +Expected file changes: + +| File | Change | +|---|---| +| `web/src/api.ts` | attachment types, upload, preview URL, `sendInput(..., attachmentIds)` | +| `web/src/lib/imageAttachments.ts` | clipboard/drop extraction and pending-image lifecycle | +| `web/src/components/ImageAttachments.tsx` | thumbnail chips, picker, progress/error states | +| `web/src/components/Sidebar.tsx` | quickstart paste/drop/picker and two-step submit | +| `web/src/components/Overview.tsx` | reply attachments and screenshot-only send | +| `web/src/components/TerminalPane.tsx` | capture-phase image paste/drop and mobile image paste | +| `web/src/styles.css` | chips, drop highlight, terminal upload overlay | +| `server/src/attachments.js` | storage, validation, lookup, preview, cleanup | +| `server/src/index.js` | routes and structured delivery | +| `server/src/runner.js` | adapter prelude sequencing/first-prompt support if needed | +| `server/src/config.js` | optional per-harness attachment-directory access flags | + +## 10. Security and privacy + +### 10.1 Treat browser metadata as untrusted + +- Detect the actual file type from bytes, not `Content-Type`, extension, or + `File.name`. +- Never accept SVG: it is active XML, not a screenshot transport format. +- Generate the stored name and attachment id server-side. +- Resolve only under a fixed session attachment root. +- Refuse symlinks and non-regular files during lookup/preview. +- Cap bytes while streaming and remove partial files on abort/error. +- Use `wx`/exclusive temporary creation and atomic rename; never overwrite. + +### 10.2 Keep screenshots private by default + +Screenshots routinely contain tokens, email addresses, internal dashboards, and +customer data. They remain on the private Agent Manager storage and are not +uploaded to the Hub by this feature. + +Session sharing should keep its current safe behavior: a trace may contain the +local attachment path, but the attachment bytes are not bundled. A future +"include attachments" option requires its own explicit consent, redaction story, +and dataset visibility review. + +### 10.3 Avoid prompt ambiguity + +Paths are data, not instructions. Format the fallback in a clearly delimited +block generated by Agent Manager, while keeping the operator's text unchanged. +The server does not derive prompt text from an original filename. + +## 11. Remote-agent phase + +Remote agents currently exchange text messages and explicitly have no remote +filesystem or file upload. Supporting screenshots requires protocol work, not +just enabling the button. + +Extend a remote user message with attachments: + +```json +{ + "seq": 42, + "role": "user", + "from": "operator", + "text": "fix this alignment", + "attachments": [ + { + "id": "att_74e0f69dc9ed772cb685999e", + "name": "screenshot.png", + "mime": "image/png", + "bytes": 184332, + "url": "/api/remote/my-agent/attachments/att_74e0f69dc9ed772cb685999e" + } + ] +} +``` + +The copied remote prompt must then: + +1. Download each attachment with the same authenticated HTTP client/HF token + used for polling. +2. Store it in a fresh local temporary directory. +3. Include the resulting local path when handing the message to the remote CLI. +4. Remove or retain it according to the remote session's lifecycle. + +The endpoint must be scoped to the remote name and attachment id. Base64 in the +message log is rejected: it exceeds the current 32 KiB message contract, bloats +Markdown logs, and needlessly injects binary material into traces. + +## 12. Failure behavior + +| Failure | User-visible result | Prompt sent? | +|---|---|---| +| unsupported clipboard type | ordinary text paste continues, or no-op | no | +| image larger than limit | chip says `too large (25 MB max)` | no | +| network/write failure | chip or terminal overlay says upload failed | no | +| one of several uploads fails | successful files remain, all chips stay for retry | no | +| attachment id missing at send | `attachment no longer exists`; retain draft | no | +| terminal socket closes after upload | say image was saved but not inserted; offer retry | no | +| remote target | explain unsupported phase | no | +| CLI cannot inspect path | agent sees explicit path and can report the limitation | yes | + +For multi-image composer sends, atomicity applies to delivery, not storage: files +may upload one by one, but `/input` runs only after all have succeeded. A retry +may reuse already uploaded ids while uploading only failed files. + +## 13. Testing + +### 13.1 Server + +- PNG, JPEG, GIF, and WebP magic bytes produce the right extension/MIME. +- MIME/extension lies do not affect detection. +- SVG, text, empty input, and malformed images are rejected. +- The byte limit trips during streaming and leaves no partial file. +- Aborted requests leave no partial file. +- Concurrent uploads produce unique files. +- Attachment ids cannot cross sessions or traverse paths. +- Preview headers include CSP and `nosniff`. +- A structured prompt rejects if any referenced attachment is absent. +- Screenshot-only prompts receive the default text. +- Deleting a session removes only its managed attachment directory. +- First-turn attachment delivery uses `pendingPrompt` rather than the boot delay. + +### 13.2 Browser/Playwright + +- A synthetic clipboard PNG on the quickstart textarea creates one chip and no + text. +- A drop containing the same file through both `items` and `files` creates one + chip. +- Plain-text paste is unchanged. +- Removing a chip revokes its preview and excludes it from upload. +- A multi-image submission waits for every upload before `/input`. +- Terminal paste uploads and inserts a path without Return. +- Terminal image drop does not trigger pane movement. +- Pane movement data does not trigger the image drop UI. +- The mobile fallback textarea handles both image and text paste. +- Clipboard API denial reaches the fallback sheet. +- Upload failure preserves the draft and shows the server's reason. + +`server/terminal-ui.test.mjs` already supplies a Chromium/xterm harness and +clipboard permissions; extend it for the live-terminal cases. Add a focused +server attachment test for storage and validation. + +### 13.3 Manual harness matrix + +For each installed CLI, verify a first turn and a later turn with one PNG: + +- the prompt arrives once; +- the CLI/model actually inspects the image; +- the terminal remains editable before submission; +- resuming the session can still inspect the same path; and +- a path containing a configurable `DATA_DIR` with spaces is quoted correctly. + +Test Chrome and Safari desktop, Chrome Android or Safari iOS, direct Space URL, +and the Hugging Face iframe. Clipboard permission behavior differs enough that +the DOM paste fallback must be exercised explicitly. + +## 14. Rollout + +1. Land storage/API plus Overview attachments. This proves durable upload and + structured delivery without touching xterm. +2. Add Sidebar quickstart and the first-prompt `pendingPrompt` path. +3. Add terminal paste/drop and the mobile fallback. +4. Add/verify native harness adapters one at a time; retain fallback paths. +5. Observe attachment sizes/counts in logs, then tune limits if real screenshots + routinely exceed them. +6. Design and land remote download transport separately. + +No migration is required. The attachment directory is created lazily on the +first successful upload. Old web clients continue sending `{text}` and old +sessions have no attachment state. + +## 15. Alternatives considered + +### Reuse `/api/files/:id/upload` + +Rejected as the public contract. It is intentionally a general workspace file +operation, accepts arbitrary names and bytes, overwrites an existing target, and +has no size cap. Its streaming/error cleanup pattern should be reused in the new +managed route. + +### Save screenshots inside the working repository + +Rejected as the default. It makes `git status` noisy, can accidentally enter a +commit, and forces cleanup policy into user source trees. A user who wants the +image as a project asset can separately save/copy it there. + +### Put base64 images in the prompt or remote Markdown log + +Rejected. It expands bytes, consumes memory and tokens, pollutes traces, exceeds +remote message limits, and makes accidental sharing more dangerous. + +### Emulate native clipboard/image terminal protocols + +Rejected. The browser still cannot place bytes on the container's clipboard, +and support differs across terminal emulators and CLIs. Uploading produces the +local path those native mechanisms eventually need anyway. + +### Integrate every CLI's structured API + +Deferred. Codex app-server, Gemini ACP, opencode serve, Hermes gateway, and other +interfaces could carry native image blocks, but adopting all of them would +replace Agent Manager's core PTY/session architecture. A path bridge is small, +observable, and works with the terminal sessions already being managed. + +## 16. References + +- Agent Manager input delivery: `server/src/index.js`, `server/src/runner.js` +- Existing browser paste behavior: `web/src/components/TerminalPane.tsx` +- Existing streaming file uploads: `server/src/index.js`, + `web/src/components/FilesPane.tsx` +- Remote filesystem scope: `docs/remote-agents.md` +- Claude Code image workflow: + +- Gemini CLI `@` commands: + +- opencode image drag/drop: + +- OpenAI Codex CLI overview (multimodal inputs): + diff --git a/server/package.json b/server/package.json index a2d6f9b..4515319 100644 --- a/server/package.json +++ b/server/package.json @@ -14,7 +14,7 @@ "start": "node src/index.js", "dev": "node --watch src/index.js", "test:ui": "node terminal-ui.test.mjs", - "test": "node test/repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node migration.test.mjs && node resize.test.mjs" + "test": "node test/attachments.test.mjs && node test/repin.test.mjs && node test/opencode-resume.test.mjs && node test/terminal-modes.test.mjs && node migration.test.mjs && node resize.test.mjs" }, "engines": { "node": ">=20.19" diff --git a/server/src/attachments.js b/server/src/attachments.js new file mode 100644 index 0000000..7981e7d --- /dev/null +++ b/server/src/attachments.js @@ -0,0 +1,232 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { STATE_DIR } from './config.js'; + +export const ATTACHMENT_LIMIT = 25 * 1024 * 1024; +export const ATTACHMENT_ID = /^att_[a-f0-9]{24}$/; +export const IMAGE_MIMES = Object.freeze([ + 'image/png', + 'image/jpeg', + 'image/webp', + 'image/gif', +]); + +const ATTACHMENTS_DIR = path.join(STATE_DIR, 'attachments'); +const EXTENSIONS = Object.freeze({ + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/webp': 'webp', + 'image/gif': 'gif', +}); +const uploadWindows = new Map(); + +function httpError(statusCode, message) { + const error = new Error(message); + error.statusCode = statusCode; + return error; +} + +function sessionDir(sessionId) { + // Session ids are server-generated slugs. Keep this check here as a second + // boundary: a future caller must not turn an attachment lookup into a path + // join with an arbitrary browser value. + if (!/^[a-z0-9][a-z0-9-]{0,100}$/.test(String(sessionId))) { + throw httpError(400, 'session cannot accept images'); + } + return path.join(ATTACHMENTS_DIR, sessionId); +} + +function checkUploadRate(sessionId) { + const now = Date.now(); + const recent = (uploadWindows.get(sessionId) || []).filter((at) => now - at < 60_000); + if (recent.length >= 20) throw httpError(429, 'too many image uploads — try again in a minute'); + recent.push(now); + uploadWindows.set(sessionId, recent); +} + +export function detectImageMime(bytes) { + if (bytes.length >= 8 + && bytes[0] === 0x89 && bytes.subarray(1, 4).toString('ascii') === 'PNG' + && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) { + return 'image/png'; + } + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg'; + } + if (bytes.length >= 6 && ['GIF87a', 'GIF89a'].includes(bytes.subarray(0, 6).toString('ascii'))) { + return 'image/gif'; + } + if (bytes.length >= 12 + && bytes.subarray(0, 4).toString('ascii') === 'RIFF' + && bytes.subarray(8, 12).toString('ascii') === 'WEBP') { + return 'image/webp'; + } + return null; +} + +function imageEnvelopeIsValid(mime, header, tail, totalBytes) { + if (mime === 'image/png') { + return totalBytes >= 45 && header.length >= 24 + && header.readUInt32BE(8) === 13 + && header.subarray(12, 16).toString('ascii') === 'IHDR' + && header.readUInt32BE(16) > 0 && header.readUInt32BE(20) > 0 + && tail.length >= 12 + && tail.readUInt32BE(tail.length - 12) === 0 + && tail.subarray(tail.length - 8, tail.length - 4).toString('ascii') === 'IEND'; + } + if (mime === 'image/jpeg') { + return totalBytes >= 6 && tail.length >= 2 + && tail[tail.length - 2] === 0xff && tail[tail.length - 1] === 0xd9; + } + if (mime === 'image/gif') { + return totalBytes >= 14 && header.length >= 10 + && header.readUInt16LE(6) > 0 && header.readUInt16LE(8) > 0 + && tail[tail.length - 1] === 0x3b; + } + if (mime === 'image/webp') { + const chunk = header.subarray(12, 16).toString('ascii'); + return totalBytes >= 20 && header.length >= 16 + && ['VP8 ', 'VP8L', 'VP8X'].includes(chunk) + && header.readUInt32LE(4) + 8 <= totalBytes; + } + return false; +} + +const responseShape = (sessionId, id, mime, bytes, filePath) => ({ + id, + kind: 'image', + name: path.basename(filePath), + mime, + bytes, + path: filePath, + previewUrl: `/api/sessions/${encodeURIComponent(sessionId)}/attachments/${id}/raw`, + insertText: `Screenshot: ${filePath} `, +}); + +/** Stream one browser image into the session-owned attachment store. */ +export async function receiveImage(readable, sessionId, contentType) { + const declared = String(contentType || '').split(';', 1)[0].trim().toLowerCase(); + if (!IMAGE_MIMES.includes(declared)) throw httpError(415, 'use PNG, JPEG, GIF, or WebP'); + checkUploadRate(sessionId); + + const dir = sessionDir(sessionId); + await fs.promises.mkdir(dir, { recursive: true }); + const id = `att_${crypto.randomBytes(12).toString('hex')}`; + const temporary = path.join(dir, `.${id}.${crypto.randomBytes(4).toString('hex')}.part`); + let bytes = 0; + const limiter = new Transform({ + transform(chunk, _encoding, callback) { + bytes += chunk.length; + if (bytes > ATTACHMENT_LIMIT) return callback(httpError(413, 'image is larger than 25 MB')); + callback(null, chunk); + }, + }); + + try { + await pipeline(readable, limiter, fs.createWriteStream(temporary, { flags: 'wx' })); + if (bytes === 0) throw httpError(413, 'image is empty'); + + const handle = await fs.promises.open(temporary, 'r'); + const header = Buffer.alloc(32); + const tail = Buffer.alloc(Math.min(16, bytes)); + let bytesRead = 0; + let tailBytesRead = 0; + try { + ({ bytesRead } = await handle.read(header, 0, header.length, 0)); + ({ bytesRead: tailBytesRead } = await handle.read(tail, 0, tail.length, Math.max(0, bytes - tail.length))); + } finally { + await handle.close(); + } + const headerBytes = header.subarray(0, bytesRead); + const tailBytes = tail.subarray(0, tailBytesRead); + const detected = detectImageMime(headerBytes); + if (!detected || detected !== declared) { + throw httpError(415, detected + ? `image bytes are ${detected}, not ${declared}` + : 'file is not a supported raster image'); + } + if (!imageEnvelopeIsValid(detected, headerBytes, tailBytes, bytes)) { + throw httpError(415, 'image is truncated or malformed'); + } + + const finalPath = path.join(dir, `${id}.${EXTENSIONS[detected]}`); + await fs.promises.rename(temporary, finalPath); + return responseShape(sessionId, id, detected, bytes, finalPath); + } catch (error) { + await fs.promises.unlink(temporary).catch(() => {}); + throw error; + } +} + +/** Resolve an untrusted attachment id within exactly one session. */ +export function resolveImage(sessionId, attachmentId) { + if (!ATTACHMENT_ID.test(String(attachmentId))) throw httpError(404, 'attachment not found'); + const dir = sessionDir(sessionId); + for (const [mime, extension] of Object.entries(EXTENSIONS)) { + const filePath = path.join(dir, `${attachmentId}.${extension}`); + try { + const stat = fs.lstatSync(filePath); + if (!stat.isFile() || stat.isSymbolicLink()) continue; + return responseShape(sessionId, attachmentId, mime, stat.size, filePath); + } catch {} + } + throw httpError(404, 'attachment not found'); +} + +export function resolveImages(sessionId, attachmentIds) { + if (!Array.isArray(attachmentIds)) throw httpError(400, 'attachmentIds must be an array'); + if (attachmentIds.length > 5) throw httpError(400, 'at most five images may be attached'); + if (new Set(attachmentIds).size !== attachmentIds.length) throw httpError(400, 'duplicate attachment id'); + return attachmentIds.map((id) => resolveImage(sessionId, id)); +} + +export function removeSessionAttachments(sessionId) { + uploadWindows.delete(sessionId); + fs.rmSync(sessionDir(sessionId), { recursive: true, force: true }); +} + +/** Remove only old orphan stores; recent crash leftovers keep a seven-day grace. */ +export function pruneAttachmentDirs(sessionIds, now = Date.now()) { + const live = new Set(sessionIds); + const cutoff = now - 7 * 24 * 60 * 60 * 1000; + let entries = []; + try { entries = fs.readdirSync(ATTACHMENTS_DIR, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const dir = path.join(ATTACHMENTS_DIR, entry.name); + // A process death can happen between exclusive temp creation and rename. + // Normal request failures remove these immediately; this is the crash + // backstop, with the same grace period as orphan session directories. + try { + for (const name of fs.readdirSync(dir)) { + const part = path.join(dir, name); + if (name.endsWith('.part') && fs.lstatSync(part).isFile() && fs.statSync(part).mtimeMs < cutoff) { + fs.unlinkSync(part); + } + } + } catch {} + if (live.has(entry.name)) continue; + let newest = 0; + try { + newest = Math.max(fs.statSync(dir).mtimeMs, ...fs.readdirSync(dir).map((name) => fs.lstatSync(path.join(dir, name)).mtimeMs)); + } catch { continue; } + if (newest < cutoff) fs.rmSync(dir, { recursive: true, force: true }); + } +} + +const quotePath = (filePath) => JSON.stringify(filePath); + +/** Keep CLI-version-specific formatting out of request handlers and React. */ +export function formatAttachmentDelivery(cli, text, images) { + const prompt = String(text || '').trim() + || `Please inspect the attached screenshot${images.length === 1 ? '' : 's'}.`; + if (!images.length) return prompt; + const paths = images.map((image) => image.path); + if (cli === 'gemini') { + return `${prompt}\n\n${paths.map((filePath) => `@${quotePath(filePath)}`).join('\n')}`; + } + return `${prompt}\n\nAttached screenshots:\n${paths.map((filePath) => `- ${quotePath(filePath)}`).join('\n')}`; +} diff --git a/server/src/index.js b/server/src/index.js index f5a2224..6f48cd3 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -31,10 +31,15 @@ import { startWatchdog } from './watchdog.js'; import { shareSession, shareNamespace, findTrace, shareAccess, grantAccess, revokeAccess, importBundle, listBundles, SHAREABLE_CLIS } from './share.js'; import * as backup from './backup.js'; +import { + formatAttachmentDelivery, pruneAttachmentDirs, receiveImage, removeSessionAttachments, + resolveImage, resolveImages, +} from './attachments.js'; ensureDirs(); refreshVersions(); store.init(); +pruneAttachmentDirs(store.list().map((session) => session.id)); groups.init(); order.init(); demo.init(); @@ -247,8 +252,9 @@ const operatorName = () => process.env.SPACE_AUTHOR_NAME || process.env.AM_USER * agent-to-agent API) go through here, which is what makes remote agents * reachable from everywhere the local ones are without duplicating either path. */ -async function deliver(session, text, from) { +async function deliver(session, { text, attachments = [] }, from) { if (isRemote(session.cli)) { + if (attachments.length) throw Object.assign(new Error('screenshots are not available for remote agents yet'), { statusCode: 400 }); const name = session.remote?.name; if (!name) throw new Error('this remote pane has no name recorded'); // Delivery does NOT un-pause: a disconnected agent isn't listening, so the @@ -257,9 +263,18 @@ async function deliver(session, text, from) { remote.append(name, { role: 'user', from: from || operatorName(), text }); return false; } + const prompt = formatAttachmentDelivery(session.cli, text, attachments); + // A session created before its first prompt can still use the CLI's launch + // argument. This is especially important for quickstart attachments: upload + // needs a session id first, but typing into a half-booted TUI loses turns. + const cli = cliById(session.cli); + if (!session.everStarted && cli?.withPrompt) { + store.update(session.id, { pendingPrompt: prompt }); + return ensureRunning(store.get(session.id) || session); + } const started = ensureRunning(session); if (started) await sleep(3500); // let the CLI boot before the keystrokes land - await sendInput(session.id, text); + await sendInput(session.id, prompt); return started; } @@ -271,12 +286,57 @@ app.post('/api/sessions/:id/input', async (req, res) => { if (!s) return res.status(404).json({ error: 'not found' }); if (PASSIVE_CLIS.includes(s.cli)) return res.status(400).json({ error: `${s.cli} pane takes no input` }); const text = typeof (req.body || {}).text === 'string' ? req.body.text.trim() : ''; - if (!text) return res.status(400).json({ error: 'empty' }); + const attachmentIds = (req.body || {}).attachmentIds ?? []; + if (!text && (!Array.isArray(attachmentIds) || attachmentIds.length === 0)) return res.status(400).json({ error: 'empty' }); try { - const started = await deliver(s, text); + const attachments = resolveImages(s.id, attachmentIds); + const started = await deliver(s, { text, attachments }); res.json({ ok: true, started }); } catch (e) { - res.status(409).json({ error: String(e.message || e) }); + res.status(e.statusCode || 409).json({ error: String(e.message || e) }); + } +}); + +const canAttachImages = (session) => session.cli !== 'shell' + && !PASSIVE_CLIS.includes(session.cli) && !isRemote(session.cli); + +// Managed screenshots live under STATE_DIR, never in the user's repository. +// The raw body is streamed and capped in attachments.js; express.json ignores +// these image content types, so no middleware buffers them first. +app.post('/api/sessions/:id/attachments', async (req, res) => { + const s = store.get(req.params.id); + if (!s) return res.status(404).json({ error: 'not found' }); + if (!canAttachImages(s)) { + const error = isRemote(s.cli) + ? 'screenshots are not available for remote agents yet — that agent cannot read files stored on this Space' + : `${s.cli} pane cannot accept screenshots`; + return res.status(400).json({ error }); + } + try { + const image = await receiveImage(req, s.id, req.headers['content-type']); + if (!res.destroyed) res.status(201).json(image); + } catch (e) { + if (!res.headersSent && !res.destroyed) res.status(e.statusCode || 500).json({ error: String(e.message || e) }); + } +}); + +app.get('/api/sessions/:id/attachments/:attachmentId/raw', (req, res) => { + const s = store.get(req.params.id); + if (!s) return res.status(404).json({ error: 'not found' }); + try { + const image = resolveImage(s.id, req.params.attachmentId); + res.set({ + 'Content-Type': image.mime, + 'Content-Length': String(image.bytes), + 'X-Content-Type-Options': 'nosniff', + 'Content-Security-Policy': 'sandbox', + 'Cache-Control': 'no-store', + }); + const stream = fs.createReadStream(image.path); + stream.on('error', () => { if (!res.headersSent) res.status(404).end(); else res.destroy(); }); + stream.pipe(res); + } catch (e) { + res.status(e.statusCode || 404).json({ error: String(e.message || e) }); } }); @@ -466,7 +526,7 @@ app.post('/api/agents/:id/prompt', promptBody, async (req, res) => { // the operator's laptop, and the [message from x:] prefix plus `from:` in // the message's frontmatter is how it can tell a peer's request from the // operator's. - const started = await deliver(s, `[message from ${from.session.name}:] ${text}`, from.session.name); + const started = await deliver(s, { text: `[message from ${from.session.name}:] ${text}` }, from.session.name); res.json({ ok: true, id: s.id, name: s.name, started }); } catch (e) { res.status(409).json({ error: String(e.message || e) }); @@ -2016,6 +2076,7 @@ app.delete('/api/sessions/:id', (req, res) => { groups.detachSession(s.id); order.drop(`s:${s.id}`); store.remove(s.id); + try { removeSessionAttachments(s.id); } catch (e) { console.error('[attachments.remove]', e && e.message); } res.json({ ok: true }); }); diff --git a/server/test/attachments.test.mjs b/server/test/attachments.test.mjs new file mode 100644 index 0000000..fad7b21 --- /dev/null +++ b/server/test/attachments.test.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'am-attachments-')); +process.env.DATA_DIR = root; + +const { + detectImageMime, formatAttachmentDelivery, pruneAttachmentDirs, receiveImage, removeSessionAttachments, + resolveImage, resolveImages, +} = await import('../src/attachments.js'); + +const png = Buffer.alloc(45); +Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(png); +png.writeUInt32BE(13, 8); Buffer.from('IHDR').copy(png, 12); +png.writeUInt32BE(1, 16); png.writeUInt32BE(1, 20); +Buffer.from('IEND').copy(png, 37); +const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0xff, 0xd9]); +const gif = Buffer.alloc(14); +Buffer.from('GIF89a', 'ascii').copy(gif); gif.writeUInt16LE(1, 6); gif.writeUInt16LE(1, 8); gif[13] = 0x3b; +const webp = Buffer.alloc(20); +Buffer.from('RIFF', 'ascii').copy(webp); webp.writeUInt32LE(12, 4); +Buffer.from('WEBPVP8 ', 'ascii').copy(webp, 8); + +try { + assert.equal(detectImageMime(png), 'image/png'); + assert.equal(detectImageMime(jpeg), 'image/jpeg'); + assert.equal(detectImageMime(gif), 'image/gif'); + assert.equal(detectImageMime(webp), 'image/webp'); + assert.equal(detectImageMime(Buffer.from('')), null); + + const stored = await receiveImage(Readable.from([png]), 'codex-123abc', 'image/png'); + assert.match(stored.id, /^att_[a-f0-9]{24}$/); + assert.equal(stored.mime, 'image/png'); + assert.equal(stored.bytes, png.length); + assert.equal(fs.readFileSync(stored.path).compare(png), 0); + assert.deepEqual(resolveImage('codex-123abc', stored.id), stored); + assert.deepEqual(resolveImages('codex-123abc', [stored.id]), [stored]); + + assert.throws(() => resolveImage('other-123abc', stored.id), /not found/); + assert.throws(() => resolveImage('codex-123abc', '../sessions.json'), /not found/); + assert.throws(() => resolveImages('codex-123abc', Array(6).fill(stored.id)), /at most five/); + + await assert.rejects( + receiveImage(Readable.from([png]), 'codex-123abc', 'image/jpeg'), + (error) => error.statusCode === 415, + ); + await assert.rejects( + receiveImage(Readable.from([png.subarray(0, 24)]), 'codex-123abc', 'image/png'), + (error) => error.statusCode === 415 && /malformed/.test(error.message), + ); + await assert.rejects( + receiveImage(Readable.from([]), 'codex-123abc', 'image/png'), + (error) => error.statusCode === 413, + ); + + const formatted = formatAttachmentDelivery('codex', 'Compare this', [stored]); + assert.match(formatted, /Compare this/); + assert.match(formatted, /Attached screenshots:/); + assert.match(formatted, new RegExp(stored.id)); + assert.match(formatAttachmentDelivery('gemini', '', [stored]), /Please inspect the attached screenshot\./); + + const old = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + const oldPart = path.join(path.dirname(stored.path), '.crashed.part'); + fs.writeFileSync(oldPart, 'partial'); + fs.utimesSync(oldPart, old, old); + const orphan = path.join(root, 'state', 'attachments', 'orphan-123abc'); + fs.mkdirSync(orphan, { recursive: true }); + fs.writeFileSync(path.join(orphan, 'old'), 'old'); + fs.utimesSync(path.join(orphan, 'old'), old, old); + fs.utimesSync(orphan, old, old); + pruneAttachmentDirs(['codex-123abc']); + assert.equal(fs.existsSync(oldPart), false); + assert.equal(fs.existsSync(orphan), false); + + removeSessionAttachments('codex-123abc'); + assert.equal(fs.existsSync(path.dirname(stored.path)), false); + console.log('attachment tests passed'); +} finally { + fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 94f7d74..239af99 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import Sidebar from './components/Sidebar'; +import type { QuickStartImageOptions } from './components/Sidebar'; import TerminalPane from './components/TerminalPane'; import FilesPane from './components/FilesPane'; import TracePane from './components/TracePane'; @@ -17,6 +18,7 @@ import * as api from './api'; import type { Cli, GridSpec, MoveTarget, OverviewFilter, Session, Tree } from './types'; import { isPassive, isRemote } from './types'; import { GridGlyph, ListGlyph } from './components/icons'; +import { uploadPendingImages } from './lib/imageAttachments'; // Phone-sized viewport: the app becomes two full-screen views (list ⇄ pane). function useIsMobile() { @@ -489,13 +491,38 @@ export default function App() { }; // Quickstart: server boots the agent and types the prompt; we jump straight // to the new pane so you watch it happen. - const quickStart = async (cli: string, prompt: string, name = '', path = '.') => { + const quickStart = async (cli: string, prompt: string, name = '', path = '.', imageOptions?: QuickStartImageOptions) => { try { - const s = await api.quickStart(cli, prompt, name, path); - rememberPath(s.path); + let sessionId: string; + let sessionPath: string | null = path; + if (imageOptions?.images.length) { + if (imageOptions.sessionId) { + sessionId = imageOptions.sessionId; + } else { + // Attachments are session-scoped, so create the stopped session first, + // then upload. If an upload fails the session remains visible and the + // sidebar retains its id for a retry. + const created = await api.createSession(name, cli, undefined, path); + sessionId = created.id; + sessionPath = created.path; + imageOptions.onSessionCreated(created.id); + rememberPath(created.path); + await refresh(); + } + const attachments = await uploadPendingImages(sessionId, imageOptions.images, imageOptions.onImageUpdate); + await api.sendInput(sessionId, prompt, attachments.map((image) => image.id)); + } else { + const created = await api.quickStart(cli, prompt, name, path); + sessionId = created.id; + sessionPath = created.path; + } + rememberPath(sessionPath); await refresh(); - setActiveRef(`s:${s.id}`); - } catch (e) { showErr('Couldn’t quickstart the agent')(e); } + setActiveRef(`s:${sessionId}`); + } catch (e) { + showErr('Couldn’t quickstart the agent')(e); + throw e; + } }; // Creations land in an explicitly targeted group (the group's + button), // else the group you're currently looking at; loose otherwise. diff --git a/web/src/api.ts b/web/src/api.ts index 1e4bd1a..8c3d4a5 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -178,8 +178,31 @@ export const getRemotePrompt = (name: string): Promise => export const setRemotePaused = (id: string, paused: boolean): Promise => fetch(`/api/sessions/${id}/remote/paused`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ paused }) }).then(json); -export const sendInput = (id: string, text: string): Promise<{ ok: boolean; started?: boolean }> => - fetch(`/api/sessions/${id}/input`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ text }) }).then(json); +export interface ImageAttachment { + id: string; + kind: 'image'; + name: string; + mime: 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif'; + bytes: number; + path: string; + previewUrl: string; + insertText: string; +} + +export const uploadImageAttachment = async (id: string, file: File): Promise => { + const response = await fetch(`/api/sessions/${id}/attachments`, { + method: 'POST', + headers: { + 'content-type': file.type, + 'x-file-name': encodeURIComponent(file.name || 'Screenshot'), + }, + body: file, + }); + return jsonOrError(response); +}; + +export const sendInput = (id: string, text: string, attachmentIds: string[] = []): Promise<{ ok: boolean; started?: boolean }> => + fetch(`/api/sessions/${id}/input`, { method: 'POST', headers: HEADERS, body: JSON.stringify({ text, attachmentIds }) }).then(jsonOrError); // ---- push notifications ---- export const getPushKey = (): Promise<{ publicKey: string; devices: number }> => diff --git a/web/src/components/ImageAttachments.tsx b/web/src/components/ImageAttachments.tsx new file mode 100644 index 0000000..51161dd --- /dev/null +++ b/web/src/components/ImageAttachments.tsx @@ -0,0 +1,60 @@ +import { useRef } from 'react'; +import type { PendingImage } from '../lib/imageAttachments'; +import { IMAGE_ACCEPT } from '../lib/imageAttachments'; + +const fmtBytes = (bytes: number) => bytes < 1024 * 1024 + ? `${Math.max(1, Math.round(bytes / 1024))} KB` + : `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + +export default function ImageAttachments({ images, disabled, disabledReason, onFiles, onRemove }: { + images: PendingImage[]; + disabled?: boolean; + disabledReason?: string; + onFiles: (files: File[]) => void; + onRemove: (key: string) => void; +}) { + const picker = useRef(null); + return ( +
+ + { + onFiles(Array.from(event.currentTarget.files || [])); + event.currentTarget.value = ''; + }} + /> + {images.map((image) => ( +
+ + + {image.file.name || 'Screenshot'} + + {image.status === 'uploading' ? 'uploading…' + : image.error || (image.status === 'uploaded' ? 'uploaded' : fmtBytes(image.file.size))} + + + +
+ ))} +
+ ); +} diff --git a/web/src/components/Overview.tsx b/web/src/components/Overview.tsx index ea8cc14..82c6085 100644 --- a/web/src/components/Overview.tsx +++ b/web/src/components/Overview.tsx @@ -3,8 +3,14 @@ import type { CSSProperties, ReactNode } from 'react'; import * as api from '../api'; import type { MetaSession } from '../api'; import type { Cli, OverviewFilter, Session, SessionState, Tree } from '../types'; -import { isPassive } from '../types'; +import { isPassive, isRemote } from '../types'; import { renderMarkdown } from '../lib/markdown'; +import { + defaultImagePrompt, imageFilesFromTransfer, pendingImagesFromFiles, revokePendingImages, + transferMayContainImage, uploadPendingImages, +} from '../lib/imageAttachments'; +import type { PendingImage } from '../lib/imageAttachments'; +import ImageAttachments from './ImageAttachments'; import Logo from './Logo'; const fmtAgo = (ts: number) => { @@ -36,14 +42,40 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { }) { const d = s.digest; const [draft, setDraft] = useState(''); + const [images, setImages] = useState([]); + const imagesRef = useRef([]); + const [imageError, setImageError] = useState(null); + const [dropActive, setDropActive] = useState(false); const [sending, setSending] = useState(false); - const [failed, setFailed] = useState(false); + const [failed, setFailed] = useState(null); // Optimistic echo: the sent text becomes the prompt line the moment the // send succeeds — the digest round-trip (CLI writes transcript → rebuild → // poll) can take seconds, and a frozen card reads as "did that get lost?". const [sent, setSent] = useState<{ text: string; at: number } | null>(null); const [histIdx, setHistIdx] = useState(0); // 0 = live view, n = n-th exchange back const inputRef = useRef(null); + const allowImages = !isRemote(s.cli); + + useEffect(() => { imagesRef.current = images; }, [images]); + useEffect(() => () => revokePendingImages(imagesRef.current), []); + + const addImages = (files: File[]) => { + if (!allowImages || !files.length) return; + const next = pendingImagesFromFiles(files, images.length); + setImages((current) => [...current, ...next.images]); + setImageError(next.error); + }; + const removeImage = (key: string) => { + setImages((current) => { + const removed = current.find((image) => image.key === key); + if (removed) URL.revokeObjectURL(removed.previewUrl); + return current.filter((image) => image.key !== key); + }); + setImageError(null); + }; + const updateImage = (key: string, patch: Partial) => { + setImages((current) => current.map((image) => image.key === key ? { ...image, ...patch } : image)); + }; // After you send (or when the transcript shows a prompt newer than the last // answer), the old answer is stale — a spinner takes its place. @@ -60,18 +92,23 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { const send = async () => { const text = draft.trim(); - if (!text || sending) return; + if ((!text && !images.length) || sending) return; setSending(true); - setFailed(false); + setFailed(null); try { - await api.sendInput(s.id, text); + const attachments = await uploadPendingImages(s.id, images, updateImage); + await api.sendInput(s.id, text, attachments.map((image) => image.id)); + const optimisticText = text || defaultImagePrompt(images.length); setDraft(''); - setSent({ text, at: Date.now() }); + revokePendingImages(images); + setImages([]); + setImageError(null); + setSent({ text: optimisticText, at: Date.now() }); setHistIdx(0); if (inputRef.current) { inputRef.current.style.height = 'auto'; inputRef.current.blur(); } - } catch { - setFailed(true); - setTimeout(() => setFailed(false), 4000); + } catch (error) { + setFailed(error instanceof Error ? error.message : 'failed to reach the agent'); + setTimeout(() => setFailed(null), 5000); } setSending(false); }; @@ -147,33 +184,57 @@ function Card({ s, color, pending, isMobile, onOpen, onClose }: { {showLiveProgress &&
running
} -
- ❯ -