From 83483360a9a9b47908fc65e61bae02d3223a2e15 Mon Sep 17 00:00:00 2001 From: Vaibhav Patil Date: Mon, 3 Aug 2026 13:45:43 +0000 Subject: [PATCH] feat(sandbox): keep mount endpoints stable --- packages/agentos-sandbox/src/provider.ts | 69 +++- .../agentos-sandbox/tests/provider.test.ts | 22 + .../tests/vm-integration.test.ts | 48 ++- packages/core/src/index.ts | 1 + packages/core/src/sandbox-relay.ts | 382 ++++++++++++++++++ packages/core/src/sandbox.ts | 357 ++++++++++++++-- packages/core/src/test/sandbox-agent.ts | 16 +- packages/core/tests/options-schema.test.ts | 195 ++++++++- .../core/tests/public-api-exports.test.ts | 2 + website/public/docs/docs/sandboxes.md | 27 +- website/src/content/docs/docs/sandboxes.mdx | 27 +- 11 files changed, 1086 insertions(+), 60 deletions(-) create mode 100644 packages/core/src/sandbox-relay.ts diff --git a/packages/agentos-sandbox/src/provider.ts b/packages/agentos-sandbox/src/provider.ts index 77f9658342..a7ebcb1ca2 100644 --- a/packages/agentos-sandbox/src/provider.ts +++ b/packages/agentos-sandbox/src/provider.ts @@ -13,7 +13,70 @@ export type SandboxAgentProviderOptions = Omit< "sandbox" | "sandboxId" >; -/** Adapt any sandbox-agent backend into a per-VM AgentOS sandbox provider. */ +interface SandboxAgentTransportInternals { + baseUrl: string; + token?: string; + defaultHeaders?: HeadersInit; + fetcher?: typeof globalThis.fetch; + awaitHealthy?(signal?: AbortSignal): Promise; +} + +function validateTransportBaseUrl(raw: string): string { + const normalized = raw.trim().replace(/\/+$/, ""); + if (!normalized) throw new Error("SandboxAgent baseUrl must not be empty"); + const url = new URL(normalized); + if (!url.hostname || url.search || url.hash) { + throw new Error( + "SandboxAgent baseUrl must include a host without a query string or fragment", + ); + } + const hostname = url.hostname + .replace(/^\[/, "") + .replace(/\]$/, "") + .toLowerCase(); + const loopback = + hostname === "localhost" || + hostname === "::1" || + /^127(?:\.|$)/.test(hostname); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("SandboxAgent baseUrl must use http or https"); + } + if (url.protocol !== "https:" && !loopback) { + throw new Error( + "SandboxAgent baseUrl must use https unless it targets localhost", + ); + } + return normalized; +} + +async function requestThroughSandboxAgent( + client: SandboxAgent, + path: string, + init: RequestInit = {}, +): Promise { + const transport = client as unknown as SandboxAgentTransportInternals; + await transport.awaitHealthy?.(init.signal ?? undefined); + if (typeof transport.fetcher !== "function") { + throw new Error( + "SandboxAgent client does not expose the authenticated fetch transport required by agentOS", + ); + } + const headers = new Headers(transport.defaultHeaders); + new Headers(init.headers).forEach((value, name) => headers.set(name, value)); + if (transport.token) { + headers.set("authorization", `Bearer ${transport.token}`); + } + return await transport.fetcher( + new URL(`${validateTransportBaseUrl(transport.baseUrl)}${path}`), + { + ...init, + headers, + redirect: "manual", + }, + ); +} + +/** Adapt any sandbox-agent backend into a per-VM agentOS sandbox provider. */ export function sandboxAgentProvider( backend: SandboxAgentBackend, options: SandboxAgentProviderOptions = {}, @@ -26,6 +89,10 @@ export function sandboxAgentProvider( if (property === "dispose") { return target.destroySandbox.bind(target); } + if (property === "request") { + return (path: string, init?: RequestInit) => + requestThroughSandboxAgent(target, path, init); + } const value = Reflect.get(target, property, target); return typeof value === "function" ? value.bind(target) : value; }, diff --git a/packages/agentos-sandbox/tests/provider.test.ts b/packages/agentos-sandbox/tests/provider.test.ts index ec1f606b6e..2264411c93 100644 --- a/packages/agentos-sandbox/tests/provider.test.ts +++ b/packages/agentos-sandbox/tests/provider.test.ts @@ -6,8 +6,17 @@ describe("sandboxAgentProvider", () => { test("starts a fresh client and destroys its backend on disposal", async () => { const destroySandbox = vi.fn(async () => {}); const runProcess = vi.fn(async () => ({ stdout: "ok", exitCode: 0 })); + const awaitHealthy = vi.fn(async () => {}); + const fetcher = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response("ok"), + ); const client = { baseUrl: "https://sandbox.example", + token: "current-token", + defaultHeaders: { "x-sandbox-provider": "test" }, + fetcher, + awaitHealthy, destroySandbox, runProcess, }; @@ -25,6 +34,19 @@ describe("sandboxAgentProvider", () => { stdout: "ok", exitCode: 0, }); + await first.request?.("/v1/fs/stat?path=%2F", { + headers: { range: "bytes=0-3" }, + }); + expect(awaitHealthy).toHaveBeenCalledTimes(1); + expect(fetcher).toHaveBeenCalledTimes(1); + const [requestUrl, requestInit] = fetcher.mock.calls[0] ?? []; + expect(String(requestUrl)).toBe( + "https://sandbox.example/v1/fs/stat?path=%2F", + ); + const headers = new Headers(requestInit?.headers); + expect(headers.get("authorization")).toBe("Bearer current-token"); + expect(headers.get("x-sandbox-provider")).toBe("test"); + expect(headers.get("range")).toBe("bytes=0-3"); await first.dispose?.(); await second.dispose?.(); expect(destroySandbox).toHaveBeenCalledTimes(2); diff --git a/packages/agentos-sandbox/tests/vm-integration.test.ts b/packages/agentos-sandbox/tests/vm-integration.test.ts index 386a2db178..b8a5569579 100644 --- a/packages/agentos-sandbox/tests/vm-integration.test.ts +++ b/packages/agentos-sandbox/tests/vm-integration.test.ts @@ -14,6 +14,7 @@ import { import { createSandboxBindings } from "../src/index.js"; let sandbox: MockSandboxAgentHandle; +let providerSandbox: MockSandboxAgentHandle; const SANDBOX_TEST_PERMISSIONS = { fs: "allow", @@ -24,7 +25,8 @@ const SANDBOX_TEST_PERMISSIONS = { } as const; beforeAll(async () => { - sandbox = await startMockSandboxAgent(); + sandbox = await startMockSandboxAgent({ token: "first-generation-token" }); + providerSandbox = sandbox; }, 150_000); afterAll(async () => { @@ -39,15 +41,17 @@ describe("VM integration", () => { beforeEach(async () => { providerStarts = 0; providerDisposals = 0; + providerSandbox = sandbox; vm = await AgentOs.create({ permissions: SANDBOX_TEST_PERMISSIONS, software: [common], sandbox: { mountPath: "/sandbox", + idleTimeoutMs: 25, provider: { start: async () => { providerStarts += 1; - return new Proxy(sandbox.client, { + return new Proxy(providerSandbox.client, { get(target, property) { if (property === "dispose") { return () => { @@ -62,12 +66,12 @@ describe("VM integration", () => { }, }, }); - expect(providerStarts).toBe(1); + expect(providerStarts).toBe(0); }, 150_000); afterEach(async () => { if (vm) await vm.dispose(); - expect(providerDisposals).toBe(1); + expect(providerDisposals).toBe(providerStarts); }); // -- Filesystem mount tests -- @@ -105,6 +109,42 @@ describe("VM integration", () => { expect(new TextDecoder().decode(content)).toBe("deep file"); }); + it("keeps the mounted path live when the backing sandbox endpoint changes", async () => { + const replacement = await startMockSandboxAgent({ + token: "second-generation-token", + }); + try { + await sandbox.client.writeFsFile( + { path: "/generation.txt" }, + new TextEncoder().encode("first"), + ); + expect( + new TextDecoder().decode( + await vm.readFile("/sandbox/generation.txt"), + ), + ).toBe("first"); + expect(providerStarts).toBe(1); + + providerSandbox = replacement; + await replacement.client.writeFsFile( + { path: "/generation.txt" }, + new TextEncoder().encode("second"), + ); + for (let attempt = 0; attempt < 50 && providerDisposals === 0; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(providerDisposals).toBe(1); + expect( + new TextDecoder().decode( + await vm.readFile("/sandbox/generation.txt"), + ), + ).toBe("second"); + expect(providerStarts).toBe(2); + } finally { + await replacement.stop(); + } + }, 150_000); + // -- Bindings direct execution (host RPC, not via CLI shim) -- it("should execute the run-command binding directly via the binding collection", async () => { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ede6c13beb..b7b18233b1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -53,5 +53,6 @@ export { createSandboxFs, getSandboxDisposeHooks, resolveSandboxOptions, + SandboxStartupError, } from "./sandbox.js"; export type * from "./types.js"; diff --git a/packages/core/src/sandbox-relay.ts b/packages/core/src/sandbox-relay.ts new file mode 100644 index 0000000000..6306d7c0aa --- /dev/null +++ b/packages/core/src/sandbox-relay.ts @@ -0,0 +1,382 @@ +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import { + createServer, + type IncomingHttpHeaders, + type Server, + type ServerResponse, +} from "node:http"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as NodeReadableStream } from "node:stream/web"; +import type { AgentOsSandboxClient } from "./sandbox.js"; + +const DEFAULT_MAX_RELAY_REQUESTS = 64; +const RELAY_WARNING_PERCENT = 80; +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); +const ALLOWED_RELAY_ROUTES = new Map>([ + ["/v1/fs/entries", new Set(["GET"])], + ["/v1/fs/file", new Set(["GET", "PUT"])], + ["/v1/fs/entry", new Set(["DELETE"])], + ["/v1/fs/mkdir", new Set(["POST"])], + ["/v1/fs/move", new Set(["POST"])], + ["/v1/fs/stat", new Set(["GET"])], + ["/v1/processes/run", new Set(["POST"])], +]); + +export interface SandboxRelayClientController { + withClient( + operation: (client: AgentOsSandboxClient) => Promise, + ): Promise; +} + +export interface SandboxRelayOptions { + controller: SandboxRelayClientController; + maxConcurrentRequests?: number; +} + +export interface SandboxRelay { + baseUrl: string; + token: string; + dispose(): Promise; +} + +interface SerializableSandboxClient { + baseUrl?: string; + token?: string; + defaultHeaders?: RequestInit["headers"]; +} + +type RelayRequestInit = RequestInit & { duplex?: "half" }; + +function problem( + response: ServerResponse, + status: number, + title: string, + detail: string, +): void { + if (response.headersSent) { + response.destroy(new Error(detail)); + return; + } + const body = Buffer.from( + JSON.stringify({ + type: "about:blank", + title, + status, + detail, + }), + ); + response.writeHead(status, { + "content-length": String(body.length), + "content-type": "application/problem+json", + }); + response.end(body); +} + +function bearerToken(headers: IncomingHttpHeaders): string | undefined { + const authorization = headers.authorization; + if (!authorization?.startsWith("Bearer ")) return undefined; + return authorization.slice("Bearer ".length); +} + +function tokenMatches(actual: string | undefined, expected: string): boolean { + if (!actual) return false; + const actualBytes = Buffer.from(actual); + const expectedBytes = Buffer.from(expected); + return ( + actualBytes.length === expectedBytes.length && + timingSafeEqual(actualBytes, expectedBytes) + ); +} + +function isAllowedRelayRoute(method: string, pathname: string): boolean { + return ALLOWED_RELAY_ROUTES.get(pathname)?.has(method) === true; +} + +function copyRequestHeaders(headers: IncomingHttpHeaders): Headers { + const copied = new Headers(); + for (const [name, value] of Object.entries(headers)) { + const lower = name.toLowerCase(); + if ( + value === undefined || + lower === "accept-encoding" || + lower === "authorization" || + lower === "host" || + HOP_BY_HOP_HEADERS.has(lower) + ) { + continue; + } + if (Array.isArray(value)) { + for (const item of value) copied.append(name, item); + } else { + copied.set(name, value); + } + } + copied.set("accept-encoding", "identity"); + return copied; +} + +function validateUpstreamBaseUrl(raw: string): string { + const normalized = raw.trim().replace(/\/+$/, ""); + if (!normalized) throw new Error("Sandbox client baseUrl must not be empty"); + const url = new URL(normalized); + if (!url.hostname || url.search || url.hash) { + throw new Error( + "Sandbox client baseUrl must include a host without a query string or fragment", + ); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("Sandbox client baseUrl must use http or https"); + } + const hostname = url.hostname + .replace(/^\[/, "") + .replace(/\]$/, "") + .toLowerCase(); + const loopback = + hostname === "localhost" || + hostname === "::1" || + /^127(?:\.|$)/.test(hostname); + if (url.protocol !== "https:" && !loopback) { + throw new Error( + "Sandbox client baseUrl must use https unless it targets localhost", + ); + } + return normalized; +} + +function mergeUpstreamHeaders( + client: AgentOsSandboxClient, + requestHeaders: Headers, +): Headers { + const serializable = client as AgentOsSandboxClient & + SerializableSandboxClient; + const headers = new Headers(serializable.defaultHeaders); + requestHeaders.forEach((value, name) => headers.set(name, value)); + if (serializable.token) { + headers.set("authorization", `Bearer ${serializable.token}`); + } + return headers; +} + +async function requestUpstream( + client: AgentOsSandboxClient, + path: string, + init: RelayRequestInit, +): Promise { + const headers = mergeUpstreamHeaders(client, new Headers(init.headers)); + const upstreamInit: RelayRequestInit = { + ...init, + headers, + redirect: "manual", + }; + if (client.request) { + return await client.request(path, upstreamInit); + } + + const serializable = client as AgentOsSandboxClient & + SerializableSandboxClient; + const rawBaseUrl = serializable.baseUrl; + if (!rawBaseUrl) { + throw new Error( + "Sandbox client does not expose request() or a serializable baseUrl", + ); + } + const baseUrl = validateUpstreamBaseUrl(rawBaseUrl); + return await fetch(`${baseUrl}${path}`, upstreamInit); +} + +function copyResponseHeaders(headers: Headers): Record { + const copied: Record = {}; + headers.forEach((value, name) => { + if (!HOP_BY_HOP_HEADERS.has(name.toLowerCase())) copied[name] = value; + }); + return copied; +} + +async function writeUpstreamResponse( + upstream: Response, + response: ServerResponse, +): Promise { + response.writeHead(upstream.status, copyResponseHeaders(upstream.headers)); + if (!upstream.body) { + response.end(); + return; + } + const body = Readable.fromWeb( + upstream.body as unknown as NodeReadableStream, + ); + await pipeline(body, response); +} + +function closeServer(server: Server): Promise { + return new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + server.closeIdleConnections(); + server.closeAllConnections(); + }); +} + +export async function createSandboxRelay( + options: SandboxRelayOptions, +): Promise { + const token = randomBytes(32).toString("base64url"); + const maxConcurrentRequests = + options.maxConcurrentRequests ?? DEFAULT_MAX_RELAY_REQUESTS; + if (!Number.isSafeInteger(maxConcurrentRequests) || maxConcurrentRequests <= 0) { + throw new Error("sandbox.maxRelayRequests must be a positive safe integer"); + } + + let activeRequests = 0; + let warnedNearCapacity = false; + let disposed = false; + const server = createServer((request, response) => { + void (async () => { + if (disposed) { + problem( + response, + 503, + "Sandbox relay unavailable", + "agentOS VM sandbox relay is shutting down", + ); + return; + } + if (!tokenMatches(bearerToken(request.headers), token)) { + problem(response, 401, "Unauthorized", "Invalid sandbox relay token"); + return; + } + + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + const method = request.method ?? "GET"; + if (!isAllowedRelayRoute(method, url.pathname)) { + problem( + response, + 404, + "Not Found", + `Sandbox relay does not expose ${method} ${url.pathname}`, + ); + return; + } + if (activeRequests >= maxConcurrentRequests) { + problem( + response, + 429, + "Sandbox relay capacity exceeded", + `Sandbox relay reached sandbox.maxRelayRequests=${maxConcurrentRequests}; raise sandbox.maxRelayRequests to allow more concurrent requests`, + ); + return; + } + + activeRequests += 1; + if ( + !warnedNearCapacity && + activeRequests * 100 >= + maxConcurrentRequests * RELAY_WARNING_PERCENT + ) { + warnedNearCapacity = true; + console.warn( + `agentOS sandbox relay near sandbox.maxRelayRequests: ${activeRequests}/${maxConcurrentRequests}`, + ); + } + try { + await options.controller.withClient(async (client) => { + const abortController = new AbortController(); + const abort = () => { + if (!response.writableEnded) abortController.abort(); + }; + request.once("aborted", abort); + response.once("close", abort); + try { + const hasBody = method !== "GET" && method !== "HEAD"; + const init: RelayRequestInit = { + method, + headers: copyRequestHeaders(request.headers), + redirect: "manual", + signal: abortController.signal, + ...(hasBody + ? { + body: Readable.toWeb(request) as unknown as BodyInit, + duplex: "half" as const, + } + : {}), + }; + const upstream = await requestUpstream( + client, + `${url.pathname}${url.search}`, + init, + ); + await writeUpstreamResponse(upstream, response); + } finally { + request.off("aborted", abort); + response.off("close", abort); + } + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + problem(response, 503, "Sandbox unavailable", detail); + } finally { + activeRequests -= 1; + if ( + activeRequests * 100 < + maxConcurrentRequests * RELAY_WARNING_PERCENT + ) { + warnedNearCapacity = false; + } + } + })().catch((error) => { + console.error("agentOS sandbox relay request failed", error); + problem( + response, + 500, + "Sandbox relay failure", + error instanceof Error ? error.message : String(error), + ); + }); + }); + server.maxConnections = Math.min( + Number.MAX_SAFE_INTEGER, + maxConcurrentRequests + 1, + ); + server.listen(0, "127.0.0.1"); + try { + await Promise.race([ + once(server, "listening"), + once(server, "error").then(([error]) => Promise.reject(error)), + ]); + } catch (error) { + await closeServer(server).catch((closeError) => { + console.error("agentOS sandbox relay cleanup failed", closeError); + }); + throw error; + } + server.unref(); + + const address = server.address(); + if (!address || typeof address === "string") { + await closeServer(server); + throw new Error("Sandbox relay failed to bind to a TCP port"); + } + + return { + baseUrl: `http://127.0.0.1:${address.port}`, + token, + async dispose() { + if (disposed) return; + disposed = true; + await closeServer(server); + }, + }; +} diff --git a/packages/core/src/sandbox.ts b/packages/core/src/sandbox.ts index d85fd7dbbb..7ac94d8d3c 100644 --- a/packages/core/src/sandbox.ts +++ b/packages/core/src/sandbox.ts @@ -5,6 +5,13 @@ import type { NativeMountPluginDescriptor, } from "./agent-os.js"; import type { Binding, Bindings } from "./bindings.js"; +import { + createSandboxRelay, + type SandboxRelayClientController, +} from "./sandbox-relay.js"; + +const DEFAULT_SANDBOX_IDLE_TIMEOUT_MS = 5 * 60_000; +const DEFAULT_SANDBOX_STARTUP_TIMEOUT_MS = 20_000; export interface AgentOsSandboxProcessResult { stdout?: string; @@ -34,6 +41,12 @@ export interface AgentOsSandboxProcessLogs { export interface AgentOsSandboxClient { dispose?(): Promise | void; + /** + * Optional authenticated raw transport used by the native filesystem relay. + * The path always starts with `/v1/` and includes its query string. Return the + * upstream Response unchanged, including non-success HTTP statuses. + */ + request?(path: string, init?: RequestInit): Promise; runProcess(options: { command: string; args?: string[]; @@ -73,6 +86,15 @@ export interface AgentOsSandboxCommonOptions { timeoutMs?: number; /** Maximum file size allowed for buffered pread/truncate fallbacks. */ maxFullReadBytes?: number; + /** + * Shut down an inactive provider sandbox after this duration. The next + * operation starts a new sandbox. Set to 0 to disable. Defaults to 5 minutes. + */ + idleTimeoutMs?: number; + /** Maximum time to wait for a provider start. Set to 0 to disable. Defaults to 20 seconds. */ + startupTimeoutMs?: number; + /** Maximum concurrent native filesystem relay requests. Defaults to 64. */ + maxRelayRequests?: number; /** Marks the VM mount read-only. Defaults to false. */ readOnly?: boolean; } @@ -114,6 +136,245 @@ type ResolvedSandboxOptions = AgentOsSandboxCommonOptions & { client: AgentOsSandboxClient; }; +export class SandboxStartupError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "SandboxStartupError"; + } +} + +class SandboxClientController implements SandboxRelayClientController { + readonly #provider?: AgentOsSandboxProvider; + readonly #startupTimeoutMs: number; + readonly #idleTimeoutMs: number; + readonly #disposeClient?: SandboxDisposeHook; + #current?: AgentOsSandboxClient; + #startPromise?: Promise; + #stopPromise?: Promise; + #idleTimer?: NodeJS.Timeout; + #activeOperations = 0; + #disposed = false; + + constructor(options: { + provider?: AgentOsSandboxProvider; + client?: AgentOsSandboxClient; + disposeClient?: SandboxDisposeHook; + startupTimeoutMs?: number; + idleTimeoutMs?: number; + }) { + this.#provider = options.provider; + this.#current = options.client; + this.#disposeClient = options.disposeClient; + this.#startupTimeoutMs = + options.startupTimeoutMs ?? DEFAULT_SANDBOX_STARTUP_TIMEOUT_MS; + this.#idleTimeoutMs = + options.idleTimeoutMs ?? DEFAULT_SANDBOX_IDLE_TIMEOUT_MS; + for (const [name, value] of [ + ["sandbox.startupTimeoutMs", this.#startupTimeoutMs], + ["sandbox.idleTimeoutMs", this.#idleTimeoutMs], + ] as const) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative safe integer`); + } + } + } + + async withClient( + operation: (client: AgentOsSandboxClient) => Promise, + ): Promise { + if (this.#disposed) { + throw new Error("agentOS VM sandbox has been disposed"); + } + this.#clearIdleTimer(); + this.#activeOperations += 1; + try { + return await operation(await this.#getClient()); + } finally { + this.#activeOperations -= 1; + this.#scheduleIdleStop(); + } + } + + async #getClient(): Promise { + if (this.#disposed) { + throw new Error("agentOS VM sandbox has been disposed"); + } + if (this.#current) return this.#current; + if (!this.#provider) { + throw new Error("Sandbox client is not available"); + } + if (this.#stopPromise) await this.#stopPromise; + if (this.#current) return this.#current; + if (this.#startPromise) return await this.#startPromise; + + const startPromise = this.#startProvider(); + this.#startPromise = startPromise; + try { + return await startPromise; + } finally { + if (this.#startPromise === startPromise) this.#startPromise = undefined; + } + } + + async #startProvider(): Promise { + const provider = this.#provider; + if (!provider) throw new Error("Sandbox provider is not configured"); + + let abandoned = false; + let timeout: NodeJS.Timeout | undefined; + let providerResult: Promise; + try { + providerResult = Promise.resolve(provider.start()); + } catch (error) { + providerResult = Promise.reject(error); + } + const providerStart = providerResult.then(async (client) => { + if (!client || typeof client !== "object") { + throw new Error("sandbox.provider.start() did not return a client"); + } + if (!abandoned && !this.#disposed) return client; + try { + await client.dispose?.(); + } catch (error) { + console.error("agentOS late sandbox startup cleanup failed", error); + } + throw new SandboxStartupError( + "Sandbox provider completed after its startup was cancelled", + ); + }); + const timeoutPromise = new Promise((_, reject) => { + if (this.#startupTimeoutMs === 0) return; + timeout = setTimeout(() => { + abandoned = true; + reject( + new SandboxStartupError( + `Sandbox provider startup exceeded sandbox.startupTimeoutMs=${this.#startupTimeoutMs}; raise sandbox.startupTimeoutMs to allow a longer startup`, + ), + ); + }, this.#startupTimeoutMs); + }); + try { + const client = await Promise.race([providerStart, timeoutPromise]); + if (this.#disposed) { + abandoned = true; + await client.dispose?.(); + throw new SandboxStartupError( + "Sandbox provider completed after the agentOS VM was disposed", + ); + } + this.#current = client; + return client; + } catch (error) { + abandoned = true; + if (error instanceof SandboxStartupError) throw error; + throw new SandboxStartupError( + `Sandbox provider startup failed: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } finally { + if (timeout) clearTimeout(timeout); + } + } + + #clearIdleTimer(): void { + if (!this.#idleTimer) return; + clearTimeout(this.#idleTimer); + this.#idleTimer = undefined; + } + + #scheduleIdleStop(): void { + this.#clearIdleTimer(); + if ( + !this.#provider || + this.#disposed || + this.#idleTimeoutMs === 0 || + this.#activeOperations !== 0 || + !this.#current + ) { + return; + } + this.#idleTimer = setTimeout(() => { + this.#idleTimer = undefined; + void this.#stopIdleClient().catch((error) => { + console.error("agentOS idle sandbox shutdown failed", error); + }); + }, this.#idleTimeoutMs); + this.#idleTimer.unref(); + } + + async #stopIdleClient(): Promise { + if ( + this.#disposed || + this.#activeOperations !== 0 || + !this.#current || + this.#stopPromise + ) { + return; + } + const client = this.#current; + this.#current = undefined; + const stopPromise = Promise.resolve(client.dispose?.()).then(() => undefined); + this.#stopPromise = stopPromise; + try { + await stopPromise; + } finally { + if (this.#stopPromise === stopPromise) this.#stopPromise = undefined; + } + } + + async dispose(): Promise { + if (this.#disposed) return; + this.#disposed = true; + this.#clearIdleTimer(); + const errors: unknown[] = []; + try { + await this.#startPromise; + } catch (error) { + errors.push(error); + } + try { + await this.#stopPromise; + } catch (error) { + errors.push(error); + } + const client = this.#current; + this.#current = undefined; + if (client) { + try { + if (this.#provider) await client.dispose?.(); + else await this.#disposeClient?.(); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError(errors, "agentOS sandbox disposal failed"); + } + } +} + +function createControllerClient( + controller: SandboxClientController, +): AgentOsSandboxClient { + return { + runProcess: (options) => + controller.withClient((client) => client.runProcess(options)), + createProcess: (options) => + controller.withClient((client) => client.createProcess(options)), + listProcesses: () => + controller.withClient((client) => client.listProcesses()), + stopProcess: (id) => + controller.withClient((client) => client.stopProcess(id)), + killProcess: (id) => + controller.withClient((client) => client.killProcess(id)), + getProcessLogs: (id, options) => + controller.withClient((client) => client.getProcessLogs(id, options)), + sendProcessInput: (id, input) => + controller.withClient((client) => client.sendProcessInput(id, input)), + }; +} + export type SandboxMountPluginConfig = MountConfigJsonObject & { baseUrl: string; token?: string; @@ -367,36 +628,40 @@ function assertNoLegacySandboxOptions(input: AgentOsSandboxInput): void { } } -async function normalizeSandboxInput(input: AgentOsSandboxInput): Promise<{ - options: ResolvedSandboxOptions; - dispose?: SandboxDisposeHook; -}> { +function createSandboxController( + input: AgentOsSandboxInput, +): SandboxClientController { assertNoLegacySandboxOptions(input); if (isProviderOptions(input)) { if (typeof input.provider?.start !== "function") { throw new Error("sandbox.provider must expose a start() function."); } - const client = await input.provider.start(); - return { - options: { ...input, client }, - dispose: () => client.dispose?.(), - }; + return new SandboxClientController({ + provider: input.provider, + idleTimeoutMs: input.idleTimeoutMs, + startupTimeoutMs: input.startupTimeoutMs, + }); } if (!isClientOptions(input)) { throw new Error( "sandbox must be configured with either { provider } or { client }.", ); } - const dispose = + if (!input.client || typeof input.client !== "object") { + throw new Error("sandbox.client must be an object."); + } + const disposeClient = typeof input.dispose === "function" ? input.dispose : input.dispose === true ? () => input.client.dispose?.() : undefined; - return { - options: input, - dispose, - }; + return new SandboxClientController({ + client: input.client, + disposeClient, + idleTimeoutMs: input.idleTimeoutMs ?? 0, + startupTimeoutMs: input.startupTimeoutMs, + }); } function attachSandboxDisposeHooks( @@ -438,25 +703,44 @@ export async function resolveSandboxOptions< return rest; } - const normalizedSandbox = await normalizeSandboxInput(sandbox); + const controller = createSandboxController(sandbox); + let relay: Awaited> | undefined; try { - const sandboxOptions = normalizedSandbox.options; + relay = await createSandboxRelay({ + controller, + maxConcurrentRequests: sandbox.maxRelayRequests, + }); const expanded = rest as Omit & { mounts?: MountConfig[]; bindings?: Bindings[]; }; - const mountPath = sandboxOptions.mountPath ?? "/mnt/sandbox"; + const mountPath = sandbox.mountPath ?? "/mnt/sandbox"; + const plugin: NativeMountPluginDescriptor = { + id: "sandbox_agent", + config: { + baseUrl: relay.baseUrl, + token: relay.token, + ...(sandbox.sandboxRoot ? { basePath: sandbox.sandboxRoot } : {}), + ...(sandbox.timeoutMs != null ? { timeoutMs: sandbox.timeoutMs } : {}), + ...(sandbox.maxFullReadBytes != null + ? { maxFullReadBytes: sandbox.maxFullReadBytes } + : {}), + }, + }; const mounts = [ ...(expanded.mounts ?? []), { path: mountPath, - plugin: createSandboxFs(sandboxOptions), - readOnly: sandboxOptions.readOnly, + plugin, + readOnly: sandbox.readOnly, }, ]; const bindings = [ ...(expanded.bindings ?? []), - createSandboxBindings(sandboxOptions), + createSandboxBindings({ + ...sandbox, + client: createControllerClient(controller), + }), ]; return attachSandboxDisposeHooks( @@ -465,15 +749,36 @@ export async function resolveSandboxOptions< mounts, bindings, }, - normalizedSandbox.dispose ? [normalizedSandbox.dispose] : [], + [ + async () => { + const results = await Promise.allSettled([ + relay?.dispose(), + controller.dispose(), + ]); + const errors = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError( + errors, + "agentOS sandbox relay cleanup failed", + ); + } + }, + ], ); } catch (error) { - if (!normalizedSandbox.dispose) throw error; - try { - await normalizedSandbox.dispose(); - } catch (disposeError) { + const cleanupResults = await Promise.allSettled([ + relay?.dispose(), + controller.dispose(), + ]); + const cleanupErrors = cleanupResults.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (cleanupErrors.length > 0) { throw new AggregateError( - [error, disposeError], + [error, ...cleanupErrors], "Sandbox configuration and cleanup failed", ); } diff --git a/packages/core/src/test/sandbox-agent.ts b/packages/core/src/test/sandbox-agent.ts index 5daa15df22..03682094b8 100644 --- a/packages/core/src/test/sandbox-agent.ts +++ b/packages/core/src/test/sandbox-agent.ts @@ -35,6 +35,10 @@ interface ManagedProcess { tty: boolean; } +export interface MockSandboxAgentOptions { + token?: string; +} + export interface MockSandboxAgentHandle { baseUrl: string; client: SandboxAgent; @@ -231,12 +235,21 @@ async function runCommand(request: { }); } -export async function startMockSandboxAgent(): Promise { +export async function startMockSandboxAgent( + options: MockSandboxAgentOptions = {}, +): Promise { const rootDir = await mkdtemp(resolve(tmpdir(), "agentos-sandbox-agent-")); const processes = new Map(); const server = createServer(async (request, response) => { try { + if ( + options.token && + request.headers.authorization !== `Bearer ${options.token}` + ) { + problem(response, 401, "Invalid sandbox token"); + return; + } const url = new URL(request.url ?? "/", "http://127.0.0.1"); const method = request.method ?? "GET"; @@ -503,6 +516,7 @@ export async function startMockSandboxAgent(): Promise { const { SandboxAgent } = await import("sandbox-agent"); const client = await SandboxAgent.connect({ baseUrl, + ...(options.token ? { token: options.token } : {}), waitForHealth: { timeoutMs: 5_000 }, }); diff --git a/packages/core/tests/options-schema.test.ts b/packages/core/tests/options-schema.test.ts index 1279163cd5..3f861ea304 100644 --- a/packages/core/tests/options-schema.test.ts +++ b/packages/core/tests/options-schema.test.ts @@ -1,5 +1,9 @@ import { describe, expect, test } from "vitest"; -import { AgentOs, agentOsOptionsSchema } from "../src/index.js"; +import { + AgentOs, + agentOsOptionsSchema, + SandboxStartupError, +} from "../src/index.js"; import { getSandboxDisposeHooks, resolveSandboxOptions, @@ -112,10 +116,12 @@ describe("AgentOsOptions validation", () => { ).toBe(false); } }); - test("provider sandbox starts a client and owns disposal", async () => { + test("provider sandbox starts lazily and owns disposal", async () => { + let started = 0; let disposed = false; const client = { baseUrl: "http://127.0.0.1:1234", + listProcesses: async () => ({ processes: [] }), dispose: () => { disposed = true; }, @@ -124,22 +130,34 @@ describe("AgentOsOptions validation", () => { const options = await resolveSandboxOptions({ sandbox: { provider: { - start: async () => client, + start: async () => { + started += 1; + return client; + }, }, }, } as never); expect(options).not.toHaveProperty("sandbox"); expect(options.mounts?.[0]?.path).toBe("/mnt/sandbox"); expect(options.bindings?.[0]?.name).toBe("sandbox"); + expect(started).toBe(0); + await options.bindings?.[0]?.bindings["list-processes"].execute({}); + expect(started).toBe(1); for (const hook of getSandboxDisposeHooks(options)) { await hook(); } expect(disposed).toBe(true); }); - test("advanced sandbox client leaves disposal manual by default", async () => { - const client = { baseUrl: "http://127.0.0.1:1234" } as never; + test("advanced sandbox client leaves client disposal manual by default", async () => { + let disposed = false; + const client = { + baseUrl: "http://127.0.0.1:1234", + dispose: () => { + disposed = true; + }, + } as never; const options = await resolveSandboxOptions({ sandbox: { client, @@ -147,25 +165,170 @@ describe("AgentOsOptions validation", () => { }, } as never); expect(options.mounts?.[0]?.path).toBe("/work"); - expect(getSandboxDisposeHooks(options)).toHaveLength(0); + const mount = options.mounts?.[0]; + if (!mount || !("plugin" in mount)) { + throw new Error("sandbox mount config is missing"); + } + expect(mount.plugin.config.baseUrl).not.toBe("http://127.0.0.1:1234"); + expect(mount.plugin.config.token).toEqual(expect.any(String)); + expect(getSandboxDisposeHooks(options)).toHaveLength(1); + for (const hook of getSandboxDisposeHooks(options)) await hook(); + expect(disposed).toBe(false); }); - test("disposes a provider client when sandbox expansion fails", async () => { + test("advanced sandbox client can transfer disposal ownership", async () => { let disposed = 0; - await expect( - resolveSandboxOptions({ - sandbox: { - provider: { - start: async () => ({ - dispose: () => { + const options = await resolveSandboxOptions({ + sandbox: { + client: { + baseUrl: "http://127.0.0.1:1234", + dispose: async () => { + disposed += 1; + }, + } as never, + dispose: true, + }, + } as never); + for (const hook of getSandboxDisposeHooks(options)) await hook(); + expect(disposed).toBe(1); + }); + + test("shares one provider startup across mount and binding calls", async () => { + let started = 0; + let releaseStart!: () => void; + const startGate = new Promise((resolve) => { + releaseStart = resolve; + }); + const options = await resolveSandboxOptions({ + sandbox: { + provider: { + start: async () => { + started += 1; + await startGate; + return { + request: async () => + new Response( + JSON.stringify({ + path: "/", + entryType: "directory", + size: 0, + }), + { headers: { "content-type": "application/json" } }, + ), + listProcesses: async () => ({ processes: [] }), + dispose: async () => {}, + } as never; + }, + }, + }, + } as never); + const execute = options.bindings?.[0]?.bindings["list-processes"].execute; + if (!execute) throw new Error("sandbox list-processes binding is missing"); + const mount = options.mounts?.[0]; + if (!mount || !("plugin" in mount)) { + throw new Error("sandbox mount config is missing"); + } + const mountConfig = mount.plugin.config; + const bindingCall = execute({}); + const mountCall = fetch( + `${String(mountConfig.baseUrl)}/v1/fs/stat?path=%2F`, + { + headers: { + authorization: `Bearer ${String(mountConfig.token)}`, + }, + }, + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(started).toBe(1); + releaseStart(); + const [, mountResponse] = await Promise.all([bindingCall, mountCall]); + expect(mountResponse.status).toBe(200); + await mountResponse.arrayBuffer(); + expect(started).toBe(1); + for (const hook of getSandboxDisposeHooks(options)) await hook(); + }); + + test("reports startup failures and retries on the next operation", async () => { + let started = 0; + const options = await resolveSandboxOptions({ + sandbox: { + provider: { + start: async () => { + started += 1; + if (started === 1) throw new Error("provider unavailable"); + return { + listProcesses: async () => ({ processes: [] }), + dispose: async () => {}, + } as never; + }, + }, + }, + } as never); + const execute = options.bindings?.[0]?.bindings["list-processes"].execute; + if (!execute) throw new Error("sandbox list-processes binding is missing"); + await expect(execute({})).rejects.toEqual( + expect.objectContaining({ + name: SandboxStartupError.name, + message: expect.stringContaining("provider unavailable"), + }), + ); + await expect(execute({})).resolves.toEqual({ processes: [] }); + expect(started).toBe(2); + for (const hook of getSandboxDisposeHooks(options)) await hook(); + }); + + test("restarts an idle provider without changing the mount endpoint", async () => { + let started = 0; + let disposed = 0; + const options = await resolveSandboxOptions({ + sandbox: { + idleTimeoutMs: 10, + provider: { + start: async () => { + started += 1; + return { + listProcesses: async () => ({ processes: [] }), + dispose: async () => { disposed += 1; }, - }), + } as never; }, }, - } as never), - ).rejects.toThrow(/serializable baseUrl/); + }, + } as never); + const mount = options.mounts?.[0]; + if (!mount || !("plugin" in mount)) { + throw new Error("sandbox mount config is missing"); + } + const relayUrl = mount.plugin.config.baseUrl; + const execute = options.bindings?.[0]?.bindings["list-processes"].execute; + if (!execute) throw new Error("sandbox list-processes binding is missing"); + await execute({}); + for (let attempt = 0; attempt < 50 && disposed === 0; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } expect(disposed).toBe(1); + await execute({}); + expect(started).toBe(2); + expect(mount.plugin.config.baseUrl).toBe(relayUrl); + for (const hook of getSandboxDisposeHooks(options)) await hook(); + }); + + test("validates sandbox relay and lifecycle limits", async () => { + for (const [field, value] of [ + ["maxRelayRequests", 0], + ["idleTimeoutMs", -1], + ["startupTimeoutMs", 1.5], + ] as const) { + await expect( + resolveSandboxOptions({ + sandbox: { + client: { baseUrl: "http://127.0.0.1:1234" } as never, + [field]: value, + }, + } as never), + ).rejects.toThrow(new RegExp(`sandbox\\.${field}`)); + } }); test("does not start a provider when VM option validation fails", async () => { diff --git a/packages/core/tests/public-api-exports.test.ts b/packages/core/tests/public-api-exports.test.ts index 2c1922c6ab..4dc41b666b 100644 --- a/packages/core/tests/public-api-exports.test.ts +++ b/packages/core/tests/public-api-exports.test.ts @@ -40,6 +40,7 @@ import { type PromptResult, parseAgentOsOptions, rootFilesystemConfigSchema, + SandboxStartupError, type SessionCapabilities, type SessionInfo, type SessionStreamEntry, @@ -105,6 +106,7 @@ describe("root public API exports", () => { defaultSoftware: false, }); expect(KernelError).toBeTypeOf("function"); + expect(SandboxStartupError).toBeTypeOf("function"); expect(createSnapshotExport).toBeTypeOf("function"); // Package dirs are the public software descriptor. expect(defineSoftware("/opt/pkg")).toBe("/opt/pkg"); diff --git a/website/public/docs/docs/sandboxes.md b/website/public/docs/docs/sandboxes.md index dd442b3352..bf66841506 100644 --- a/website/public/docs/docs/sandboxes.md +++ b/website/public/docs/docs/sandboxes.md @@ -44,9 +44,9 @@ npm install @rivet-dev/agentos-sandbox sandbox-agent - `createSandboxFs`, `createSandboxBindings` — from `@rivet-dev/agentos-sandbox`. - `SandboxAgent` + provider helpers (e.g. `docker`) — from `sandbox-agent`. -- Pass a provider as `sandbox: { provider: docker() }`. agentOS starts the - client, mounts it at `/mnt/sandbox`, registers process bindings, and disposes - it with the VM. +- Pass a provider as `sandbox: { provider: docker() }`. agentOS mounts it at + `/mnt/sandbox`, registers process bindings, starts the sandbox on first use, + and disposes it with the VM. - In RivetKit actors, pass the provider to `agentOS(...)` — a fresh client per actor VM. @@ -62,6 +62,15 @@ accepts these options alongside `provider` or `client`: | `readOnly` | Prevents the VM from modifying files through the mount. Defaults to `false`. | | `timeoutMs` | Sets the per-request timeout for Sandbox Agent filesystem calls. | | `maxFullReadBytes` | Bounds files buffered by full-read and truncate fallbacks. | +| `idleTimeoutMs` | Stops an inactive provider sandbox so the next operation starts a fresh one. Defaults to five minutes; set to `0` to disable. | +| `startupTimeoutMs` | Bounds provider startup. Defaults to 20 seconds; set to `0` to disable. | +| `maxRelayRequests` | Bounds concurrent filesystem requests through the per-VM relay. Defaults to 64. | + +The native mount talks to a stable, authenticated loopback endpoint owned by +that agentOS VM. When an idle sandbox is recreated or resumed with a different +provider URL or token, the mount and process bindings switch to the current +client without remounting or rebooting the VM. Concurrent first operations +share one provider startup. The server example above changes `mountPath` to `/home/agentos/sandbox`. Paths used inside the external sandbox remain relative @@ -112,8 +121,12 @@ additional provider SDK. For another backend, adapt any [Sandbox Agent](https://sandboxagent.dev) provider with `sandboxAgentProvider` from `@rivet-dev/agentos-sandbox`. Provider -mode is the preferred lifecycle: each VM gets a fresh sandbox, and disposal of -the VM destroys it. +mode is the preferred lifecycle: each VM gets a fresh sandbox on first use, +inactive sandboxes are replaced transparently, and disposal of the VM destroys +the current sandbox. A process ID belongs to the sandbox generation that +created it and is not valid after a destroy-and-recreate cycle. Providers that +pause and reconnect may preserve their provider sandbox ID, but callers must +not assume that the network address or credentials remain unchanged. ## Advanced: mount an existing client @@ -121,7 +134,9 @@ Standalone `AgentOs.create()` can mount an already-connected, Sandbox-Agent-compatible client. Install `sandbox-agent` directly for this manual path. The caller owns the client by default; set `dispose` to `true` when the client implements disposal, or provide a callback to transfer lifecycle -ownership to the VM. +ownership to the VM. A custom compatible client must expose either a +serializable `baseUrl` plus its current token/headers or an authenticated +`request(path, init)` transport so the native mount relay can reach it. RivetKit `agentOS()` intentionally rejects the `client` form because one client cannot be shared safely across actor VMs. Pass a `provider` there so every actor diff --git a/website/src/content/docs/docs/sandboxes.mdx b/website/src/content/docs/docs/sandboxes.mdx index ec1150703b..12847ba77a 100644 --- a/website/src/content/docs/docs/sandboxes.mdx +++ b/website/src/content/docs/docs/sandboxes.mdx @@ -46,9 +46,9 @@ npm install @rivet-dev/agentos-sandbox sandbox-agent - `createSandboxFs`, `createSandboxBindings` — from `@rivet-dev/agentos-sandbox`. - `SandboxAgent` + provider helpers (e.g. `docker`) — from `sandbox-agent`. -- Pass a provider as `sandbox: { provider: docker() }`. agentOS starts the - client, mounts it at `/mnt/sandbox`, registers process bindings, and disposes - it with the VM. +- Pass a provider as `sandbox: { provider: docker() }`. agentOS mounts it at + `/mnt/sandbox`, registers process bindings, starts the sandbox on first use, + and disposes it with the VM. - In RivetKit actors, pass the provider to `agentOS(...)` — a fresh client per actor VM. @@ -68,6 +68,15 @@ accepts these options alongside `provider` or `client`: | `readOnly` | Prevents the VM from modifying files through the mount. Defaults to `false`. | | `timeoutMs` | Sets the per-request timeout for Sandbox Agent filesystem calls. | | `maxFullReadBytes` | Bounds files buffered by full-read and truncate fallbacks. | +| `idleTimeoutMs` | Stops an inactive provider sandbox so the next operation starts a fresh one. Defaults to five minutes; set to `0` to disable. | +| `startupTimeoutMs` | Bounds provider startup. Defaults to 20 seconds; set to `0` to disable. | +| `maxRelayRequests` | Bounds concurrent filesystem requests through the per-VM relay. Defaults to 64. | + +The native mount talks to a stable, authenticated loopback endpoint owned by +that agentOS VM. When an idle sandbox is recreated or resumed with a different +provider URL or token, the mount and process bindings switch to the current +client without remounting or rebooting the VM. Concurrent first operations +share one provider startup. The server example above changes `mountPath` to `/home/agentos/sandbox`. Paths used inside the external sandbox remain relative @@ -120,8 +129,12 @@ additional provider SDK. For another backend, adapt any [Sandbox Agent](https://sandboxagent.dev) provider with `sandboxAgentProvider` from `@rivet-dev/agentos-sandbox`. Provider -mode is the preferred lifecycle: each VM gets a fresh sandbox, and disposal of -the VM destroys it. +mode is the preferred lifecycle: each VM gets a fresh sandbox on first use, +inactive sandboxes are replaced transparently, and disposal of the VM destroys +the current sandbox. A process ID belongs to the sandbox generation that +created it and is not valid after a destroy-and-recreate cycle. Providers that +pause and reconnect may preserve their provider sandbox ID, but callers must +not assume that the network address or credentials remain unchanged. ## Advanced: mount an existing client @@ -129,7 +142,9 @@ Standalone `AgentOs.create()` can mount an already-connected, Sandbox-Agent-compatible client. Install `sandbox-agent` directly for this manual path. The caller owns the client by default; set `dispose` to `true` when the client implements disposal, or provide a callback to transfer lifecycle -ownership to the VM. +ownership to the VM. A custom compatible client must expose either a +serializable `baseUrl` plus its current token/headers or an authenticated +`request(path, init)` transport so the native mount relay can reach it.