From 00d75c6cce00a14339510bacb44cb8b72aef93d8 Mon Sep 17 00:00:00 2001 From: Filipe Forattini Date: Tue, 8 Sep 2026 10:52:20 -0300 Subject: [PATCH 1/2] feat(usage): mirror token usage into a sidecar the reporters already read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Token accounting lived only inside the session store, so nothing outside redcode could report on it — and the store is ours to change. This adds a second, tiny SQLite file that carries only what a usage report needs: one row per assistant turn with tokens, cost, model, provider and timestamps, and no message content at all. ~/.red/code/data/usage/opencode.db The layout is OpenCode's, which `ccusage` reads today: OPENCODE_DATA_DIR=~/.red/code/data/usage ccusage opencode daily That variable takes a comma-separated list, so a machine that also runs OpenCode keeps both sources. Nothing here depends on ccusage and ccusage needs no redcode-specific code — which is the point: when the session store moves off SQLite, this file is the contract that stays. The mirror is written by the same projection that persists the message, so it tracks the store row by row, and a re-published message updates its row rather than adding one. It is best-effort: any failure turns the sidecar off for the process, is reported once through the projector's log, and never fails a turn. `REDCODE_DISABLE_USAGE_SIDECAR=1` opts out. Drivers follow the repo's `#sqlite`/`#pty` convention with a bun and a node variant behind `#usage-sidecar`. Verified end to end: a sidecar written by this code, read back by the published `ccusage`, reports the right days, tokens, cost and model. Claude-Session: https://claude.ai/code/session_01MyZkjLGy7k8uQdRWzU3aSB --- packages/core/package.json | 5 + packages/core/src/session/projector.ts | 10 ++ packages/core/src/usage/sidecar-store.bun.ts | 20 +++ packages/core/src/usage/sidecar-store.node.ts | 20 +++ packages/core/src/usage/sidecar-store.ts | 29 ++++ packages/core/src/usage/usage.ts | 161 ++++++++++++++++++ packages/core/test/usage.test.ts | 119 +++++++++++++ 7 files changed, 364 insertions(+) create mode 100644 packages/core/src/usage/sidecar-store.bun.ts create mode 100644 packages/core/src/usage/sidecar-store.node.ts create mode 100644 packages/core/src/usage/sidecar-store.ts create mode 100644 packages/core/src/usage/usage.ts create mode 100644 packages/core/test/usage.test.ts diff --git a/packages/core/package.json b/packages/core/package.json index 738c0b2e03fd..f0b5c11a064b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -35,6 +35,11 @@ "bun": "./src/filesystem/fff.bun.ts", "node": "./src/filesystem/fff.node.ts", "default": "./src/filesystem/fff.bun.ts" + }, + "#usage-sidecar": { + "bun": "./src/usage/sidecar-store.bun.ts", + "node": "./src/usage/sidecar-store.node.ts", + "default": "./src/usage/sidecar-store.bun.ts" } }, "devDependencies": { diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 792067017d14..9752d36050b5 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -12,6 +12,7 @@ import { SessionMessage } from "./message" import { SessionMessageUpdater } from "./message-updater" import { SessionInput } from "./input" import { WorkspaceV2 } from "../workspace" +import { Usage } from "../usage/usage" import { MessageTable, PartTable, SessionInputTable, SessionMessageTable, SessionTable } from "./sql" import type { DeepMutable } from "../schema" @@ -74,6 +75,8 @@ function sessionRow(info: SessionV1.SessionInfo): typeof SessionTable.$inferInse } } +let warnedAboutSidecar = false + function messageData( info: (typeof SessionV1.Event.MessageUpdated.Type)["data"]["info"], ): typeof MessageTable.$inferInsert.data { @@ -269,6 +272,13 @@ const layer = Layer.effectDiscard( .onConflictDoUpdate({ target: MessageTable.id, set: { data } }) .run() .pipe(Effect.orDie) + // The usage sidecar mirrors the same row, minus the content: it is the file usage reporters read, and it + // outlives whatever stores the session itself (Usage.record swallows its own failures). + const mirrored = Usage.recordMessage({ id, sessionID, timeCreated: time_created, info: event.data.info }) + if (mirrored === false && !warnedAboutSidecar) { + warnedAboutSidecar = true + yield* Effect.logWarning("usage sidecar disabled after a write failure", { error: Usage.lastError() }) + } }), ) yield* events.project(SessionV1.Event.MessageRemoved, (event) => diff --git a/packages/core/src/usage/sidecar-store.bun.ts b/packages/core/src/usage/sidecar-store.bun.ts new file mode 100644 index 000000000000..52b269144b9b --- /dev/null +++ b/packages/core/src/usage/sidecar-store.bun.ts @@ -0,0 +1,20 @@ +import { Database } from "bun:sqlite" +import { CREATE_INDEX, CREATE_TABLE, UPSERT, type SidecarRow, type SidecarStore } from "./sidecar-store" + +export function open(filename: string): SidecarStore { + const database = new Database(filename, { create: true }) + database.exec("PRAGMA journal_mode = WAL") + database.exec("PRAGMA synchronous = NORMAL") + database.exec("PRAGMA busy_timeout = 5000") + database.exec(CREATE_TABLE) + database.exec(CREATE_INDEX) + const statement = database.query(UPSERT) + return { + upsert(row: SidecarRow) { + statement.run(row.id, row.sessionID, row.timeCreated, row.data) + }, + close() { + database.close() + }, + } +} diff --git a/packages/core/src/usage/sidecar-store.node.ts b/packages/core/src/usage/sidecar-store.node.ts new file mode 100644 index 000000000000..4c05290b7827 --- /dev/null +++ b/packages/core/src/usage/sidecar-store.node.ts @@ -0,0 +1,20 @@ +import { DatabaseSync } from "node:sqlite" +import { CREATE_INDEX, CREATE_TABLE, UPSERT, type SidecarRow, type SidecarStore } from "./sidecar-store" + +export function open(filename: string): SidecarStore { + const database = new DatabaseSync(filename) + database.exec("PRAGMA journal_mode = WAL") + database.exec("PRAGMA synchronous = NORMAL") + database.exec("PRAGMA busy_timeout = 5000") + database.exec(CREATE_TABLE) + database.exec(CREATE_INDEX) + const statement = database.prepare(UPSERT) + return { + upsert(row: SidecarRow) { + statement.run(row.id, row.sessionID, row.timeCreated, row.data) + }, + close() { + database.close() + }, + } +} diff --git a/packages/core/src/usage/sidecar-store.ts b/packages/core/src/usage/sidecar-store.ts new file mode 100644 index 000000000000..8222da8dcb90 --- /dev/null +++ b/packages/core/src/usage/sidecar-store.ts @@ -0,0 +1,29 @@ +/** + * The shape both platform drivers implement. The sidecar is a tiny append-mostly table, so the driver surface is + * two calls: upsert one row, close the handle. + */ +export interface SidecarStore { + upsert(row: SidecarRow): void + close(): void +} + +/** One usage row, in the column layout ccusage's OpenCode reader expects. */ +export interface SidecarRow { + readonly id: string + readonly sessionID: string + readonly timeCreated: number + readonly data: string +} + +export const CREATE_TABLE = `CREATE TABLE IF NOT EXISTS message ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + time_created INTEGER NOT NULL, + data TEXT NOT NULL +)` + +export const CREATE_INDEX = `CREATE INDEX IF NOT EXISTS message_time_created ON message (time_created)` + +export const UPSERT = `INSERT INTO message (id, session_id, time_created, data) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET data = excluded.data, time_created = excluded.time_created` diff --git a/packages/core/src/usage/usage.ts b/packages/core/src/usage/usage.ts new file mode 100644 index 000000000000..4e68ad75cb87 --- /dev/null +++ b/packages/core/src/usage/usage.ts @@ -0,0 +1,161 @@ +export * as Usage from "./usage" + +import { mkdirSync } from "fs" +import { join } from "path" +import { open } from "#usage-sidecar" +import { Global } from "../global" +import type { SidecarStore } from "./sidecar-store" + +/** + * The USAGE SIDECAR: a second, tiny SQLite file carrying only what a usage report needs — tokens, cost, model, + * provider and timestamps, one row per assistant message. + * + * It exists so token accounting survives the storage underneath it. The session store is redcode's own and will + * change; this file is a stable, documented contract with the outside world, and it is written in the layout + * OpenCode uses, which `ccusage` already reads: + * + * ~/.red/code/data/usage/opencode.db → ccusage opencode daily + * + * with `OPENCODE_DATA_DIR` pointing at the directory (it takes a comma-separated list, so a machine that also runs + * OpenCode keeps both sources). Nothing here depends on ccusage, and ccusage needs no redcode-specific code. + * + * The sidecar never carries message content: prompts, tool output and file paths stay in the session store. + */ +const DIRECTORY = "usage" +const FILENAME = "opencode.db" + +/** Every field the reader consumes, and nothing else. */ +export interface Entry { + readonly id: string + readonly sessionID: string + readonly timeCreated: number + readonly timeCompleted?: number | undefined + readonly modelID?: string | undefined + readonly providerID?: string | undefined + readonly cost?: number | undefined + readonly tokens?: + | { + readonly input?: number | undefined + readonly output?: number | undefined + readonly reasoning?: number | undefined + readonly cache?: { readonly read?: number | undefined; readonly write?: number | undefined } | undefined + } + | undefined +} + +export function path() { + return join(Global.Path.data, DIRECTORY, FILENAME) +} + +/** + * On unless `REDCODE_DISABLE_USAGE_SIDECAR` says otherwise. Read per call rather than captured at import like + * `Flag`: the sidecar is opened lazily, long after start-up, and a test that flips the variable should be obeyed. + */ +export function enabled() { + const flag = process.env["REDCODE_DISABLE_USAGE_SIDECAR"]?.toLowerCase() + return flag !== "1" && flag !== "true" +} + +let store: SidecarStore | undefined +let broken = false +let failure: unknown + +/** The error that turned the sidecar off, for a caller that wants to report it once. Cleared by `reset`. */ +export function lastError() { + return failure +} + +function handle() { + if (broken || !enabled()) return undefined + if (store) return store + try { + mkdirSync(join(Global.Path.data, DIRECTORY), { recursive: true }) + store = open(path()) + return store + } catch (error) { + // A usage mirror is never worth failing a turn over: give up for the process and keep going. + broken = true + failure = error + return undefined + } +} + +/** + * Mirror one assistant message. Called on the same event that persists the message, so the sidecar tracks the + * session store row by row; a re-published message updates its row instead of adding one. + */ +export function record(entry: Entry) { + const target = handle() + if (!target) return + try { + target.upsert({ + id: entry.id, + sessionID: entry.sessionID, + timeCreated: entry.timeCreated, + data: JSON.stringify({ + id: entry.id, + sessionID: entry.sessionID, + role: "assistant", + modelID: entry.modelID, + providerID: entry.providerID, + cost: entry.cost, + tokens: entry.tokens, + time: { created: entry.timeCreated, completed: entry.timeCompleted }, + }), + }) + } catch (error) { + broken = true + failure = error + } +} + +/** + * Mirror a persisted message when it carries usage. Answers `false` when the sidecar just gave up (so a caller can + * report it once), `true` when the row was written, and `undefined` when there was nothing to mirror — a user + * message, or an assistant turn that reported no tokens. + */ +export function recordMessage(input: { + id: string + sessionID: string + timeCreated: number + info: { + readonly role?: string | undefined + readonly cost?: number | undefined + readonly tokens?: Entry["tokens"] + readonly modelID?: string | undefined + readonly providerID?: string | undefined + readonly time?: { readonly created?: number | undefined; readonly completed?: number | undefined } | undefined + } +}) { + const info = input.info + if (info.role !== "assistant") return undefined + const tokens = info.tokens + const used = + (tokens?.input ?? 0) + (tokens?.output ?? 0) + (tokens?.reasoning ?? 0) + + (tokens?.cache?.read ?? 0) + (tokens?.cache?.write ?? 0) + // A turn is mirrored once it has numbers: the same message is published several times while it streams, and the + // early publications carry no usage yet. + if (used === 0 && (info.cost ?? 0) === 0) return undefined + if (!enabled() || broken) return broken ? false : undefined + record({ + id: input.id, + sessionID: input.sessionID, + timeCreated: input.timeCreated, + timeCompleted: info.time?.completed, + modelID: info.modelID, + providerID: info.providerID, + cost: info.cost, + tokens, + }) + return !broken +} + +/** Test seam: drop the cached handle so a later call reopens (and re-reads the configured path). */ +export function reset() { + try { + store?.close() + } catch {} + store = undefined + broken = false + failure = undefined +} diff --git a/packages/core/test/usage.test.ts b/packages/core/test/usage.test.ts new file mode 100644 index 000000000000..4a92821094ce --- /dev/null +++ b/packages/core/test/usage.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, test, beforeEach, afterEach } from "bun:test" +import { Database } from "bun:sqlite" +import fs from "fs" +import path from "path" +import { Global } from "@reddb-io/redcode-core/global" +import { Usage } from "@reddb-io/redcode-core/usage/usage" + +/** + * The sidecar is a contract with the outside world: a usage reporter opens this file and reads these columns. + * The tests pin the layout and the record shape, not the implementation. + */ +describe("usage sidecar", () => { + // `Global.Path.data` resolves at module scope (see test/preload.ts), so the suite works inside the shared test + // home and clears the sidecar between cases instead of moving it. + beforeEach(() => { + Usage.reset() + fs.rmSync(path.dirname(Usage.path()), { recursive: true, force: true }) + }) + + afterEach(() => { + Usage.reset() + fs.rmSync(path.dirname(Usage.path()), { recursive: true, force: true }) + }) + + const assistant = (over: Record = {}) => ({ + role: "assistant", + modelID: "claude-sonnet-5", + providerID: "anthropic", + cost: 0.25, + tokens: { input: 100, output: 20, reasoning: 5, cache: { read: 7, write: 3 } }, + time: { created: 1_700_000_000_000, completed: 1_700_000_001_000 }, + ...over, + }) + + type Row = { id: string; session_id: string; time_created: number; data: string } + + const read = (): Row[] => { + const database = new Database(Usage.path(), { readonly: true }) + const rows = database.query("SELECT id, session_id, time_created, data FROM message ORDER BY time_created").all() + database.close() + return rows + } + + test("writes one row per assistant message, in the reader's column layout", () => { + Usage.recordMessage({ id: "msg_1", sessionID: "ses_1", timeCreated: 1_700_000_000_000, info: assistant() }) + + const rows = read() + expect(rows).toHaveLength(1) + expect(rows[0].id).toBe("msg_1") + expect(rows[0].session_id).toBe("ses_1") + expect(rows[0].time_created).toBe(1_700_000_000_000) + + const data = JSON.parse(rows[0].data) + expect(data).toEqual({ + id: "msg_1", + sessionID: "ses_1", + role: "assistant", + modelID: "claude-sonnet-5", + providerID: "anthropic", + cost: 0.25, + tokens: { input: 100, output: 20, reasoning: 5, cache: { read: 7, write: 3 } }, + time: { created: 1_700_000_000_000, completed: 1_700_000_001_000 }, + }) + }) + + test("carries no message content", () => { + Usage.recordMessage({ + id: "msg_2", + sessionID: "ses_1", + timeCreated: 1_700_000_000_000, + info: assistant({ path: { cwd: "/secret/project" }, agent: "build", summary: "a private prompt" }), + }) + + expect(read()[0]!.data).not.toContain("secret") + expect(read()[0]!.data).not.toContain("private") + }) + + test("re-publishing a message updates its row instead of adding one", () => { + Usage.recordMessage({ id: "msg_3", sessionID: "ses_1", timeCreated: 1_700_000_000_000, info: assistant() }) + Usage.recordMessage({ + id: "msg_3", + sessionID: "ses_1", + timeCreated: 1_700_000_000_000, + info: assistant({ cost: 0.5, tokens: { input: 200, output: 40, reasoning: 0, cache: { read: 0, write: 0 } } }), + }) + + const rows = read() + expect(rows).toHaveLength(1) + expect(JSON.parse(rows[0].data).cost).toBe(0.5) + }) + + test("skips user messages and turns that reported no usage", () => { + expect(Usage.recordMessage({ id: "u", sessionID: "s", timeCreated: 1, info: { role: "user" } })).toBeUndefined() + expect( + Usage.recordMessage({ + id: "empty", + sessionID: "s", + timeCreated: 1, + info: assistant({ cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } }), + }), + ).toBeUndefined() + expect(fs.existsSync(Usage.path())).toBe(false) + }) + + test("REDCODE_DISABLE_USAGE_SIDECAR writes nothing", () => { + process.env.REDCODE_DISABLE_USAGE_SIDECAR = "1" + try { + Usage.recordMessage({ id: "msg_4", sessionID: "ses_1", timeCreated: 1_700_000_000_000, info: assistant() }) + expect(fs.existsSync(Usage.path())).toBe(false) + } finally { + delete process.env.REDCODE_DISABLE_USAGE_SIDECAR + } + }) + + test("lives beside the session store, under the name a usage reader looks for", () => { + expect(Usage.path()).toBe(path.join(Global.Path.data, "usage", "opencode.db")) + expect(Usage.path().endsWith(path.join("data", "usage", "opencode.db"))).toBe(true) + }) +}) From 1471c27599c040f71673c8cbce836a5951bf47b2 Mon Sep 17 00:00:00 2001 From: Filipe Forattini Date: Tue, 8 Sep 2026 11:14:15 -0300 Subject: [PATCH 2/2] feat(usage): fan token usage out to OpenCode's database as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sidecar this builds on is ours and needs `OPENCODE_DATA_DIR` on the command line to be read. This adds the second target that needs nothing: when an OpenCode database exists, each assistant turn is mirrored into it too, so `ccusage opencode daily` reports redcode usage with no setup. Writing into another application's file is only safe if it honours that application's schema, and OpenCode's does not stand alone: `message` points at `session`, and `session` at `project`. The fan-out therefore writes all three, and every id it introduces is marked — `prj_redcode_…`, `ses_redcode_…`, `redcode_…` — so a row this process wrote is recognizable there and removable by hand. Rows go in with INSERT OR IGNORE for the session and project: we add, we never edit what OpenCode owns. The file is opened read-write and never created: a machine without OpenCode gets no fan-out and no invented database. Verified against a copy of a real OpenCode database (1,783 existing messages): `PRAGMA foreign_key_check` clean afterwards, the existing rows untouched, and the published `ccusage` reporting redcode's days beside OpenCode's own. The test suite pins the same properties against the real schema, copied into `test/fixtures/opencode-schema.sql`. Costs of this choice, accepted deliberately: redcode's sessions become visible in OpenCode's own session list (empty, since no parts are written), and deleting one there takes its usage rows with it through the foreign key's ON DELETE CASCADE. `REDCODE_DISABLE_USAGE_FANOUT=1` turns the second target off and keeps the sidecar; `REDCODE_DISABLE_USAGE_SIDECAR=1` turns off both. Claude-Session: https://claude.ai/code/session_01MyZkjLGy7k8uQdRWzU3aSB --- packages/core/src/usage/sidecar-store.bun.ts | 46 +++- packages/core/src/usage/sidecar-store.node.ts | 42 +++- packages/core/src/usage/sidecar-store.ts | 59 ++++- packages/core/src/usage/usage.ts | 230 ++++++++++++------ .../core/test/fixtures/opencode-schema.sql | 46 ++++ packages/core/test/usage.test.ts | 60 +++++ 6 files changed, 385 insertions(+), 98 deletions(-) create mode 100644 packages/core/test/fixtures/opencode-schema.sql diff --git a/packages/core/src/usage/sidecar-store.bun.ts b/packages/core/src/usage/sidecar-store.bun.ts index 52b269144b9b..d7cf45b15b25 100644 --- a/packages/core/src/usage/sidecar-store.bun.ts +++ b/packages/core/src/usage/sidecar-store.bun.ts @@ -1,17 +1,43 @@ import { Database } from "bun:sqlite" -import { CREATE_INDEX, CREATE_TABLE, UPSERT, type SidecarRow, type SidecarStore } from "./sidecar-store" +import { + CREATE_MESSAGE_INDEX, + CREATE_MESSAGE_TABLE, + INSERT_PROJECT, + INSERT_SESSION, + UPSERT_MESSAGE, + type MessageRow, + type ProjectRow, + type SessionRow, + type SidecarStore, +} from "./sidecar-store" -export function open(filename: string): SidecarStore { - const database = new Database(filename, { create: true }) - database.exec("PRAGMA journal_mode = WAL") - database.exec("PRAGMA synchronous = NORMAL") +/** + * `own` creates the message table; the fan-out target must never do that — the tables there belong to the other + * application, and creating them would mean inventing a schema it did not choose. + */ +export function open(filename: string, options: { own: boolean }): SidecarStore { + // `{ create }` alone is not a valid open mode in bun: the fan-out opens an existing file read-write, and only + // our own sidecar may bring a file into existence. + const database = options.own ? new Database(filename, { create: true }) : new Database(filename, { readwrite: true }) database.exec("PRAGMA busy_timeout = 5000") - database.exec(CREATE_TABLE) - database.exec(CREATE_INDEX) - const statement = database.query(UPSERT) + if (options.own) { + database.exec("PRAGMA journal_mode = WAL") + database.exec("PRAGMA synchronous = NORMAL") + database.exec(CREATE_MESSAGE_TABLE) + database.exec(CREATE_MESSAGE_INDEX) + } + const message = database.query(UPSERT_MESSAGE) + const session = options.own ? undefined : database.query(INSERT_SESSION) + const project = options.own ? undefined : database.query(INSERT_PROJECT) return { - upsert(row: SidecarRow) { - statement.run(row.id, row.sessionID, row.timeCreated, row.data) + message(row: MessageRow) { + message.run(row.id, row.sessionID, row.timeCreated, row.timeUpdated, row.data) + }, + session(row: SessionRow) { + session?.run(row.id, row.projectID, row.slug, row.directory, row.title, row.version, row.timeCreated, row.timeUpdated) + }, + project(row: ProjectRow) { + project?.run(row.id, row.worktree, row.timeCreated, row.timeUpdated) }, close() { database.close() diff --git a/packages/core/src/usage/sidecar-store.node.ts b/packages/core/src/usage/sidecar-store.node.ts index 4c05290b7827..94a4d750f6ac 100644 --- a/packages/core/src/usage/sidecar-store.node.ts +++ b/packages/core/src/usage/sidecar-store.node.ts @@ -1,17 +1,39 @@ import { DatabaseSync } from "node:sqlite" -import { CREATE_INDEX, CREATE_TABLE, UPSERT, type SidecarRow, type SidecarStore } from "./sidecar-store" +import { + CREATE_MESSAGE_INDEX, + CREATE_MESSAGE_TABLE, + INSERT_PROJECT, + INSERT_SESSION, + UPSERT_MESSAGE, + type MessageRow, + type ProjectRow, + type SessionRow, + type SidecarStore, +} from "./sidecar-store" -export function open(filename: string): SidecarStore { - const database = new DatabaseSync(filename) - database.exec("PRAGMA journal_mode = WAL") - database.exec("PRAGMA synchronous = NORMAL") +/** See the bun driver: `own` decides whether this process may create the schema. */ +export function open(filename: string, options: { own: boolean }): SidecarStore { + // Only our own sidecar may create a file; the fan-out opens what is already there. + const database = new DatabaseSync(filename, { readOnly: false, ...(options.own ? {} : { open: true }) }) database.exec("PRAGMA busy_timeout = 5000") - database.exec(CREATE_TABLE) - database.exec(CREATE_INDEX) - const statement = database.prepare(UPSERT) + if (options.own) { + database.exec("PRAGMA journal_mode = WAL") + database.exec("PRAGMA synchronous = NORMAL") + database.exec(CREATE_MESSAGE_TABLE) + database.exec(CREATE_MESSAGE_INDEX) + } + const message = database.prepare(UPSERT_MESSAGE) + const session = options.own ? undefined : database.prepare(INSERT_SESSION) + const project = options.own ? undefined : database.prepare(INSERT_PROJECT) return { - upsert(row: SidecarRow) { - statement.run(row.id, row.sessionID, row.timeCreated, row.data) + message(row: MessageRow) { + message.run(row.id, row.sessionID, row.timeCreated, row.timeUpdated, row.data) + }, + session(row: SessionRow) { + session?.run(row.id, row.projectID, row.slug, row.directory, row.title, row.version, row.timeCreated, row.timeUpdated) + }, + project(row: ProjectRow) { + project?.run(row.id, row.worktree, row.timeCreated, row.timeUpdated) }, close() { database.close() diff --git a/packages/core/src/usage/sidecar-store.ts b/packages/core/src/usage/sidecar-store.ts index 8222da8dcb90..8b0a5788e723 100644 --- a/packages/core/src/usage/sidecar-store.ts +++ b/packages/core/src/usage/sidecar-store.ts @@ -1,29 +1,70 @@ /** - * The shape both platform drivers implement. The sidecar is a tiny append-mostly table, so the driver surface is - * two calls: upsert one row, close the handle. + * The shape both platform drivers implement. Two targets share it: our own sidecar, and the fan-out into + * OpenCode's database, which needs the session and project rows its foreign keys demand. */ export interface SidecarStore { - upsert(row: SidecarRow): void + message(row: MessageRow): void + session(row: SessionRow): void + project(row: ProjectRow): void close(): void } /** One usage row, in the column layout ccusage's OpenCode reader expects. */ -export interface SidecarRow { +export interface MessageRow { readonly id: string readonly sessionID: string readonly timeCreated: number + readonly timeUpdated: number readonly data: string } -export const CREATE_TABLE = `CREATE TABLE IF NOT EXISTS message ( +/** The session a usage row hangs off. OpenCode's `message.session_id` is a foreign key into this table. */ +export interface SessionRow { + readonly id: string + readonly projectID: string + readonly slug: string + readonly directory: string + readonly title: string + readonly version: string + readonly timeCreated: number + readonly timeUpdated: number +} + +/** The project a session hangs off — the other end of OpenCode's foreign-key chain. */ +export interface ProjectRow { + readonly id: string + readonly worktree: string + readonly timeCreated: number + readonly timeUpdated: number +} + +/** + * Our own sidecar carries the message table alone: nothing reads it through a foreign key, and a usage reporter + * only needs these four columns. + */ +export const CREATE_MESSAGE_TABLE = `CREATE TABLE IF NOT EXISTS message ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, time_created INTEGER NOT NULL, + time_updated INTEGER NOT NULL, data TEXT NOT NULL )` -export const CREATE_INDEX = `CREATE INDEX IF NOT EXISTS message_time_created ON message (time_created)` +export const CREATE_MESSAGE_INDEX = `CREATE INDEX IF NOT EXISTS message_time_created ON message (time_created)` + +export const UPSERT_MESSAGE = `INSERT INTO message (id, session_id, time_created, time_updated, data) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET data = excluded.data, time_updated = excluded.time_updated` + +/** + * The session and project rows exist for the fan-out target only, so they are written with the columns OpenCode's + * schema makes NOT NULL and nothing else — every other column there either allows null or carries a default. + * `INSERT OR IGNORE`: the row is written once and never overwrites what the other application owns. + */ +export const INSERT_SESSION = `INSERT OR IGNORE INTO session + (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` -export const UPSERT = `INSERT INTO message (id, session_id, time_created, data) - VALUES (?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET data = excluded.data, time_created = excluded.time_created` +export const INSERT_PROJECT = `INSERT OR IGNORE INTO project + (id, worktree, time_created, time_updated, sandboxes) + VALUES (?, ?, ?, ?, '[]')` diff --git a/packages/core/src/usage/usage.ts b/packages/core/src/usage/usage.ts index 4e68ad75cb87..7554bf9cd938 100644 --- a/packages/core/src/usage/usage.ts +++ b/packages/core/src/usage/usage.ts @@ -1,35 +1,40 @@ export * as Usage from "./usage" -import { mkdirSync } from "fs" +import { existsSync, mkdirSync } from "fs" +import { homedir } from "os" import { join } from "path" import { open } from "#usage-sidecar" import { Global } from "../global" import type { SidecarStore } from "./sidecar-store" /** - * The USAGE SIDECAR: a second, tiny SQLite file carrying only what a usage report needs — tokens, cost, model, - * provider and timestamps, one row per assistant message. + * USAGE MIRRORS: token accounting written a second time, outside the session store, so a usage reporter can read + * it without knowing anything about redcode. * - * It exists so token accounting survives the storage underneath it. The session store is redcode's own and will - * change; this file is a stable, documented contract with the outside world, and it is written in the layout - * OpenCode uses, which `ccusage` already reads: + * Two targets, both carrying only what a report needs — tokens, cost, model, provider, timestamps — and never a + * prompt, a tool result or a file path: * - * ~/.red/code/data/usage/opencode.db → ccusage opencode daily + * - OUR SIDECAR, `~/.red/code/data/usage/opencode.db`, in OpenCode's layout, which `ccusage` reads when pointed + * at the directory. It is ours, so it survives whatever the session store becomes. + * - THE FAN-OUT into OpenCode's own database, when one exists, so `ccusage opencode` finds redcode's usage with + * no configuration at all. OpenCode's `message` rows are keyed to `session`, and `session` to `project`, so a + * fan-out writes those two rows as well; every id it introduces there carries the `redcode` marker below, and + * the writes are INSERT OR IGNORE — this process adds rows and never edits the other application's. * - * with `OPENCODE_DATA_DIR` pointing at the directory (it takes a comma-separated list, so a machine that also runs - * OpenCode keeps both sources). Nothing here depends on ccusage, and ccusage needs no redcode-specific code. - * - * The sidecar never carries message content: prompts, tool output and file paths stay in the session store. + * The fan-out is why the mirrors exist at all in another app's file, and it is the part to switch off first if it + * ever misbehaves: `REDCODE_DISABLE_USAGE_FANOUT=1`. `REDCODE_DISABLE_USAGE_SIDECAR=1` turns off both. */ const DIRECTORY = "usage" const FILENAME = "opencode.db" -/** Every field the reader consumes, and nothing else. */ +/** Every id this module introduces into a foreign database starts with it, so the rows are identifiable there. */ +export const MARKER = "redcode" + export interface Entry { readonly id: string readonly sessionID: string readonly timeCreated: number - readonly timeCompleted?: number | undefined + readonly timeUpdated?: number | undefined readonly modelID?: string | undefined readonly providerID?: string | undefined readonly cost?: number | undefined @@ -41,6 +46,10 @@ export interface Entry { readonly cache?: { readonly read?: number | undefined; readonly write?: number | undefined } | undefined } | undefined + /** The working directory the turn ran in, when the message carries one: the fan-out's project and session need it. */ + readonly directory?: string | undefined + readonly title?: string | undefined + readonly version?: string | undefined } export function path() { @@ -48,69 +57,145 @@ export function path() { } /** - * On unless `REDCODE_DISABLE_USAGE_SIDECAR` says otherwise. Read per call rather than captured at import like - * `Flag`: the sidecar is opened lazily, long after start-up, and a test that flips the variable should be obeyed. + * OpenCode's own database, discovered the way OpenCode itself resolves it. Answers a path only when the file is + * already there: creating it would mean inventing a schema for an application that is not installed. */ -export function enabled() { - const flag = process.env["REDCODE_DISABLE_USAGE_SIDECAR"]?.toLowerCase() +export function fanoutPath() { + const configured = process.env["OPENCODE_DATA_DIR"] + const dataHome = process.env["XDG_DATA_HOME"] + const home = configured ?? join(dataHome && dataHome.startsWith("/") ? dataHome : join(homedir(), ".local", "share"), "opencode") + const file = join(home, FILENAME) + return existsSync(file) ? file : undefined +} + +function on(name: string) { + const flag = process.env[name]?.toLowerCase() return flag !== "1" && flag !== "true" } -let store: SidecarStore | undefined -let broken = false +/** On unless disabled. Read per call rather than captured at import: the mirrors open lazily, long after start-up. */ +export function enabled() { + return on("REDCODE_DISABLE_USAGE_SIDECAR") +} + +export function fanoutEnabled() { + return enabled() && on("REDCODE_DISABLE_USAGE_FANOUT") +} + +interface Target { + store: SidecarStore + own: boolean + sessions: Set +} + +let targets: Target[] | undefined let failure: unknown -/** The error that turned the sidecar off, for a caller that wants to report it once. Cleared by `reset`. */ +/** The error that turned a mirror off, for a caller that wants to report it once. Cleared by `reset`. */ export function lastError() { return failure } -function handle() { - if (broken || !enabled()) return undefined - if (store) return store - try { - mkdirSync(join(Global.Path.data, DIRECTORY), { recursive: true }) - store = open(path()) - return store - } catch (error) { - // A usage mirror is never worth failing a turn over: give up for the process and keep going. - broken = true - failure = error - return undefined +function openTargets() { + if (targets) return targets + const opened: Target[] = [] + if (enabled()) { + try { + mkdirSync(join(Global.Path.data, DIRECTORY), { recursive: true }) + opened.push({ store: open(path(), { own: true }), own: true, sessions: new Set() }) + } catch (error) { + failure = error + } + } + if (fanoutEnabled()) { + const file = fanoutPath() + if (file) { + try { + opened.push({ store: open(file, { own: false }), own: false, sessions: new Set() }) + } catch (error) { + failure = error + } + } } + targets = opened + return targets +} + +/** `redcode` in front of the foreign id, so a row this module wrote is recognizable in the other application. */ +function marked(kind: "ses" | "prj", value: string) { + return `${kind}_${MARKER}_${value.replace(/^[a-z]+_/, "")}` } -/** - * Mirror one assistant message. Called on the same event that persists the message, so the sidecar tracks the - * session store row by row; a re-published message updates its row instead of adding one. - */ export function record(entry: Entry) { - const target = handle() - if (!target) return - try { - target.upsert({ - id: entry.id, - sessionID: entry.sessionID, - timeCreated: entry.timeCreated, - data: JSON.stringify({ - id: entry.id, - sessionID: entry.sessionID, - role: "assistant", - modelID: entry.modelID, - providerID: entry.providerID, - cost: entry.cost, - tokens: entry.tokens, - time: { created: entry.timeCreated, completed: entry.timeCompleted }, - }), - }) - } catch (error) { - broken = true - failure = error + const opened = openTargets() + if (opened.length === 0) return + const timeUpdated = entry.timeUpdated ?? entry.timeCreated + const directory = entry.directory ?? Global.Path.data + const data = JSON.stringify({ + id: entry.id, + sessionID: entry.sessionID, + role: "assistant", + modelID: entry.modelID, + providerID: entry.providerID, + cost: entry.cost, + tokens: entry.tokens, + time: { created: entry.timeCreated, completed: entry.timeUpdated }, + }) + for (const target of opened) { + try { + if (target.own) { + target.store.message({ + id: entry.id, + sessionID: entry.sessionID, + timeCreated: entry.timeCreated, + timeUpdated, + data, + }) + continue + } + // The fan-out target keys messages to a session and a project it does not have: write those first, once per + // session per process. `INSERT OR IGNORE` keeps a re-run from touching rows that are already there. + const sessionID = marked("ses", entry.sessionID) + if (!target.sessions.has(sessionID)) { + const projectID = marked("prj", MARKER) + target.store.project({ + id: projectID, + worktree: directory, + timeCreated: entry.timeCreated, + timeUpdated, + }) + target.store.session({ + id: sessionID, + projectID, + slug: "", + directory, + title: entry.title ?? `${MARKER}: ${entry.sessionID}`, + version: entry.version ?? MARKER, + timeCreated: entry.timeCreated, + timeUpdated, + }) + target.sessions.add(sessionID) + } + target.store.message({ + id: `${MARKER}_${entry.id.replace(/^[a-z]+_/, "")}`, + sessionID, + timeCreated: entry.timeCreated, + timeUpdated, + data: data.replace(entry.sessionID, sessionID), + }) + } catch (error) { + // A usage mirror is never worth failing a turn over: drop this target and keep the others. + failure = error + targets = opened.filter((candidate) => candidate !== target) + try { + target.store.close() + } catch {} + } } } /** - * Mirror a persisted message when it carries usage. Answers `false` when the sidecar just gave up (so a caller can + * Mirror a persisted message when it carries usage. Answers `false` when a mirror just gave up (so a caller can * report it once), `true` when the row was written, and `undefined` when there was nothing to mirror — a user * message, or an assistant turn that reported no tokens. */ @@ -125,37 +210,44 @@ export function recordMessage(input: { readonly modelID?: string | undefined readonly providerID?: string | undefined readonly time?: { readonly created?: number | undefined; readonly completed?: number | undefined } | undefined + readonly path?: { readonly cwd?: string | undefined; readonly root?: string | undefined } | undefined } }) { const info = input.info if (info.role !== "assistant") return undefined const tokens = info.tokens const used = - (tokens?.input ?? 0) + (tokens?.output ?? 0) + (tokens?.reasoning ?? 0) + - (tokens?.cache?.read ?? 0) + (tokens?.cache?.write ?? 0) + (tokens?.input ?? 0) + + (tokens?.output ?? 0) + + (tokens?.reasoning ?? 0) + + (tokens?.cache?.read ?? 0) + + (tokens?.cache?.write ?? 0) // A turn is mirrored once it has numbers: the same message is published several times while it streams, and the // early publications carry no usage yet. if (used === 0 && (info.cost ?? 0) === 0) return undefined - if (!enabled() || broken) return broken ? false : undefined + if (!enabled()) return undefined + const before = failure record({ id: input.id, sessionID: input.sessionID, timeCreated: input.timeCreated, - timeCompleted: info.time?.completed, + timeUpdated: info.time?.completed, modelID: info.modelID, providerID: info.providerID, cost: info.cost, tokens, + directory: info.path?.root ?? info.path?.cwd, }) - return !broken + return failure === before } -/** Test seam: drop the cached handle so a later call reopens (and re-reads the configured path). */ +/** Test seam: drop the cached handles so a later call reopens (and re-reads the configured paths). */ export function reset() { - try { - store?.close() - } catch {} - store = undefined - broken = false + for (const target of targets ?? []) { + try { + target.store.close() + } catch {} + } + targets = undefined failure = undefined } diff --git a/packages/core/test/fixtures/opencode-schema.sql b/packages/core/test/fixtures/opencode-schema.sql new file mode 100644 index 000000000000..4d71708315f6 --- /dev/null +++ b/packages/core/test/fixtures/opencode-schema.sql @@ -0,0 +1,46 @@ +-- The tables an OpenCode database keeps for messages, copied from a real one so the fan-out is tested +-- against the foreign keys it actually has to satisfy. +CREATE TABLE `project` ( + `id` text PRIMARY KEY, + `worktree` text NOT NULL, + `vcs` text, + `name` text, + `icon_url` text, + `icon_color` text, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `time_initialized` integer, + `sandboxes` text NOT NULL +, `commands` text, `icon_url_override` text); + +CREATE TABLE `session` ( + `id` text PRIMARY KEY, + `project_id` text NOT NULL, + `parent_id` text, + `slug` text NOT NULL, + `directory` text NOT NULL, + `title` text NOT NULL, + `version` text NOT NULL, + `share_url` text, + `summary_additions` integer, + `summary_deletions` integer, + `summary_files` integer, + `summary_diffs` text, + `revert` text, + `permission` text, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `time_compacting` integer, + `time_archived` integer, `workspace_id` text, `path` text, `agent` text, `model` text, `cost` real DEFAULT 0 NOT NULL, `tokens_input` integer DEFAULT 0 NOT NULL, `tokens_output` integer DEFAULT 0 NOT NULL, `tokens_reasoning` integer DEFAULT 0 NOT NULL, `tokens_cache_read` integer DEFAULT 0 NOT NULL, `tokens_cache_write` integer DEFAULT 0 NOT NULL, `metadata` text, + CONSTRAINT `fk_session_project_id_project_id_fk` FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON DELETE CASCADE +); + +CREATE TABLE `message` ( + `id` text PRIMARY KEY, + `session_id` text NOT NULL, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `data` text NOT NULL, + CONSTRAINT `fk_message_session_id_session_id_fk` FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON DELETE CASCADE +); + diff --git a/packages/core/test/usage.test.ts b/packages/core/test/usage.test.ts index 4a92821094ce..809fe8352762 100644 --- a/packages/core/test/usage.test.ts +++ b/packages/core/test/usage.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test, beforeEach, afterEach } from "bun:test" import { Database } from "bun:sqlite" import fs from "fs" +import os from "os" import path from "path" import { Global } from "@reddb-io/redcode-core/global" import { Usage } from "@reddb-io/redcode-core/usage/usage" @@ -102,6 +103,65 @@ describe("usage sidecar", () => { expect(fs.existsSync(Usage.path())).toBe(false) }) + test("the fan-out satisfies OpenCode's foreign keys, and marks every row it introduces", () => { + const foreign = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "redcode-fanout-")), "opencode") + fs.mkdirSync(foreign, { recursive: true }) + const file = path.join(foreign, "opencode.db") + const database = new Database(file, { create: true }) + database.exec(fs.readFileSync(path.join(import.meta.dir, "fixtures", "opencode-schema.sql"), "utf8")) + database.close() + + process.env.OPENCODE_DATA_DIR = foreign + try { + Usage.reset() + Usage.recordMessage({ + id: "msg_5", + sessionID: "ses_abc", + timeCreated: 1_700_000_000_000, + info: assistant({ path: { cwd: "/work/project", root: "/work/project" } }), + }) + Usage.reset() + + const check = new Database(file, { readonly: true }) + // Enforcement is per connection: turning it on here is what proves the rows satisfy the constraints. + check.exec("PRAGMA foreign_keys = ON") + const violations = check.query("PRAGMA foreign_key_check").all() + const messages = check.query<{ id: string; session_id: string }, []>("SELECT id, session_id FROM message").all() + const sessions = check.query<{ id: string; directory: string }, []>("SELECT id, directory FROM session").all() + const projects = check.query<{ id: string; worktree: string }, []>("SELECT id, worktree FROM project").all() + check.close() + + expect(violations).toEqual([]) + expect(messages).toHaveLength(1) + expect(messages[0].id).toContain(Usage.MARKER) + expect(messages[0].session_id).toContain(Usage.MARKER) + expect(sessions).toHaveLength(1) + expect(sessions[0].id).toContain(Usage.MARKER) + expect(sessions[0].directory).toBe("/work/project") + expect(projects).toHaveLength(1) + expect(projects[0].id).toContain(Usage.MARKER) + } finally { + delete process.env.OPENCODE_DATA_DIR + Usage.reset() + fs.rmSync(path.dirname(foreign), { recursive: true, force: true }) + } + }) + + test("no OpenCode database means no fan-out, and never a created one", () => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "redcode-nofanout-")) + process.env.OPENCODE_DATA_DIR = empty + try { + Usage.reset() + expect(Usage.fanoutPath()).toBeUndefined() + Usage.recordMessage({ id: "msg_6", sessionID: "ses_1", timeCreated: 1_700_000_000_000, info: assistant() }) + expect(fs.readdirSync(empty)).toEqual([]) + } finally { + delete process.env.OPENCODE_DATA_DIR + Usage.reset() + fs.rmSync(empty, { recursive: true, force: true }) + } + }) + test("REDCODE_DISABLE_USAGE_SIDECAR writes nothing", () => { process.env.REDCODE_DISABLE_USAGE_SIDECAR = "1" try {