diff --git a/src/utils/__tests__/jsonl-lines.test.ts b/src/utils/__tests__/jsonl-lines.test.ts new file mode 100644 index 00000000..85fec379 --- /dev/null +++ b/src/utils/__tests__/jsonl-lines.test.ts @@ -0,0 +1,136 @@ +import * as fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { + afterEach, + describe, + expect, + it +} from 'vitest'; + +import { + iterateJsonlLines, + iterateJsonlLinesSync, + readJsonlLines, + readJsonlLinesSync +} from '../jsonl-lines'; + +describe('jsonl line streaming', () => { + const tempRoots: string[] = []; + + afterEach(() => { + while (tempRoots.length > 0) { + const root = tempRoots.pop(); + if (root) { + fs.rmSync(root, { recursive: true, force: true }); + } + } + }); + + function writeTranscript(name: string, content: string): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-')); + tempRoots.push(root); + const filePath = path.join(root, name); + fs.writeFileSync(filePath, content); + return filePath; + } + + it('reads lf and crlf lines without requiring a trailing newline', async () => { + const filePath = writeTranscript('mixed.jsonl', [ + '{"id":1}', + '{"id":2}\r', + '{"id":3}' + ].join('\n')); + + await expect(readJsonlLines(filePath)).resolves.toEqual([ + '{"id":1}', + '{"id":2}', + '{"id":3}' + ]); + expect(readJsonlLinesSync(filePath)).toEqual([ + '{"id":1}', + '{"id":2}', + '{"id":3}' + ]); + }); + + it('skips empty lines like the previous whole-file trim/split path', async () => { + const filePath = writeTranscript('empty-lines.jsonl', '\n{"a":1}\n\n{"b":2}\n\n'); + + await expect(readJsonlLines(filePath)).resolves.toEqual([ + '{"a":1}', + '{"b":2}' + ]); + expect(readJsonlLinesSync(filePath)).toEqual([ + '{"a":1}', + '{"b":2}' + ]); + }); + + it('handles multi-byte utf-8 sequences that span sync read chunks', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-')); + tempRoots.push(root); + const filePath = path.join(root, 'utf8.jsonl'); + + // Force the sync reader across many tiny chunks by writing a long prefix + // so the multi-byte character is unlikely to land on a single natural boundary + // only — the reader itself uses 1MB chunks; put the character mid-file. + const emoji = '😀'; // 4-byte UTF-8 + const prefix = `{"n":"${'x'.repeat(100)}"}`; + const mid = `{"emoji":"${emoji}"}`; + const suffix = `{"n":"${'y'.repeat(100)}"}`; + fs.writeFileSync(filePath, [prefix, mid, suffix].join('\n'), 'utf8'); + + const lines = readJsonlLinesSync(filePath); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[1]!).emoji).toBe(emoji); + }); + + it('streams via async iterator without loading the full file as one string', async () => { + const filePath = writeTranscript('stream.jsonl', [ + '{"line":1}', + '{"line":2}', + '{"line":3}' + ].join('\n')); + + const seen: string[] = []; + for await (const line of iterateJsonlLines(filePath)) { + seen.push(line); + } + expect(seen).toEqual([ + '{"line":1}', + '{"line":2}', + '{"line":3}' + ]); + + expect(Array.from(iterateJsonlLinesSync(filePath))).toEqual(seen); + }); + + it('can read files larger than Node max string length via streaming', async () => { + // Node refuses to create a single string longer than ~0x1fffffe8 (~512MB). + // Building a real 512MB+ fixture is too heavy for unit tests, so we prove + // the streaming path never calls readFile/readFileSync for the payload and + // still aggregates many chunks correctly by reading a multi-chunk file. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-')); + tempRoots.push(root); + const filePath = path.join(root, 'chunked.jsonl'); + + const lineCount = 5000; + const handle = fs.openSync(filePath, 'w'); + try { + for (let i = 0; i < lineCount; i++) { + fs.writeSync(handle, `{"i":${i},"pad":"${'z'.repeat(200)}"}\n`); + } + } finally { + fs.closeSync(handle); + } + + const lines = await readJsonlLines(filePath); + expect(lines).toHaveLength(lineCount); + expect(JSON.parse(lines[0]!).i).toBe(0); + expect(JSON.parse(lines[lineCount - 1]!).i).toBe(lineCount - 1); + + const syncLines = readJsonlLinesSync(filePath); + expect(syncLines).toHaveLength(lineCount); + }, 30000); +}); diff --git a/src/utils/__tests__/jsonl-metrics.test.ts b/src/utils/__tests__/jsonl-metrics.test.ts index d1d5e5e7..860d1173 100644 --- a/src/utils/__tests__/jsonl-metrics.test.ts +++ b/src/utils/__tests__/jsonl-metrics.test.ts @@ -513,6 +513,44 @@ describe('jsonl transcript metrics', () => { }); }); + it('aggregates token metrics by streaming many usage lines without a whole-file string read', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-')); + tempRoots.push(root); + const transcriptPath = path.join(root, 'streamed-tokens.jsonl'); + + // Many small lines so the streamer crosses chunk boundaries while the + // cumulative totals stay easy to assert. Regression for #550: full-file + // utf-8 reads throw once transcripts exceed Node's max string length. + const lineCount = 2500; + const handle = fs.openSync(transcriptPath, 'w'); + try { + for (let i = 0; i < lineCount; i++) { + fs.writeSync(handle, `${makeUsageLine({ + timestamp: `2026-01-01T10:${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}.000Z`, + input: 2, + output: 3, + cacheRead: 4, + cacheCreate: 1, + stopReason: 'end_turn' + })}\n`); + } + } finally { + fs.closeSync(handle); + } + + const metrics = await getTokenMetrics(transcriptPath); + + expect(metrics).toEqual({ + inputTokens: lineCount * 2, + outputTokens: lineCount * 3, + cachedTokens: lineCount * 5, + cacheReadTokens: lineCount * 4, + cacheCreationTokens: lineCount * 1, + totalTokens: lineCount * 10, + contextLength: 2 + 4 + 1 + }); + }, 30000); + it('calculates speed metrics from user-to-assistant processing windows', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-speed-')); tempRoots.push(root); diff --git a/src/utils/jsonl-lines.ts b/src/utils/jsonl-lines.ts index 3663a800..5b54a42f 100644 --- a/src/utils/jsonl-lines.ts +++ b/src/utils/jsonl-lines.ts @@ -1,21 +1,98 @@ import * as fs from 'fs'; -import { promisify } from 'util'; +import { createInterface } from 'node:readline'; -const readFile = promisify(fs.readFile); -const readFileSync = fs.readFileSync; +const SYNC_READ_CHUNK_BYTES = 1024 * 1024; -function splitJsonlContent(content: string): string[] { - return content.trim().split('\n').filter(line => line.length > 0); +/** + * Stream a JSONL file line-by-line without materializing the whole file as one + * string. Claude Code session transcripts can exceed Node's max string length + * (~512MB / 0x1fffffe8), so `fs.readFile(..., 'utf-8')` throws and callers that + * catch the error end up reporting zeros. + */ +export async function* iterateJsonlLines(filePath: string): AsyncGenerator { + const stream = fs.createReadStream(filePath, { encoding: 'utf8' }); + const reader = createInterface({ + input: stream, + crlfDelay: Infinity + }); + + try { + for await (const line of reader) { + if (line.length > 0) { + yield line; + } + } + } finally { + reader.close(); + stream.destroy(); + } +} + +/** + * Synchronous line iterator for call sites that cannot be async. + * Completes each line in a Buffer before decoding so multi-byte UTF-8 sequences + * are never split across chunk boundaries. + */ +export function* iterateJsonlLinesSync(filePath: string): Generator { + const fd = fs.openSync(filePath, 'r'); + try { + const scratch = Buffer.allocUnsafe(SYNC_READ_CHUNK_BYTES); + let pending = Buffer.alloc(0); + + for (;;) { + const bytesRead = fs.readSync(fd, scratch, 0, scratch.length, null); + if (bytesRead === 0) { + break; + } + + const chunk = scratch.subarray(0, bytesRead); + const combined = pending.length > 0 ? Buffer.concat([pending, chunk]) : chunk; + let start = 0; + + for (let i = 0; i < combined.length; i++) { + if (combined[i] !== 0x0a) { + continue; + } + + let lineBuf = combined.subarray(start, i); + if (lineBuf.length > 0 && lineBuf[lineBuf.length - 1] === 0x0d) { + lineBuf = lineBuf.subarray(0, lineBuf.length - 1); + } + if (lineBuf.length > 0) { + yield lineBuf.toString('utf8'); + } + start = i + 1; + } + + pending = start === 0 + ? Buffer.from(combined) + : Buffer.from(combined.subarray(start)); + } + + if (pending.length > 0) { + let lineBuf = pending; + if (lineBuf[lineBuf.length - 1] === 0x0d) { + lineBuf = lineBuf.subarray(0, lineBuf.length - 1); + } + if (lineBuf.length > 0) { + yield lineBuf.toString('utf8'); + } + } + } finally { + fs.closeSync(fd); + } } export async function readJsonlLines(filePath: string): Promise { - const content = await readFile(filePath, 'utf-8'); - return splitJsonlContent(content); + const lines: string[] = []; + for await (const line of iterateJsonlLines(filePath)) { + lines.push(line); + } + return lines; } export function readJsonlLinesSync(filePath: string): string[] { - const content = readFileSync(filePath, 'utf-8'); - return splitJsonlContent(content); + return Array.from(iterateJsonlLinesSync(filePath)); } export function parseJsonlLine(line: string): unknown { diff --git a/src/utils/jsonl-metrics.ts b/src/utils/jsonl-metrics.ts index 2321bf41..e233e24b 100644 --- a/src/utils/jsonl-metrics.ts +++ b/src/utils/jsonl-metrics.ts @@ -12,6 +12,7 @@ import { isCompactBoundary } from './compaction'; import { + iterateJsonlLines, parseJsonlLine, readJsonlLines } from './jsonl-lines'; @@ -91,39 +92,29 @@ export async function getSessionDuration(transcriptPath: string): Promise= 0; i--) { - const line = lines[i]; - if (!line) { + const timestamp = new Date(data.timestamp); + if (Number.isNaN(timestamp.getTime())) { continue; } - const data = parseJsonlLine(line) as { timestamp?: string } | null; - if (data?.timestamp) { - lastTimestamp = new Date(data.timestamp); - break; + if (!firstTimestamp) { + firstTimestamp = timestamp; } + lastTimestamp = timestamp; } - if (!firstTimestamp || !lastTimestamp) { + if (!sawAnyLine || !firstTimestamp || !lastTimestamp) { return null; } @@ -154,13 +145,12 @@ export async function getSessionDuration(transcriptPath: string): Promise { try { - // Use Node.js-compatible file reading + // Stream line-by-line. Full-file readFile('utf-8') throws once a session + // transcript exceeds Node's max string length (~512MB). if (!fs.existsSync(transcriptPath)) { return { inputTokens: 0, outputTokens: 0, cachedTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, totalTokens: 0, contextLength: 0 }; } - const lines = await readJsonlLines(transcriptPath); - let inputTokens = 0; let outputTokens = 0; let cacheReadTokens = 0; @@ -187,10 +177,13 @@ export async function getTokenMetrics(transcriptPath: string): Promise lastCompactBoundaryLineIndex + if (entryLineIndex > lastCompactBoundaryLineIndex && (!mostRecentPostCompactionTimestamp || entryTime > mostRecentPostCompactionTimestamp)) { mostRecentPostCompactionTimestamp = entryTime; mostRecentPostCompactionEntry = data;