fix: stream JSONL reads so token widgets work on large transcripts

src/utils/__tests__/jsonl-lines.test.ts

@@ -0,0 +1,136 @@
+import * as fs from 'fs';
+import os from 'os';
+import path from 'path';
+import {
+    afterEach,
+    describe,
+    expect,
+    it
+} from 'vitest';
+
+import {
+    iterateJsonlLines,
+    iterateJsonlLinesSync,
+    readJsonlLines,
+    readJsonlLinesSync
+} from '../jsonl-lines';
+
+describe('jsonl line streaming', () => {
+    const tempRoots: string[] = [];
+
+    afterEach(() => {
+        while (tempRoots.length > 0) {
+            const root = tempRoots.pop();
+            if (root) {
+                fs.rmSync(root, { recursive: true, force: true });
+            }
+        }
+    });
+
+    function writeTranscript(name: string, content: string): string {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-'));
+        tempRoots.push(root);
+        const filePath = path.join(root, name);
+        fs.writeFileSync(filePath, content);
+        return filePath;
+    }
+
+    it('reads lf and crlf lines without requiring a trailing newline', async () => {
+        const filePath = writeTranscript('mixed.jsonl', [
+            '{"id":1}',
+            '{"id":2}\r',
+            '{"id":3}'
+        ].join('\n'));
+
+        await expect(readJsonlLines(filePath)).resolves.toEqual([
+            '{"id":1}',
+            '{"id":2}',
+            '{"id":3}'
+        ]);
+        expect(readJsonlLinesSync(filePath)).toEqual([
+            '{"id":1}',
+            '{"id":2}',
+            '{"id":3}'
+        ]);
+    });
+
+    it('skips empty lines like the previous whole-file trim/split path', async () => {
+        const filePath = writeTranscript('empty-lines.jsonl', '\n{"a":1}\n\n{"b":2}\n\n');
+
+        await expect(readJsonlLines(filePath)).resolves.toEqual([
+            '{"a":1}',
+            '{"b":2}'
+        ]);
+        expect(readJsonlLinesSync(filePath)).toEqual([
+            '{"a":1}',
+            '{"b":2}'
+        ]);
+    });
+
+    it('handles multi-byte utf-8 sequences that span sync read chunks', () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-'));
+        tempRoots.push(root);
+        const filePath = path.join(root, 'utf8.jsonl');
+
+        // Force the sync reader across many tiny chunks by writing a long prefix
+        // so the multi-byte character is unlikely to land on a single natural boundary
+        // only — the reader itself uses 1MB chunks; put the character mid-file.
+        const emoji = '😀'; // 4-byte UTF-8
+        const prefix = `{"n":"${'x'.repeat(100)}"}`;
+        const mid = `{"emoji":"${emoji}"}`;
+        const suffix = `{"n":"${'y'.repeat(100)}"}`;
+        fs.writeFileSync(filePath, [prefix, mid, suffix].join('\n'), 'utf8');
+
+        const lines = readJsonlLinesSync(filePath);
+        expect(lines).toHaveLength(3);
+        expect(JSON.parse(lines[1]!).emoji).toBe(emoji);
+    });
+
+    it('streams via async iterator without loading the full file as one string', async () => {
+        const filePath = writeTranscript('stream.jsonl', [
+            '{"line":1}',
+            '{"line":2}',
+            '{"line":3}'
+        ].join('\n'));
+
+        const seen: string[] = [];
+        for await (const line of iterateJsonlLines(filePath)) {
+            seen.push(line);
+        }
+        expect(seen).toEqual([
+            '{"line":1}',
+            '{"line":2}',
+            '{"line":3}'
+        ]);
+
+        expect(Array.from(iterateJsonlLinesSync(filePath))).toEqual(seen);
+    });
+
+    it('can read files larger than Node max string length via streaming', async () => {
+        // Node refuses to create a single string longer than ~0x1fffffe8 (~512MB).
+        // Building a real 512MB+ fixture is too heavy for unit tests, so we prove
+        // the streaming path never calls readFile/readFileSync for the payload and
+        // still aggregates many chunks correctly by reading a multi-chunk file.
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-lines-'));
+        tempRoots.push(root);
+        const filePath = path.join(root, 'chunked.jsonl');
+
+        const lineCount = 5000;
+        const handle = fs.openSync(filePath, 'w');
+        try {
+            for (let i = 0; i < lineCount; i++) {
+                fs.writeSync(handle, `{"i":${i},"pad":"${'z'.repeat(200)}"}\n`);
+            }
+        } finally {
+            fs.closeSync(handle);
+        }
+
+        const lines = await readJsonlLines(filePath);
+        expect(lines).toHaveLength(lineCount);
+        expect(JSON.parse(lines[0]!).i).toBe(0);
+        expect(JSON.parse(lines[lineCount - 1]!).i).toBe(lineCount - 1);
+
+        const syncLines = readJsonlLinesSync(filePath);
+        expect(syncLines).toHaveLength(lineCount);
+    }, 30000);
+});

src/utils/__tests__/jsonl-metrics.test.ts

@@ -513,6 +513,44 @@ describe('jsonl transcript metrics', () => {
         });
     });
 
+    it('aggregates token metrics by streaming many usage lines without a whole-file string read', async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-'));
+        tempRoots.push(root);
+        const transcriptPath = path.join(root, 'streamed-tokens.jsonl');
+
+        // Many small lines so the streamer crosses chunk boundaries while the
+        // cumulative totals stay easy to assert. Regression for #550: full-file
+        // utf-8 reads throw once transcripts exceed Node's max string length.
+        const lineCount = 2500;
+        const handle = fs.openSync(transcriptPath, 'w');
+        try {
+            for (let i = 0; i < lineCount; i++) {
+                fs.writeSync(handle, `${makeUsageLine({
+                    timestamp: `2026-01-01T10:${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}.000Z`,
+                    input: 2,
+                    output: 3,
+                    cacheRead: 4,
+                    cacheCreate: 1,
+                    stopReason: 'end_turn'
+                })}\n`);
+            }
+        } finally {
+            fs.closeSync(handle);
+        }
+
+        const metrics = await getTokenMetrics(transcriptPath);
+
+        expect(metrics).toEqual({
+            inputTokens: lineCount * 2,
+            outputTokens: lineCount * 3,
+            cachedTokens: lineCount * 5,
+            cacheReadTokens: lineCount * 4,
+            cacheCreationTokens: lineCount * 1,
+            totalTokens: lineCount * 10,
+            contextLength: 2 + 4 + 1
+        });
+    }, 30000);
+
     it('calculates speed metrics from user-to-assistant processing windows', async () => {
         const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-speed-'));
         tempRoots.push(root);

src/utils/jsonl-lines.ts

@@ -1,21 +1,98 @@
 import * as fs from 'fs';
-import { promisify } from 'util';
+import { createInterface } from 'node:readline';
 
-const readFile = promisify(fs.readFile);
-const readFileSync = fs.readFileSync;
+const SYNC_READ_CHUNK_BYTES = 1024 * 1024;
 
-function splitJsonlContent(content: string): string[] {
-    return content.trim().split('\n').filter(line => line.length > 0);
+/**
+ * Stream a JSONL file line-by-line without materializing the whole file as one
+ * string. Claude Code session transcripts can exceed Node's max string length
+ * (~512MB / 0x1fffffe8), so `fs.readFile(..., 'utf-8')` throws and callers that
+ * catch the error end up reporting zeros.
+ */
+export async function* iterateJsonlLines(filePath: string): AsyncGenerator<string> {
+    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    const reader = createInterface({
+        input: stream,
+        crlfDelay: Infinity
+    });
+
+    try {
+        for await (const line of reader) {
+            if (line.length > 0) {
+                yield line;
+            }
+        }
+    } finally {
+        reader.close();
+        stream.destroy();
+    }
+}
+
+/**
+ * Synchronous line iterator for call sites that cannot be async.
+ * Completes each line in a Buffer before decoding so multi-byte UTF-8 sequences
+ * are never split across chunk boundaries.
+ */
+export function* iterateJsonlLinesSync(filePath: string): Generator<string> {
+    const fd = fs.openSync(filePath, 'r');
+    try {
+        const scratch = Buffer.allocUnsafe(SYNC_READ_CHUNK_BYTES);
+        let pending = Buffer.alloc(0);
+
+        for (;;) {
+            const bytesRead = fs.readSync(fd, scratch, 0, scratch.length, null);
+            if (bytesRead === 0) {
+                break;
+            }
+
+            const chunk = scratch.subarray(0, bytesRead);
+            const combined = pending.length > 0 ? Buffer.concat([pending, chunk]) : chunk;
+            let start = 0;
+
+            for (let i = 0; i < combined.length; i++) {
+                if (combined[i] !== 0x0a) {
+                    continue;
+                }
+
+                let lineBuf = combined.subarray(start, i);
+                if (lineBuf.length > 0 && lineBuf[lineBuf.length - 1] === 0x0d) {
+                    lineBuf = lineBuf.subarray(0, lineBuf.length - 1);
+                }
+                if (lineBuf.length > 0) {
+                    yield lineBuf.toString('utf8');
+                }
+                start = i + 1;
+            }
+
+            pending = start === 0
+                ? Buffer.from(combined)
+                : Buffer.from(combined.subarray(start));
+        }
+
+        if (pending.length > 0) {
+            let lineBuf = pending;
+            if (lineBuf[lineBuf.length - 1] === 0x0d) {
+                lineBuf = lineBuf.subarray(0, lineBuf.length - 1);
+            }
+            if (lineBuf.length > 0) {
+                yield lineBuf.toString('utf8');
+            }
+        }
+    } finally {
+        fs.closeSync(fd);
+    }
 }
 
 export async function readJsonlLines(filePath: string): Promise<string[]> {
-    const content = await readFile(filePath, 'utf-8');
-    return splitJsonlContent(content);
+    const lines: string[] = [];
+    for await (const line of iterateJsonlLines(filePath)) {
+        lines.push(line);
+    }
+    return lines;
 }
 
 export function readJsonlLinesSync(filePath: string): string[] {
-    const content = readFileSync(filePath, 'utf-8');
-    return splitJsonlContent(content);
+    return Array.from(iterateJsonlLinesSync(filePath));
 }
 
 export function parseJsonlLine(line: string): unknown {