From f62c70cb9989148fad1b330f20bf7666781ae730 Mon Sep 17 00:00:00 2001 From: Arik Levinsky Date: Sun, 26 Jul 2026 18:39:23 -0600 Subject: [PATCH] feat(adapter): read Claude Code transcripts into the neutral representation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real implementation of the capture port, and the first thing to put the neutral representation under load. Two corrections to it fell out of building this, both measured rather than argued: - no transcript record in any sampled version carries a price. 401 files and 135,001 records, zero occurrences. Even the reference harness cannot supply money from a transcript, so the golden session's cost profile cannot be read from the thing being blessed. - a tool result records whether the call errored and never the code it returned, so ExitCode becomes a pointer. Filling it with zero would have reported success beside an outcome saying the call failed, and zero is itself a meaningful code. The capability declaration is measured, not assumed. Cost, the workspace patch, file mutations and their content are not claimed. Model identity is partial because a transcript records what served a message and never the alias asked for. Tool results are partial because large ones are shortened in place. Delegation is partial, and that was nearly an overclaim: it read correctly from the synthetic fixture and produced nothing at all from 300 real transcripts. A scan of 601 project-root files found no inline delegation — every one keeps it in sidecar files this adapter does not read. Declared because the inline form is handled, marked partial because on the real store the answer is always empty, so an assertion resolves to an error rather than to a false negative. Verified against the real store: 300 of 300 transcripts parsed, no refusals, 15,022 turns, and 5,112 tool calls matched to 5,112 results — the exact pairing the earlier survey measured. Every committed fixture is synthetic; no transcript content is committed, and the redaction pass that would make an export safe is not trustworthy yet. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ew64uotJqtvVDN3twMAVEQ --- .golangci.yml | 7 + .../adapter/claudecode/capture/capture.go | 246 +++++++++++ .../claudecode/capture/capture_test.go | 345 +++++++++++++++ internal/adapter/claudecode/capture/parse.go | 409 ++++++++++++++++++ internal/adapter/conformance/coverage.go | 2 +- internal/adapter/conformance/reference.go | 4 +- internal/arch/classify_test.go | 3 +- internal/assay/capability.go | 6 +- internal/assay/session.go | 17 +- 9 files changed, 1026 insertions(+), 13 deletions(-) create mode 100644 internal/adapter/claudecode/capture/capture.go create mode 100644 internal/adapter/claudecode/capture/capture_test.go create mode 100644 internal/adapter/claudecode/capture/parse.go diff --git a/.golangci.yml b/.golangci.yml index dbd96ff..d998188 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,6 +33,13 @@ linters: deny: - pkg: github.com/JumpMasters/assayer desc: "cmd/assayer is class cmd: within this module it may import only internal/cli (ADR-0005)" + adapter-claudecode-capture: + list-mode: lax + files: ["**/internal/adapter/claudecode/capture/*.go", "!$test"] + allow: [github.com/JumpMasters/assayer/internal/assay, github.com/JumpMasters/assayer/internal/port] + deny: + - pkg: github.com/JumpMasters/assayer + desc: "internal/adapter/claudecode/capture is class adapter: within this module it may import only internal/assay, internal/port (ADR-0005)" adapter-conformance: list-mode: lax files: ["**/internal/adapter/conformance/*.go", "!$test"] diff --git a/internal/adapter/claudecode/capture/capture.go b/internal/adapter/claudecode/capture/capture.go new file mode 100644 index 0000000..951df13 --- /dev/null +++ b/internal/adapter/claudecode/capture/capture.go @@ -0,0 +1,246 @@ +// Package capture reads Claude Code's own session transcripts and translates +// them into the neutral representation. +// +// This is one of the two places in the program allowed to know a harness +// exists; everything downstream works from what this produces without knowing +// where it came from. +// +// What it can observe is measured rather than assumed. A survey of the local +// store found timestamps, working directories and branches, per-message model +// identity and token counts, tool calls with their inputs, and tool results — +// and found no cost anywhere, in any record, in any version. So this adapter +// does not declare that it can see money, and a caller asking for a spending +// comparison gets an honest refusal rather than a zero. +package capture + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// Adapter reads transcripts from a Claude Code store. +type Adapter struct { + // Root is the directory holding per-project transcript directories. Empty + // means the default location under the user's home directory. + Root string +} + +var _ port.Capture = Adapter{} + +// ID implements port.Capture. +func (Adapter) ID() string { return "claude-code" } + +// Tier implements port.Capture. Transcripts are the harness's own record of +// what it did, which is the strongest capture this project has. +func (Adapter) Tier() assay.Tier { return assay.TierNative } + +// Capabilities implements port.Capture. +// +// Four absences are deliberate and each was checked against the store rather +// than assumed: +// +// - cost. No record in any sampled version carries a price. The harness +// reports one when it finishes a run, but a transcript is not that report, +// and deriving a price from token counts and a table would be presenting a +// calculation as an observation. +// - the workspace patch. A transcript records what the agent did, not the +// state of the tree before it started. +// - file mutations and their content. The store does carry a file-history +// stream, which an earlier survey classified as bookkeeping and skipped; +// reading it is worthwhile and is not done here, so the capability is not +// claimed. +func (Adapter) Capabilities() assay.CapabilitySet { + return assay.CapabilitySet(0).With( + assay.CanSeeLineage, + assay.CanSeeModelIdentity, + assay.CanSeeWorkspace, + assay.CanSeeToolCalls, + assay.CanSeeToolInputs, + assay.CanSeeToolResults, + assay.CanSeeToolOutcome, + assay.CanSeeTurnText, + assay.CanSeeReasoning, + assay.CanSeeTokens, + assay.CanSeeTiming, + assay.CanSeeDelegation, + ) +} + +// partialCapabilities are observed but known to be incomplete for every session +// this adapter produces. +// +// Model identity is partial because a transcript records the model that served +// a message and never the alias that was asked for, so a silent remapping of an +// alias is invisible from here alone. +// +// Tool results are partial because large ones are written elsewhere and +// shortened in place, so a result body may be a prefix of what the tool +// actually returned. +// +// Delegation is partial for a reason worth stating plainly, because it was +// nearly an overclaim. Delegated work is recorded two ways: inline in the +// session's own transcript, which this adapter reads, and offloaded into +// separate files beside it, which it does not. Scanning 601 project-root +// transcripts found no inline delegation at all — every one of them keeps it in +// the files this adapter skips. So the capability is declared, because the +// inline form is read correctly when it appears, and marked partial, because on +// the store as it exists today the answer is always empty. An assertion about +// delegated work that would fail on this evidence resolves to an error instead, +// which is the right answer: nothing was seen because nothing was looked at. +func partialCapabilities() assay.CapabilitySet { + return assay.CapabilitySet(0).With( + assay.CanSeeModelIdentity, + assay.CanSeeToolResults, + assay.CanSeeDelegation, + ) +} + +func (a Adapter) root() (string, error) { + if a.Root != "" { + return a.Root, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("locating the transcript store: %w", err) + } + return filepath.Join(home, ".claude", "projects"), nil +} + +// Discover implements port.Capture. +// +// Only transcripts sitting directly in a project directory are listed. Anything +// deeper is a record of delegated work, which belongs to the session that +// spawned it rather than standing on its own. +// +// The label is built from the path rather than from the session's contents, so +// that listing does not mean opening every file. One local store held 2,485 of +// them. +func (a Adapter) Discover(ctx context.Context, q port.Query) ([]port.Ref, error) { + root, err := a.root() + if err != nil { + return nil, err + } + + entries, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return []port.Ref{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading the transcript store: %w", err) + } + + refs := []port.Ref{} + for _, project := range entries { + if err := ctx.Err(); err != nil { + return nil, err + } + if !project.IsDir() { + continue + } + dir := filepath.Join(root, project.Name()) + files, err := os.ReadDir(dir) + if err != nil { + continue // a project directory we cannot read is not a fatal condition + } + for _, f := range files { + if f.IsDir() || !strings.HasSuffix(f.Name(), ".jsonl") { + continue + } + info, err := f.Info() + if err != nil { + continue + } + ref := port.Ref{ + ID: filepath.Join(dir, f.Name()), + Label: project.Name() + "/" + strings.TrimSuffix(f.Name(), ".jsonl"), + At: info.ModTime(), + } + if !q.Since.IsZero() && ref.At.Before(q.Since) { + continue + } + if q.Dir != "" && !strings.Contains(project.Name(), encodeProjectDir(q.Dir)) { + continue + } + refs = append(refs, ref) + } + } + + sort.Slice(refs, func(i, j int) bool { return refs[i].At.After(refs[j].At) }) + if q.Limit > 0 && len(refs) > q.Limit { + refs = refs[:q.Limit] + } + return refs, nil +} + +// encodeProjectDir renders a filesystem path the way the store names the +// directory it keeps that project's transcripts in. +func encodeProjectDir(dir string) string { + return strings.ReplaceAll(strings.TrimSuffix(dir, "/"), "/", "-") +} + +// Load implements port.Capture. +func (a Adapter) Load(ctx context.Context, ref port.Ref) (assay.Session, error) { + if ref.ID == "" { + return assay.Session{}, port.ErrNotFound + } + + f, err := os.Open(ref.ID) + if errors.Is(err, fs.ErrNotExist) { + // Stores are live: a session listed a moment ago can be pruned before + // it is read. + return assay.Session{}, port.ErrNotFound + } + if err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrUnsupported, err) + } + defer f.Close() + + s, err := parse(ctx, f) + if err != nil { + return assay.Session{}, err + } + + s.Fidelity = assay.Fidelity{ + Adapter: a.ID(), + Version: s.Fidelity.Version, + Tier: a.Tier(), + Observed: a.Capabilities(), + Partial: partialCapabilities(), + Verified: s.Fidelity.Verified, + } + return s, nil +} + +// knownVersions are the harness releases this adapter has been read against. +// +// An unknown release is parsed rather than refused, and the session is marked +// unverified. Releases arrived every few days across the measured period while +// the keys this adapter reads never moved, so refusing them would break capture +// roughly twice a week to guard against a change that has not yet happened. +// Saying "read, but not from a release anyone has checked" is the honest third +// option between refusing and pretending. +func knownVersions() []string { + return []string{ + "2.1.142", "2.1.149", "2.1.156", "2.1.170", "2.1.177", "2.1.181", + "2.1.190", "2.1.191", "2.1.197", "2.1.198", "2.1.199", "2.1.201", + "2.1.209", "2.1.210", "2.1.211", "2.1.217", "2.1.219", "2.1.220", + } +} + +func isKnownVersion(v string) bool { + for _, known := range knownVersions() { + if v == known { + return true + } + } + return false +} diff --git a/internal/adapter/claudecode/capture/capture_test.go b/internal/adapter/claudecode/capture/capture_test.go new file mode 100644 index 0000000..41d206c --- /dev/null +++ b/internal/adapter/claudecode/capture/capture_test.go @@ -0,0 +1,345 @@ +package capture + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/JumpMasters/assayer/internal/adapter/conformance" + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// Every fixture here is synthetic. +// +// No real transcript is committed, and none should be: the store holds working +// directories, file contents, commands and whatever a session happened to type, +// and the redaction pass that would make an export safe is not trustworthy yet +// — a survey found it firing on one local file in two. These fixtures reproduce +// the structure the survey measured without carrying anything from it. + +const transcriptWithATool = ` +{"type":"user","version":"2.1.220","sessionId":"s1","cwd":"/work","gitBranch":"main","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"add a test"}} +{"type":"assistant","version":"2.1.220","sessionId":"s1","cwd":"/work","timestamp":"2026-01-01T00:00:10Z","message":{"role":"assistant","model":"claude-x-1","usage":{"input_tokens":100,"output_tokens":50},"content":[{"type":"thinking","text":"consider the options"},{"type":"text","text":"running the tests"},{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"go test ./..."}}]}} +{"type":"user","version":"2.1.220","sessionId":"s1","cwd":"/work","timestamp":"2026-01-01T00:00:20Z","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"ok","is_error":false}]}} +{"type":"queue-operation","note":"bookkeeping, no version, must be skipped"} +{"type":"assistant","version":"2.1.220","sessionId":"s1","cwd":"/other","timestamp":"2026-01-01T00:01:00Z","message":{"role":"assistant","model":"claude-x-2","usage":{"input_tokens":10,"output_tokens":5},"content":[{"type":"text","text":"done"}]}} +` + +func writeStore(t *testing.T, name, body string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, "-work-project") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.TrimSpace(body)+"\n"), 0o644); err != nil { + t.Fatalf("write transcript: %v", err) + } + return root +} + +func loadOnly(t *testing.T, body string) assay.Session { + t.Helper() + a := Adapter{Root: writeStore(t, "session.jsonl", body)} + refs, err := a.Discover(context.Background(), port.Query{}) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(refs) != 1 { + t.Fatalf("Discover returned %d refs, want 1", len(refs)) + } + s, err := a.Load(context.Background(), refs[0]) + if err != nil { + t.Fatalf("Load: %v", err) + } + return s +} + +// TestSatisfiesTheSharedContract runs the kit every capture adapter must pass. +func TestSatisfiesTheSharedContract(t *testing.T) { + conformance.Verify(t, Adapter{Root: writeStore(t, "session.jsonl", transcriptWithATool)}) +} + +func TestTranslatesASession(t *testing.T) { + s := loadOnly(t, transcriptWithATool) + + t.Run("turns keep their order and roles", func(t *testing.T) { + if len(s.Turns) != 3 { + t.Fatalf("got %d turns, want 3", len(s.Turns)) + } + if s.Turns[0].Role != assay.RoleHuman { + t.Errorf("first turn role = %v, want human", s.Turns[0].Role) + } + if s.Turns[0].Text != "add a test" { + t.Errorf("first turn text = %q", s.Turns[0].Text) + } + if s.Turns[1].Role != assay.RoleAgent { + t.Errorf("second turn role = %v, want agent", s.Turns[1].Role) + } + }) + + t.Run("reasoning is kept apart from prose", func(t *testing.T) { + if s.Turns[1].Reasoning != "consider the options" { + t.Errorf("reasoning = %q", s.Turns[1].Reasoning) + } + if strings.Contains(s.Turns[1].Text, "consider the options") { + t.Error("reasoning leaked into the turn's text; a rubric would grade a scratchpad") + } + }) + + t.Run("a tool call carries its command and neutral kind", func(t *testing.T) { + if len(s.Turns[1].Tools) != 1 { + t.Fatalf("got %d tool calls, want 1", len(s.Turns[1].Tools)) + } + c := s.Turns[1].Tools[0] + if c.Kind != assay.ToolExec { + t.Errorf("kind = %v, want exec", c.Kind) + } + if len(c.Argv) != 1 || c.Argv[0] != "go test ./..." { + t.Errorf("argv = %v", c.Argv) + } + if c.Input == "" { + t.Error("tool input was dropped; distillation needs what was put to the tool") + } + }) + + t.Run("a result is matched to the call it answers", func(t *testing.T) { + c := s.Turns[1].Tools[0] + if c.Result == nil { + t.Fatal("result is nil; it arrives on a later record and must be paired back") + } + if c.Result.Text != "ok" { + t.Errorf("result text = %q", c.Result.Text) + } + if c.Result.Outcome != assay.OutcomeOK { + t.Errorf("outcome = %v, want ok", c.Result.Outcome) + } + if c.Result.ExitCode != nil { + t.Error("an exit code was reported; transcripts do not carry one, " + + "and a zero here would read as success beside a failing outcome") + } + }) + + t.Run("every working directory is kept", func(t *testing.T) { + if len(s.Workspace.Dirs) != 2 { + t.Fatalf("dirs = %v, want both", s.Workspace.Dirs) + } + if s.Workspace.Dirs[0] != "/work" || s.Workspace.Dirs[1] != "/other" { + t.Errorf("dirs = %v, want first-use order", s.Workspace.Dirs) + } + }) + + t.Run("the model is per turn", func(t *testing.T) { + models := s.Models() + if len(models) != 2 { + t.Fatalf("Models() = %v, want two distinct", models) + } + if models[0].Canonical != "claude-x-1" || models[1].Canonical != "claude-x-2" { + t.Errorf("Models() = %v", models) + } + if models[0].Alias != "" { + t.Error("an alias was reported; a transcript records only what served the message") + } + }) + + t.Run("tokens are summed and money is not invented", func(t *testing.T) { + if s.Usage.InputTokens != 110 || s.Usage.OutputTokens != 55 { + t.Errorf("usage = %+v", s.Usage) + } + if s.Usage.CostMicroUSD != 0 { + t.Error("a price was reported; no transcript record carries one") + } + }) + + t.Run("fidelity names the adapter and the releases read", func(t *testing.T) { + if s.Fidelity.Adapter != "claude-code" || s.Fidelity.Tier != assay.TierNative { + t.Errorf("fidelity = %+v", s.Fidelity) + } + if s.Fidelity.Version != "2.1.220" { + t.Errorf("version = %q", s.Fidelity.Version) + } + if !s.Fidelity.Verified { + t.Error("a release this adapter has been read against was marked unverified") + } + }) +} + +// TestLineageKeepsTheParentSeparate covers the misattribution a single +// identifier would cause: the second field names the session this one +// continues, not a duplicate of the first. +func TestLineageKeepsTheParentSeparate(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"2.1.220","sessionId":"child","session_id":"parent","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if s.Lineage.ID != "child" { + t.Errorf("lineage id = %q, want the session's own", s.Lineage.ID) + } + if s.Lineage.Parent != "parent" { + t.Errorf("lineage parent = %q, want the session it continues", s.Lineage.Parent) + } +} + +// TestForkRecordNamesTheParent covers the record type an earlier survey +// classified as bookkeeping and skipped, which is where a fork's parent is +// actually written down. +func TestForkRecordNamesTheParent(t *testing.T) { + s := loadOnly(t, ` +{"type":"fork-context-ref","parentSessionId":"ancestor","parentLastUuid":"u1"} +{"type":"user","version":"2.1.220","sessionId":"forked","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if s.Lineage.Parent != "ancestor" { + t.Errorf("lineage parent = %q, want the fork record's parent", s.Lineage.Parent) + } +} + +// TestDelegatedWorkIsCarriedAndOrderIsNotClaimed covers work handed to a +// sub-agent: it is kept rather than dropped, and because nothing links it back +// to the call that spawned it, the session says its order is incomplete. +func TestDelegatedWorkIsCarriedAndOrderIsNotClaimed(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"2.1.220","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +{"type":"assistant","version":"2.1.220","sessionId":"s","cwd":"/w","isSidechain":true,"timestamp":"2026-01-01T00:00:05Z","message":{"role":"assistant","model":"m","content":[{"type":"text","text":"sub-task"}]}} +`) + if len(s.Delegated) != 1 || len(s.Delegated[0].Turns) != 1 { + t.Fatalf("delegated = %+v, want one delegation with one turn", s.Delegated) + } + if s.OrderComplete { + t.Error("order was claimed complete while delegated work is unattributed") + } + for _, turn := range s.Turns { + if turn.Text == "sub-task" { + t.Error("delegated work was folded into the main turns") + } + } +} + +// TestUnknownReleaseIsReadButNotVerified covers the choice not to refuse a +// release nobody has checked. Refusing would break capture roughly twice a week +// at the rate releases were measured arriving. +func TestUnknownReleaseIsReadButNotVerified(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"9.9.999","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if len(s.Turns) != 1 { + t.Fatalf("an unknown release was refused; got %d turns", len(s.Turns)) + } + if s.Fidelity.Verified { + t.Error("a release nobody has read against was reported as verified") + } +} + +// TestUnknownRecordTypeIsRefused is the other half of that choice: a new kind +// of content record must fail loudly rather than be skipped into a session that +// looks whole and is not. +func TestUnknownRecordTypeIsRefused(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", ` +{"type":"user","version":"2.1.220","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +{"type":"a-record-type-nobody-has-seen","version":"2.1.220"} +`)} + refs, _ := a.Discover(context.Background(), port.Query{}) + _, err := a.Load(context.Background(), refs[0]) + if !errors.Is(err, port.ErrUnsupported) { + t.Errorf("Load returned %v, want ErrUnsupported", err) + } +} + +func TestMalformedTranscriptIsReported(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", `{"type":"user"`+"\n")} + refs, _ := a.Discover(context.Background(), port.Query{}) + _, err := a.Load(context.Background(), refs[0]) + if !errors.Is(err, port.ErrMalformed) { + t.Errorf("Load returned %v, want ErrMalformed", err) + } +} + +func TestMissingSessionReportsNotFound(t *testing.T) { + a := Adapter{Root: t.TempDir()} + _, err := a.Load(context.Background(), port.Ref{ID: filepath.Join(t.TempDir(), "gone.jsonl")}) + if !errors.Is(err, port.ErrNotFound) { + t.Errorf("Load of a pruned session returned %v, want ErrNotFound", err) + } + if _, err := a.Load(context.Background(), port.Ref{}); !errors.Is(err, port.ErrNotFound) { + t.Errorf("Load of an empty ref returned %v, want ErrNotFound", err) + } +} + +func TestDiscoverOnAnAbsentStoreIsEmptyNotAnError(t *testing.T) { + a := Adapter{Root: filepath.Join(t.TempDir(), "no-such-store")} + refs, err := a.Discover(context.Background(), port.Query{}) + if err != nil { + t.Fatalf("Discover on an absent store returned %v", err) + } + if len(refs) != 0 { + t.Errorf("got %d refs from an absent store", len(refs)) + } +} + +func TestDiscoverHonoursTheQuery(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", transcriptWithATool)} + ctx := context.Background() + + if refs, _ := a.Discover(ctx, port.Query{Since: time.Now().Add(time.Hour)}); len(refs) != 0 { + t.Errorf("Since in the future returned %d refs", len(refs)) + } + if refs, _ := a.Discover(ctx, port.Query{Dir: "/nowhere/at/all"}); len(refs) != 0 { + t.Errorf("a non-matching Dir returned %d refs", len(refs)) + } + if refs, _ := a.Discover(ctx, port.Query{Limit: 1}); len(refs) != 1 { + t.Errorf("Limit 1 returned %d refs", len(refs)) + } +} + +// TestToolNamesMapToNeutralKinds is what lets core reason about behaviour +// without matching on the names one harness happens to use. +func TestToolNamesMapToNeutralKinds(t *testing.T) { + for name, want := range map[string]assay.ToolKind{ + "Read": assay.ToolRead, + "Grep": assay.ToolRead, + "Edit": assay.ToolMutate, + "Write": assay.ToolMutate, + "Bash": assay.ToolExec, + "Task": assay.ToolDelegate, + "Unheard": assay.ToolOther, + "": assay.ToolUnknown, + } { + if got := classify(name); got != want { + t.Errorf("classify(%q) = %v, want %v", name, got, want) + } + } +} + +// TestCapabilitiesAreNotOverclaimed pins the four this adapter must not assert. +// Each was checked against the store rather than assumed. +func TestCapabilitiesAreNotOverclaimed(t *testing.T) { + caps := Adapter{}.Capabilities() + for _, c := range []assay.Capability{ + assay.CanSeeCost, + assay.CanSeeWorkspacePatch, + assay.CanSeeFileMutations, + assay.CanSeeMutationContent, + } { + if caps.Has(c) { + t.Errorf("adapter declares %v, which no transcript record supplies", c) + } + } + + partial := partialCapabilities() + if !partial.Has(assay.CanSeeModelIdentity) { + t.Error("model identity is not marked partial; the alias is never recorded") + } + if !partial.Has(assay.CanSeeToolResults) { + t.Error("tool results are not marked partial; large ones are shortened in place") + } + if !partial.Has(assay.CanSeeDelegation) { + t.Error("delegation is not marked partial; a scan of 601 project-root transcripts " + + "found none recorded inline, so on the real store the answer is always empty") + } + if !caps.Contains(partial) { + t.Error("a capability is marked partial that the adapter does not claim at all") + } +} diff --git a/internal/adapter/claudecode/capture/parse.go b/internal/adapter/claudecode/capture/parse.go new file mode 100644 index 0000000..5b68d91 --- /dev/null +++ b/internal/adapter/claudecode/capture/parse.go @@ -0,0 +1,409 @@ +package capture + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// record is one line of a transcript, in the shape this adapter reads. +// +// Only the fields the neutral representation needs are declared. A transcript +// carries a great deal more, and anything not needed downstream stays here +// rather than travelling as an opaque payload. +type record struct { + Type string `json:"type"` + UUID string `json:"uuid"` + SessionID string `json:"sessionId"` + ResumedFrom string `json:"session_id"` + CWD string `json:"cwd"` + GitBranch string `json:"gitBranch"` + Version string `json:"version"` + Timestamp string `json:"timestamp"` + IsSidechain bool `json:"isSidechain"` + Message message `json:"message"` + + // ParentSessionID appears on the record that marks a fork. The field that + // names a parent lives on its own record type rather than on every line. + ParentSessionID string `json:"parentSessionId"` +} + +type message struct { + Role string `json:"role"` + Model string `json:"model"` + Usage usage `json:"usage"` + Content json.RawMessage `json:"content"` +} + +type usage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// block is one element of a structured message body. +type block struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` + + // Tool results arrive on a following user message rather than beside the + // call, so they are matched back by identifier. + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` + IsError bool `json:"is_error"` +} + +// contentTypes carry a session's substance and always declare a version. +func isContentType(t string) bool { + switch t { + case "assistant", "user", "attachment", "system": + return true + } + return false +} + +// bookkeepingTypes never declare a version and hold no conversation. +// +// They are listed rather than inferred from a missing version field. Roughly a +// third of all records carry no version, so treating absence as the signal +// would reject every real transcript; and inferring it the other way would let +// a genuinely new kind of content record be skipped in silence, producing a +// session that looks complete and is not. +func isBookkeepingType(t string) bool { + switch t { + case "queue-operation", "last-prompt", "ai-title", "mode", "pr-link", + "file-history-snapshot", "bridge-session", "permission-mode", + "file-history-delta", "worktree-state", "custom-title", "fork-context-ref": + return true + } + return false +} + +// parse reads a transcript into the neutral representation. +func parse(ctx context.Context, r io.Reader) (assay.Session, error) { + var s assay.Session + + sc := bufio.NewScanner(r) + // Transcript lines carry whole tool results and can be far longer than the + // scanner's default limit. + sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + + var ( + versions = map[string]bool{} + dirs []string + seenDir = map[string]bool{} + pending = map[string]*pendingCall{} + delegated []assay.Turn + ) + + for sc.Scan() { + if err := ctx.Err(); err != nil { + return assay.Session{}, err + } + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + + var rec record + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrMalformed, err) + } + + switch { + case rec.Type == "fork-context-ref": + // The one record that names a parent explicitly. + if rec.ParentSessionID != "" { + s.Lineage.Parent = rec.ParentSessionID + } + continue + case isBookkeepingType(rec.Type): + continue + case !isContentType(rec.Type): + // Refused rather than skipped: an unrecognised kind of content + // record would otherwise vanish into a session that reads as whole. + return assay.Session{}, fmt.Errorf("%w: unknown record type %q", + port.ErrUnsupported, rec.Type) + } + + if rec.Version != "" { + versions[rec.Version] = true + } + if rec.SessionID != "" && s.Lineage.ID == "" { + s.Lineage.ID = rec.SessionID + } + // The second identifier names the session this one continues, and is + // not a duplicate of the first. Preferring the wrong one attributes a + // session's work to its predecessor. + if rec.ResumedFrom != "" && rec.ResumedFrom != rec.SessionID && s.Lineage.Parent == "" { + s.Lineage.Parent = rec.ResumedFrom + } + if rec.CWD != "" && !seenDir[rec.CWD] { + seenDir[rec.CWD] = true + dirs = append(dirs, rec.CWD) + } + if rec.GitBranch != "" && s.Workspace.Branch == "" { + s.Workspace.Branch = rec.GitBranch + } + + turn, results := recordToTurn(&rec) + + // A result arrives on a later message than the call it answers. + for id, res := range results { + if p, ok := pending[id]; ok { + p.call.Result = res + } + } + + if turn == nil { + continue + } + + if rec.IsSidechain { + delegated = append(delegated, *turn) + continue + } + + s.Turns = append(s.Turns, *turn) + last := &s.Turns[len(s.Turns)-1] + // Index each call so a result arriving on a later message can be + // attached to it. The identifiers are gathered once: recomputing them + // per call would re-parse the whole message body each time. + ids := toolIDs(&rec) + for i := range last.Tools { + if i < len(ids) && ids[i] != "" { + pending[ids[i]] = &pendingCall{call: &last.Tools[i]} + } + } + + s.Usage.InputTokens += rec.Message.Usage.InputTokens + s.Usage.OutputTokens += rec.Message.Usage.OutputTokens + } + if err := sc.Err(); err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrMalformed, err) + } + + s.Workspace.Dirs = dirs + if len(delegated) > 0 { + // Delegated work is carried flat. Nothing here links it back to the + // call that spawned it, so the order of the whole session is not fully + // known and assertions about ordering must say so rather than guess. + s.Delegated = []assay.Delegation{{Turns: delegated}} + s.OrderComplete = false + } else { + s.OrderComplete = true + } + + if len(s.Turns) > 0 { + s.Usage.Wall = s.Turns[len(s.Turns)-1].At.Sub(s.Turns[0].At) + } + + s.Fidelity.Version = strings.Join(sortedKeys(versions), ",") + s.Fidelity.Verified = allKnown(versions) + return s, nil +} + +type pendingCall struct{ call *assay.ToolCall } + +// recordToTurn translates one content record, returning any tool results it +// carries so they can be matched to the calls they answer. +func recordToTurn(rec *record) (turn *assay.Turn, results map[string]*assay.ToolResult) { + results = map[string]*assay.ToolResult{} + + role := assay.RoleUnknown + switch rec.Message.Role { + case "user": + role = assay.RoleHuman + case "assistant": + role = assay.RoleAgent + } + + t := assay.Turn{ + Role: role, + Model: modelOf(rec), + At: parseTime(rec.Timestamp), + } + + blocks := blocksOf(rec.Message.Content) + for i := range blocks { + b := &blocks[i] + switch b.Type { + case "text": + t.Text += b.Text + case "thinking": + t.Reasoning += b.Text + case "tool_use": + t.Tools = append(t.Tools, assay.ToolCall{ + Name: b.Name, + Kind: classify(b.Name), + Argv: argvOf(b), + Input: string(b.Input), + }) + case "tool_result": + results[b.ToolUseID] = toolResult(b) + } + } + + // A record carrying only tool results is not a turn in the conversation. + // The protocol returns results on a message with the user's role, so + // admitting it would insert an empty human turn between an agent's call and + // its answer — inflating any count of turns, and putting a human turn where + // the person said nothing. + if t.Text == "" && t.Reasoning == "" && len(t.Tools) == 0 { + return nil, results + } + return &t, results +} + +func toolResult(b *block) *assay.ToolResult { + outcome := assay.OutcomeOK + if b.IsError { + outcome = assay.OutcomeNonZero + } + return &assay.ToolResult{ + Text: textOf(b.Content), + Outcome: outcome, + // No exit code: transcripts record that a call errored and never the + // code it returned. Filling this with zero would report success beside + // an outcome saying otherwise. + ExitCode: nil, + } +} + +// classify maps a harness's tool name onto the neutral vocabulary, so that no +// package outside an adapter has to match on a name. +func classify(name string) assay.ToolKind { + switch name { + case "Read", "Glob", "Grep", "NotebookRead", "WebFetch", "WebSearch": + return assay.ToolRead + case "Edit", "Write", "NotebookEdit", "MultiEdit": + return assay.ToolMutate + case "Bash", "BashOutput", "KillShell": + return assay.ToolExec + case "Task", "Agent": + return assay.ToolDelegate + case "": + return assay.ToolUnknown + } + return assay.ToolOther +} + +// argvOf extracts a command line when the call ran one. +func argvOf(b *block) []string { + if classify(b.Name) != assay.ToolExec || len(b.Input) == 0 { + return nil + } + var in struct { + Command string `json:"command"` + } + if err := json.Unmarshal(b.Input, &in); err != nil || in.Command == "" { + return nil + } + // The harness records a command as one string. It is kept whole in a + // single element rather than split here: splitting a shell command + // correctly needs a shell, and guessing produces an argv that never ran. + return []string{in.Command} +} + +func modelOf(rec *record) assay.Model { + if rec.Message.Model == "" { + return assay.Model{} + } + // The transcript records what served the message, never the alias that was + // asked for, so a quiet remapping of an alias cannot be seen from here. + return assay.Model{Canonical: rec.Message.Model, Provider: "anthropic"} +} + +// blocksOf reads a message body, which is either a plain string or a list of +// structured blocks depending on the record. +func blocksOf(raw json.RawMessage) []block { + if len(raw) == 0 { + return nil + } + var blocks []block + if err := json.Unmarshal(raw, &blocks); err == nil { + return blocks + } + var text string + if err := json.Unmarshal(raw, &text); err == nil && text != "" { + return []block{{Type: "text", Text: text}} + } + return nil +} + +// textOf renders a result body, which has the same two shapes as a message. +func textOf(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var text string + if err := json.Unmarshal(raw, &text); err == nil { + return text + } + var blocks []block + if err := json.Unmarshal(raw, &blocks); err == nil { + var b strings.Builder + for i := range blocks { + b.WriteString(blocks[i].Text) + } + return b.String() + } + return "" +} + +// toolIDs lists the identifiers of the calls a record makes, in order. +func toolIDs(rec *record) []string { + blocks := blocksOf(rec.Message.Content) + ids := make([]string, 0, len(blocks)) + for i := range blocks { + if blocks[i].Type == "tool_use" { + ids = append(ids, blocks[i].ID) + } + } + return ids +} + +func parseTime(s string) time.Time { + if s == "" { + return time.Time{} + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{} + } + return t +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j] < out[j-1]; j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out +} + +func allKnown(versions map[string]bool) bool { + if len(versions) == 0 { + return false + } + for v := range versions { + if !isKnownVersion(v) { + return false + } + } + return true +} diff --git a/internal/adapter/conformance/coverage.go b/internal/adapter/conformance/coverage.go index a347382..af364bd 100644 --- a/internal/adapter/conformance/coverage.go +++ b/internal/adapter/conformance/coverage.go @@ -63,7 +63,7 @@ func governed(key string) (assay.Capability, bool) { case "ToolResult.Text", "ToolResult.Truncated": return assay.CanSeeToolResults, true case "ToolResult.Outcome", "ToolResult.ExitCode": - return assay.CanSeeExitStatus, true + return assay.CanSeeToolOutcome, true case "Usage.CostMicroUSD": return assay.CanSeeCost, true diff --git a/internal/adapter/conformance/reference.go b/internal/adapter/conformance/reference.go index 1333495..3c83b9f 100644 --- a/internal/adapter/conformance/reference.go +++ b/internal/adapter/conformance/reference.go @@ -58,7 +58,7 @@ func (Reference) Capabilities() assay.CapabilitySet { assay.CanSeeToolCalls, assay.CanSeeToolInputs, assay.CanSeeToolResults, - assay.CanSeeExitStatus, + assay.CanSeeToolOutcome, assay.CanSeeFileMutations, assay.CanSeeModelIdentity, assay.CanSeeTokens, @@ -121,7 +121,7 @@ func (r Reference) Load(_ context.Context, ref port.Ref) (assay.Session, error) Result: &assay.ToolResult{ Text: "ok\n", Outcome: assay.OutcomeOK, - ExitCode: exit, + ExitCode: &exit, }, }, { Name: "put", diff --git a/internal/arch/classify_test.go b/internal/arch/classify_test.go index ed5e557..a18f1be 100644 --- a/internal/arch/classify_test.go +++ b/internal/arch/classify_test.go @@ -67,7 +67,8 @@ var classification = map[string]class{ "internal/cli": classRootCLI, "cmd/assayer": classCmd, - "internal/adapter/conformance": classAdapter, + "internal/adapter/conformance": classAdapter, + "internal/adapter/claudecode/capture": classAdapter, } // allowedInternal returns the in-module packages a class may import. diff --git a/internal/assay/capability.go b/internal/assay/capability.go index 3841eac..eef4379 100644 --- a/internal/assay/capability.go +++ b/internal/assay/capability.go @@ -31,7 +31,7 @@ const ( CanSeeToolCalls CanSeeToolInputs CanSeeToolResults - CanSeeExitStatus + CanSeeToolOutcome CanSeeFileMutations CanSeeMutationContent CanSeeTurnText @@ -69,8 +69,8 @@ func (c Capability) String() string { return "tool-inputs" case CanSeeToolResults: return "tool-results" - case CanSeeExitStatus: - return "exit-status" + case CanSeeToolOutcome: + return "tool-outcome" case CanSeeFileMutations: return "file-mutations" case CanSeeMutationContent: diff --git a/internal/assay/session.go b/internal/assay/session.go index dd6dd16..e9d516d 100644 --- a/internal/assay/session.go +++ b/internal/assay/session.go @@ -178,13 +178,18 @@ type ToolResult struct { Outcome Outcome - // ExitCode is meaningful when Outcome is OutcomeOK or OutcomeNonZero. + // ExitCode is nil when no code was observed, which is the ordinary case + // rather than the exception. // - // The distinction it preserves is load-bearing rather than decorative: test - // runners use separate non-zero codes for "tests failed" and "the thing you - // named does not exist", and the second is a pin that has rotted rather than - // a regression. - ExitCode int + // A pointer because zero is a meaningful exit code and would otherwise be + // indistinguishable from absence. The reference harness records whether a + // call errored and never records the code itself, so an adapter that filled + // this with 0 would be reporting success beside an Outcome saying the call + // failed. The distinction the field preserves when it IS observed is + // load-bearing: test runners use separate non-zero codes for "tests failed" + // and "the thing you named does not exist", and the second is a pin that has + // rotted rather than a regression. + ExitCode *int } // Workspace is the state of the files a session worked on.