diff --git a/.golangci.yml b/.golangci.yml index dbd96ff..d998188 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -33,6 +33,13 @@ linters: deny: - pkg: github.com/JumpMasters/assayer desc: "cmd/assayer is class cmd: within this module it may import only internal/cli (ADR-0005)" + adapter-claudecode-capture: + list-mode: lax + files: ["**/internal/adapter/claudecode/capture/*.go", "!$test"] + allow: [github.com/JumpMasters/assayer/internal/assay, github.com/JumpMasters/assayer/internal/port] + deny: + - pkg: github.com/JumpMasters/assayer + desc: "internal/adapter/claudecode/capture is class adapter: within this module it may import only internal/assay, internal/port (ADR-0005)" adapter-conformance: list-mode: lax files: ["**/internal/adapter/conformance/*.go", "!$test"] diff --git a/internal/adapter/claudecode/capture/capture.go b/internal/adapter/claudecode/capture/capture.go new file mode 100644 index 0000000..951df13 --- /dev/null +++ b/internal/adapter/claudecode/capture/capture.go @@ -0,0 +1,246 @@ +// Package capture reads Claude Code's own session transcripts and translates +// them into the neutral representation. +// +// This is one of the two places in the program allowed to know a harness +// exists; everything downstream works from what this produces without knowing +// where it came from. +// +// What it can observe is measured rather than assumed. A survey of the local +// store found timestamps, working directories and branches, per-message model +// identity and token counts, tool calls with their inputs, and tool results — +// and found no cost anywhere, in any record, in any version. So this adapter +// does not declare that it can see money, and a caller asking for a spending +// comparison gets an honest refusal rather than a zero. +package capture + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// Adapter reads transcripts from a Claude Code store. +type Adapter struct { + // Root is the directory holding per-project transcript directories. Empty + // means the default location under the user's home directory. + Root string +} + +var _ port.Capture = Adapter{} + +// ID implements port.Capture. +func (Adapter) ID() string { return "claude-code" } + +// Tier implements port.Capture. Transcripts are the harness's own record of +// what it did, which is the strongest capture this project has. +func (Adapter) Tier() assay.Tier { return assay.TierNative } + +// Capabilities implements port.Capture. +// +// Four absences are deliberate and each was checked against the store rather +// than assumed: +// +// - cost. No record in any sampled version carries a price. The harness +// reports one when it finishes a run, but a transcript is not that report, +// and deriving a price from token counts and a table would be presenting a +// calculation as an observation. +// - the workspace patch. A transcript records what the agent did, not the +// state of the tree before it started. +// - file mutations and their content. The store does carry a file-history +// stream, which an earlier survey classified as bookkeeping and skipped; +// reading it is worthwhile and is not done here, so the capability is not +// claimed. +func (Adapter) Capabilities() assay.CapabilitySet { + return assay.CapabilitySet(0).With( + assay.CanSeeLineage, + assay.CanSeeModelIdentity, + assay.CanSeeWorkspace, + assay.CanSeeToolCalls, + assay.CanSeeToolInputs, + assay.CanSeeToolResults, + assay.CanSeeToolOutcome, + assay.CanSeeTurnText, + assay.CanSeeReasoning, + assay.CanSeeTokens, + assay.CanSeeTiming, + assay.CanSeeDelegation, + ) +} + +// partialCapabilities are observed but known to be incomplete for every session +// this adapter produces. +// +// Model identity is partial because a transcript records the model that served +// a message and never the alias that was asked for, so a silent remapping of an +// alias is invisible from here alone. +// +// Tool results are partial because large ones are written elsewhere and +// shortened in place, so a result body may be a prefix of what the tool +// actually returned. +// +// Delegation is partial for a reason worth stating plainly, because it was +// nearly an overclaim. Delegated work is recorded two ways: inline in the +// session's own transcript, which this adapter reads, and offloaded into +// separate files beside it, which it does not. Scanning 601 project-root +// transcripts found no inline delegation at all — every one of them keeps it in +// the files this adapter skips. So the capability is declared, because the +// inline form is read correctly when it appears, and marked partial, because on +// the store as it exists today the answer is always empty. An assertion about +// delegated work that would fail on this evidence resolves to an error instead, +// which is the right answer: nothing was seen because nothing was looked at. +func partialCapabilities() assay.CapabilitySet { + return assay.CapabilitySet(0).With( + assay.CanSeeModelIdentity, + assay.CanSeeToolResults, + assay.CanSeeDelegation, + ) +} + +func (a Adapter) root() (string, error) { + if a.Root != "" { + return a.Root, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("locating the transcript store: %w", err) + } + return filepath.Join(home, ".claude", "projects"), nil +} + +// Discover implements port.Capture. +// +// Only transcripts sitting directly in a project directory are listed. Anything +// deeper is a record of delegated work, which belongs to the session that +// spawned it rather than standing on its own. +// +// The label is built from the path rather than from the session's contents, so +// that listing does not mean opening every file. One local store held 2,485 of +// them. +func (a Adapter) Discover(ctx context.Context, q port.Query) ([]port.Ref, error) { + root, err := a.root() + if err != nil { + return nil, err + } + + entries, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return []port.Ref{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading the transcript store: %w", err) + } + + refs := []port.Ref{} + for _, project := range entries { + if err := ctx.Err(); err != nil { + return nil, err + } + if !project.IsDir() { + continue + } + dir := filepath.Join(root, project.Name()) + files, err := os.ReadDir(dir) + if err != nil { + continue // a project directory we cannot read is not a fatal condition + } + for _, f := range files { + if f.IsDir() || !strings.HasSuffix(f.Name(), ".jsonl") { + continue + } + info, err := f.Info() + if err != nil { + continue + } + ref := port.Ref{ + ID: filepath.Join(dir, f.Name()), + Label: project.Name() + "/" + strings.TrimSuffix(f.Name(), ".jsonl"), + At: info.ModTime(), + } + if !q.Since.IsZero() && ref.At.Before(q.Since) { + continue + } + if q.Dir != "" && !strings.Contains(project.Name(), encodeProjectDir(q.Dir)) { + continue + } + refs = append(refs, ref) + } + } + + sort.Slice(refs, func(i, j int) bool { return refs[i].At.After(refs[j].At) }) + if q.Limit > 0 && len(refs) > q.Limit { + refs = refs[:q.Limit] + } + return refs, nil +} + +// encodeProjectDir renders a filesystem path the way the store names the +// directory it keeps that project's transcripts in. +func encodeProjectDir(dir string) string { + return strings.ReplaceAll(strings.TrimSuffix(dir, "/"), "/", "-") +} + +// Load implements port.Capture. +func (a Adapter) Load(ctx context.Context, ref port.Ref) (assay.Session, error) { + if ref.ID == "" { + return assay.Session{}, port.ErrNotFound + } + + f, err := os.Open(ref.ID) + if errors.Is(err, fs.ErrNotExist) { + // Stores are live: a session listed a moment ago can be pruned before + // it is read. + return assay.Session{}, port.ErrNotFound + } + if err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrUnsupported, err) + } + defer f.Close() + + s, err := parse(ctx, f) + if err != nil { + return assay.Session{}, err + } + + s.Fidelity = assay.Fidelity{ + Adapter: a.ID(), + Version: s.Fidelity.Version, + Tier: a.Tier(), + Observed: a.Capabilities(), + Partial: partialCapabilities(), + Verified: s.Fidelity.Verified, + } + return s, nil +} + +// knownVersions are the harness releases this adapter has been read against. +// +// An unknown release is parsed rather than refused, and the session is marked +// unverified. Releases arrived every few days across the measured period while +// the keys this adapter reads never moved, so refusing them would break capture +// roughly twice a week to guard against a change that has not yet happened. +// Saying "read, but not from a release anyone has checked" is the honest third +// option between refusing and pretending. +func knownVersions() []string { + return []string{ + "2.1.142", "2.1.149", "2.1.156", "2.1.170", "2.1.177", "2.1.181", + "2.1.190", "2.1.191", "2.1.197", "2.1.198", "2.1.199", "2.1.201", + "2.1.209", "2.1.210", "2.1.211", "2.1.217", "2.1.219", "2.1.220", + } +} + +func isKnownVersion(v string) bool { + for _, known := range knownVersions() { + if v == known { + return true + } + } + return false +} diff --git a/internal/adapter/claudecode/capture/capture_test.go b/internal/adapter/claudecode/capture/capture_test.go new file mode 100644 index 0000000..41d206c --- /dev/null +++ b/internal/adapter/claudecode/capture/capture_test.go @@ -0,0 +1,345 @@ +package capture + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/JumpMasters/assayer/internal/adapter/conformance" + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// Every fixture here is synthetic. +// +// No real transcript is committed, and none should be: the store holds working +// directories, file contents, commands and whatever a session happened to type, +// and the redaction pass that would make an export safe is not trustworthy yet +// — a survey found it firing on one local file in two. These fixtures reproduce +// the structure the survey measured without carrying anything from it. + +const transcriptWithATool = ` +{"type":"user","version":"2.1.220","sessionId":"s1","cwd":"/work","gitBranch":"main","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"add a test"}} +{"type":"assistant","version":"2.1.220","sessionId":"s1","cwd":"/work","timestamp":"2026-01-01T00:00:10Z","message":{"role":"assistant","model":"claude-x-1","usage":{"input_tokens":100,"output_tokens":50},"content":[{"type":"thinking","text":"consider the options"},{"type":"text","text":"running the tests"},{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"go test ./..."}}]}} +{"type":"user","version":"2.1.220","sessionId":"s1","cwd":"/work","timestamp":"2026-01-01T00:00:20Z","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"ok","is_error":false}]}} +{"type":"queue-operation","note":"bookkeeping, no version, must be skipped"} +{"type":"assistant","version":"2.1.220","sessionId":"s1","cwd":"/other","timestamp":"2026-01-01T00:01:00Z","message":{"role":"assistant","model":"claude-x-2","usage":{"input_tokens":10,"output_tokens":5},"content":[{"type":"text","text":"done"}]}} +` + +func writeStore(t *testing.T, name, body string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, "-work-project") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(strings.TrimSpace(body)+"\n"), 0o644); err != nil { + t.Fatalf("write transcript: %v", err) + } + return root +} + +func loadOnly(t *testing.T, body string) assay.Session { + t.Helper() + a := Adapter{Root: writeStore(t, "session.jsonl", body)} + refs, err := a.Discover(context.Background(), port.Query{}) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(refs) != 1 { + t.Fatalf("Discover returned %d refs, want 1", len(refs)) + } + s, err := a.Load(context.Background(), refs[0]) + if err != nil { + t.Fatalf("Load: %v", err) + } + return s +} + +// TestSatisfiesTheSharedContract runs the kit every capture adapter must pass. +func TestSatisfiesTheSharedContract(t *testing.T) { + conformance.Verify(t, Adapter{Root: writeStore(t, "session.jsonl", transcriptWithATool)}) +} + +func TestTranslatesASession(t *testing.T) { + s := loadOnly(t, transcriptWithATool) + + t.Run("turns keep their order and roles", func(t *testing.T) { + if len(s.Turns) != 3 { + t.Fatalf("got %d turns, want 3", len(s.Turns)) + } + if s.Turns[0].Role != assay.RoleHuman { + t.Errorf("first turn role = %v, want human", s.Turns[0].Role) + } + if s.Turns[0].Text != "add a test" { + t.Errorf("first turn text = %q", s.Turns[0].Text) + } + if s.Turns[1].Role != assay.RoleAgent { + t.Errorf("second turn role = %v, want agent", s.Turns[1].Role) + } + }) + + t.Run("reasoning is kept apart from prose", func(t *testing.T) { + if s.Turns[1].Reasoning != "consider the options" { + t.Errorf("reasoning = %q", s.Turns[1].Reasoning) + } + if strings.Contains(s.Turns[1].Text, "consider the options") { + t.Error("reasoning leaked into the turn's text; a rubric would grade a scratchpad") + } + }) + + t.Run("a tool call carries its command and neutral kind", func(t *testing.T) { + if len(s.Turns[1].Tools) != 1 { + t.Fatalf("got %d tool calls, want 1", len(s.Turns[1].Tools)) + } + c := s.Turns[1].Tools[0] + if c.Kind != assay.ToolExec { + t.Errorf("kind = %v, want exec", c.Kind) + } + if len(c.Argv) != 1 || c.Argv[0] != "go test ./..." { + t.Errorf("argv = %v", c.Argv) + } + if c.Input == "" { + t.Error("tool input was dropped; distillation needs what was put to the tool") + } + }) + + t.Run("a result is matched to the call it answers", func(t *testing.T) { + c := s.Turns[1].Tools[0] + if c.Result == nil { + t.Fatal("result is nil; it arrives on a later record and must be paired back") + } + if c.Result.Text != "ok" { + t.Errorf("result text = %q", c.Result.Text) + } + if c.Result.Outcome != assay.OutcomeOK { + t.Errorf("outcome = %v, want ok", c.Result.Outcome) + } + if c.Result.ExitCode != nil { + t.Error("an exit code was reported; transcripts do not carry one, " + + "and a zero here would read as success beside a failing outcome") + } + }) + + t.Run("every working directory is kept", func(t *testing.T) { + if len(s.Workspace.Dirs) != 2 { + t.Fatalf("dirs = %v, want both", s.Workspace.Dirs) + } + if s.Workspace.Dirs[0] != "/work" || s.Workspace.Dirs[1] != "/other" { + t.Errorf("dirs = %v, want first-use order", s.Workspace.Dirs) + } + }) + + t.Run("the model is per turn", func(t *testing.T) { + models := s.Models() + if len(models) != 2 { + t.Fatalf("Models() = %v, want two distinct", models) + } + if models[0].Canonical != "claude-x-1" || models[1].Canonical != "claude-x-2" { + t.Errorf("Models() = %v", models) + } + if models[0].Alias != "" { + t.Error("an alias was reported; a transcript records only what served the message") + } + }) + + t.Run("tokens are summed and money is not invented", func(t *testing.T) { + if s.Usage.InputTokens != 110 || s.Usage.OutputTokens != 55 { + t.Errorf("usage = %+v", s.Usage) + } + if s.Usage.CostMicroUSD != 0 { + t.Error("a price was reported; no transcript record carries one") + } + }) + + t.Run("fidelity names the adapter and the releases read", func(t *testing.T) { + if s.Fidelity.Adapter != "claude-code" || s.Fidelity.Tier != assay.TierNative { + t.Errorf("fidelity = %+v", s.Fidelity) + } + if s.Fidelity.Version != "2.1.220" { + t.Errorf("version = %q", s.Fidelity.Version) + } + if !s.Fidelity.Verified { + t.Error("a release this adapter has been read against was marked unverified") + } + }) +} + +// TestLineageKeepsTheParentSeparate covers the misattribution a single +// identifier would cause: the second field names the session this one +// continues, not a duplicate of the first. +func TestLineageKeepsTheParentSeparate(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"2.1.220","sessionId":"child","session_id":"parent","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if s.Lineage.ID != "child" { + t.Errorf("lineage id = %q, want the session's own", s.Lineage.ID) + } + if s.Lineage.Parent != "parent" { + t.Errorf("lineage parent = %q, want the session it continues", s.Lineage.Parent) + } +} + +// TestForkRecordNamesTheParent covers the record type an earlier survey +// classified as bookkeeping and skipped, which is where a fork's parent is +// actually written down. +func TestForkRecordNamesTheParent(t *testing.T) { + s := loadOnly(t, ` +{"type":"fork-context-ref","parentSessionId":"ancestor","parentLastUuid":"u1"} +{"type":"user","version":"2.1.220","sessionId":"forked","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if s.Lineage.Parent != "ancestor" { + t.Errorf("lineage parent = %q, want the fork record's parent", s.Lineage.Parent) + } +} + +// TestDelegatedWorkIsCarriedAndOrderIsNotClaimed covers work handed to a +// sub-agent: it is kept rather than dropped, and because nothing links it back +// to the call that spawned it, the session says its order is incomplete. +func TestDelegatedWorkIsCarriedAndOrderIsNotClaimed(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"2.1.220","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +{"type":"assistant","version":"2.1.220","sessionId":"s","cwd":"/w","isSidechain":true,"timestamp":"2026-01-01T00:00:05Z","message":{"role":"assistant","model":"m","content":[{"type":"text","text":"sub-task"}]}} +`) + if len(s.Delegated) != 1 || len(s.Delegated[0].Turns) != 1 { + t.Fatalf("delegated = %+v, want one delegation with one turn", s.Delegated) + } + if s.OrderComplete { + t.Error("order was claimed complete while delegated work is unattributed") + } + for _, turn := range s.Turns { + if turn.Text == "sub-task" { + t.Error("delegated work was folded into the main turns") + } + } +} + +// TestUnknownReleaseIsReadButNotVerified covers the choice not to refuse a +// release nobody has checked. Refusing would break capture roughly twice a week +// at the rate releases were measured arriving. +func TestUnknownReleaseIsReadButNotVerified(t *testing.T) { + s := loadOnly(t, ` +{"type":"user","version":"9.9.999","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +`) + if len(s.Turns) != 1 { + t.Fatalf("an unknown release was refused; got %d turns", len(s.Turns)) + } + if s.Fidelity.Verified { + t.Error("a release nobody has read against was reported as verified") + } +} + +// TestUnknownRecordTypeIsRefused is the other half of that choice: a new kind +// of content record must fail loudly rather than be skipped into a session that +// looks whole and is not. +func TestUnknownRecordTypeIsRefused(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", ` +{"type":"user","version":"2.1.220","sessionId":"s","cwd":"/w","timestamp":"2026-01-01T00:00:00Z","message":{"role":"user","content":"go"}} +{"type":"a-record-type-nobody-has-seen","version":"2.1.220"} +`)} + refs, _ := a.Discover(context.Background(), port.Query{}) + _, err := a.Load(context.Background(), refs[0]) + if !errors.Is(err, port.ErrUnsupported) { + t.Errorf("Load returned %v, want ErrUnsupported", err) + } +} + +func TestMalformedTranscriptIsReported(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", `{"type":"user"`+"\n")} + refs, _ := a.Discover(context.Background(), port.Query{}) + _, err := a.Load(context.Background(), refs[0]) + if !errors.Is(err, port.ErrMalformed) { + t.Errorf("Load returned %v, want ErrMalformed", err) + } +} + +func TestMissingSessionReportsNotFound(t *testing.T) { + a := Adapter{Root: t.TempDir()} + _, err := a.Load(context.Background(), port.Ref{ID: filepath.Join(t.TempDir(), "gone.jsonl")}) + if !errors.Is(err, port.ErrNotFound) { + t.Errorf("Load of a pruned session returned %v, want ErrNotFound", err) + } + if _, err := a.Load(context.Background(), port.Ref{}); !errors.Is(err, port.ErrNotFound) { + t.Errorf("Load of an empty ref returned %v, want ErrNotFound", err) + } +} + +func TestDiscoverOnAnAbsentStoreIsEmptyNotAnError(t *testing.T) { + a := Adapter{Root: filepath.Join(t.TempDir(), "no-such-store")} + refs, err := a.Discover(context.Background(), port.Query{}) + if err != nil { + t.Fatalf("Discover on an absent store returned %v", err) + } + if len(refs) != 0 { + t.Errorf("got %d refs from an absent store", len(refs)) + } +} + +func TestDiscoverHonoursTheQuery(t *testing.T) { + a := Adapter{Root: writeStore(t, "session.jsonl", transcriptWithATool)} + ctx := context.Background() + + if refs, _ := a.Discover(ctx, port.Query{Since: time.Now().Add(time.Hour)}); len(refs) != 0 { + t.Errorf("Since in the future returned %d refs", len(refs)) + } + if refs, _ := a.Discover(ctx, port.Query{Dir: "/nowhere/at/all"}); len(refs) != 0 { + t.Errorf("a non-matching Dir returned %d refs", len(refs)) + } + if refs, _ := a.Discover(ctx, port.Query{Limit: 1}); len(refs) != 1 { + t.Errorf("Limit 1 returned %d refs", len(refs)) + } +} + +// TestToolNamesMapToNeutralKinds is what lets core reason about behaviour +// without matching on the names one harness happens to use. +func TestToolNamesMapToNeutralKinds(t *testing.T) { + for name, want := range map[string]assay.ToolKind{ + "Read": assay.ToolRead, + "Grep": assay.ToolRead, + "Edit": assay.ToolMutate, + "Write": assay.ToolMutate, + "Bash": assay.ToolExec, + "Task": assay.ToolDelegate, + "Unheard": assay.ToolOther, + "": assay.ToolUnknown, + } { + if got := classify(name); got != want { + t.Errorf("classify(%q) = %v, want %v", name, got, want) + } + } +} + +// TestCapabilitiesAreNotOverclaimed pins the four this adapter must not assert. +// Each was checked against the store rather than assumed. +func TestCapabilitiesAreNotOverclaimed(t *testing.T) { + caps := Adapter{}.Capabilities() + for _, c := range []assay.Capability{ + assay.CanSeeCost, + assay.CanSeeWorkspacePatch, + assay.CanSeeFileMutations, + assay.CanSeeMutationContent, + } { + if caps.Has(c) { + t.Errorf("adapter declares %v, which no transcript record supplies", c) + } + } + + partial := partialCapabilities() + if !partial.Has(assay.CanSeeModelIdentity) { + t.Error("model identity is not marked partial; the alias is never recorded") + } + if !partial.Has(assay.CanSeeToolResults) { + t.Error("tool results are not marked partial; large ones are shortened in place") + } + if !partial.Has(assay.CanSeeDelegation) { + t.Error("delegation is not marked partial; a scan of 601 project-root transcripts " + + "found none recorded inline, so on the real store the answer is always empty") + } + if !caps.Contains(partial) { + t.Error("a capability is marked partial that the adapter does not claim at all") + } +} diff --git a/internal/adapter/claudecode/capture/parse.go b/internal/adapter/claudecode/capture/parse.go new file mode 100644 index 0000000..5b68d91 --- /dev/null +++ b/internal/adapter/claudecode/capture/parse.go @@ -0,0 +1,409 @@ +package capture + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "github.com/JumpMasters/assayer/internal/assay" + "github.com/JumpMasters/assayer/internal/port" +) + +// record is one line of a transcript, in the shape this adapter reads. +// +// Only the fields the neutral representation needs are declared. A transcript +// carries a great deal more, and anything not needed downstream stays here +// rather than travelling as an opaque payload. +type record struct { + Type string `json:"type"` + UUID string `json:"uuid"` + SessionID string `json:"sessionId"` + ResumedFrom string `json:"session_id"` + CWD string `json:"cwd"` + GitBranch string `json:"gitBranch"` + Version string `json:"version"` + Timestamp string `json:"timestamp"` + IsSidechain bool `json:"isSidechain"` + Message message `json:"message"` + + // ParentSessionID appears on the record that marks a fork. The field that + // names a parent lives on its own record type rather than on every line. + ParentSessionID string `json:"parentSessionId"` +} + +type message struct { + Role string `json:"role"` + Model string `json:"model"` + Usage usage `json:"usage"` + Content json.RawMessage `json:"content"` +} + +type usage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// block is one element of a structured message body. +type block struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` + + // Tool results arrive on a following user message rather than beside the + // call, so they are matched back by identifier. + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` + IsError bool `json:"is_error"` +} + +// contentTypes carry a session's substance and always declare a version. +func isContentType(t string) bool { + switch t { + case "assistant", "user", "attachment", "system": + return true + } + return false +} + +// bookkeepingTypes never declare a version and hold no conversation. +// +// They are listed rather than inferred from a missing version field. Roughly a +// third of all records carry no version, so treating absence as the signal +// would reject every real transcript; and inferring it the other way would let +// a genuinely new kind of content record be skipped in silence, producing a +// session that looks complete and is not. +func isBookkeepingType(t string) bool { + switch t { + case "queue-operation", "last-prompt", "ai-title", "mode", "pr-link", + "file-history-snapshot", "bridge-session", "permission-mode", + "file-history-delta", "worktree-state", "custom-title", "fork-context-ref": + return true + } + return false +} + +// parse reads a transcript into the neutral representation. +func parse(ctx context.Context, r io.Reader) (assay.Session, error) { + var s assay.Session + + sc := bufio.NewScanner(r) + // Transcript lines carry whole tool results and can be far longer than the + // scanner's default limit. + sc.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + + var ( + versions = map[string]bool{} + dirs []string + seenDir = map[string]bool{} + pending = map[string]*pendingCall{} + delegated []assay.Turn + ) + + for sc.Scan() { + if err := ctx.Err(); err != nil { + return assay.Session{}, err + } + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + + var rec record + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrMalformed, err) + } + + switch { + case rec.Type == "fork-context-ref": + // The one record that names a parent explicitly. + if rec.ParentSessionID != "" { + s.Lineage.Parent = rec.ParentSessionID + } + continue + case isBookkeepingType(rec.Type): + continue + case !isContentType(rec.Type): + // Refused rather than skipped: an unrecognised kind of content + // record would otherwise vanish into a session that reads as whole. + return assay.Session{}, fmt.Errorf("%w: unknown record type %q", + port.ErrUnsupported, rec.Type) + } + + if rec.Version != "" { + versions[rec.Version] = true + } + if rec.SessionID != "" && s.Lineage.ID == "" { + s.Lineage.ID = rec.SessionID + } + // The second identifier names the session this one continues, and is + // not a duplicate of the first. Preferring the wrong one attributes a + // session's work to its predecessor. + if rec.ResumedFrom != "" && rec.ResumedFrom != rec.SessionID && s.Lineage.Parent == "" { + s.Lineage.Parent = rec.ResumedFrom + } + if rec.CWD != "" && !seenDir[rec.CWD] { + seenDir[rec.CWD] = true + dirs = append(dirs, rec.CWD) + } + if rec.GitBranch != "" && s.Workspace.Branch == "" { + s.Workspace.Branch = rec.GitBranch + } + + turn, results := recordToTurn(&rec) + + // A result arrives on a later message than the call it answers. + for id, res := range results { + if p, ok := pending[id]; ok { + p.call.Result = res + } + } + + if turn == nil { + continue + } + + if rec.IsSidechain { + delegated = append(delegated, *turn) + continue + } + + s.Turns = append(s.Turns, *turn) + last := &s.Turns[len(s.Turns)-1] + // Index each call so a result arriving on a later message can be + // attached to it. The identifiers are gathered once: recomputing them + // per call would re-parse the whole message body each time. + ids := toolIDs(&rec) + for i := range last.Tools { + if i < len(ids) && ids[i] != "" { + pending[ids[i]] = &pendingCall{call: &last.Tools[i]} + } + } + + s.Usage.InputTokens += rec.Message.Usage.InputTokens + s.Usage.OutputTokens += rec.Message.Usage.OutputTokens + } + if err := sc.Err(); err != nil { + return assay.Session{}, fmt.Errorf("%w: %w", port.ErrMalformed, err) + } + + s.Workspace.Dirs = dirs + if len(delegated) > 0 { + // Delegated work is carried flat. Nothing here links it back to the + // call that spawned it, so the order of the whole session is not fully + // known and assertions about ordering must say so rather than guess. + s.Delegated = []assay.Delegation{{Turns: delegated}} + s.OrderComplete = false + } else { + s.OrderComplete = true + } + + if len(s.Turns) > 0 { + s.Usage.Wall = s.Turns[len(s.Turns)-1].At.Sub(s.Turns[0].At) + } + + s.Fidelity.Version = strings.Join(sortedKeys(versions), ",") + s.Fidelity.Verified = allKnown(versions) + return s, nil +} + +type pendingCall struct{ call *assay.ToolCall } + +// recordToTurn translates one content record, returning any tool results it +// carries so they can be matched to the calls they answer. +func recordToTurn(rec *record) (turn *assay.Turn, results map[string]*assay.ToolResult) { + results = map[string]*assay.ToolResult{} + + role := assay.RoleUnknown + switch rec.Message.Role { + case "user": + role = assay.RoleHuman + case "assistant": + role = assay.RoleAgent + } + + t := assay.Turn{ + Role: role, + Model: modelOf(rec), + At: parseTime(rec.Timestamp), + } + + blocks := blocksOf(rec.Message.Content) + for i := range blocks { + b := &blocks[i] + switch b.Type { + case "text": + t.Text += b.Text + case "thinking": + t.Reasoning += b.Text + case "tool_use": + t.Tools = append(t.Tools, assay.ToolCall{ + Name: b.Name, + Kind: classify(b.Name), + Argv: argvOf(b), + Input: string(b.Input), + }) + case "tool_result": + results[b.ToolUseID] = toolResult(b) + } + } + + // A record carrying only tool results is not a turn in the conversation. + // The protocol returns results on a message with the user's role, so + // admitting it would insert an empty human turn between an agent's call and + // its answer — inflating any count of turns, and putting a human turn where + // the person said nothing. + if t.Text == "" && t.Reasoning == "" && len(t.Tools) == 0 { + return nil, results + } + return &t, results +} + +func toolResult(b *block) *assay.ToolResult { + outcome := assay.OutcomeOK + if b.IsError { + outcome = assay.OutcomeNonZero + } + return &assay.ToolResult{ + Text: textOf(b.Content), + Outcome: outcome, + // No exit code: transcripts record that a call errored and never the + // code it returned. Filling this with zero would report success beside + // an outcome saying otherwise. + ExitCode: nil, + } +} + +// classify maps a harness's tool name onto the neutral vocabulary, so that no +// package outside an adapter has to match on a name. +func classify(name string) assay.ToolKind { + switch name { + case "Read", "Glob", "Grep", "NotebookRead", "WebFetch", "WebSearch": + return assay.ToolRead + case "Edit", "Write", "NotebookEdit", "MultiEdit": + return assay.ToolMutate + case "Bash", "BashOutput", "KillShell": + return assay.ToolExec + case "Task", "Agent": + return assay.ToolDelegate + case "": + return assay.ToolUnknown + } + return assay.ToolOther +} + +// argvOf extracts a command line when the call ran one. +func argvOf(b *block) []string { + if classify(b.Name) != assay.ToolExec || len(b.Input) == 0 { + return nil + } + var in struct { + Command string `json:"command"` + } + if err := json.Unmarshal(b.Input, &in); err != nil || in.Command == "" { + return nil + } + // The harness records a command as one string. It is kept whole in a + // single element rather than split here: splitting a shell command + // correctly needs a shell, and guessing produces an argv that never ran. + return []string{in.Command} +} + +func modelOf(rec *record) assay.Model { + if rec.Message.Model == "" { + return assay.Model{} + } + // The transcript records what served the message, never the alias that was + // asked for, so a quiet remapping of an alias cannot be seen from here. + return assay.Model{Canonical: rec.Message.Model, Provider: "anthropic"} +} + +// blocksOf reads a message body, which is either a plain string or a list of +// structured blocks depending on the record. +func blocksOf(raw json.RawMessage) []block { + if len(raw) == 0 { + return nil + } + var blocks []block + if err := json.Unmarshal(raw, &blocks); err == nil { + return blocks + } + var text string + if err := json.Unmarshal(raw, &text); err == nil && text != "" { + return []block{{Type: "text", Text: text}} + } + return nil +} + +// textOf renders a result body, which has the same two shapes as a message. +func textOf(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var text string + if err := json.Unmarshal(raw, &text); err == nil { + return text + } + var blocks []block + if err := json.Unmarshal(raw, &blocks); err == nil { + var b strings.Builder + for i := range blocks { + b.WriteString(blocks[i].Text) + } + return b.String() + } + return "" +} + +// toolIDs lists the identifiers of the calls a record makes, in order. +func toolIDs(rec *record) []string { + blocks := blocksOf(rec.Message.Content) + ids := make([]string, 0, len(blocks)) + for i := range blocks { + if blocks[i].Type == "tool_use" { + ids = append(ids, blocks[i].ID) + } + } + return ids +} + +func parseTime(s string) time.Time { + if s == "" { + return time.Time{} + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{} + } + return t +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j] < out[j-1]; j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out +} + +func allKnown(versions map[string]bool) bool { + if len(versions) == 0 { + return false + } + for v := range versions { + if !isKnownVersion(v) { + return false + } + } + return true +} diff --git a/internal/adapter/conformance/coverage.go b/internal/adapter/conformance/coverage.go index a347382..af364bd 100644 --- a/internal/adapter/conformance/coverage.go +++ b/internal/adapter/conformance/coverage.go @@ -63,7 +63,7 @@ func governed(key string) (assay.Capability, bool) { case "ToolResult.Text", "ToolResult.Truncated": return assay.CanSeeToolResults, true case "ToolResult.Outcome", "ToolResult.ExitCode": - return assay.CanSeeExitStatus, true + return assay.CanSeeToolOutcome, true case "Usage.CostMicroUSD": return assay.CanSeeCost, true diff --git a/internal/adapter/conformance/reference.go b/internal/adapter/conformance/reference.go index 1333495..3c83b9f 100644 --- a/internal/adapter/conformance/reference.go +++ b/internal/adapter/conformance/reference.go @@ -58,7 +58,7 @@ func (Reference) Capabilities() assay.CapabilitySet { assay.CanSeeToolCalls, assay.CanSeeToolInputs, assay.CanSeeToolResults, - assay.CanSeeExitStatus, + assay.CanSeeToolOutcome, assay.CanSeeFileMutations, assay.CanSeeModelIdentity, assay.CanSeeTokens, @@ -121,7 +121,7 @@ func (r Reference) Load(_ context.Context, ref port.Ref) (assay.Session, error) Result: &assay.ToolResult{ Text: "ok\n", Outcome: assay.OutcomeOK, - ExitCode: exit, + ExitCode: &exit, }, }, { Name: "put", diff --git a/internal/arch/classify_test.go b/internal/arch/classify_test.go index ed5e557..a18f1be 100644 --- a/internal/arch/classify_test.go +++ b/internal/arch/classify_test.go @@ -67,7 +67,8 @@ var classification = map[string]class{ "internal/cli": classRootCLI, "cmd/assayer": classCmd, - "internal/adapter/conformance": classAdapter, + "internal/adapter/conformance": classAdapter, + "internal/adapter/claudecode/capture": classAdapter, } // allowedInternal returns the in-module packages a class may import. diff --git a/internal/assay/capability.go b/internal/assay/capability.go index 3841eac..eef4379 100644 --- a/internal/assay/capability.go +++ b/internal/assay/capability.go @@ -31,7 +31,7 @@ const ( CanSeeToolCalls CanSeeToolInputs CanSeeToolResults - CanSeeExitStatus + CanSeeToolOutcome CanSeeFileMutations CanSeeMutationContent CanSeeTurnText @@ -69,8 +69,8 @@ func (c Capability) String() string { return "tool-inputs" case CanSeeToolResults: return "tool-results" - case CanSeeExitStatus: - return "exit-status" + case CanSeeToolOutcome: + return "tool-outcome" case CanSeeFileMutations: return "file-mutations" case CanSeeMutationContent: diff --git a/internal/assay/session.go b/internal/assay/session.go index dd6dd16..e9d516d 100644 --- a/internal/assay/session.go +++ b/internal/assay/session.go @@ -178,13 +178,18 @@ type ToolResult struct { Outcome Outcome - // ExitCode is meaningful when Outcome is OutcomeOK or OutcomeNonZero. + // ExitCode is nil when no code was observed, which is the ordinary case + // rather than the exception. // - // The distinction it preserves is load-bearing rather than decorative: test - // runners use separate non-zero codes for "tests failed" and "the thing you - // named does not exist", and the second is a pin that has rotted rather than - // a regression. - ExitCode int + // A pointer because zero is a meaningful exit code and would otherwise be + // indistinguishable from absence. The reference harness records whether a + // call errored and never records the code itself, so an adapter that filled + // this with 0 would be reporting success beside an Outcome saying the call + // failed. The distinction the field preserves when it IS observed is + // load-bearing: test runners use separate non-zero codes for "tests failed" + // and "the thing you named does not exist", and the second is a pin that has + // rotted rather than a regression. + ExitCode *int } // Workspace is the state of the files a session worked on.