Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 63 additions & 50 deletions cmd/entire/cli/checkpoint/remote/checkpoint_ref.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,15 @@ package remote
import (
"bytes"
"context"
"errors"
"fmt"
"log/slog"
"os/exec"
"strings"
"time"

"github.com/entireio/cli/cmd/entire/cli/logging"
"github.com/entireio/cli/cmd/entire/cli/settings"

"github.com/go-git/go-git/v6/plumbing"
)

Expand All @@ -29,47 +33,22 @@ const readFetchTimeout = 2 * time.Minute
// resolution fails, it falls back to the origin remote name so callers can
// still attempt a fetch.
func CheckpointFetchTarget(ctx context.Context) string {
target, _, _ := checkpointFetchTarget(ctx)
target, _ := checkpointFetchTarget(ctx)
return target
}

// checkpointFetchTarget is CheckpointFetchTarget plus two facts about the
// returned target:
// - authoritative: whether the target is certified as the host of checkpoint
// refs (see fetchURLAuthoritative). The bare "origin" fallbacks are
// non-authoritative: they exist so a fetch can still be attempted, not to
// certify where checkpoint refs live.
// - resolved: whether a concrete fetch URL/remote was resolved at all. It is
// false ONLY when no checkpoint remote is configured (no origin remote and
// no configured checkpoint_remote) and we fell back to the bare "origin"
// name. That literal "origin" is not guaranteed to be a real git remote, so
// a probe failure against it says only "there is nowhere to fetch from". If
// a checkpoint_remote IS configured but its URL can't be derived this call,
// resolved is true so a probe failure surfaces as a real error, not absence.
func checkpointFetchTarget(ctx context.Context) (target string, authoritative, resolved bool) {
url, auth, err := fetchURLAuthoritative(ctx)
// checkpointFetchTarget is CheckpointFetchTarget plus whether the target is
// authoritative for checkpoint refs (see fetchURLAuthoritative). The bare
// "origin" fallbacks are non-authoritative: they exist so a fetch can still
// be attempted, not to certify where checkpoint refs live.
func checkpointFetchTarget(ctx context.Context) (string, bool) {
url, authoritative, err := fetchURLAuthoritative(ctx)
if err == nil && url != "" {
return url, auth, true
}
if errors.Is(err, errNoCheckpointRemoteConfigured) {
// Genuinely no checkpoint remote configured: a probe failure against the
// bare "origin" fallback proves only "nowhere to fetch from", so callers
// classify it as checkpoint absence and fall back to the v1-branch store.
return "origin", false, false
return url, authoritative
}
// A checkpoint remote IS configured but its URL could not be derived this
// call (transient GetRemoteURL error, provider-host derivation failure). Mark
// resolved so a probe failure surfaces as a real error rather than being
// silently reclassified as checkpoint absence.
return "origin", false, true
return "origin", false
}

// errNoCheckpointRemoteConfigured marks the "no checkpoint remote configured at
// all" case (no origin remote and no configured checkpoint_remote) — the only
// situation in which a probe failure against the bare "origin" fallback is
// classified as checkpoint absence rather than surfaced as a real error.
var errNoCheckpointRemoteConfigured = errors.New("no checkpoint remote configured")

// FetchCheckpointRef fetches a single per-checkpoint ref
// (refs/entire/checkpoints/<shard>/<id>) from the checkpoint remote into the
// local ref of the same name, so the git-refs store can resolve a checkpoint
Expand All @@ -84,28 +63,52 @@ var errNoCheckpointRemoteConfigured = errors.New("no checkpoint remote configure
// - Any transport-level failure (probe or fetch) is surfaced as a real
// error, never mapped to absence — a false "absent" would misdirect a
// backfill onto another backend instead of retrying.
// - A repository with no git remotes at all and no checkpoint_remote
// configured also returns an error wrapping plumbing.ErrReferenceNotFound
// without probing: there is no remote that could host the ref.
func FetchCheckpointRef(ctx context.Context, ref plumbing.ReferenceName) error {
ctx, cancel := context.WithTimeout(ctx, readFetchTimeout)
defer cancel()

fetchTarget, authoritative, resolved := checkpointFetchTarget(ctx)
fetchTarget, authoritative := checkpointFetchTarget(ctx)

// A fully local repository — no git remotes at all and no
// checkpoint_remote configured — has no remote that could host checkpoint
// refs, so the ref's local absence is the final verdict. Without this,
// the origin-name fallback below probes a remote git cannot resolve
// ("'origin' does not appear to be a git repository", exit 128) and a
// remoteless repo is misreported as a transport outage. Absence is only
// ever classified on positive evidence; each escape below keeps today's
// hard-failure probe:
// - dead caller context: every subprocess fails for reasons that say
// nothing about the repository
// - unreadable settings: a configured checkpoint remote cannot be
// ruled out
// - checkpoint_remote key present in any form, valid or malformed
// (see settings.HasCheckpointRemoteKey for why key presence, not
// GetCheckpointRemote, is the signal)
// - any remote listed, or the listing itself failing: checkpoint refs
// are pushed to whatever remote the pre-push hook fires for, so a
// repo with only non-origin remotes is NOT remoteless
// The skipped-guard cases surface through the ordinary probe paths: a
// missing origin fails the ls-remote probe as a transport error, and an
// origin that merely lacks the ref hits the fallback-emptiness refusal
// below — neither classifies as absence.
if fetchTarget == originRemote && !authoritative && ctx.Err() == nil {
s, loadErr := settings.Load(ctx)
switch {
case loadErr != nil:
logging.Warn(ctx, "checkpoint probe: settings unreadable; cannot rule out a configured checkpoint remote, probing anyway",
slog.String("error", loadErr.Error()))
case !s.HasCheckpointRemoteKey() && repoHasNoRemotes(ctx):
logging.Debug(ctx, "checkpoint probe: repository has no git remotes; classifying ref as absent",
slog.String("ref", ref.String()))
return fmt.Errorf("checkpoint ref %s: repository has no git remotes to fetch from: %w", ref, plumbing.ErrReferenceNotFound)
}
}
Comment thread
peyton-alt marked this conversation as resolved.

out, err := LsRemoteInDir(ctx, "", fetchTarget, ref.String())
if err != nil {
if !resolved {
// No checkpoint remote is configured or reachable — resolution fell
// back to the bare "origin" name, which need not be a real git
// remote (the local repo may have no origin at all). A probe failure
// here proves only that there is nowhere to fetch from, not that the
// checkpoint is missing on a real remote. Classify as absence so the
// git-refs store maps it to ErrCheckpointNotFound and write routing
// falls back to the v1-branch store, mirroring the branch-existence
// path (BranchExistsOnRemote treats an ls-remote failure as "not
// found"). A genuine transport failure against a *resolved* remote
// still propagates below, so real offline/network loss is never
// silently swallowed as absence.
return fmt.Errorf("probe checkpoint ref %s: no checkpoint remote configured: %w", ref, plumbing.ErrReferenceNotFound)
}
// Redact: fetchTarget can be a remote URL with embedded credentials
// (CI origin URLs), and this error is logged and shown to users.
return fmt.Errorf("probe checkpoint ref %s on %s: %w", ref, RedactURL(fetchTarget), err)
Expand Down Expand Up @@ -140,6 +143,16 @@ func FetchCheckpointRef(ctx context.Context, ref plumbing.ReferenceName) error {
return nil
}

// repoHasNoRemotes reports whether the repository at the current directory
// definitively has no git remotes configured. Only a successful, empty
// `git remote` listing counts as proof; any error (dead context, missing git
// binary, not a repository) returns false so the caller falls through to the
// probe instead of classifying absence off an undifferentiated failure.
func repoHasNoRemotes(ctx context.Context) bool {
out, err := exec.CommandContext(ctx, "git", "remote").Output()
return err == nil && len(bytes.TrimSpace(out)) == 0
}

// HookCheckpointRefFetcher returns the write-probe fetcher for git-hook
// contexts (post-commit attribution, stop-time transcript finalize): the
// bounded budget plus BatchMode SSH, so a passphrase-protected key can never
Expand Down
116 changes: 100 additions & 16 deletions cmd/entire/cli/checkpoint/remote/checkpoint_ref_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,16 @@ func TestFetchCheckpointRef_UnreachableRemoteIsFailure(t *testing.T) {
"a transport failure must stay distinguishable from absence")
}

// TestFetchCheckpointRef_NoConfiguredRemoteIsAbsence: with no origin remote and
// no configured checkpoint_remote, fetch-target resolution can resolve NOTHING
// and falls back to the bare "origin" name, which is not a real git remote. The
// ls-remote probe then fails — but that failure means only "there is nowhere to
// fetch from", not "the checkpoint is missing on a real remote". It must
// classify as absence (wrap plumbing.ErrReferenceNotFound) so the git-refs
// store maps it to ErrCheckpointNotFound and write routing falls back to the
// v1-branch store, instead of hard-erroring the whole save (the refs-primary
// regression this fixes). This is the mirror of
// TestFetchCheckpointRef_UnreachableRemoteIsFailure, where origin IS configured
// (a real, resolved remote) and the same probe failure must propagate.
func TestFetchCheckpointRef_NoConfiguredRemoteIsAbsence(t *testing.T) {
t.Setenv("ENTIRE_CONFIG_DIR", t.TempDir())

// TestFetchCheckpointRef_NoRemoteAtAllIsAbsence: a fully local repository —
// no origin remote and no checkpoint_remote configured — has no remote that
// could host checkpoint refs, so the ref's local absence is the final
// verdict, not a transport failure. Regression: the origin-name fallback
// probe used to run `git ls-remote origin` in a remoteless repo and surface
// exit 128, which broke backfill routing (and `explain --generate`) in fully
// local repos.
func TestFetchCheckpointRef_NoRemoteAtAllIsAbsence(t *testing.T) {
workDir := t.TempDir()
testutil.InitRepo(t, workDir) // no origin remote, no checkpoint_remote
testutil.InitRepo(t, workDir)
testutil.WriteFile(t, workDir, "f.txt", "content")
testutil.GitAdd(t, workDir, "f.txt")
testutil.GitCommit(t, workDir, "init")
Expand All @@ -120,5 +114,95 @@ func TestFetchCheckpointRef_NoConfiguredRemoteIsAbsence(t *testing.T) {
err := FetchCheckpointRef(context.Background(), ref)
require.Error(t, err)
require.ErrorIs(t, err, plumbing.ErrReferenceNotFound,
"a probe failure with no configured/reachable checkpoint remote must classify as absence so callers fall back")
"a repo with no remotes must classify a locally absent ref as absence")
}

// TestFetchCheckpointRef_UnreadableSettingsNeverClassifiesAbsence: when the
// checkpoint_remote configuration CANNOT BE READ (corrupt settings), whether a
// checkpoint remote exists is undeterminable. The no-remotes absence shortcut
// must not fire on a load error — the run falls through to the ls-remote
// probe, which surfaces the missing origin as a transport error, never as
// absence.
func TestFetchCheckpointRef_UnreadableSettingsNeverClassifiesAbsence(t *testing.T) {
workDir := t.TempDir()
testutil.InitRepo(t, workDir)
testutil.WriteFile(t, workDir, "f.txt", "content")
testutil.GitAdd(t, workDir, "f.txt")
testutil.GitCommit(t, workDir, "init")
testutil.WriteFile(t, workDir, ".entire/settings.json", "{not valid json")
t.Chdir(workDir)

ref := plumbing.ReferenceName("refs/entire/checkpoints/Z9/01KVBJCWYA4YW6J5M9GP655HZ9")
err := FetchCheckpointRef(context.Background(), ref)
require.Error(t, err)
require.NotErrorIs(t, err, plumbing.ErrReferenceNotFound,
"an unreadable checkpoint_remote configuration must not classify as absence")
}

// TestFetchCheckpointRef_MalformedCheckpointRemoteNeverClassifiesAbsence: a
// checkpoint_remote entry that is present but malformed (here: missing the
// required repo field) means the user configured a checkpoint remote and
// botched it. Combined with a missing origin, that must stay a failure —
// classifying it as absence would misroute backfills for checkpoints that
// live on the remote the user intended.
func TestFetchCheckpointRef_MalformedCheckpointRemoteNeverClassifiesAbsence(t *testing.T) {
workDir := t.TempDir()
testutil.InitRepo(t, workDir)
testutil.WriteFile(t, workDir, "f.txt", "content")
testutil.GitAdd(t, workDir, "f.txt")
testutil.GitCommit(t, workDir, "init")
testutil.WriteFile(t, workDir, ".entire/settings.json",
`{"enabled": true, "strategy_options": {"checkpoint_remote": {"provider": "github"}}}`)
t.Chdir(workDir)

ref := plumbing.ReferenceName("refs/entire/checkpoints/Z9/01KVBJCWYA4YW6J5M9GP655HZ9")
err := FetchCheckpointRef(context.Background(), ref)
require.Error(t, err)
require.NotErrorIs(t, err, plumbing.ErrReferenceNotFound,
"a present-but-malformed checkpoint_remote must not classify as absence")
}

// TestFetchCheckpointRef_NonOriginRemoteNeverClassifiesAbsence: a repo whose
// only remote is not named origin (git clone -o upstream is a common shape)
// is NOT remoteless — checkpoint refs are pushed to whatever remote the
// pre-push hook fires for, so they can legitimately live on a non-origin
// remote. Classifying this repo as absence would misroute backfills; it must
// stay a failure.
func TestFetchCheckpointRef_NonOriginRemoteNeverClassifiesAbsence(t *testing.T) {
bareDir := t.TempDir()
out, err := exec.CommandContext(t.Context(), "git", "init", "--bare", bareDir).CombinedOutput()
require.NoError(t, err, "git init --bare: %s", out)

workDir := t.TempDir()
testutil.InitRepo(t, workDir)
testutil.WriteFile(t, workDir, "f.txt", "content")
testutil.GitAdd(t, workDir, "f.txt")
testutil.GitCommit(t, workDir, "init")
out, err = exec.CommandContext(t.Context(), "git", "-C", workDir, "remote", "add", "upstream", bareDir).CombinedOutput()
require.NoError(t, err, "git remote add upstream: %s", out)
t.Chdir(workDir)

ref := plumbing.ReferenceName("refs/entire/checkpoints/Z9/01KVBJCWYA4YW6J5M9GP655HZ9")
err = FetchCheckpointRef(context.Background(), ref)
require.Error(t, err)
require.NotErrorIs(t, err, plumbing.ErrReferenceNotFound,
"a repo with a non-origin remote must not classify as absence")
}

// TestFetchCheckpointRef_CanceledContextNeverClassifiesAbsence: a dead caller
// context makes every git subprocess fail, which must surface as a transport
// failure — never as absence. Regression: the no-remotes guard once inferred
// "no origin" from a GetRemoteURL failure, which a canceled context also
// produces, converting Ctrl-C in a healthy repo into a false "checkpoint does
// not exist" verdict that write routing acts on.
func TestFetchCheckpointRef_CanceledContextNeverClassifiesAbsence(t *testing.T) {
_, ref := checkpointRefFixture(t, true)

ctx, cancel := context.WithCancel(context.Background())
cancel()

err := FetchCheckpointRef(ctx, ref)
require.Error(t, err)
require.NotErrorIs(t, err, plumbing.ErrReferenceNotFound,
"a canceled context must stay a failure, never absence")
}
5 changes: 1 addition & 4 deletions cmd/entire/cli/checkpoint/remote/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,7 @@ func fetchURLAuthoritative(ctx context.Context, opts ...FetchURLOptions) (string
config := s.GetCheckpointRemote()
if config == nil {
if originURL == "" {
// Genuinely no remote to fetch checkpoints from: no origin remote and
// no configured checkpoint_remote. Tagged with the sentinel so callers
// classify a probe failure as absence rather than a real error.
return "", false, fmt.Errorf("no fetch URL found: %w: %w", originErr, errNoCheckpointRemoteConfigured)
return "", false, fmt.Errorf("no fetch URL found: %w", originErr)
}
// No checkpoint_remote configured: origin IS the checkpoint host.
return originURL, true, nil
Expand Down
7 changes: 4 additions & 3 deletions cmd/entire/cli/checkpoint/routing_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,10 @@ func (s *kindRoutingStore) ReadSessionMetadataAndPrompts(ctx context.Context, ch
// (which falls through on absent OR any error): only the not-found sentinel
// falls through here. Redirecting a write to another backend after a transient
// primary failure could fork the data, so a hard error aborts and surfaces.
// Note the stores' backfill absence probes are local-only (the refs store's
// refBase does not on-demand fetch like its read path does); a checkpoint
// whose ref exists only remotely backfills to the fallback store.
// Note the refs store's backfill absence probe fetches a locally-missing ref
// on demand when a fetcher is wired (refBaseForBackfill), so a checkpoint
// whose ref exists only remotely is fetched and backfilled in place rather
// than falling through to the fallback store.
func (s *kindRoutingStore) Write(ctx context.Context, req WriteRequest) error {
checkpointID, isBackfill := backfillTarget(req)
if !isBackfill {
Expand Down
13 changes: 13 additions & 0 deletions cmd/entire/cli/settings/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -1407,6 +1407,19 @@ func (c *CheckpointRemoteConfig) Owner() string {
return parts[0]
}

// HasCheckpointRemoteKey reports whether a checkpoint_remote entry exists in
// strategy options at all — deliberately including malformed entries that
// GetCheckpointRemote rejects (it returns nil for absent AND malformed, so it
// cannot distinguish "no intent" from "botched intent"). Presence in any form
// means the user intends a checkpoint remote.
func (s *EntireSettings) HasCheckpointRemoteKey() bool {
if s.StrategyOptions == nil {
return false
}
_, ok := s.StrategyOptions["checkpoint_remote"]
return ok
}

// GetCheckpointRemote returns the configured checkpoint remote.
// Expects a structured object: {"provider": "github", "repo": "org/repo"}.
// Returns nil if not configured, wrong type, or missing required fields.
Expand Down
18 changes: 18 additions & 0 deletions cmd/entire/cli/settings/settings_checkpoint_remote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,21 @@ func TestCheckpointRemoteConfig_Owner(t *testing.T) {
})
}
}

func TestHasCheckpointRemoteKey(t *testing.T) {
t.Parallel()

assert.False(t, (&EntireSettings{}).HasCheckpointRemoteKey(), "nil strategy options")
assert.False(t, (&EntireSettings{StrategyOptions: map[string]any{}}).HasCheckpointRemoteKey(), "empty strategy options")
assert.True(t, (&EntireSettings{StrategyOptions: map[string]any{
"checkpoint_remote": map[string]any{"provider": "github", "repo": "org/repo"},
}}).HasCheckpointRemoteKey(), "well-formed entry")
// The reason this method exists: a malformed entry still counts as
// present even though GetCheckpointRemote rejects it.
assert.True(t, (&EntireSettings{StrategyOptions: map[string]any{
"checkpoint_remote": map[string]any{"provider": "github"},
}}).HasCheckpointRemoteKey(), "malformed entry still counts as present")
assert.True(t, (&EntireSettings{StrategyOptions: map[string]any{
"checkpoint_remote": nil,
}}).HasCheckpointRemoteKey(), "null entry still counts as present")
}
Loading