diff --git a/README.md b/README.md index 5d414e7..102f3e3 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,13 @@ sure-cli imports list --type TransactionImport sure-cli imports rows sure-cli imports create --file data.csv --date-col-label Date --amount-col-label Amount --name-col-label Name sure-cli imports create --file backup.ndjson --type SureImport --publish --apply + +# Chunked Sure NDJSON import (create -> add-chunk... -> publish) +sure-cli import-sessions create --expected-chunks 2 --apply +sure-cli import-sessions add-chunk --sequence 1 --file part-1.ndjson --apply +sure-cli import-sessions add-chunk --sequence 2 --raw-content "$(cat part-2.ndjson)" --apply +sure-cli import-sessions show +sure-cli import-sessions publish --apply sure-cli family-exports create sure-cli family-exports create --apply sure-cli family-exports download --out sure-export.zip diff --git a/cmd/sure-cli/root/import_sessions_cmd.go b/cmd/sure-cli/root/import_sessions_cmd.go new file mode 100644 index 0000000..b300a18 --- /dev/null +++ b/cmd/sure-cli/root/import_sessions_cmd.go @@ -0,0 +1,222 @@ +package root + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/go-resty/resty/v2" + "github.com/spf13/cobra" + "github.com/we-promise/sure-cli/internal/api" +) + +// newImportSessionsCmd wires the chunked Sure NDJSON import flow: +// +// create -> POST /api/v1/import_sessions +// show -> GET /api/v1/import_sessions/:id +// add-chunk -> POST /api/v1/import_sessions/:id/chunks +// publish -> POST /api/v1/import_sessions/:id/publish +// +// Unlike `imports` (CSV with column mapping), import_sessions uploads a Sure +// NDJSON export in one or more sequenced chunks and then publishes the whole +// session for processing. All params are top-level (the controller reads +// params[:type], params[:sequence], ... directly — no wrapping key). +func newImportSessionsCmd() *cobra.Command { + cmd := &cobra.Command{Use: "import-sessions", Short: "Import sessions (chunked Sure NDJSON import)"} + + cmd.AddCommand(&cobra.Command{ + Use: "show ", + Short: "Show import session", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + printGet(fmt.Sprintf("/api/v1/import_sessions/%s", url.PathEscape(args[0]))) + }, + }) + + cmd.AddCommand(newImportSessionsCreateCmd()) + cmd.AddCommand(newImportSessionsAddChunkCmd()) + cmd.AddCommand(newImportSessionsPublishCmd()) + + return cmd +} + +type importSessionCreateOpts struct { + Type string + ClientSessionID string + ExpectedChunks int + Apply bool +} + +// buildImportSessionCreatePayload builds the top-level create body. The +// controller does `params[:type].to_s` and treats blank as an empty import +// type, so we reject an empty type client-side for fast feedback. +func buildImportSessionCreatePayload(o importSessionCreateOpts) (map[string]any, error) { + t := strings.TrimSpace(o.Type) + if t == "" { + return nil, errors.New("type is required (e.g. SureImport)") + } + if o.ExpectedChunks < 0 { + return nil, fmt.Errorf("expected-chunks must be >= 0, got %d", o.ExpectedChunks) + } + body := map[string]any{"type": t} + if o.ClientSessionID != "" { + body["client_session_id"] = o.ClientSessionID + } + if o.ExpectedChunks > 0 { + body["expected_chunks"] = o.ExpectedChunks + } + return body, nil +} + +func newImportSessionsCreateCmd() *cobra.Command { + var o importSessionCreateOpts + cmd := &cobra.Command{ + Use: "create", + Short: "Create an import session (default dry-run; use --apply to execute)", + Run: func(cmd *cobra.Command, args []string) { + payload, err := buildImportSessionCreatePayload(o) + if err != nil { + failValidation(err) + } + dispatchWrite(o.Apply, "POST", "/api/v1/import_sessions", payload) + }, + } + cmd.Flags().StringVar(&o.Type, "type", "SureImport", "import type") + cmd.Flags().StringVar(&o.ClientSessionID, "client-session-id", "", "idempotency key; reuses an existing session for the same id") + cmd.Flags().IntVar(&o.ExpectedChunks, "expected-chunks", 0, "number of chunks you plan to upload (optional)") + cmd.Flags().BoolVar(&o.Apply, "apply", false, "execute the create (otherwise dry-run)") + return cmd +} + +type chunkCreateOpts struct { + Sequence int + ClientChunkID string + File string + RawContent string + Apply bool +} + +// chunkCreate is the resolved upload plan: either a multipart file part or a +// JSON body carrying raw_file_content. Exactly one is set. +type chunkCreate struct { + Multipart bool + // multipart fields (string values), used when Multipart is true + Fields map[string]string + FilePath string + // JSON body, used when Multipart is false (raw content path) + JSONBody map[string]any +} + +// buildChunkCreate validates and resolves a chunk upload. The controller +// requires `sequence` and exactly one content source — a Sure NDJSON `file` +// (multipart) or `raw_file_content` (string). Extension is checked +// client-side because the controller only accepts NDJSON/JSON content types. +func buildChunkCreate(o chunkCreateOpts) (chunkCreate, error) { + if o.Sequence < 1 { + return chunkCreate{}, fmt.Errorf("sequence is required and must be >= 1, got %d", o.Sequence) + } + if o.File == "" && o.RawContent == "" { + return chunkCreate{}, errors.New("file or raw-content is required") + } + if o.File != "" && o.RawContent != "" { + return chunkCreate{}, errors.New("provide only one of file or raw-content") + } + + if o.File != "" { + ext := strings.ToLower(filepath.Ext(o.File)) + if ext != ".ndjson" && ext != ".json" { + return chunkCreate{}, fmt.Errorf("file must be a Sure NDJSON file (.ndjson or .json), got %q", filepath.Ext(o.File)) + } + info, err := os.Stat(o.File) + if err != nil { + return chunkCreate{}, fmt.Errorf("file not accessible: %w", err) + } + if info.IsDir() { + return chunkCreate{}, errors.New("file must be a regular file") + } + fields := map[string]string{"sequence": strconv.Itoa(o.Sequence)} + if o.ClientChunkID != "" { + fields["client_chunk_id"] = o.ClientChunkID + } + return chunkCreate{Multipart: true, Fields: fields, FilePath: o.File}, nil + } + + body := map[string]any{ + "sequence": o.Sequence, + "raw_file_content": o.RawContent, + } + if o.ClientChunkID != "" { + body["client_chunk_id"] = o.ClientChunkID + } + return chunkCreate{Multipart: false, JSONBody: body}, nil +} + +func newImportSessionsAddChunkCmd() *cobra.Command { + var o chunkCreateOpts + cmd := &cobra.Command{ + Use: "add-chunk ", + Short: "Upload a chunk to an import session (default dry-run; use --apply to execute)", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + plan, err := buildChunkCreate(o) + if err != nil { + failValidation(err) + } + path := fmt.Sprintf("/api/v1/import_sessions/%s/chunks", url.PathEscape(args[0])) + + if !o.Apply { + printDryRun("POST", path, chunkDryRunBody(plan)) + return + } + + client := api.New() + var res any + var r *resty.Response + if plan.Multipart { + r, err = client.PostMultipart(path, plan.Fields, "file", plan.FilePath, mimeForImportFile(plan.FilePath), &res) + } else { + r, err = client.Post(path, plan.JSONBody, &res) + } + respond(r, err, res) + }, + } + cmd.Flags().IntVar(&o.Sequence, "sequence", 0, "chunk sequence number (required, >= 1)") + cmd.Flags().StringVar(&o.ClientChunkID, "client-chunk-id", "", "idempotency key for this chunk (optional)") + cmd.Flags().StringVar(&o.File, "file", "", "path to a Sure NDJSON file (.ndjson/.json)") + cmd.Flags().StringVar(&o.RawContent, "raw-content", "", "raw Sure NDJSON content (alternative to --file)") + cmd.Flags().BoolVar(&o.Apply, "apply", false, "execute the upload (otherwise dry-run)") + return cmd +} + +// chunkDryRunBody renders the planned request for a dry-run without leaking a +// whole NDJSON file into the envelope: the multipart file is shown as a path +// reference, raw content is passed through. +func chunkDryRunBody(plan chunkCreate) map[string]any { + if plan.Multipart { + body := map[string]any{"file": plan.FilePath} + for k, v := range plan.Fields { + body[k] = v + } + return body + } + return plan.JSONBody +} + +func newImportSessionsPublishCmd() *cobra.Command { + var apply bool + cmd := &cobra.Command{ + Use: "publish ", + Short: "Publish an import session for processing (default dry-run; use --apply to execute)", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + path := fmt.Sprintf("/api/v1/import_sessions/%s/publish", url.PathEscape(args[0])) + dispatchWrite(apply, "POST", path, nil) + }, + } + cmd.Flags().BoolVar(&apply, "apply", false, "execute the publish (otherwise dry-run)") + return cmd +} diff --git a/cmd/sure-cli/root/import_sessions_test.go b/cmd/sure-cli/root/import_sessions_test.go new file mode 100644 index 0000000..aa0baae --- /dev/null +++ b/cmd/sure-cli/root/import_sessions_test.go @@ -0,0 +1,149 @@ +package root + +import ( + "os" + "path/filepath" + "testing" +) + +func TestBuildImportSessionCreatePayload_RequiresType(t *testing.T) { + if _, err := buildImportSessionCreatePayload(importSessionCreateOpts{Type: " "}); err == nil { + t.Fatal("expected blank type to be rejected") + } +} + +func TestBuildImportSessionCreatePayload_Minimal(t *testing.T) { + body, err := buildImportSessionCreatePayload(importSessionCreateOpts{Type: "SureImport"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body["type"] != "SureImport" { + t.Fatalf("type = %v", body["type"]) + } + if _, has := body["client_session_id"]; has { + t.Fatal("client_session_id should be omitted when empty") + } + if _, has := body["expected_chunks"]; has { + t.Fatal("expected_chunks should be omitted when zero") + } +} + +func TestBuildImportSessionCreatePayload_OptionalFields(t *testing.T) { + body, err := buildImportSessionCreatePayload(importSessionCreateOpts{ + Type: "SureImport", + ClientSessionID: "sess-1", + ExpectedChunks: 3, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body["client_session_id"] != "sess-1" { + t.Fatalf("client_session_id = %v", body["client_session_id"]) + } + if body["expected_chunks"] != 3 { + t.Fatalf("expected_chunks = %v", body["expected_chunks"]) + } +} + +func TestBuildImportSessionCreatePayload_NegativeExpectedChunks(t *testing.T) { + if _, err := buildImportSessionCreatePayload(importSessionCreateOpts{Type: "SureImport", ExpectedChunks: -1}); err == nil { + t.Fatal("expected negative expected-chunks to be rejected") + } +} + +func TestBuildChunkCreate_RequiresSequence(t *testing.T) { + if _, err := buildChunkCreate(chunkCreateOpts{RawContent: "{}"}); err == nil { + t.Fatal("expected missing sequence to be rejected") + } + if _, err := buildChunkCreate(chunkCreateOpts{Sequence: 0, RawContent: "{}"}); err == nil { + t.Fatal("expected sequence 0 to be rejected") + } +} + +func TestBuildChunkCreate_RequiresExactlyOneContentSource(t *testing.T) { + if _, err := buildChunkCreate(chunkCreateOpts{Sequence: 1}); err == nil { + t.Fatal("expected missing content to be rejected") + } + if _, err := buildChunkCreate(chunkCreateOpts{Sequence: 1, File: "a.ndjson", RawContent: "{}"}); err == nil { + t.Fatal("expected both file and raw-content to be rejected") + } +} + +func TestBuildChunkCreate_RawContentBuildsJSONBody(t *testing.T) { + plan, err := buildChunkCreate(chunkCreateOpts{Sequence: 2, RawContent: `{"a":1}`, ClientChunkID: "c-2"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if plan.Multipart { + t.Fatal("raw content should not be multipart") + } + if plan.JSONBody["sequence"] != 2 { + t.Fatalf("sequence = %v", plan.JSONBody["sequence"]) + } + if plan.JSONBody["raw_file_content"] != `{"a":1}` { + t.Fatalf("raw_file_content = %v", plan.JSONBody["raw_file_content"]) + } + if plan.JSONBody["client_chunk_id"] != "c-2" { + t.Fatalf("client_chunk_id = %v", plan.JSONBody["client_chunk_id"]) + } +} + +func TestBuildChunkCreate_RejectsNonNDJSONExtension(t *testing.T) { + if _, err := buildChunkCreate(chunkCreateOpts{Sequence: 1, File: "data.csv"}); err == nil { + t.Fatal("expected .csv file to be rejected (NDJSON only)") + } +} + +func TestBuildChunkCreate_FileBuildsMultipart(t *testing.T) { + dir := t.TempDir() + fp := filepath.Join(dir, "chunk.ndjson") + if err := os.WriteFile(fp, []byte(`{"x":1}`+"\n"), 0o600); err != nil { + t.Fatalf("write temp file: %v", err) + } + plan, err := buildChunkCreate(chunkCreateOpts{Sequence: 1, File: fp}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !plan.Multipart { + t.Fatal("file path should be multipart") + } + if plan.FilePath != fp { + t.Fatalf("FilePath = %q", plan.FilePath) + } + if plan.Fields["sequence"] != "1" { + t.Fatalf("sequence field = %q", plan.Fields["sequence"]) + } +} + +func TestBuildChunkCreate_MissingFileIsRejected(t *testing.T) { + if _, err := buildChunkCreate(chunkCreateOpts{Sequence: 1, File: filepath.Join(t.TempDir(), "nope.ndjson")}); err == nil { + t.Fatal("expected missing file to be rejected") + } +} + +func TestImportSessionsCommandsRegistered(t *testing.T) { + root := New() + cases := []struct { + path []string + flags []string + }{ + {[]string{"import-sessions", "create"}, []string{"type", "client-session-id", "expected-chunks", "apply"}}, + {[]string{"import-sessions", "show"}, nil}, + {[]string{"import-sessions", "add-chunk"}, []string{"sequence", "client-chunk-id", "file", "raw-content", "apply"}}, + {[]string{"import-sessions", "publish"}, []string{"apply"}}, + } + for _, c := range cases { + got, _, err := root.Find(c.path) + if err != nil { + t.Fatalf("%v not registered: %v", c.path, err) + } + if got.Name() != c.path[len(c.path)-1] { + t.Fatalf("resolved to %q, want %q", got.Name(), c.path[len(c.path)-1]) + } + for _, f := range c.flags { + if got.Flags().Lookup(f) == nil { + t.Fatalf("%v missing --%s", c.path, f) + } + } + } +} diff --git a/cmd/sure-cli/root/root.go b/cmd/sure-cli/root/root.go index d9aff77..58a8f9a 100644 --- a/cmd/sure-cli/root/root.go +++ b/cmd/sure-cli/root/root.go @@ -47,6 +47,7 @@ func New() *cobra.Command { cmd.AddCommand(newTagsCmd()) cmd.AddCommand(newTransactionsCmd()) cmd.AddCommand(newImportsCmd()) + cmd.AddCommand(newImportSessionsCmd()) cmd.AddCommand(newFamilyExportsCmd()) cmd.AddCommand(newRulesCmd()) cmd.AddCommand(newRuleRunsCmd())