diff --git a/bin/test.sh b/bin/test.sh index 8ca5a3d2e..c015b6ddd 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -18,10 +18,15 @@ main() { set -e go test ./... "$@" - # commented while they don't work on github for now + + # tests_e2e is its own Go module, so `go test ./...` above never reaches it. It stays + # out of this script until it passes on Linux: the node writes root-owned files into + # the bind-mounted t.TempDir, which fails every container test's cleanup on a CI + # runner. See the fix/tests-e2e-on-linux branch. + # # pushd tests_e2e &> /dev/null # go test ./... "$@" - # popd + # popd &> /dev/null } usage_error() { diff --git a/cmd/substreams/init_test.go b/cmd/substreams/init_test.go index e29ed19a7..c8a9a2d98 100644 --- a/cmd/substreams/init_test.go +++ b/cmd/substreams/init_test.go @@ -3,9 +3,9 @@ package main import ( "testing" + pbconvo "github.com/streamingfast/substreams/pb/sf/codegen/conversation/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - pbconvo "github.com/streamingfast/substreams/pb/sf/codegen/conversation/v1" ) func TestProtocolOrderPreservation(t *testing.T) { diff --git a/cmd/substreams/sink_clickhouse.go b/cmd/substreams/sink_clickhouse.go index 4d13e6937..1086cde2f 100644 --- a/cmd/substreams/sink_clickhouse.go +++ b/cmd/substreams/sink_clickhouse.go @@ -17,19 +17,20 @@ var sinkClickhouseCmd = &cobra.Command{ } var sinkClickhouseSetupCmd = &cobra.Command{ - Use: "setup ", + Use: "setup []", Short: "Setup the required infrastructure to deploy a Substreams SQL deployable unit", Long: cli.Dedent(` Setup the database for the Substreams SQL sink, auto-detecting the mode from the output module type, exactly like the run action: - - DatabaseChanges output: creates the system tables (cursors, history) and applies - the 'schema.sql' bundled in the manifest sink config. - - Any other output type (from-proto): resolves the schema from the module's output - proto and creates the database schema and tables, then exits. This step is - idempotent and can be run again safely. + - Database Changes Mode ('DatabaseChanges' output): creates the system tables + (cursors, history) and applies the 'schema.sql' bundled in the manifest sink + config. + - Relational Mappings Mode (any other output type): resolves the schema from the + module's output proto and creates the database schema and tables, then exits. + This step is idempotent and can be run again safely. `), - Args: cobra.ExactArgs(1), + Args: cobra.RangeArgs(1, 2), RunE: newSinkSetupE(sinkClickhouseDriver), } @@ -39,15 +40,18 @@ func init() { addOperatorFlags(persistent) addSinkRunFlags(sinkClickhouseCmd.Flags(), sinkClickhouseDriver) + setModeGroupedUsage(sinkClickhouseCmd) setupFlags := sinkClickhouseSetupCmd.Flags() addCursorTableFlags(setupFlags) addClusterFlag(setupFlags) addOnModuleHashMismatchFlag(setupFlags) - setupFlags.Bool("system-tables-only", false, "[DatabaseChanges mode] will only create/update the systems tables (cursors, substreams_history) and ignore the schema from the manifest") - setupFlags.Bool("ignore-duplicate-table-errors", false, "[DatabaseChanges mode][Dev] Use this if you want to ignore duplicate table errors, take caution that this means the 'schema.sql' file will not have run fully!") + setupFlags.Bool("system-tables-only", false, "will only create/update the systems tables (cursors, substreams_history) and ignore the schema from the manifest") + setupFlags.Bool("ignore-duplicate-table-errors", false, "[Dev] Use this if you want to ignore duplicate table errors, take caution that this means the 'schema.sql' file will not have run fully!") addBytesEncodingFlag(setupFlags) - addFromProtoModeRunFlags(setupFlags, sinkClickhouseDriver) + addFromProtoSchemaFlags(setupFlags) + addConstraintTimingFlag(setupFlags) + addClickhouseStateFlags(setupFlags) sinkClickhouseCmd.AddCommand(sinkClickhouseSetupCmd) sinkClickhouseCmd.AddCommand(newSinkToolsCmd(sinkClickhouseDriver)) diff --git a/cmd/substreams/sink_postgres.go b/cmd/substreams/sink_postgres.go index dfec0f14f..7ec714e73 100644 --- a/cmd/substreams/sink_postgres.go +++ b/cmd/substreams/sink_postgres.go @@ -17,39 +17,110 @@ var sinkPostgresCmd = &cobra.Command{ } var sinkPostgresSetupCmd = &cobra.Command{ - Use: "setup ", + Use: "setup []", Short: "Setup the required infrastructure to deploy a Substreams SQL deployable unit", Long: cli.Dedent(` Setup the database for the Substreams SQL sink, auto-detecting the mode from the output module type, exactly like the run action: - - DatabaseChanges output: creates the system tables (cursors, history) and applies - the 'schema.sql' bundled in the manifest sink config. - - Any other output type (from-proto): resolves the schema from the module's output - proto and creates the database schema and tables, then exits. This step is - idempotent and can be run again safely. + - Database Changes Mode ('DatabaseChanges' output): creates the system tables + (cursors, history) and applies the 'schema.sql' bundled in the manifest sink + config. + - Relational Mappings Mode (any other output type): resolves the schema from the + module's output proto and creates the database schema and tables, then exits. + This step is idempotent and can be run again safely. `), - Args: cobra.ExactArgs(1), + Args: cobra.RangeArgs(1, 2), RunE: newSinkSetupE(sinkPostgresDriver), } +var sinkPostgresConstraintsCmd = &cobra.Command{ + Use: "constraints", + Short: "Create or drop the schema's constraints on an already loaded database", +} + +var sinkPostgresConstraintsApplyCmd = &cobra.Command{ + Use: "apply []", + Short: "Create the schema's constraints on an already loaded database", + Long: cli.Dedent(` + Create the primary keys, unique and foreign key constraints of a Relational + Mappings schema on a database the sink has already loaded, skipping the ones + already in place. + + The sink loads without them on purpose: measured through binary COPY, loading with + foreign keys in place runs 27x slower than loading without, where building the very + same constraints afterwards costs 3.3x. Creating them is a stop-the-world + operation, though — every index is built and every foreign key validated, with the + tables locked while it runs — so on a large database this belongs in a maintenance + window, which is what --apply-constraints=manual leaves it to this command for. + + The index on _block_number_ is not created here. The sink creates that one when it + starts, concurrently: this command is yours to schedule, and the reorg path cannot + wait for a maintenance window. + + Running it again is safe: constraints already in place are left alone. + + The module is inferred from the package when it is left out. A package with more + than one candidate has to be told which, or the schema this derives will not be the + one the run created. + `), + Args: cobra.RangeArgs(1, 2), + RunE: newSinkConstraintsE(sinkPostgresDriver, constraintsApply), +} + +var sinkPostgresConstraintsDropCmd = &cobra.Command{ + Use: "drop []", + Short: "Drop the schema's constraints", + Long: cli.Dedent(` + Drop the primary keys, unique and foreign key constraints of a Relational Mappings + schema, leaving anything the sink did not create alone — the index on _block_number_ + included, that one being the sink's own and recreated when it next starts. + + This is the escape hatch after --apply-constraints=always, and what makes a + backfill that has to be resumed fast again without setting the schema up afresh: + loading with foreign keys in place measured 27x slower than loading without them. + + Running it again is safe: anything already absent is skipped. + `), + Args: cobra.RangeArgs(1, 2), + RunE: newSinkConstraintsE(sinkPostgresDriver, constraintsDrop), +} + func init() { persistent := sinkPostgresCmd.PersistentFlags() addDSNFlag(persistent) addOperatorFlags(persistent) addSinkRunFlags(sinkPostgresCmd.Flags(), sinkPostgresDriver) + setModeGroupedUsage(sinkPostgresCmd) setupFlags := sinkPostgresSetupCmd.Flags() addCursorTableFlags(setupFlags) addOnModuleHashMismatchFlag(setupFlags) - setupFlags.Bool("postgraphile", false, "[DatabaseChanges mode] Will append the necessary 'comments' on cursors table to fully support postgraphile") - setupFlags.Bool("system-tables-only", false, "[DatabaseChanges mode] will only create/update the systems tables (cursors, substreams_history) and ignore the schema from the manifest") - setupFlags.Bool("ignore-duplicate-table-errors", false, "[DatabaseChanges mode][Dev] Use this if you want to ignore duplicate table errors, take caution that this means the 'schema.sql' file will not have run fully!") + setupFlags.Bool("postgraphile", false, "Will append the necessary 'comments' on cursors table to fully support postgraphile") + setupFlags.Bool("system-tables-only", false, "will only create/update the systems tables (cursors, substreams_history) and ignore the schema from the manifest") + setupFlags.Bool("ignore-duplicate-table-errors", false, "[Dev] Use this if you want to ignore duplicate table errors, take caution that this means the 'schema.sql' file will not have run fully!") addBytesEncodingFlag(setupFlags) - addFromProtoModeRunFlags(setupFlags, sinkPostgresDriver) + addFromProtoSchemaFlags(setupFlags) + addConstraintTimingFlag(setupFlags) + + applyFlags := sinkPostgresConstraintsApplyCmd.Flags() + addBytesEncodingFlag(applyFlags) + addFromProtoSchemaFlags(applyFlags) + addConstraintPassFlags(applyFlags) + + // Drop always removes every constraint managed by the sink. The disable-* and + // --no-constraints flags describe what should be created, so exposing them here would + // suggest that drop honors a policy it cannot apply. + dropFlags := sinkPostgresConstraintsDropCmd.Flags() + addBytesEncodingFlag(dropFlags) + dropFlags.String("proto-file-override", "", "Override protobuf file to use instead of extracting from substreams package") + addConstraintPassFlags(dropFlags) + sinkPostgresConstraintsCmd.AddCommand(sinkPostgresConstraintsApplyCmd) + sinkPostgresConstraintsCmd.AddCommand(sinkPostgresConstraintsDropCmd) sinkPostgresCmd.AddCommand(sinkPostgresSetupCmd) + sinkPostgresCmd.AddCommand(sinkPostgresConstraintsCmd) sinkPostgresCmd.AddCommand(newSinkToolsCmd(sinkPostgresDriver)) SinkCmd.AddCommand(sinkPostgresCmd) diff --git a/cmd/substreams/sink_postgres_generate_csv.go b/cmd/substreams/sink_postgres_generate_csv.go index 6d843e4bf..6e07c5916 100644 --- a/cmd/substreams/sink_postgres_generate_csv.go +++ b/cmd/substreams/sink_postgres_generate_csv.go @@ -104,7 +104,7 @@ func sinkPostgresGenerateCSVE(cmd *cobra.Command, args []string) error { supportedOutputTypes, manifestPath, outputModule, - "sink_database_changes", + sinkUserAgent("sink_database_changes", sinkPostgresDriver), zlog, tracer, ) diff --git a/cmd/substreams/sink_sql_common.go b/cmd/substreams/sink_sql_common.go index 07922f7ec..beccdf1e3 100644 --- a/cmd/substreams/sink_sql_common.go +++ b/cmd/substreams/sink_sql_common.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/desc/protoparse" "github.com/spf13/cobra" @@ -24,6 +25,8 @@ import ( "github.com/streamingfast/substreams/sink/sql/db_changes/sinker" "github.com/streamingfast/substreams/sink/sql/db_proto" "github.com/streamingfast/substreams/sink/sql/db_proto/proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" "github.com/streamingfast/substreams/sink/sql/services" "go.uber.org/zap" "google.golang.org/protobuf/types/descriptorpb" @@ -125,6 +128,63 @@ func boolFlag(cmd *cobra.Command, name string) bool { return false } +// stringSliceFlag mirrors boolFlag for the per-table constraint switches, which are only +// registered for the from-proto commands. +func stringFlag(cmd *cobra.Command, name string) string { + if sflags.FlagDefined(cmd, name) { + return sflags.MustGetString(cmd, name) + } + + return "" +} + +// intFlag returns the value of an int flag, or zero when the flag is not registered on +// the command. +func intFlag(cmd *cobra.Command, name string) int { + if sflags.FlagDefined(cmd, name) { + return sflags.MustGetInt(cmd, name) + } + + return 0 +} + +func stringSliceFlag(cmd *cobra.Command, name string) []string { + if sflags.FlagDefined(cmd, name) { + return sflags.MustGetStringSlice(cmd, name) + } + + return nil +} + +// sinkManifestAndModule reads the positional arguments the schema-side commands take. +// +// The module is optional and inferred from the package when it is left out, exactly as the +// run command does — but a package with more than one candidate has to be told which, or +// `setup` and `constraints` would derive a schema from a different module than the run +// they are meant to accompany. +func sinkManifestAndModule(args []string) (manifestPath string, outputModule string) { + if len(args) > 1 && args[1] != "" { + return args[0], args[1] + } + + return args[0], sink.InferOutputModuleFromPackage +} + +// sinkUserAgent is what the server sees as the gRPC User-Agent of a sink run. It names the +// mode, which decides how rows are written, and the engine they are written to: both +// engines share every command here, so the mode alone said nothing about where the data +// went. +func sinkUserAgent(mode, driver string) string { + switch driver { + case sinkClickhouseDriver: + return mode + "_ch" + case sinkPostgresDriver: + return mode + "_pg" + default: + return mode + } +} + func isDatabaseChangesType(outputType string) bool { unprefixed := strings.TrimPrefix(outputType, "proto:") for _, t := range strings.Split(supportedOutputTypes, ",") { @@ -138,43 +198,115 @@ func isDatabaseChangesType(outputType string) bool { // addDatabaseChangesModeRunFlags registers the run flags that only apply when the // selected module outputs DatabaseChanges. func addDatabaseChangesModeRunFlags(flags *pflag.FlagSet) { - flags.Int("undo-buffer-size", 0, "[DatabaseChanges mode] If non-zero, handling of reorgs in the database is disabled. Instead, a buffer is introduced to only process blocks once they have been confirmed by that many blocks, introducing a latency but slightly reducing the load on the database when close to head. Set to 0 to enable reorg handling in the database (required for some databases like Postgres).") - flags.Int("batch-block-flush-interval", 1_000, "[DatabaseChanges mode] When in catch up mode, flush every N blocks or after batch-row-flush-interval, whichever comes first. Set to 0 to disable and only use batch-row-flush-interval. Ineffective if the sink is now in the live portion of the chain where only 'live-block-flush-interval' applies.") - flags.Int("batch-row-flush-interval", 100_000, "[DatabaseChanges mode] When in catch up mode, flush every N rows or after batch-block-flush-interval, whichever comes first. Set to 0 to disable and only use batch-block-flush-interval. Ineffective if the sink is now in the live portion of the chain where only 'live-block-flush-interval' applies.") - flags.Int("live-block-flush-interval", 1, "[DatabaseChanges mode] When processing in live mode, flush every N blocks.") - flags.Int("flush-retry-count", 3, "[DatabaseChanges mode] Number of retry attempts for flush operations") - flags.Duration("flush-retry-delay", 1*time.Second, "[DatabaseChanges mode] Base delay for incremental retry backoff on flush failures") - flags.String("cursors-table", "cursors", "[DatabaseChanges mode] Name of the table to use for storing cursors") - flags.String("history-table", "substreams_history", "[DatabaseChanges mode] Name of the table to use for storing block history, used to handle reorgs") - flags.String(onModuleHashMismatchFlag, "error", "[DatabaseChanges mode] What to do when the module hash in the manifest does not match the one in the database, can be 'error', 'warn' or 'ignore'") -} - -// addFromProtoModeRunFlags registers the run flags that only apply when the selected -// module outputs an arbitrary protobuf message (relational mappings). The ClickHouse -// specific flags are only registered for the clickhouse engine. -func addFromProtoModeRunFlags(flags *pflag.FlagSet, driver string) { - flags.Bool("no-constraints", false, "[from-proto mode] Do not add any constraints to the database. This is useful to speed up the initial import of a large dataset.") - flags.Int("block-batch-size", 25, "[from-proto mode] number of blocks to process at a time") - flags.String("proto-file-override", "", "[from-proto mode] Override protobuf file to use instead of extracting from substreams package") + flags.Int("undo-buffer-size", 0, "If non-zero, handling of reorgs in the database is disabled. Instead, a buffer is introduced to only process blocks once they have been confirmed by that many blocks, introducing a latency but slightly reducing the load on the database when close to head. Set to 0 to enable reorg handling in the database (required for some databases like Postgres).") + flags.Int("batch-block-flush-interval", 1_000, "When in catch up mode, flush every N blocks or after batch-row-flush-interval, whichever comes first. Set to 0 to disable and only use batch-row-flush-interval. Ineffective if the sink is now in the live portion of the chain where only 'live-block-flush-interval' applies.") + flags.Int("batch-row-flush-interval", 100_000, "When in catch up mode, flush every N rows or after batch-block-flush-interval, whichever comes first. Set to 0 to disable and only use batch-block-flush-interval. Ineffective if the sink is now in the live portion of the chain where only 'live-block-flush-interval' applies.") + flags.Int("live-block-flush-interval", 1, "When processing in live mode, flush every N blocks.") + flags.Int("flush-retry-count", 3, "Number of retry attempts for flush operations") + flags.Duration("flush-retry-delay", 1*time.Second, "Base delay for incremental retry backoff on flush failures") + flags.String("cursors-table", "cursors", "Name of the table to use for storing cursors") + flags.String("history-table", "substreams_history", "Name of the table to use for storing block history, used to handle reorgs") + flags.String(onModuleHashMismatchFlag, "error", "What to do when the module hash in the manifest does not match the one in the database, can be 'error', 'warn' or 'ignore'") +} + +// defaultSpoolDir is where the from-proto sink spools rows when nothing else is asked +// for: a data folder under the working directory, as the other sink commands use for +// their own local state. +const defaultSpoolDir = "./localdata/spool" + +// fromProtoRunFlagNames are the from-proto flags that only mean something while the sink +// is running. Registering them anywhere else puts knobs on commands that never decode a +// block or write a row. +var fromProtoRunFlagNames = []string{ + "apply-constraints", + "write-mode", + "decode-workers", + "decode-batch-size", + "db-write-target-duration", + "db-write-max-size", + "spool-dir", + "spool-max-size", + "spool-max-idle", + "block-batch-size", + "sink-info-folder", + "cursor-file-path", + "query-retry-count", + "query-retry-sleep", +} + +// databaseChangesFlagNames are the run flags that only mean something when the module +// outputs DatabaseChanges. +var databaseChangesFlagNames = []string{ + "undo-buffer-size", + "batch-block-flush-interval", + "batch-row-flush-interval", + "live-block-flush-interval", + "flush-retry-count", + "flush-retry-delay", + "cursors-table", + "history-table", +} + +// addFromProtoSchemaFlags registers the from-proto flags that describe the schema. They +// go on the run command, on `setup` and on `constraints`, which all have to agree on +// which constraints the schema is meant to have. See sink_sql_constraints.go. + +// addFromProtoRunFlags registers the from-proto flags that only apply while the sink is +// running. The ClickHouse specific ones are only registered for the clickhouse engine. +func addFromProtoRunFlags(flags *pflag.FlagSet, driver string) { + addConstraintTimingFlag(flags) + + flags.String("write-mode", string(protosql.WriteModeAuto), "How a sealed spool segment reaches the database: 'copy' loads each table file with binary COPY, 'batch-insert' builds one multi-row INSERT per table, 'row-insert' issues one prepared INSERT per row. 'auto' picks copy on PostgreSQL, batch-insert on ClickHouse, and row-insert for a schema whose foreign keys cannot be ordered. An explicit mode the driver or the schema cannot support is an error rather than a downgrade. Ignored once the stream reaches chain HEAD, where the sink always inserts directly.") + + flags.Int("decode-workers", 0, "How many blocks are unmarshalled and walked concurrently. Zero takes one per core less one for the goroutine draining the stream, capped at 8: measured at 4.13x on eight workers and only 4.24x on fifteen, the work being allocator-bound well before it runs out of cores. This is CPU work only and does not change what the database sees.") + flags.Int("decode-batch-size", 0, "How many blocks are held in memory and decoded together. Zero takes four per decode worker. Larger batches keep the workers fed; smaller ones cost less memory, since every held block keeps its payload and its decoded rows. This sizes the CPU stage, not the database write — see --db-write-target-duration for that.") + + flags.Duration("db-write-target-duration", 3*time.Second, "How long one commit to the database should take. A commit is one spooled segment, whichever write mode applies it; each is measured and the next segment sized toward this. Raise it for fewer, larger commits; lower it to keep the sink from occupying a database that is shared with something else. Sizing by measured duration rather than by a block count is what keeps this stable across chains, where block payloads differ by orders of magnitude. Backfill only.") + flags.String("db-write-max-size", "512MiB", "Ceiling for the segment size the sizer may choose, whatever the target duration would allow. Backfill only.") + + flags.Int("block-batch-size", 25, "Deprecated, use --decode-batch-size.") + _ = flags.MarkDeprecated("block-batch-size", "use --decode-batch-size") + + flags.String("spool-dir", defaultSpoolDir, "Directory the pending segments are written to. Rows land here first and a background goroutine loads them, so the stream never waits on the database and blocks already downloaded survive a restart instead of being streamed, and paid for, twice. Backfill only.") + flags.String("spool-max-size", "8GiB", "Disk budget for --spool-dir, and the only bound on how far ahead of the database the stream may run. The stream is held once pending segments reach it, which is what turns a slow database into backpressure rather than a full disk. Backfill only.") + flags.Duration("spool-max-idle", 10*time.Second, "Write the open segment to the database once no new row has reached it for this long, short of its size target. A stream that stalls would otherwise sit on those rows indefinitely, leaving the cursor where it was and the blocks to be streamed, and paid for, again on restart. Zero disables it. Backfill only.") if driver == "clickhouse" { - flags.String("sink-info-folder", "", "[from-proto mode] folder where to store the clickhouse sink info") - flags.String("cursor-file-path", "cursor.txt", "[from-proto mode] file name where to store the clickhouse cursor") - flags.Int("query-retry-count", 3, "[from-proto mode] Number of retries for ClickHouse queries when an error occurs") - flags.Duration("query-retry-sleep", time.Second, "[from-proto mode] Sleep duration between ClickHouse query retries (e.g. 1s, 500ms)") + addClickhouseStateFlags(flags) } } +// addClickhouseStateFlags registers where the ClickHouse sink keeps the state PostgreSQL +// keeps in the database itself. `setup` needs them as much as the run does: it reads the +// schema hash from that folder to decide whether the database is already set up. +func addClickhouseStateFlags(flags *pflag.FlagSet) { + flags.String("sink-info-folder", "", "Folder where to store the clickhouse sink info") + flags.String("cursor-file-path", "cursor.txt", "File name where to store the clickhouse cursor") + flags.Int("query-retry-count", 3, "Number of retries for ClickHouse queries when an error occurs") + flags.Duration("query-retry-sleep", time.Second, "Sleep duration between ClickHouse query retries (e.g. 1s, 500ms)") +} + // addSinkRunFlags registers the full set of flags used by the sink process of an // engine command. +// +// Both mode vocabularies land on the same command, and that is not fixable: the mode is +// read from the module's output type at run time, long after init() has registered +// everything. What the run command can do is say which half applies — see +// setModeGroupedUsage — and fail on a flag typed for the other one. func addSinkRunFlags(flags *pflag.FlagSet, driver string) { - sink.AddFlagsToSet(flags, sink.FlagExcludeDefault(sink.FlagUndoBufferSize)) + // --live-block-time-delta is excluded because this sink overrides the liveness checker + // with the cursor-based one (see runFromProtoSink): the flag would be accepted and then + // silently ignored. It is also what the spool's safety rests on — liveness has to turn + // on the first undo-able block, not on a wall-clock guess. The database-changes sink + // does not look at liveness at all. + sink.AddFlagsToSet(flags, sink.FlagExcludeDefault(sink.FlagUndoBufferSize, sink.FlagLiveBlockTimeDelta)) addBytesEncodingFlag(flags) if driver == "clickhouse" { addClusterFlag(flags) } addDatabaseChangesModeRunFlags(flags) - addFromProtoModeRunFlags(flags, driver) + addFromProtoSchemaFlags(flags) + addFromProtoRunFlags(flags, driver) } // sinkBytesEncoding resolves the --bytes-encoding flag into a concrete encoding. @@ -240,14 +372,22 @@ func newSinkRunE(driver string) func(*cobra.Command, []string) error { } if isDatabaseChangesType(module.Output.Type) { - return runDatabaseChangesSink(cmd, manifestPath, outputModule, dsnString) + if err := rejectFromProtoFlags(cmd); err != nil { + return err + } + + return runDatabaseChangesSink(cmd, driver, manifestPath, outputModule, dsnString) + } + + if err := rejectDatabaseChangesFlags(cmd); err != nil { + return err } return runFromProtoSink(cmd, driver, manifestPath, dsnString, spkg, module.Name) } } -func runDatabaseChangesSink(cmd *cobra.Command, manifestPath, outputModule, dsnString string) error { +func runDatabaseChangesSink(cmd *cobra.Command, driver, manifestPath, outputModule, dsnString string) error { sinkDBPreStart(cmd) app := cli.NewApplication(cmd.Context()) @@ -259,7 +399,7 @@ func runDatabaseChangesSink(cmd *cobra.Command, manifestPath, outputModule, dsnS supportedOutputTypes, manifestPath, outputModule, - "sink_database_changes", + sinkUserAgent("sink_dbchanges", driver), zlog, tracer, ) @@ -291,8 +431,15 @@ func runDatabaseChangesSink(cmd *cobra.Command, manifestPath, outputModule, dsnS } func runFromProtoSink(cmd *cobra.Command, driver, manifestPath, dsnString string, spkg *pbsubstreams.Package, outputModuleName string) error { - useConstraints := !sflags.MustGetBool(cmd, "no-constraints") - blockBatchSize := sflags.MustGetInt(cmd, "block-batch-size") + constraints, err := fromProtoConstraintPolicy(cmd) + if err != nil { + return err + } + + writeMode, err := protosql.ParseWriteMode(sflags.MustGetString(cmd, "write-mode")) + if err != nil { + return err + } encoding, err := sinkBytesEncoding(cmd) if err != nil { @@ -320,15 +467,22 @@ func runFromProtoSink(cmd *cobra.Command, driver, manifestPath, dsnString string return err } if !useProtoOption { - useConstraints = false + // Without the schema annotations there are no relations to constrain. The index on + // _block_number_ is not declared by them either, so it stays. + constraints = protosql.DisableAllConstraints().WithBlockNumberIndex(constraints) + + zlog.Warn("the module's output carries no schema.proto annotations, so its tables get no primary keys, no unique constraints and no foreign keys. " + + "Queries against them are sequential scans and duplicate ids are not rejected. " + + "`substreams tools extract-proto --sql` writes the annotated proto to start from, and --proto-file-override points the sink at it") } + warnAboutConstraints(constraints) baseSink, err := sink.NewFromViper( cmd, outputType, manifestPath, outputModuleName, - "sink_from_proto", + sinkUserAgent("sink_relational", driver), zlog, tracer, ) @@ -336,14 +490,32 @@ func runFromProtoSink(cmd *cobra.Command, driver, manifestPath, dsnString string return fmt.Errorf("new base sinker: %w", err) } - factory := db_proto.SinkerFactory(baseSink, outputModuleName, rootMessageDescriptor.UnwrapMessage(), db_proto.SinkerFactoryOptions{ + // Nothing told the sinker when the stream reached the chain head, so nothing could + // act on it — the local buffer kept holding blocks that a live sink wants in the + // database as they arrive. The cursor-based checker turns live as soon as an + // undo-able block shows up, which is exactly that moment. + sink.WithLivenessChecker(sink.NewCursorBasedLivenessChecker())(baseSink) + + spool, err := fromProtoSpoolOptions(cmd) + if err != nil { + return err + } + + options := db_proto.SinkerFactoryOptions{ + Spool: spool, UseProtoOption: useProtoOption, - UseConstraints: useConstraints, + Constraints: constraints, UseTransactions: true, - BlockBatchSize: blockBatchSize, + WriteMode: writeMode, + DecodeWorkers: sflags.MustGetInt(cmd, "decode-workers"), + DecodeBatchSize: fromProtoDecodeBatchSize(cmd), Encoding: encoding, Clickhouse: fromProtoClickhouseOptions(cmd, driver), - }) + }.Defaults() + + logFromProtoSettings(cmd, driver, options, constraints) + + factory := db_proto.SinkerFactory(baseSink, outputModuleName, rootMessageDescriptor.UnwrapMessage(), options) sqlSinker, err := factory(cmd.Context(), dsnString, dsn.Schema(), zlog, tracer) if err != nil { @@ -453,25 +625,79 @@ func newSinkSetupE(driver string) func(*cobra.Command, []string) error { return err } - manifestPath := args[0] + manifestPath, outputModule := sinkManifestAndModule(args) sinkDBPreStart(cmd) sink.LoadSubstreamsAuthEnvFile(manifestPath) - spkg, module, _, err := sink.ReadManifestAndModule(manifestPath, "", nil, sink.InferOutputModuleFromPackage, sink.IgnoreOutputModuleType, false, nil, zlog) + spkg, module, _, err := sink.ReadManifestAndModule(manifestPath, "", nil, outputModule, sink.IgnoreOutputModuleType, false, nil, zlog) if err != nil { return fmt.Errorf("reading manifest: %w", err) } if isDatabaseChangesType(module.Output.Type) { + names := append(append([]string{}, fromProtoSchemaFlagNames...), "apply-constraints") + if err := rejectFlags(cmd, names, "Relational Mappings Mode", + "This one runs in Database Changes Mode, where the SQL schema is yours: it is created from the "+ + "'schema.sql' bundled in the manifest, and the sink neither derives it nor manages its constraints"); err != nil { + return err + } + return runDatabaseChangesSetup(cmd, dsnString, spkg) } + if err := rejectDatabaseChangesSetupFlags(cmd); err != nil { + return err + } + return runFromProtoSetup(cmd, driver, dsnString, spkg, module.Name) } } +// newSinkApplyConstraintsE creates the schema's constraints on a database the sink has +// already loaded, which is deliberately a separate command. See sink_sql_constraints.go +// for newSinkConstraintsE, the constraintsAction type and its apply/drop constants. + +// rejectFromProtoFlags fails on a from-proto flag typed for a Substreams that outputs +// DatabaseChanges. +// +// The mode is read from the module, not chosen by a flag, so both vocabularies are +// registered on the same command and half of them are inert for any given run. They used +// to be inert silently; a flag typed here means the operator expects the sink to be doing +// something it will not do, and a warning in a log that scrolls past is not how they +// should find that out. +func rejectFromProtoFlags(cmd *cobra.Command) error { + names := append(append([]string{}, fromProtoSchemaFlagNames...), fromProtoRunFlagNames...) + + return rejectFlags(cmd, names, "Relational Mappings Mode", + "This one runs in Database Changes Mode, where rows are written from the module's own database changes "+ + "and the schema is the 'schema.sql' bundled in the manifest") +} + +// rejectDatabaseChangesFlags is the same guard in the other direction. +func rejectDatabaseChangesFlags(cmd *cobra.Command) error { + return rejectFlags(cmd, databaseChangesFlagNames, "Database Changes Mode", + "This one outputs an arbitrary protobuf message, which the sink maps to relational tables of its own") +} + +func rejectFlags(cmd *cobra.Command, names []string, appliesTo string, because string) error { + for _, name := range names { + if flagChanged(cmd, name) { + return fmt.Errorf("--%s only applies to %s. %s", name, appliesTo, because) + } + } + + return nil +} + +// errDatabaseChangesOwnsSchema is what every from-proto-only entry point says when the +// module turns out to output DatabaseChanges. +func errDatabaseChangesOwnsSchema(what string) error { + return fmt.Errorf("%s only applies to Relational Mappings Mode. This one runs in Database Changes Mode, where the SQL schema is yours: "+ + "it is created from the 'schema.sql' bundled in the manifest, and the sink neither derives it nor manages its constraints", what) +} + func runDatabaseChangesSetup(cmd *cobra.Command, dsnString string, spkg *pbsubstreams.Package) error { options := sinker.SinkerSetupOptions{ CursorTableName: sflags.MustGetString(cmd, "cursors-table"), @@ -491,9 +717,10 @@ func runDatabaseChangesSetup(cmd *cobra.Command, dsnString string, spkg *pbsubst // schema exists via db_proto.SetupDatabaseSchema, then exits without starting a sinker. It // is idempotent thanks to the sink-info guard inside SetupDatabaseSchema. func runFromProtoSetup(cmd *cobra.Command, driver, dsnString string, spkg *pbsubstreams.Package, outputModuleName string) error { - warnIgnoredDatabaseChangesSetupFlags(cmd) - - useConstraints := !boolFlag(cmd, "no-constraints") + constraints, err := fromProtoConstraintPolicy(cmd) + if err != nil { + return err + } encoding, err := sinkBytesEncoding(cmd) if err != nil { @@ -511,32 +738,43 @@ func runFromProtoSetup(cmd *cobra.Command, driver, dsnString string, spkg *pbsub return err } if !useProtoOption { - useConstraints = false + // Without the schema annotations there are no relations to constrain. The index on + // _block_number_ is not declared by them either, so it stays. + constraints = protosql.DisableAllConstraints().WithBlockNumberIndex(constraints) + + zlog.Warn("the module's output carries no schema.proto annotations, so its tables get no primary keys, no unique constraints and no foreign keys. " + + "Queries against them are sequential scans and duplicate ids are not rejected. " + + "`substreams tools extract-proto --sql` writes the annotated proto to start from, and --proto-file-override points the sink at it") } + warnAboutConstraints(constraints) options := db_proto.SinkerFactoryOptions{ UseProtoOption: useProtoOption, - UseConstraints: useConstraints, + Constraints: constraints, Encoding: encoding, Clickhouse: fromProtoClickhouseOptions(cmd, driver), } - if _, err := db_proto.SetupDatabaseSchema(cmd.Context(), dsnString, dsn.Schema(), outputModuleName, rootMessageDescriptor.UnwrapMessage(), options, zlog, tracer); err != nil { + database, err := db_proto.SetupDatabaseSchema(cmd.Context(), dsnString, dsn.Schema(), outputModuleName, rootMessageDescriptor.UnwrapMessage(), options, zlog, tracer) + if err != nil { return fmt.Errorf("setting up database schema: %w", err) } + defer database.Close(cmd.Context()) - zlog.Info("database schema setup completed", zap.String("schema", dsn.Schema()), zap.Bool("constraints", useConstraints)) + zlog.Info("database schema setup completed", zap.String("schema", dsn.Schema()), zap.String("constraints", constraints.Describe())) return nil } // warnIgnoredDatabaseChangesSetupFlags logs a warning for each DatabaseChanges-only setup // flag that the user explicitly set, since those flags have no effect in from-proto mode. -func warnIgnoredDatabaseChangesSetupFlags(cmd *cobra.Command) { - for _, name := range []string{"postgraphile", "system-tables-only", "ignore-duplicate-table-errors"} { - if flag := cmd.Flags().Lookup(name); flag != nil && flag.Changed { - zlog.Warn("flag has no effect in from-proto setup mode and is ignored", zap.String("flag", name)) - } - } +// rejectDatabaseChangesSetupFlags fails on a DatabaseChanges-only setup flag typed for a +// from-proto setup. It used to warn; the two directions now agree, because a flag typed +// for the wrong mode means the operator expects something that will not happen. +func rejectDatabaseChangesSetupFlags(cmd *cobra.Command) error { + names := append([]string{"postgraphile", "system-tables-only", "ignore-duplicate-table-errors"}, databaseChangesFlagNames...) + + return rejectFlags(cmd, names, "Database Changes Mode", + "This one outputs an arbitrary protobuf message, whose schema the sink derives from the module's own descriptors") } // newSinkToolsCmd builds the `tools` command subtree for the given engine. The @@ -767,3 +1005,98 @@ func cursorToShortString(in *sink.Cursor) string { return cursor } + +// fromProtoDecodeBatchSize resolves how many blocks are decoded together, honouring the +// flag --block-batch-size shipped under before it named the database write it no longer +// sizes. +func fromProtoDecodeBatchSize(cmd *cobra.Command) int { + if flagChanged(cmd, "decode-batch-size") { + return sflags.MustGetInt(cmd, "decode-batch-size") + } + if flagChanged(cmd, "block-batch-size") { + return sflags.MustGetInt(cmd, "block-batch-size") + } + + return 0 +} + +// fromProtoSpoolOptions resolves the --spool-* and --db-write-* flags. +func fromProtoSpoolOptions(cmd *cobra.Command) (*spool.Options, error) { + dir := sflags.MustGetString(cmd, "spool-dir") + if dir == "" { + return nil, fmt.Errorf("--spool-dir cannot be empty; the spool is what makes a restart resume rather than re-stream") + } + + maxBytes, err := parseByteSize(cmd, "spool-max-size") + if err != nil { + return nil, err + } + + segmentMaxBytes, err := parseByteSize(cmd, "db-write-max-size") + if err != nil { + return nil, err + } + + maxIdle := sflags.MustGetDuration(cmd, "spool-max-idle") + if maxIdle == 0 { + // Zero disables it, which the Options zero value cannot say on its own. + maxIdle = -1 + } + + return &spool.Options{ + Dir: dir, + MaxBytes: maxBytes, + WriteTargetDuration: sflags.MustGetDuration(cmd, "db-write-target-duration"), + // A segment is one indivisible database write, so keep its configured ceiling + // within the total spool budget rather than allowing an invalid combination to + // deadlock when the segment is sealed. + SegmentMaxBytes: min(segmentMaxBytes, maxBytes), + MaxIdle: maxIdle, + }, nil +} + +func parseByteSize(cmd *cobra.Command, name string) (int64, error) { + raw := sflags.MustGetString(cmd, name) + parsed, err := humanize.ParseBytes(raw) + if err != nil { + return 0, fmt.Errorf("invalid --%s %q: %w", name, raw, err) + } + + return int64(parsed), nil +} + +// flagChanged reports whether the operator actually typed the flag, as opposed to it +// carrying its default. Every mode and deprecation check keys off this: a default value +// must never be read as an instruction. +func flagChanged(cmd *cobra.Command, name string) bool { + flag := cmd.Flags().Lookup(name) + + return flag != nil && flag.Changed +} + +// logFromProtoSettings states, in one line, every knob that applies to this run. +// +// The alternative is an operator reading a help screen that lists both mode vocabularies +// and guessing which half took effect. The write mode here is what was asked for; the +// database logs what it resolved to, and why, once it has seen the schema. +func logFromProtoSettings(cmd *cobra.Command, driver string, options db_proto.SinkerFactoryOptions, constraints protosql.ConstraintPolicy) { + fields := []zap.Field{ + zap.String("driver", driver), + zap.String("write_mode_requested", string(options.WriteMode)), + zap.Int("decode_workers", options.DecodeWorkers), + zap.Int("decode_batch_size", options.DecodeBatchSize), + zap.String("constraints", constraints.Describe()), + } + + if options.Spool != nil { + fields = append(fields, + zap.String("spool_dir", options.Spool.Dir), + zap.String("spool_max_size", humanize.IBytes(uint64(options.Spool.MaxBytes))), + zap.Duration("spool_max_idle", options.Spool.MaxIdle), + zap.Duration("db_write_target_duration", options.Spool.WriteTargetDuration), + zap.String("db_write_max_size", humanize.IBytes(uint64(options.Spool.SegmentMaxBytes))), + ) + } + + zlog.Info("Relational Mappings Mode sink settings", fields...) +} diff --git a/cmd/substreams/sink_sql_common_test.go b/cmd/substreams/sink_sql_common_test.go new file mode 100644 index 000000000..6ceef86c9 --- /dev/null +++ b/cmd/substreams/sink_sql_common_test.go @@ -0,0 +1,17 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestSinkUserAgent pins what the server sees: the two engines run the same commands, so +// the mode alone did not say where the rows went. +func TestSinkUserAgent(t *testing.T) { + assert.Equal(t, "sink_relational_ch", sinkUserAgent("sink_relational", sinkClickhouseDriver)) + assert.Equal(t, "sink_relational_pg", sinkUserAgent("sink_relational", sinkPostgresDriver)) + assert.Equal(t, "sink_dbchanges_ch", sinkUserAgent("sink_dbchanges", sinkClickhouseDriver)) + assert.Equal(t, "sink_dbchanges_pg", sinkUserAgent("sink_dbchanges", sinkPostgresDriver)) + assert.Equal(t, "sink_dbchanges", sinkUserAgent("sink_dbchanges", "duckdb")) +} diff --git a/cmd/substreams/sink_sql_constraints.go b/cmd/substreams/sink_sql_constraints.go new file mode 100644 index 000000000..5adbfdc6f --- /dev/null +++ b/cmd/substreams/sink_sql_constraints.go @@ -0,0 +1,254 @@ +package main + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/streamingfast/cli/sflags" + "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_changes/db" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "go.uber.org/zap" +) + +// fromProtoSchemaFlagNames are the from-proto flags that describe the schema itself, as +// opposed to how the sink writes to it. They are the ones `setup` and `constraints` need +// too: all three commands have to agree on which constraints the schema is meant to have. +var fromProtoSchemaFlagNames = []string{ + "disable-foreign-keys", + "disable-primary-keys", + "disable-unique-constraints", + "disable-block-number-index", + "no-constraints", + "proto-file-override", +} + +// addConstraintPassFlags registers what governs how the constraint pass runs, as opposed +// to which constraints it creates. Only the `constraints` commands carry it: the run +// creates them one at a time, which is the safe shape, and anything more deliberate is +// what the command is for. +func addConstraintPassFlags(flags *pflag.FlagSet) { + flags.Int("constraints-parallelism", 1, "How many constraints are created or dropped at once. They go on independent relations, so the server can build them side by side, and the only ordering that matters is that a foreign key needs the key it references to exist — which the pass handles by running the keys first. One at a time by default, since each build takes its own --constraints-work-mem and holds a lock on its table. On a schema whose rows sit in one dominant table this buys little; on one with several large tables it is close to linear.") + flags.String("constraints-work-mem", "", "What maintenance_work_mem is set to for the duration of each constraint statement, e.g. 1GB. Empty leaves the server's own setting alone, which on most is 64MB — at which an index build over a large table spills to an external merge sort. Raising it for the pass alone is the cheapest thing that makes it faster. It multiplies with --constraints-parallelism, each concurrent build taking its own.") + + flags.Int("constraints-per-transaction", 1, "Deprecated, use --constraints-parallelism.") + _ = flags.MarkDeprecated("constraints-per-transaction", "use --constraints-parallelism") +} + +// constraintsParallelism reads the parallelism, honouring the name it shipped under. The +// old flag was documented as an execution knob and only ever bundled statements into one +// transaction, which is not what it says. +func constraintsParallelism(cmd *cobra.Command) int { + if flagChanged(cmd, "constraints-parallelism") { + return intFlag(cmd, "constraints-parallelism") + } + if flagChanged(cmd, "constraints-per-transaction") { + return intFlag(cmd, "constraints-per-transaction") + } + + return intFlag(cmd, "constraints-parallelism") +} + +func addFromProtoSchemaFlags(flags *pflag.FlagSet) { + flags.Bool("disable-foreign-keys", false, "Never create foreign keys, including the one every table has to the block table. They are what a load pays most for, and without them a reorg is undone by deleting from each table rather than by a cascade.") + flags.StringSlice("disable-primary-keys", nil, "Tables that go without a primary key, or 'all'. No primary key also means no index on the entity id.") + flags.StringSlice("disable-unique-constraints", nil, "Tables whose unique constraints are left out, or 'all'.") + flags.Bool("disable-block-number-index", false, "Leave out the index on _block_number_. Every table carries that column and every reorg deletes from every table by it, so without the index each undo is a sequential scan per table — a foreign key indexes its referenced side only. It is created when the sink starts, concurrently and outside the constraint pass: --apply-constraints describes the schema and is yours to schedule, where this one the sink depends on to undo a reorg. Measured over 10GiB it costs 2.6s to build and 2% of the table. Only dead weight on a run that can never reorg, such as --final-blocks-only.") + flags.String("proto-file-override", "", "Override protobuf file to use instead of extracting from substreams package") + + flags.Bool("no-constraints", false, "Deprecated, use --disable-foreign-keys --disable-primary-keys=all --disable-unique-constraints=all.") + _ = flags.MarkDeprecated("no-constraints", "use --disable-foreign-keys --disable-primary-keys=all --disable-unique-constraints=all") +} + +// addConstraintTimingFlag registers --apply-constraints, which says when the constraints +// are created. +// +// It is on `setup` as well as the run. There it answers the one question `setup` can act +// on: 'always' creates them with the schema, while 'auto' and 'manual' both leave the +// tables bare, the first for the run to constrain when it reaches chain HEAD and the +// second for `constraints apply`. +func addConstraintTimingFlag(flags *pflag.FlagSet) { + flags.String("apply-constraints", string(protosql.ConstraintsAuto), "When the schema's constraints are created: 'auto' has the sink create them once the stream reaches chain HEAD — and only there, a stop block ending a run without saying the backfill is done — 'manual' leaves it to the 'sink postgres constraints apply' command, 'always' creates them before the load. Creating them is a stop-the-world operation — indexes to build, foreign keys to validate, tables locked throughout — so on a large database 'manual' is how that pass goes into a maintenance window instead. Loading with them already in place is the expensive option: measured through binary COPY, 27x slower than loading without, where building the same constraints afterwards costs 3.3x. The index on _block_number_ is not one of these: the sink creates it when it starts, see --disable-block-number-index.") +} + +// newSinkApplyConstraintsE creates the schema's constraints on a database the sink has +// already loaded, which is deliberately a separate command. +// +// It is a stop-the-world operation: every index is built and every foreign key validated, +// with the tables locked while it runs. On a large database that is a maintenance window, +// so it is the operator who picks the moment, not the sink. +// constraintsAction says which way `sink postgres constraints` runs. +type constraintsAction string + +const ( + constraintsApply constraintsAction = "apply" + constraintsDrop constraintsAction = "drop" +) + +func newSinkConstraintsE(driver string, action constraintsAction) func(*cobra.Command, []string) error { + return func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + dsnString, err := resolveSinkDSN(cmd) + if err != nil { + return err + } + if _, err := validateDSNEngine(dsnString, driver); err != nil { + return err + } + + manifestPath, outputModule := sinkManifestAndModule(args) + + sinkDBPreStart(cmd) + sink.LoadSubstreamsAuthEnvFile(manifestPath) + + spkg, module, _, err := sink.ReadManifestAndModule(manifestPath, "", nil, outputModule, sink.IgnoreOutputModuleType, false, nil, zlog) + if err != nil { + return fmt.Errorf("reading manifest: %w", err) + } + + if isDatabaseChangesType(module.Output.Type) { + return errDatabaseChangesOwnsSchema("sink " + driver + " constraints") + } + + constraints, err := fromProtoConstraintPolicy(cmd) + if err != nil { + return err + } + if constraints.SkipsEverything() { + return fmt.Errorf("every constraint is disabled by the flags, so there is nothing to %s", action) + } + + dsn, err := db.ParseDSN(dsnString) + if err != nil { + return fmt.Errorf("parsing dsn: %w", err) + } + + protoFileOverride := sflags.MustGetString(cmd, "proto-file-override") + rootMessageDescriptor, _, useProtoOption, err := resolveFromProtoRootMessage(spkg, module.Name, protoFileOverride) + if err != nil { + return err + } + if !useProtoOption { + // Without annotations the sink has no declared keys or relations, but the + // index it creates for its own reorg path is not declared either — so there is + // still something to do, and refusing would leave every table unindexed. + constraints = protosql.DisableAllConstraints().WithBlockNumberIndex(constraints) + + zlog.Warn("the module's output carries no schema.proto annotations, so there are no primary keys, unique constraints or foreign keys to " + string(action) + ". " + + "The index on _block_number_ is not affected: the sink creates that one when it starts. " + + "`substreams tools extract-proto --sql` writes the annotated proto to start from") + } + + encoding, err := sinkBytesEncoding(cmd) + if err != nil { + return err + } + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: useProtoOption, + Constraints: constraints, + Encoding: encoding, + Clickhouse: fromProtoClickhouseOptions(cmd, driver), + } + + if action == constraintsApply { + zlog.Info("creating the schema's constraints, this locks every table while indexes are built and foreign keys validated", + zap.String("schema", dsn.Schema()), + zap.String("constraints", constraints.Describe())) + } else { + zlog.Info("dropping the schema's constraints", + zap.String("schema", dsn.Schema()), + zap.String("constraints", constraints.Describe())) + } + + startAt := time.Now() + database, err := db_proto.SetupDatabaseSchema(cmd.Context(), dsnString, dsn.Schema(), module.Name, rootMessageDescriptor.UnwrapMessage(), options, zlog, tracer) + if err != nil { + return fmt.Errorf("setting up database schema: %w", err) + } + defer database.Close(cmd.Context()) + + if action == constraintsApply { + if err := applyDatabaseConstraints(database); err != nil { + return err + } + zlog.Info("constraints created", zap.Duration("duration", time.Since(startAt))) + + return nil + } + + if err := dropDatabaseConstraints(database); err != nil { + return err + } + zlog.Info("constraints dropped", zap.Duration("duration", time.Since(startAt))) + + return nil + } +} + +// applyDatabaseConstraints and dropDatabaseConstraints do not wrap the pass in a +// transaction: it owns its own, committing every --constraints-per-transaction statements +// so a killed run keeps what it finished. +func applyDatabaseConstraints(database protosql.Database) error { + if err := database.ApplyConstraints(); err != nil { + return fmt.Errorf("applying constraints: %w", err) + } + + return nil +} + +func dropDatabaseConstraints(database protosql.Database) error { + if err := database.DropConstraints(); err != nil { + return fmt.Errorf("dropping constraints: %w", err) + } + + return nil +} + +// fromProtoConstraintPolicy resolves the constraint flags. Everything is created by +// default, once the backfill is done rather than before it. +func fromProtoConstraintPolicy(cmd *cobra.Command) (protosql.ConstraintPolicy, error) { + timing, err := protosql.ParseConstraintTiming(stringFlag(cmd, "apply-constraints")) + if err != nil { + return protosql.ConstraintPolicy{}, err + } + + policy := protosql.ConstraintPolicy{ + Timing: timing, + DisableForeignKeys: boolFlag(cmd, "disable-foreign-keys"), + DisablePrimaryKeys: stringSliceFlag(cmd, "disable-primary-keys"), + DisableUniques: stringSliceFlag(cmd, "disable-unique-constraints"), + + DisableBlockNumberIndex: boolFlag(cmd, "disable-block-number-index"), + Parallelism: constraintsParallelism(cmd), + WorkMem: stringFlag(cmd, "constraints-work-mem"), + } + + // --no-constraints shipped in v1.21.0 and said "none of them at all", which is exactly + // what the three switches say together. It stays honoured for a release. + if flagChanged(cmd, "no-constraints") && boolFlag(cmd, "no-constraints") { + disabled := protosql.DisableAllConstraints() + policy.DisableForeignKeys = true + policy.DisablePrimaryKeys = disabled.DisablePrimaryKeys + policy.DisableUniques = disabled.DisableUniques + } + + return policy, nil +} + +// warnAboutConstraints says out loud what applying constraints before the backfill costs, +// since the flag makes the load an order of magnitude slower and its effect on an already +// populated database is not instantaneous. +func warnAboutConstraints(constraints protosql.ConstraintPolicy) { + if !constraints.ApplyUpfront() { + return + } + + zlog.Warn("constraints are being created before the backfill rather than after it. Loading with foreign keys in place measured 27x slower than loading without them, " + + "where building the same constraints afterwards cost 3.3x, so a large initial sync is considerably slower this way. " + + "Creating the constraints on an already populated database can also take a long time, and locks the tables while it runs") +} diff --git a/cmd/substreams/sink_sql_constraints_test.go b/cmd/substreams/sink_sql_constraints_test.go new file mode 100644 index 000000000..ea37622bf --- /dev/null +++ b/cmd/substreams/sink_sql_constraints_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFromProtoConstraintPolicyDisableAll covers the deprecated --no-constraints, which +// has to keep meaning exactly what the three switches mean together. +func TestFromProtoConstraintPolicyDisableAll(t *testing.T) { + newCmd := func() *cobra.Command { + cmd := &cobra.Command{} + addFromProtoSchemaFlags(cmd.Flags()) + addConstraintTimingFlag(cmd.Flags()) + + return cmd + } + + t.Run("nothing disabled by default", func(t *testing.T) { + policy, err := fromProtoConstraintPolicy(newCmd()) + require.NoError(t, err) + assert.False(t, policy.SkipsEverything()) + }) + + t.Run("the deprecated flag still means the same", func(t *testing.T) { + cmd := newCmd() + require.NoError(t, cmd.Flags().Set("no-constraints", "true")) + + policy, err := fromProtoConstraintPolicy(cmd) + require.NoError(t, err) + assert.True(t, policy.SkipsEverything()) + }) +} + +// TestSetupCommandsCarryConstraintTiming pins that `setup` reads --apply-constraints: it +// builds its policy from that flag, so leaving it unregistered had setup read the empty +// string and create a bare schema whatever else was asked for. +func TestSetupCommandsCarryConstraintTiming(t *testing.T) { + for _, cmd := range []*cobra.Command{sinkPostgresSetupCmd, sinkClickhouseSetupCmd} { + assert.NotNil(t, cmd.Flags().Lookup("apply-constraints"), "%s must carry --apply-constraints", cmd.Use) + } +} + +// TestSetupOnlyCreatesConstraintsWhenAlways pins which of the three values `setup` acts on. +// It creates the schema and exits, so 'auto' and 'manual' both leave the tables bare and +// only 'always' puts the constraints on with them. +func TestSetupOnlyCreatesConstraintsWhenAlways(t *testing.T) { + for value, wantUpfront := range map[string]bool{"auto": false, "manual": false, "always": true} { + cmd := &cobra.Command{} + addFromProtoSchemaFlags(cmd.Flags()) + addConstraintTimingFlag(cmd.Flags()) + require.NoError(t, cmd.Flags().Set("apply-constraints", value)) + + policy, err := fromProtoConstraintPolicy(cmd) + require.NoError(t, err) + assert.Equal(t, wantUpfront, policy.ApplyUpfront(), "--apply-constraints=%s", value) + } +} diff --git a/cmd/substreams/sink_sql_usage.go b/cmd/substreams/sink_sql_usage.go new file mode 100644 index 000000000..531978617 --- /dev/null +++ b/cmd/substreams/sink_sql_usage.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// setModeGroupedUsage partitions the run command's flags by the mode they apply to. +// +// Both vocabularies have to be registered on the same command: the mode is read from the +// module's output type at run time, long after init() has registered anything. So the run +// command lists knobs that are inert for any given Substreams, and a per-line `[mode]` +// prefix is left as the only thing separating them — which the operator has to scan for. +// +// Headings do that job better, and free every help string of its prefix. +func setModeGroupedUsage(cmd *cobra.Command) { + groups := []flagGroup{ + { + title: "Relational Mappings Mode flags (module outputs an arbitrary protobuf message)", + names: append(append([]string{}, fromProtoSchemaFlagNames...), fromProtoRunFlagNames...), + }, + { + title: "Database Changes Mode flags (module outputs 'DatabaseChanges')", + names: append(databaseChangesFlagNames, onModuleHashMismatchFlag), + }, + } + + cmd.SetUsageFunc(func(cmd *cobra.Command) error { + cmd.OutOrStderr().Write([]byte(groupedUsage(cmd, groups))) //nolint:errcheck // usage output + + return nil + }) +} + +type flagGroup struct { + title string + names []string +} + +func groupedUsage(cmd *cobra.Command, groups []flagGroup) string { + var out strings.Builder + + fmt.Fprintf(&out, "Usage:\n %s\n", cmd.UseLine()) + if cmd.HasAvailableSubCommands() { + fmt.Fprintf(&out, "\nAvailable Commands:\n") + for _, sub := range cmd.Commands() { + if sub.IsAvailableCommand() { + fmt.Fprintf(&out, " %-16s %s\n", sub.Name(), sub.Short) + } + } + } + + claimed := map[string]bool{} + for _, group := range groups { + set := pflag.NewFlagSet(group.title, pflag.ContinueOnError) + for _, name := range group.names { + if flag := cmd.Flags().Lookup(name); flag != nil && !flag.Hidden { + set.AddFlag(flag) + claimed[name] = true + } + } + if !set.HasAvailableFlags() { + continue + } + + fmt.Fprintf(&out, "\n%s:\n%s", group.title, set.FlagUsages()) + } + + common := pflag.NewFlagSet("common", pflag.ContinueOnError) + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + if !claimed[flag.Name] && !flag.Hidden { + common.AddFlag(flag) + } + }) + if common.HasAvailableFlags() { + fmt.Fprintf(&out, "\nFlags (both modes):\n%s", common.FlagUsages()) + } + + if inherited := cmd.InheritedFlags(); inherited.HasAvailableFlags() { + fmt.Fprintf(&out, "\nGlobal Flags:\n%s", inherited.FlagUsages()) + } + + if cmd.HasAvailableSubCommands() { + fmt.Fprintf(&out, "\nUse \"%s [command] --help\" for more information about a command.\n", cmd.CommandPath()) + } + + return out.String() +} diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index e5cf96fd9..69e650715 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -67,8 +67,8 @@ * [Neon](how-to-guides/sinks/hosted-sinks/neon.md) * [ClickHouse Cloud](how-to-guides/sinks/hosted-sinks/clickhouse-cloud.md) * [Substreams:SQL](how-to-guides/sinks/sql/sql.md) - * [Using Relational Mappings](how-to-guides/sinks/sql/relational-mappings.md) - * [Using Database Changes](how-to-guides/sinks/sql/db_out.md) + * [Using Relational Mappings Mode](how-to-guides/sinks/sql/relational-mappings.md) + * [Using Database Changes Mode](how-to-guides/sinks/sql/db_out.md) * [Migrating from substreams-sink-sql](how-to-guides/sinks/sql/migration.md) * [Substreams:Stream](how-to-guides/sinks/stream/stream.md) * [JavaScript](how-to-guides/sinks/stream/javascript.md) diff --git a/docs/how-to-guides/develop-your-own-substreams/generic/agent-skills.md b/docs/how-to-guides/develop-your-own-substreams/generic/agent-skills.md index df7275939..1f8467a09 100644 --- a/docs/how-to-guides/develop-your-own-substreams/generic/agent-skills.md +++ b/docs/how-to-guides/develop-your-own-substreams/generic/agent-skills.md @@ -18,8 +18,8 @@ Expert knowledge for developing, building, and debugging Substreams projects on Expert knowledge for building SQL database sinks from Substreams data: -- **Database Changes (CDC)** - Stream individual row changes for real-time consistency -- **Relational Mappings** - Transform data into normalized tables with proper relationships +- **Database Changes Mode (CDC)** - Stream individual row changes for real-time consistency +- **Relational Mappings Mode** - Transform data into normalized tables with proper relationships - **PostgreSQL** - Advanced patterns, indexing strategies, and performance optimization - **ClickHouse** - Analytics-optimized schemas, materialized views, and time-series patterns - **Schema Design** - Best practices for blockchain data modeling diff --git a/docs/how-to-guides/sinks/hosted-sinks/hosted-sinks.md b/docs/how-to-guides/sinks/hosted-sinks/hosted-sinks.md index dc4819121..aa52c4396 100644 --- a/docs/how-to-guides/sinks/hosted-sinks/hosted-sinks.md +++ b/docs/how-to-guides/sinks/hosted-sinks/hosted-sinks.md @@ -17,7 +17,7 @@ Hosted Sinks is available to organizations on The Graph Market. Your database mu Before creating a sink, you need: - An account on [The Graph Market](https://thegraph.market) with an active organization. -- A Substreams package (`.spkg`) that outputs data using a `db_out` module or relational mappings. See [Substreams:SQL](../sql/sql.md) for how to build one. +- A Substreams package (`.spkg`) that outputs data in Database Changes Mode (a `db_out` module) or Relational Mappings Mode. See [Substreams:SQL](../sql/sql.md) for how to build one. - A running Postgres (default port `5432`) or ClickHouse (default port `9000`) database that is network-accessible from the internet. - Your database schema already applied, or a schema that your Substreams package will create automatically. See [Sink Config](../../../references/sql/sink-config.md). diff --git a/docs/how-to-guides/sinks/sinks.md b/docs/how-to-guides/sinks/sinks.md index f70148ec0..0d0622296 100644 --- a/docs/how-to-guides/sinks/sinks.md +++ b/docs/how-to-guides/sinks/sinks.md @@ -1,7 +1,7 @@ Once you find a package that fits your needs, you can choose how you want to consume the data. Sinks are integrations that allow you to send the extracted data to different destinations, such as a SQL database, or a file. {% hint style="info" %} -**Tip**: Building a SQL sink? The [substreams-sql agent skill](../develop-your-own-substreams/generic/agent-skills.md) gives your AI coding assistant expert knowledge on database change (CDC) patterns, relational mappings, PostgreSQL, and ClickHouse schema design. +**Tip**: Building a SQL sink? The [substreams-sql agent skill](../develop-your-own-substreams/generic/agent-skills.md) gives your AI coding assistant expert knowledge on Database Changes Mode (CDC) patterns, Relational Mappings Mode, PostgreSQL, and ClickHouse schema design. {% endhint %} {% hint style="info" %} diff --git a/docs/how-to-guides/sinks/sql/db_out.md b/docs/how-to-guides/sinks/sql/db_out.md index 99798d5a0..ae616f1ad 100644 --- a/docs/how-to-guides/sinks/sql/db_out.md +++ b/docs/how-to-guides/sinks/sql/db_out.md @@ -2,7 +2,7 @@ If you require more control over the tables and the data that you want to store into the database, then creating a `db_out` module would be the best option. -You will create a new module, `db_out`, which maps the output of your Substreams to the [DatabaseChanges data model](https://docs.rs/substreams-database-change/latest/substreams_database_change/pb/database/struct.DatabaseChanges.html), which is a format that the SQL sink understands. +You will create a new module, `db_out`, which maps the output of your Substreams to the [DatabaseChanges data model](https://docs.rs/substreams-database-change/latest/substreams_database_change/pb/database/struct.DatabaseChanges.html) — the format the SQL sink consumes in Database Changes Mode. ## Running the Sink diff --git a/docs/how-to-guides/sinks/sql/migration.md b/docs/how-to-guides/sinks/sql/migration.md index 398e9f803..bcba8ebad 100644 --- a/docs/how-to-guides/sinks/sql/migration.md +++ b/docs/how-to-guides/sinks/sql/migration.md @@ -18,7 +18,7 @@ The engine is now part of the command name and must match your DSN scheme. There For ClickHouse targets, replace `postgres` with `clickhouse` in every command. `generate-csv` and `inject-csv` are PostgreSQL-only. -There is no separate `from-proto` command anymore: the engine command (and `setup`) detects the mode from the output module's type. A module producing `sf.substreams.sink.database.v1.DatabaseChanges` uses your `schema.sql`; any other output type uses relational mappings derived from the protobuf definition. +There is no separate `from-proto` command anymore: the engine command (and `setup`) detects the mode from the output module's type. A module producing `sf.substreams.sink.database.v1.DatabaseChanges` runs in Database Changes Mode and uses your `schema.sql`; any other output type runs in Relational Mappings Mode, whose schema is derived from the protobuf definition. ## Flag changes @@ -52,7 +52,7 @@ There is no separate `from-proto` command anymore: the engine command (and `setu ## Cursor compatibility -- DatabaseChanges mode stores its cursor in the same `cursors` table, keyed by module hash. Point the new CLI at the same database and it resumes from the stored cursor. +- Database Changes Mode stores its cursor in the same `cursors` table, keyed by module hash. Point the new CLI at the same database and it resumes from the stored cursor. - Relational-mappings mode is also unchanged: `_cursor_` table on PostgreSQL, cursor file on ClickHouse (`--cursor-file-path`, previously `--clickhouse-cursor-file-path`, same `cursor.txt` default). ## Environment variables diff --git a/docs/how-to-guides/sinks/sql/relational-mappings.md b/docs/how-to-guides/sinks/sql/relational-mappings.md index 90d60df94..4d9da4a7c 100644 --- a/docs/how-to-guides/sinks/sql/relational-mappings.md +++ b/docs/how-to-guides/sinks/sql/relational-mappings.md @@ -1,11 +1,11 @@ -# Relational Mapping +# Relational Mappings Mode If you want to use a relational model (e.g., creating one-to-many), you can annotate your Protobuf to indicate the primary and foreign keys in your database. To map your Protobuf definitions directly to database tables and establish relationships between objects, you need to annotate your Protobuf messages with table names, primary keys, and relationship metadata. {% hint style="warning" %} -Relational mappings from Protobuf are currently in beta. Postgres support is stable, but ClickHouse support is still under development. [Reference releases](https://github.com/streamingfast/substreams-sink-sql/releases) +Relational Mappings Mode is currently in beta. Postgres support is stable, but ClickHouse support is still under development. [Reference releases](https://github.com/streamingfast/substreams-sink-sql/releases) {% endhint %} ```proto diff --git a/docs/how-to-guides/sinks/sql/sql.md b/docs/how-to-guides/sinks/sql/sql.md index 59a4f7b18..989acc9f3 100644 --- a/docs/how-to-guides/sinks/sql/sql.md +++ b/docs/how-to-guides/sinks/sql/sql.md @@ -12,26 +12,25 @@ Before you begin, make sure you have: The core function of the SQL sink is to translate your Substreams output (Protobuf data) into SQL tables. Choose one of the following methods depending on your needs: -- [Using Relational Mappings "from-proto"](./relational-mappings.md) +- [Using Relational Mappings Mode](./relational-mappings.md) * Enables foreign key relationships in your SQL schema. * Requires adding annotations to your Protobuf messages (e.g., primary and foreign keys). * Currently insert-only. -- [Using Database Changes](./db_out.md) +- [Using Database Changes Mode](./db_out.md) * Gives you full control over the output. * Supports insert, update, and upsert operations. * Ideal for advanced use cases with evolving or mutable data. * **NOTE:** In ClickHouse, reorgs are currently supported with delay. -| | Relational Mappings | `db_out` module | -|-------------------------------|---------------------|-----------------| -| SQL relationships | Yes | No | -| Direct Protobuf<>SQL mappings | Yes | No | -| `INSERT` supported | Yes | Yes | -| `UPDATE` supported | No | Yes | -| `UPSERT` supported | No | Yes | +| | Relational Mappings Mode | Database Changes Mode (`db_out`) | +|-------------------------------|--------------------------|----------------------------------| +| SQL relationships | Yes | No | +| Direct Protobuf<>SQL mappings | Yes | No | +| `INSERT` supported | Yes | Yes | +| `UPDATE` supported | No | Yes | +| `UPSERT` supported | No | Yes | ## Installation The SQL sink is included in the [Substreams CLI](../../cli/installing-the-cli.md) — there is no separate binary to install. Once `substreams` is installed, the `substreams sink postgres` and `substreams sink clickhouse` commands are available. - diff --git a/docs/references/sql/proto-annotations.md b/docs/references/sql/proto-annotations.md index 7c3248cfa..69112af4e 100644 --- a/docs/references/sql/proto-annotations.md +++ b/docs/references/sql/proto-annotations.md @@ -1,6 +1,6 @@ # Proto Annotations Reference -When the output module of a SQL sink does not produce `DatabaseChanges`, the sink derives the database schema from the module's protobuf definition (relational mappings). This page is the reference for the annotations and type mappings driving that schema generation. For a guided walkthrough, see [Using Relational Mappings](../../how-to-guides/sinks/sql/relational-mappings.md). +When the output module of a SQL sink does not produce `DatabaseChanges`, the sink runs in Relational Mappings Mode and derives the database schema from the module's protobuf definition. This page is the reference for the annotations and type mappings driving that schema generation. For a guided walkthrough, see [Using Relational Mappings Mode](../../how-to-guides/sinks/sql/relational-mappings.md). Annotations come from `sf/substreams/sink/sql/schema/v1/schema.proto`: @@ -105,7 +105,9 @@ Typical uses: token amounts exceeding the uint64 range, high-precision decimal a ## ClickHouse-Specific Options -ClickHouse tables require the `clickhouse_table_options` annotation on each table: at minimum `order_by_fields`, optionally `partition_fields` and `index_fields`. +ClickHouse tables accept a `clickhouse_table_options` annotation: `order_by_fields`, `partition_fields` and `index_fields`. + +A table that declares no `order_by_fields` is sorted on `(_block_number_, _row_id_)` and partitioned by `toYYYYMM(_block_timestamp_)`, which is what lets a package with no annotations at all be sinked as it is. `_row_id_` numbers the rows a block writes to a table, starting at zero; the tables are `ReplacingMergeTree`, so without it every row of a block would collapse into one. It is added only to the tables that need it, and a table created with it cannot later be annotated in place — the sink refuses to start on a database whose tables disagree with the package, since the sorting key of an existing table cannot be changed. ```proto message Transfer { @@ -163,18 +165,300 @@ FROM transfers.transfers > **📚 Deep Dive**: See the [ClickHouse Showcase](https://github.com/streamingfast/substreams-sink-clickhouse-showcase) and its [Deep Dive](https://github.com/streamingfast/substreams-sink-clickhouse-showcase/blob/main/DEEP_DIVE.md) for a production-grade example with materialized views and partitioning strategies. -## Performance Flags +## Constraints and performance + +The sink loads without database constraints and creates them afterwards, because the +difference is not marginal. Measured through binary COPY over 500,000 rows +(`TestConstraintCost` in `sink/sql/db_proto/benchmarks`): + +| what the load runs with | duration | vs bare | +|---|--:|--:| +| no constraints | 137ms | 1.0x | +| primary keys | 554ms | 4.0x slower | +| primary keys + unique | 809ms | 5.9x slower | +| primary keys + unique + foreign keys | 3.798s | **27.7x slower** | +| bare load, then all of them created | 449ms | 3.3x slower | + +Building the constraints after the load costs 8.5x less than loading with them in place, +for exactly the same schema. So the default is: load bare, then create them when you say +so. + +```bash +# the load, creating the constraints itself once it reaches chain HEAD +substreams sink postgres substreams.yaml --dsn $DSN + +# or: load bare, and create them yourself in a maintenance window +substreams sink postgres substreams.yaml --dsn $DSN --apply-constraints=manual +substreams sink postgres constraints apply substreams.yaml --dsn $DSN +``` + +`--apply-constraints` says when they are created: + +| value | when | +|---|---| +| `auto` *(default)* | the sink creates them once the stream reaches chain HEAD. A `--stop-block` run leaves them alone: it ends the run without saying the backfill is done | +| `manual` | left to `sink postgres constraints apply` | +| `always` | created before the load, which is the 27.7x row above | + +`auto` is the default even though the pass is stop-the-world — every index built and every +foreign key validated, tables locked while it runs. A backfill that ends with no +constraints is a silent wrong result that looks like success; a stall is at least a +visible one. Use `manual` to put that pass in a maintenance window instead. + +An index on `_block_number_` is created in the same pass, on every table. It is the sink's +own rather than the schema's: every table carries that column and every reorg deletes from +every table by it, and a foreign key indexes only its referenced side — so without it each +undo is a sequential scan per table. Measured over 10GiB of `erc20-balance-changes`-shaped +rows (`TestBlockNumberIndexCost`), building it costs 2.6s on a 44s load and 218MiB against +10GiB, and takes one table's undo from 1.3s to 15ms. `--disable-block-number-index` leaves +it out, which only makes sense for a run that can never reorg. It is also the one thing the pass has to +do for an output with no schema annotations at all. + +Constraints are created one per transaction, committing as it goes: building an index and +validating a foreign key are the two most memory-hungry things the sink asks of the server, +and holding them all open at once is what turns a large schema into an OOM that loses the +whole pass. A run that is killed keeps what it finished, and the next one carries on. +`--constraints-per-transaction` on the `constraints` commands trades that back for fewer +round trips when the constraints are small. + +Running `constraints apply` again is safe — the constraints already in place are left +alone — so it also back-fills a schema an earlier run created without them. +`constraints drop` is the inverse, and the escape hatch after `--apply-constraints=always` +or before resuming a backfill. + +Until it has run, the tables carry no primary keys, no unique constraints, no foreign keys +and therefore **no indexes at all**, so queries are sequential scans and duplicate ids are +not rejected. A reorg is still undone correctly: the rows of the undone blocks are deleted +from each table explicitly rather than through a cascade. + +- `--apply-constraints`: when they are created — `manual` (default, the command above), + `head` (the sink creates them itself once the backfill reaches chain HEAD), or `upfront` + (before the load, paying the 27.7x). +- `--disable-foreign-keys`: never create foreign keys, including the one every table has to + the block table. +- `--disable-primary-keys`: tables that go without a primary key, or `all`. +- `--disable-unique-constraints`: tables whose unique constraints are left out, or `all`. + +## Which strategy for which database + +The constraints are the same either way; what changes is when you pay for them and what +the database can do in the meantime. Pick by how long the backfill is. + +### A development database, or anything that loads in a few minutes + +Create the constraints before the load and forget about them. The database is correct from +the first block, and on a small dataset the 32x is a couple of seconds. + +```bash +substreams sink postgres substreams.yaml --dsn $DSN \ + --apply-constraints=upfront \ + -e mainnet.eth.streamingfast.io:443 -s 20000000 -t +50000 +``` + +One command, nothing to do afterwards. This is also the right choice for a chain that is +already near its head, where there is no backfill to speak of. + +### A production backfill you can babysit — millions of rows + +Load bare, and let the sink create the constraints when it catches up to the chain head. +You get the fast path for the whole backfill and a correct database at the end, at the +cost of one pause when it switches over. + +```bash +substreams sink postgres substreams.yaml --dsn $DSN \ + --apply-constraints=auto \ + -e mainnet.eth.streamingfast.io:443 -s 20000000 +``` + +What happens, in order: rows are buffered on disk and loaded with binary COPY; the first +live block drains the buffer and switches to direct inserts; the constraints are then +created, locking each table while its indexes are built and its foreign keys validated; +the sink resumes following the chain. Expect that pause to last minutes on a table of tens +of millions of rows — the sink stops consuming the stream while it happens. + +### A large or busy production database — tens of millions of rows and up + +Keep the two apart, so the lock window is yours to schedule. + +**Step 1 — create the schema:** + +```bash +substreams sink postgres setup substreams.yaml --dsn $DSN +``` + +**Step 2 — backfill, bare** (this is the default, no flag needed): + +```bash +substreams sink postgres substreams.yaml --dsn $DSN \ + -e mainnet.eth.streamingfast.io:443 -s 20000000 +``` + +Let it run to the chain head. It logs, once the backfill is done, that the schema has no +constraints yet and names the command below. Queries against the tables work but are +sequential scans until step 3. + +**Step 3 — create the constraints, in a maintenance window:** + +```bash +substreams sink postgres constraints apply substreams.yaml --dsn $DSN +``` + +Every table is locked while its indexes are built and its foreign keys validated. Run it +when a stall is acceptable. It is idempotent, so a run that was interrupted can simply be +repeated, and it also back-fills a schema that was created without constraints long ago. + +The sink can keep running during step 3 — it will be blocked on the locks and resume when +they are released — but on a large database it is calmer to stop it, apply, and restart. + +### A database you query but never join — analytics, exports + +Foreign keys are what a load pays most for, and referential integrity is what you are +least likely to need in an append-only table. Keep the primary keys, which is what gives +you an index on the entity id, and drop the rest: + +```bash +# during the backfill: nothing to do, the default already loads bare + +# afterwards: +substreams sink postgres constraints apply substreams.yaml --dsn $DSN \ + --disable-foreign-keys +``` -For fast initial imports: +Loading with primary keys and uniques in place costs 5.9x against 32x with foreign keys, +so if you would rather have them from the start: ```bash substreams sink postgres substreams.yaml --dsn $DSN \ - --no-constraints \ - --block-batch-size 100 + --apply-constraints=upfront --disable-foreign-keys +``` + +### A bulk load into a throwaway database + +No constraints at all, ever. The fastest the sink goes, and the least the database can +tell you. + +```bash +substreams sink postgres substreams.yaml --dsn $DSN \ + --disable-foreign-keys --disable-primary-keys=all --disable-unique-constraints=all +``` + +Nothing is indexed, so add whatever index your queries actually need by hand afterwards +rather than paying for the ones the schema would have implied. + +### Per-table exceptions + +The switches take table names, so a single hot table can be treated differently from the +rest — here every table keeps its constraints except one whose primary key would be +expensive and is never queried by id: + +```bash +substreams sink postgres constraints apply substreams.yaml --dsn $DSN \ + --disable-primary-keys=order_items +``` + +### The block number index is not one of them + +Every table carries `_block_number_` and every reorg deletes from every table by it, and a +foreign key indexes only its referenced side — so without an index each undo is a +sequential scan per table. + +The sink creates that index itself, **when it starts**, and `--apply-constraints` does not +govern it. The constraints describe the schema and are yours to schedule; this one the sink +depends on, so waiting for a maintenance window would mean running without it for as long +as you like. It is built `CONCURRENTLY`, so a restart onto an already-loaded table neither +waits for a lock nor takes one, and it is why the index cannot be part of the constraint +pass at all — that runs in transactions, which a concurrent build cannot. + +Measured over 10GiB of `erc20-balance-changes`-shaped rows (`TestBlockNumberIndexCost`): +2.6s to build, 218MiB against 10GiB, and one table's undo goes from 1.3s to 15ms. + +`--disable-block-number-index` leaves it out, which only makes sense for a run that can +never reorg, such as `--final-blocks-only`. + +## Starting from a package with no annotations + +An output with no `schema.proto` annotations still sinks: the tables come from the message +structure. What it has no way to declare is which field is the primary key, which are +unique and which reference another table — so those are not created, and the sink says so +on every start. + +`substreams tools extract-proto --sql` writes the module's output proto back out with +every option commented out beside the message and field it applies to, and the annotations +file next to it so the result parses as-is: + +```bash +substreams tools extract-proto --sql substreams.yaml map_events +# uncomment the options that describe the schema, then: +substreams sink postgres setup substreams.yaml --proto-file-override=./events.proto --dsn=$DSN ``` -- `--no-constraints`: skip creating database constraints -- `--block-batch-size`: number of blocks to process at a time (default: 25) +Pass the same `--proto-file-override` to the run and to `constraints apply`, or they derive +the unannotated schema again. + +## Performance flags + +A fast initial import is the default: the sink creates no constraints during the load, +spools rows to local disk and loads them with binary COPY from a background goroutine, so +the stream never waits on the database. + +The flags are grouped by what they spend, and `substreams sink postgres --help` prints +them under those headings. + +**`--decode-*` — CPU.** These size the decode stage and change nothing about what the +database sees. + +| flag | default | what it does | +|---|---|---| +| `--decode-workers` | one per core less one, capped at 8 | blocks unmarshalled and walked concurrently | +| `--decode-batch-size` | 4 × workers | blocks held in memory and decoded together | + +`--block-batch-size` is the old name of `--decode-batch-size` and still works for one +release. It used to size the database transaction; the spool took that job, and the two +are separate concerns now. + +**`--db-write-*` — one commit to the database.** A commit is one spooled segment. The sink +measures each one and sizes the next toward the target, so the number of blocks or rows in +a commit is a result rather than a setting — block payloads differ by orders of magnitude +across chains, which is exactly what makes a fixed count give wildly variable durations. + +| flag | default | what it does | +|---|---|---| +| `--db-write-target-duration` | `3s` | how long one commit should take | +| `--db-write-max-size` | `512MiB` | ceiling on the segment size the sizer may choose | + +Lower the target for smaller, more frequent commits: gentler on a database shared with +something else, cursor advances more often, less re-streamed after a crash. + +**`--spool-*` — how far ahead of the database the stream may run.** Both PostgreSQL and +ClickHouse spool. + +| flag | default | what it does | +|---|---|---| +| `--spool-dir` | `./localdata/spool` | where pending segments are written | +| `--spool-max-size` | `8GiB` | disk budget; the stream is held once it is reached | +| `--spool-max-idle` | `10s` | commit the open segment when no new row has reached it for this long | + +`--spool-max-size` is the only bound on the backlog. `--spool-max-idle` is what keeps a +stalled stream from sitting on rows indefinitely: without it the cursor would not advance +and those blocks would be streamed, and paid for, again on restart. + +**`--write-mode` — how a segment reaches the database.** + +| value | how | +|---|---| +| `copy` | `COPY ... FROM STDIN (FORMAT BINARY)` per table — measured at ~7x the multi-row INSERT path | +| `batch-insert` | one multi-row INSERT per table | +| `row-insert` | one prepared INSERT per row | +| `auto` *(default)* | `copy` on PostgreSQL, `batch-insert` on ClickHouse, `row-insert` for a schema whose foreign keys form a cycle | + +A cycle has no table order, so grouping rows by table cannot keep a parent ahead of its +children, while the walk itself always does — hence `row-insert` as the fallback. An +explicit mode the driver or the schema cannot support is an error, not a downgrade. + +The spool is a backfill tool, so the sink stops using it on its own once the stream reaches +the chain head: on the first live block everything spooled is applied and the inserts go +straight to the database from then on, where a block is queryable as soon as it arrives. +Every flag above stops applying at that point. ## Troubleshooting diff --git a/docs/references/sql/reorg-handling.md b/docs/references/sql/reorg-handling.md index 36ecd81df..b9d138323 100644 --- a/docs/references/sql/reorg-handling.md +++ b/docs/references/sql/reorg-handling.md @@ -6,17 +6,17 @@ This document describes how the SQL sink (`substreams sink postgres` / `substrea Blockchain networks can experience reorganizations where previously accepted blocks are replaced by a different chain. When this happens, database changes that were based on the replaced blocks must be reverted to maintain consistency with the canonical chain. -The sink implements different re-org handling strategies depending on the data processing model used: +The sink implements different re-org handling strategies depending on the mode the sink runs in: -- **DatabaseChanges Model** (`db_out` modules) - Tracks individual operations in a history table -- **Relational Mappings** (from Protobuf directly) - _Documentation to come soon_ +- **Database Changes Mode** (`db_out` modules) - Tracks individual operations in a history table +- **Relational Mappings Mode** (from Protobuf directly) - _Documentation to come soon_ - **Delayed block signalling** (`undo-buffer` flags) - _Documentation to come soon_ -This document currently focuses on the **DatabaseChanges model** implementation. +This document currently focuses on the **Database Changes Mode** implementation. -## DatabaseChanges Model Re-org Handling +## Database Changes Mode Re-org Handling -The DatabaseChanges model (used by `db_out` modules) handles re-orgs through a four-phase process: +Database Changes Mode (used by `db_out` modules) handles re-orgs through a four-phase process: 1. **Tracking changes** - Storing a record of all database operations in a history table, only in the reversible segment of the chain, this means there are no operations happening when backfilling historical segments. 2. **Detecting re-orgs** - Receiving undo signals when reorganizations occur @@ -40,7 +40,7 @@ CREATE TABLE substreams_history ( ### Example Walkthrough -Let's trace through a complete re-org scenario using a simple `transfer` table with the DatabaseChanges model: +Let's trace through a complete re-org scenario using a simple `transfer` table in Database Changes Mode: ```sql CREATE TABLE transfer ( diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index 739bda32e..f3e2b81b4 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -11,87 +11,240 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased -### Added - -- CLI: Ethereum Hoodi testnet (`hoodi`) StreamingFast endpoints (`hoodi.eth.streamingfast.io:443`). -- RPC: `ModulesProgress.stages` entries now expose per-stage squash visibility, so a client can tell "segment produced" apart from "segment actually usable". - - `Stage` gained two additive fields: `ready_up_to_exclusive` (field 3), the chain block number, exclusive, up to which the stage is immediately usable, and `squash_wait_segment_count` (field 4), the number of segments whose store partial exists but has not been squashed into the store yet. - - `completed_ranges` counts a store segment as soon as its partial has been produced, before tier1 has merged it, whereas `ready_up_to_exclusive` stops at the last squashed segment. A stage could therefore render as 100% covered while a substantial part of the work was still outstanding, and since squashing runs on tier1 rather than on a worker it schedules no job and advances no `processed_blocks` — the request looked frozen at 100% with a rate of zero. A non-zero `squash_wait_segment_count` now names that state explicitly. - - For a stage that executes no store module the two notions coincide, as mapper and index output is read straight from the partial files, and `squash_wait_segment_count` stays 0. A stage that has not started reports the block its modules begin at, floored at the chain's first streamable block, so 0 is a valid value meaning "nothing usable yet" and not a sentinel for "unknown". -- RPC: `SessionInit` gained `segment_block_count` (field 11), the width in blocks of one parallel processing segment, constant for the whole session. - - The segment width was previously not exchanged at all, so a client had no way to turn a segment count such as `Stage.squash_wait_segment_count` into a number of blocks. It could only be inferred from `Job.stop_block - Job.start_block`, which is unavailable exactly when it is needed, since no job runs while tier1 squashes. - - It is an upper bound rather than an exact multiplier: the first and last segment of a run are narrower when a module's initial block or the request's stop block falls inside a segment. -- Server: tier1 request logs now explain how many parallel workers a request actually got, and why. A client asking for 300 workers and getting 15 previously left no trace of the negotiation anywhere in the logs. - - The `incoming Substreams Blocks request` entry gained a `parallelism` object (`requested_workers` as asked by the client, `granted_workers` as allowed by the authentication layer, the effective `workers`, `workers_source` telling which of the two applied, plus `plan_tier` and `stage_layer_executors`), along with `parallel_segment_count` and `stage_count` — a request with fewer segments than workers can never use them all. - - The `substreams request stats` entry gained a `workers` object (`requested`, `granted`, `effective`, `peak`, `pool_exhausted_count`, `pool_rampup_deferred_count`), reported on tier1 only. A high `pool_exhausted_count` with a `peak` well below `effective` means the shared worker pool ran dry, a case that was previously only visible at debug level. - - The periodic `substreams request progress` entry gained its own `workers` object (`requested`, `granted`, `effective`, `running`, `idle`, and `pool_exhausted_5m` when jobs failed to get a worker over the window), so the same question can be answered while the request is still running rather than only once it ends. When jobs repeatedly find no free worker while the request still has idle capacity, a hint now says so and names the three ceilings that can cause it — the tier2 fleet being full, the organization's worker quota, or the server's per-session worker cap — since the pool reports a single error for all three. - -- CLI: `substreams tools devenv` boots a complete local stack — a dummy blockchain in a container plus a tier1 and a tier2 built from the current source tree — prints the endpoint and stays up until interrupted. Requires Docker. `--burst` sets how many blocks exist at genesis and `--bundle-size` the segment size, which together decide how much parallel work a request has to do; the state store lives under `--data-dir`, so deleting it is what gives a cold backprocess again. The command waits for the merger to catch up with the burst before starting tier1, since tier1 bootstraps its block hub from merged blocks and cannot become ready until they exist. The end-to-end tests now share this same setup code, so what a developer watches locally is the path CI exercises. - - Running the tiers in-process pulls in the wasmtime runtime, whose bindings are cgo-only, so the command is built only when cgo is enabled — that is, in a local `go build` or `go install`, but not in the released Docker image, which is deliberately static. That image could not run it anyway, having no Docker daemon of its own. - -### Changed - -- Dependencies: bumped the `all` group, notably `github.com/ClickHouse/clickhouse-go/v2` to v2.48.0, `github.com/AfterShip/clickhouse-sql-parser` to v0.5.5, `google.golang.org/grpc` to v1.83.0 and the OpenTelemetry SDK to v1.45.0. - - `clickhouse-sql-parser` v0.5.x replaced the `String()` method on AST nodes with a `Formatter`; the ClickHouse `ON CLUSTER` schema rewriting path was updated to `parser.Format()`, which emits the same single-line SQL as before. - -- Sink: `substreams sink postgres` and `substreams sink clickhouse` in from-proto mode now unmarshal and walk blocks on a worker pool instead of one at a time at flush. Decoding a block was where the sink spent its time — around 7.8µs of the 9.3µs a wide entity cost — while PostgreSQL sat on roughly eight times that in headroom, so the sink itself was the throughput limit. Measured at 4.1x on a wide entity. Workers only fill a per-block buffer; the inserts are still replayed in block order, in the same transaction as before, so the resulting database is unchanged. - - The pool defaults to one worker per core less one, capped at eight, since the work is allocator-bound before it runs out of cores. `SinkerFactoryOptions.DecodeWorkers` overrides it. - -- Sink: **Breaking** `db_proto.NewSinker` takes a `decodeWorkers int` parameter, and `SinkerFactoryOptions.Parallel` is gone along with `sql.Database.Clone()`. The parallel flush path they served was unreachable — `Parallel` was hardcoded to false at every call site — and unsound had it run: `Clone()` returned the receiver, so every goroutine shared one `*sql.Tx`. - -- CLI: `substreams run` now reports backprocessing as progress and rates instead of a list of block ranges. A four-line session header (trace ID, module, chain, work to do, including how much was already cached) is printed once and stays in the scrollback, followed by a compact live block: overall percentage, blocks per second, ETA, running jobs against the worker limit, then one bar per stage with its job count and oldest job age, and an `out` row tracking the output frontier towards the requested start block. - - Percentages come from the work counts the server already reported in `SessionInit` and were never displayed, and the in-flight job progress is added to the completed-job count so the bar advances continuously rather than in steps. A run whose stores are fully cached now says so in one line instead of rendering an empty progress skeleton. +### Sink - `Slowest modules` is kept as its own section, ranked across all stages, now showing both a recent (30s window) and a lifetime per-block cost, tagged with the stage each module belongs to. Modules under 10ms per block no longer earn a line. +- Fixed: **BREAKING** `substreams sink postgres` in Relational Mappings Mode stores `bytes` fields as binary in their + `BYTEA` columns. Under the default `--bytes-encoding=raw` they were corrupted: a 7-byte value became the 14 + characters of its base64 form including quotes, or of its hex form with `--no-constraints`. The same confusion stored + repeated scalar elements with their SQL quotes (`'alpha'` rather than `alpha`). Nothing failed loudly — the rows were + there and every query against those columns matched nothing. Databases populated by an affected version need the + affected range re-synced, and anything downstream built against the corrupted form breaks. + +- Fixed: **BREAKING** `substreams sink postgres` in Relational Mappings Mode no longer doubles backslashes when + rendering string literals. With `standard_conforming_strings` on — the server default since PostgreSQL 9.1 — the + INSERT paths stored two backslashes where the value had one, affecting `string` columns, enum names and every + JSON-rendered message column (protojson output is backslash-heavy). New rows store the value verbatim, so a database + populated by an affected version holds the doubled form below the resume point and the correct form above it, and a + consumer written to un-escape the old form breaks at that boundary. Re-sync the affected range to converge on one + form. + +- **BREAKING** Timestamp columns written through binary COPY — the default write mode on PostgreSQL — keep their full + microsecond precision. Previous versions rendered timestamps as RFC3339 and truncated them to the whole second, and + the rendered `batch-insert` / `row-insert` modes still do. Rows written before the upgrade (or through a rendered + mode) are second-precision while COPY-written rows are not, so equality joins or comparisons against pre-existing + rows on a `TIMESTAMP` column can stop matching on sub-second values. + +- **automatic postgres index**: **BREAKING** Added an automatic index on `_block_number_` for every postgres table + (Relational Mappings Mode) for undo performance. It is built when the sink starts — `CREATE INDEX CONCURRENTLY`, one + table at a time — including on a pre-existing database populated by an earlier version, so the first start after the + upgrade builds the indexes over the existing data before streaming, and a failed build stops the sink. Pass + `--disable-block-number-index` to keep the previous behavior of not having these indexes. + **Running this new version on a db populated by an earlier version will create the indexes on startup** + +- **spool**: Major speed improvement (+10x) for Relational Mappings Mode (`substreams sink postgres` and `substreams sink clickhouse`): + rows are now written to disk (spool) first and loaded them from a background goroutine, one segment at a time. + The stream no longer waits on the database (up to the size of the spool directory). + Each write mode spools directly in the format it sends: binary COPY files, rendered SQL tuples per table, an interleaved + log replayed in walk order for `row-insert`, typed values on ClickHouse. + - `--spool-dir` (default `./localdata/spool`) is where they land + - `--spool-max-size` (8GiB) limits the size on disk (pushing back to the stream) + - `--spool-max-idle` (10s) commits the open segment when the stream goes quiet for that duration + - `--db-write-target-duration` (3s) says how long one commit to DB should take, adjusted so segment size adjusted dynamically + - `--db-write-max-size` (512MiB) limits the size of each commit to DB + +- **write-mode**: (postgres) in Relational Mappings Mode new flag + - `--write-mode`: + - `copy`: binary COPY) + - `batch-insert`: one multi-row INSERT per table + - `row-insert`: one prepared INSERT per row + - `auto`: try to use `copy` if available, otherwise `batch-insert` or `row-insert` depending on the schema + +- **hyperpb**: Relational Mappings Mode now parses block payloads with [hyperpb](https://buf.build/go/hyperpb) instead of + `dynamicpb`. Both are driven by the module descriptor and read through `protoreflect`, so rows are identical, but + hyperpb compiles the descriptor once and parses into an arena: 18x on the parse, one allocation per block instead of + thousands. Unmarshalling is now done on a worker pool instead of one at a time at flush. Defaults to one worker per core less one, + capped at 8; `SinkerFactoryOptions.DecodeWorkers` overrides. + +- **constraints**: Relational Mappings Mode now loads without database constraints and creates them afterwards: +- - `--apply-constraints`: + - `auto` (default) creates them when the stream reaches chain HEAD + - `manual` never creates them + - `always` creates them before the load (inducing 27x slower load time on postgres) + Manual creation is done with : `substreams sink postgres constraints apply `, which can be tweaked with: + `--constraints-parallelism`, `--constraints-work-mem`. + Both automatic and manual creation can be controlled with `--disable-foreign-keys`, `--disable-primary-keys=`, `--disable-unique-constraints=`. + +- **automatic clickhouse schema**: `substreams sink clickhouse` now accepts a package whose output proto carries no schema annotations. + Both `setup` and the run refused it outright ("clickhouse table options for table X don't have any 'order_by_fields'"), they now default to: + `ORDER BY (_block_number_, _row_id_)`, `PRIMARY KEY (_block_number_)`, `PARTITION BY (toYYYYMM(_block_timestamp_))`. + `_row_id_` is a column automatically generated with the row number for a given block. + A database whose tables disagree with the package is now refused at start rather than + written into, since `CREATE TABLE IF NOT EXISTS` would have kept the old table and written into the wrong columns. + +- Flags are grouped in "Relational Mappings Mode" VS "Database Changes Mode" in `--help` and invalid flags for one mode are now rejected by the other. + A Database Changes Mode Substreams owns its SQL schema, so `sink postgres constraints` refuses it. + +- Removed flag `--live-block-time-delta` is no longer accepted by `substreams sink postgres` and `substreams sink clickhouse`. The + SQL sink installs the cursor-based liveness checker itself, based on the type of message received from the stream. + +- Fixed: Relational Mappings Mode now writes the blocks up to the stop block (previously, last segment was left out.) +- `substreams sink postgres` in Relational Mappings Mode now performs UNDO on a reorg even without database constraints. +- Fixed: Relational Mappings Mode no longer crashes on a module whose output carries an `enum` field. Covers plain, `repeated`, `inline` and key fields. + +### Observability + +- RPC: `ModulesProgress.stages` entries now expose per-stage squash visibility, so a client can tell "segment produced" + apart from "segment actually usable". `Stage` gained `ready_up_to_exclusive` (field 3), the chain block number, + exclusive, up to which the stage is immediately usable, and `squash_wait_segment_count` (field 4), the number of + segments whose partial exists but has not been squashed in yet. `completed_ranges` counts a segment as soon as its + partial is produced, so a stage could render 100% covered with substantial work outstanding — and since squashing + runs on tier1 it schedules no job and advances no `processed_blocks`, leaving the request looking frozen at 100% with + a rate of zero. For a stage with no store module the two notions coincide and the count stays 0. A stage that has not + started reports where its modules begin, floored at the chain's first streamable block, so 0 means "nothing usable + yet" and is not a sentinel for "unknown". + +- RPC: `SessionInit` gained `segment_block_count` (field 11), the width in blocks of one parallel segment, constant for + the session. Without it a client could not turn `Stage.squash_wait_segment_count` into blocks: it could only be + inferred from `Job.stop_block - Job.start_block`, unavailable exactly when needed since no job runs while tier1 + squashes. It is an upper bound — the first and last segment of a run are narrower. + +- Server: tier1 request logs now explain how many parallel workers a request got, and why; asking for 300 and getting + 15 previously left no trace. `incoming Substreams Blocks request` gained a `parallelism` object + (`requested_workers`, `granted_workers`, effective `workers`, `workers_source`, `plan_tier`, `stage_layer_executors`) + plus `parallel_segment_count` and `stage_count`. `substreams request stats` gained a tier1-only `workers` object + (`requested`, `granted`, `effective`, `peak`, `pool_exhausted_count`, `pool_rampup_deferred_count`): a high + `pool_exhausted_count` with `peak` well below `effective` means the shared pool ran dry, previously visible only at + debug level. The periodic `substreams request progress` gained its own `workers` object (`requested`, `granted`, + `effective`, `running`, `idle`, `pool_exhausted_5m`) so the question can be answered while the request still runs, + with a hint naming the three ceilings that cause it — tier2 fleet full, organization quota, per-session cap — since + the pool reports a single error for all three. + +- Sink: the Relational Mappings Mode periodic stats report how far the download is ahead of the database: + `downloaded_through`, `applied_through`, `blocks_ahead`, `blocks_buffered`, `peak_blocks_ahead`. Substreams + throughput is paid for, so a run should be limited by the stream and not by the database. Once the buffer stops + looking like a working set the line is logged at warning level: `database is falling behind the stream, the buffer is + over half of what it was given`. With a spool that threshold is a share of `--spool-max-size`, since a block count + says nothing once rows are on disk — the sparse start of a large backfill used to warn continuously; without a spool + the blocks are in memory and their count is what is reported. + +- `Sinker.PrintStats` collapses to a single `📊 Usage Report: no data received` line when a request produced nothing. + Affects `substreams run`, `substreams sink webhook` and `substreams sink noop`. + +- The gRPC User-Agent of a SQL sink run now names the engine as well as the mode: `sink_from_proto_pg`, + `sink_from_proto_ch`, `sink_database_changes_pg`, `sink_database_changes_ch`. - Stage progress is measured from `Stage.ready_up_to_exclusive` rather than from `completed_ranges`. The latter counts a store segment as soon as its partial has been produced, so a stage rendered as 100% while tier1 was still squashing — and since squashing schedules no job and advances no `processed_blocks`, the request looked frozen at 100% with a rate of zero. That state is now named on the stage row as `squashing N segments`. Reading the squashed frontier also fixes the bar's low end: it is the lowest *contiguous* block across the stage's modules, so a stage with a gap in its produced ranges no longer counts the ranges beyond that gap, and a stage that has not started reports where it begins instead of being invisible. - The `Longest-running jobs` section is gone: it fired on a fixed 5 second threshold, which flickered in and out on healthy runs where jobs normally take 5 to 8 seconds. Job age is now always visible on the stage rows instead. Also removed: the `Progress messages received` counter, which only tracked the server's deliberately decreasing progress interval, and the `m` key toggling between bar and block-range rendering, which no longer has two modes to switch between. +### CLI -- CLI: `substreams run` output on failure is no longer one undivided wall of text. The session header, the progress block, the usage report and the error are separated, the progress block is closed with `Backprocessing aborted` rather than trailing off at `starting…`, and a request refused for exceeding `--limit-processed-blocks` now reports the figures it was given at session init and names the fix: +- `substreams run` reports backprocessing as progress and rates instead of a list of block ranges. A four-line session + header (trace ID, module, chain, work to do including what was already cached) is printed once and stays in the + scrollback, followed by a compact live block: overall percentage, blocks per second, ETA, running jobs against the + worker limit, one bar per stage with its job count and oldest job age, and an `out` row tracking the output frontier + towards the requested start block. Percentages come from work counts the server already reported in `SessionInit` and + never displayed, with in-flight job progress added so bars advance continuously. A run whose stores are fully cached + says so in one line instead of rendering an empty skeleton. + + Stage progress is measured from `Stage.ready_up_to_exclusive` rather than `completed_ranges`, so a stage no longer + renders 100% while tier1 is still squashing — that state is now named on the row as `squashing N segments`. The + squashed frontier also fixes the bar's low end: it is the lowest *contiguous* block across the stage's modules, so + ranges beyond a gap no longer count and a stage that has not started reports where it begins instead of being + invisible. + + `Slowest modules` is kept as its own section, ranked across all stages, showing a recent (30s) and a lifetime + per-block cost tagged with the stage; modules under 10ms per block no longer earn a line. Removed: `Longest-running + jobs` (a fixed 5s threshold that flickered on healthy runs where jobs take 5 to 8 seconds — job age is now always on + the stage rows), the `Progress messages received` counter, and the `m` key toggling bar vs block-range rendering. + +- `substreams run` output on failure is no longer one undivided wall of text: session header, progress block, usage + report and error are separated, the progress block is closed with `Backprocessing aborted` rather than trailing off + at `starting…`, and a request refused for exceeding `--limit-processed-blocks` reports the figures from session init + and names the fix: ``` Error: --limit-processed-blocks is set to 10,000, but this request needs to process 3,394,913 blocks (3,394,000 of them to prepare the stores): raise it with --limit-processed-blocks 3400000, or remove the guard entirely with --limit-processed-blocks 0 ``` -- Sink: `Sinker.PrintStats` collapses to a single `📊 Usage Report: no data received` line when a request produced nothing, instead of a header followed by three zeroed counters. Affects `substreams run`, `substreams sink webhook` and `substreams sink noop`. - -- Sink: `substreams sink postgres` and `substreams sink clickhouse` in from-proto mode now parse block payloads with [hyperpb](https://buf.build/go/hyperpb) instead of protobuf-go's `dynamicpb`. Both are driven only by the module's descriptor, which is all the sink has at runtime, and both are read through `protoreflect` by the descriptor walk, so the rows are identical — but hyperpb compiles the descriptor into a parser once and parses into an arena, at 18x the speed and one allocation per block instead of thousands. +- Fixed: `substreams run` needed two Ctrl-C to stop while the progress view was on screen. The view puts the terminal + in raw mode, so the first Ctrl-C arrived as a key press: the UI quit but the request kept streaming. The key press + now cancels the request directly. - End to end that is 1.9x on the full decode path for a wide entity (88k to 168k rows/s on one core) and 2.7x across the decoder's worker pool (517k to 1.40M rows/s on eight). The parse is no longer the expensive half of decoding: the descriptor walk is now around 90% of it. For wide entities the sink is no longer the throughput limit either — eight workers now produce more rows per second than binary COPY absorbs. +- Fixed: `substreams run` never printed the `Backprocessing history up to requested target block` line, nor the head + block, stage count and cached-blocks summary the non-TTY output has always shown — the line was guarded on a field + nothing ever set and rendered as blank lines. - hyperpb builds on amd64 and arm64 only. Every release target already qualifies, and 32-bit was unbuildable for unrelated reasons well before this. +- Added Ethereum Hoodi testnet (`hoodi`) StreamingFast endpoints (`hoodi.eth.streamingfast.io:443`). -- Sink: **Breaking** `sql.Database.WalkMessageDescriptorAndInsert`, `WalkMessageDescriptorAndInsertInto`, `BaseDatabase.WalkMessageDescriptorAndInsertWithDialect` and `sql.Dialect.AppendInlineFieldValues` take a `protoreflect.Message` where they took a `*dynamicpb.Message`. The walk only ever read the message, so this is what lets the parser be chosen by the caller. - -### Fixed - -- Sink: `substreams sink postgres` and `substreams sink clickhouse` in from-proto mode now write the blocks still held when a bounded run reaches its stop block. Blocks accumulate until `--block-batch-size` of them have gone by, and nothing drained that buffer at the end of the range, so a run ending mid-batch silently dropped its last blocks along with the cursor covering them — and still reported success. With a batch size larger than the requested range, nothing was written at all. - -- Sink: `substreams sink postgres` and `substreams sink clickhouse` in from-proto mode no longer crash on a module whose output carries an `enum` field. protoreflect hands an enum out as a `protoreflect.EnumNumber`, a named `int32` that neither dialect's type switch matched: PostgreSQL panicked with `unsupported type: protoreflect.EnumNumber` and ClickHouse on a failed `value.(int32)` assertion. Since the two disagree on the column type — PostgreSQL declares it `TEXT`, ClickHouse `Int32` — the walk now carries both renderings and each dialect takes the one matching the column it created, so PostgreSQL stores the enum's name and ClickHouse its number. A value absent from the descriptor falls back to its number. This covers an enum wherever it appears: as a plain field, as a `repeated` one, inside an `inline` nested message, and as a table's primary key. - -- Sink: **BREAKING** — `substreams sink postgres` in from-proto mode now stores `bytes` fields as binary in their `BYTEA` columns. Under the default `--bytes-encoding=raw` both inserters corrupted them, each differently: without `--no-constraints` a 7-byte value was stored as the 14 characters of its base64 form including the surrounding quotes, and with it as the 14 characters of its hex form. The same confusion also broke repeated scalar fields without `--no-constraints`, where each array element was stored with SQL quotes as part of its value (`'alpha'` rather than `alpha`). +### Server - Nothing failed loudly for any of these: the rows were all there, and every query against those columns simply matched nothing. Databases already populated by an affected version hold corrupted values in those columns and need the affected block range re-synced. - This changes what lands in those columns, so it breaks anything downstream built against the corrupted form. A consumer reading a `bytes` column as base64 or hex text now gets the binary value instead. A query written to work around the array corruption — matching the element `'alpha'`, quote characters and all, because that is what was stored — now has to match `alpha`. Re-syncing an existing database leaves both forms in the same table until the affected range is rewritten. +- `substreams-tier1` restarts when its block hub can no longer link incoming live blocks, instead of hanging every + request at a frozen head indefinitely. A live-source gap whose one-block files were already merged away can never be + linked, and the head-block metrics keep tracking the live source, so the process looked healthy throughout. -- CLI: `substreams run` needed two Ctrl-C to stop while the progress view was on screen. The view puts the terminal in raw mode, so the first Ctrl-C arrived as a key press rather than as SIGINT: the UI quit but the request kept streaming, and only the second one — delivered because the first had released the terminal — actually stopped it. The key press now cancels the request directly. +### Library -- CLI: `substreams run` was never printing the `Backprocessing history up to requested target block` line, nor the head block, stage count and cached-blocks summary the non-TTY output has always shown. The line was guarded on a field that nothing in the codebase ever set, and rendered as blank lines instead. +- **Breaking** `db_proto.NewSinker` takes a `decodeWorkers int` parameter, and `SinkerFactoryOptions.Parallel` is gone + along with `sql.Database.Clone()`. The parallel flush path they served was unreachable (`Parallel` hardcoded to false + at every call site) and unsound had it run: `Clone()` returned the receiver, so every goroutine shared one `*sql.Tx`. + +- **Breaking** `sql.Database.WalkMessageDescriptorAndInsert`, `WalkMessageDescriptorAndInsertInto`, + `BaseDatabase.WalkMessageDescriptorAndInsertWithDialect` and `sql.Dialect.AppendInlineFieldValues` take a + `protoreflect.Message` where they took a `*dynamicpb.Message`, which is what lets the caller choose the parser. + +### Tools + +- `substreams tools devenv` boots a complete local stack — a dummy blockchain in a container plus a tier1 and a tier2 + built from the current source tree — prints the endpoint and stays up until interrupted. Requires Docker. `--burst` + sets how many blocks exist at genesis and `--bundle-size` the segment size, which together decide how much parallel + work a request has; the state store lives under `--data-dir`, so deleting it is what gives a cold backprocess again. + The command waits for the merger to catch up with the burst before starting tier1, which bootstraps its block hub + from merged blocks. The end-to-end tests share this setup code. Running the tiers in-process pulls in wasmtime, whose + bindings are cgo-only, so the command is built only with cgo enabled — a local `go build` or `go install`, not the + released static Docker image, which has no Docker daemon of its own anyway. + +- `substreams tools extract-proto [ []]` writes a module's output protobuf definition back out. With + `--sql` it comes annotated for the SQL sink's Relational Mappings Mode — every message and field carrying its option + commented out — and the annotations file is written beside it so the result parses as-is, which is what + `--proto-file-override` needs. Starting from an unannotated package was otherwise a matter of finding the right proto + and extension names by hand. + +### Dependencies + +- Bumped notably `github.com/ClickHouse/clickhouse-go/v2` to v2.48.0, `github.com/AfterShip/clickhouse-sql-parser` to + v0.5.5, `google.golang.org/grpc` to v1.83.0 and the OpenTelemetry SDK to v1.45.0. + +### Summary of changed flags and subcommands + +#### ADDED (relational-mappings `run` unless noted) +--write-mode auto|copy|batch-insert|row-insert; auto = copy on PG, batch-insert on CH +--decode-workers 0 = auto (min(8, cores-1)) +--decode-batch-size 0 = auto (4x workers); successor of --block-batch-size +--db-write-target-duration 3s; sizes each DB commit by measured duration +--db-write-max-size 512MiB; segment size ceiling +--spool-dir ./localdata/spool; spool ON by default, "" rejected +--spool-max-size 8GiB; local-disk budget, backpressures stream +--spool-max-idle 10s; idle seal, 0 disables +--apply-constraints auto|manual|always (run, setup, constraints apply); auto = build at HEAD +--disable-foreign-keys per-table list or 'all' +--disable-primary-keys per-table list or 'all' +--disable-unique-constraints per-table list or 'all' +--disable-block-number-index default false: _block_number_ index built at every startup +--constraints-parallelism 1 (constraints apply|drop) +--constraints-work-mem "" = server default (constraints apply|drop) +(new subcommands: constraints apply / constraints drop; setup gains optional [module] arg) + +#### DEPRECATED (warn, still honored) +--no-constraints -> the three --disable-* flags; hard error on DatabaseChanges module +--block-batch-size -> --decode-batch-size; removed from setup entirely = unknown flag +--constraints-per-transaction -> --constraints-parallelism (born deprecated) + +#### REMOVED (hard 'unknown flag') +--live-block-time-delta both engines; hits DatabaseChanges users too + +#### CHANGED BEHAVIOR (same names) +setup no longer creates constraints by default; needs --apply-constraints=always +cross-mode flags wrong-mode flags now hard-error at startup (develop warned/ignored) +clickhouse state flags --cursor-file-path etc. now also on setup; defaults unchanged -- Server: `substreams-tier1` now restarts when its block hub can no longer link incoming live blocks, instead of hanging every request at a frozen head indefinitely. A live-source gap whose one-block files were already merged away can never be linked, and the head-block metrics keep tracking the live source, so the process looked healthy throughout. ## v1.21.0 diff --git a/go.mod b/go.mod index 6d42ae2ea..2611223f3 100644 --- a/go.mod +++ b/go.mod @@ -100,8 +100,8 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 - golang.org/x/mod v0.38.0 - golang.org/x/net v0.57.0 + golang.org/x/mod v0.40.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 google.golang.org/grpc v1.83.0 gopkg.in/yaml.v2 v2.4.0 @@ -243,7 +243,7 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.49.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect ) @@ -316,11 +316,11 @@ require ( go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.54.0 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/api v0.274.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect rogchap.com/v8go v0.9.0 diff --git a/go.sum b/go.sum index 3fb604257..87498eaff 100644 --- a/go.sum +++ b/go.sum @@ -842,8 +842,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= @@ -860,8 +860,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -879,8 +879,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -932,8 +932,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -957,8 +957,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go.work.sum b/go.work.sum index 6bd5f851a..4b3350fb5 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1378,6 +1378,7 @@ github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -1576,6 +1577,7 @@ golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1718,6 +1720,7 @@ golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= @@ -1781,6 +1784,7 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= @@ -1831,6 +1835,7 @@ google.golang.org/api v0.249.0/go.mod h1:dGk9qyI0UYPwO/cjt2q06LG/EhUpwZGdAbYF14w google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0= google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA= google.golang.org/api v0.273.1/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= diff --git a/plans/2026-08-07-fast-local-buffer-copy.md b/plans/2026-08-07-fast-local-buffer-copy.md new file mode 100644 index 000000000..df9ae3d26 --- /dev/null +++ b/plans/2026-08-07-fast-local-buffer-copy.md @@ -0,0 +1,380 @@ +> **Superseded.** The design below shipped, but its vocabulary did not: the local buffer +> is the spool, `--local-buffer*` became `--spool-*`, and the knobs it left hardcoded are +> now flags. See `plans/2026-08-12-coherent-sink-sql-flags.md` for the current shape. + +# Local buffer + binary COPY for the from-proto Postgres sink + +Status: proposal, revised after measurement +Scope: `sink/sql/db_proto/**` (from-proto mode, PostgreSQL, insert-only). DatabaseChanges +mode and ClickHouse are out of scope. + +Two goals, in priority order: + +1. **Get data out of Substreams as fast as possible.** The user pays for Substreams + throughput. The stream must never wait on PostgreSQL, and blocks already paid for + must never be downloaded twice. +2. **Reach maximum end-to-end performance with very few knobs.** Everything that can be + auto-tuned from a measurement should be, and the sink should always be able to say + *which* of the three stages is the limiter. + +## 1. What the measurements say + +Two benchmark suites in `sink/sql/db_proto/benchmarks/`. All numbers are +`postgres:17-alpine` in a testcontainer on an M-series laptop; ratios matter, absolutes +do not. + +### Server side — `TestCopyVsInsert`, 250k rows, 13 columns, ~88 MiB + +| strategy | duration | rows/s | vs current | +|---|--:|--:|--:| +| per-row prepared INSERT (`RowInserter`) | 22.199s | 11k | 0.10x | +| multi-row INSERT built at flush (`AccumulatorInserter`, current) | 2.281s | 110k | 1.00x | +| multi-row INSERT prebuilt on disk | 1.989s | 126k | 1.15x | +| COPY CSV from disk | 724ms | 345k | 3.15x | +| **COPY BINARY from disk** | **313ms** | **799k** | **7.29x** | +| `pgx.CopyFrom` over in-memory rows | 339ms | 737k | 6.73x | + +### Client side — `TestClientEncodeCeiling`, single core, no database + +Production path: `proto.Unmarshal` into `dynamicpb`, then +`WalkMessageDescriptorAndInsertWithDialect` with the real Postgres dialect. + +| stage | narrow entity (2 cols) | wide entity (60+ cols) | +|---|--:|--:| +| unmarshal only | 1.79M rows/s | 203k rows/s | +| + walk (discard values) | 855k rows/s | 128k rows/s | +| + encode to binary COPY | **745k rows/s** | **96k rows/s** | +| + encode to text literals (current) | 767k rows/s | 108k rows/s | + +Per wide row, single core: 4.9 µs unmarshal, 2.9 µs walk, 2.6 µs encode = 10.4 µs. + +### The conclusion that reorders the whole plan + +**For wide entities the sink's own CPU is the bottleneck, not PostgreSQL.** One core +produces 96k rows/s; binary COPY absorbs 799k rows/s. PostgreSQL has ~8x headroom that +the current single-threaded handler cannot use. For narrow entities the two are within +10% of each other. + +So the ranked levers are: + +1. **Parallelise decode + walk + encode across cores.** Blocks are independent. On 10 + cores a wide-entity workload goes from ~96k to ~900k rows/s, at which point + PostgreSQL becomes the limiter again — which is the correct place for it to be. +2. **Binary COPY** — 7.3x on the server side, and the only strategy that can absorb the + parallelised client. +3. **Decouple the stream from the flush** — so a slow or stalled PostgreSQL never stops + the download the user is paying for. +4. Pre-encoding to disk is worth only ~8% of raw throughput on its own (313ms vs 339ms). + Its value is entirely in (3) and in never re-downloading. + +### Two hypotheses tested and rejected + +- **"NUMERIC encoding via big.Int is the hot spot."** Mapping every unsigned integer to + `BIGINT` instead of `NUMERIC` moves wide-entity throughput from 96k to 100k rows/s, + 4%. Not worth a schema change or a hand-rolled numeric encoder. (`uint32` → + `NUMERIC` at `sql/postgres/types.go:75` is still arguably wrong on its own merits, + since `uint32` fits `BIGINT` exactly — but it is not a performance argument.) +- **"Building SQL strings is the current bottleneck."** Prebuilt-on-disk statements are + only 1.15x faster than building them at flush time, so `ValueToString` plus + `strings.Builder` are ~13% of the current cost. The other ~87% is the server parsing + a multi-megabyte statement. Optimising the Go-side string building is a dead end. + +One optimisation was found and kept: caching the pgtype encode plan per column in +`pgcopy.Writer` rather than letting `Map.Encode` resolve one per value. Worth +8% on +wide entities, +3% on narrow, no downside. + +## 2. Architecture + +``` + substreams gRPC stream + │ raw BlockScopedData, handler returns immediately + ▼ + [block queue, bounded] + │ + ├──▶ decode worker 1 ─┐ unmarshal (dynamicpb) + ├──▶ decode worker 2 ─┤ walk message descriptor + └──▶ decode worker N ─┘ encode to PGCOPY binary + │ + [segment slots, filled in stream order] + │ + ▼ + local buffer: sealed segments on disk ◀── download cursor lives here + │ (PGCOPY binary + manifest.json) + │ + ├──▶ COPY worker 1 ─┐ + └──▶ COPY worker M ─┘ COPY ... FROM STDIN (FORMAT BINARY) + │ + ▼ + PostgreSQL ◀── apply cursor lives here +``` + +Each stage is bounded and instrumented, which is what makes the bottleneck verdict in §6 +possible. + +### 2.1 Two cursors + +This is the change that serves goal 1 directly. + +- **Download cursor** — persisted in the local buffer next to the segments. On restart the + stream resumes from here. +- **Apply cursor** — in PostgreSQL, in `_cursor_`, as today. Only ever advances to the + last block of a committed segment. + +Today there is only the apply cursor, so a PostgreSQL outage or a sink crash means +re-streaming — and re-paying for — every block since the last successful commit. With a +local buffer, downloaded blocks are on disk and are never fetched twice. If the buffer is +lost or torn, the sink falls back to the apply cursor and re-streams, which is correct, +just slower and billable. + +### 2.2 Ordering with parallel decode + +Rows are insert-only with no foreign keys in this mode, so the *data* has no ordering +requirement. Only the segment→cursor mapping does. + +Assign each block a slot in the current segment at receive time, in stream order. Workers +fill their slots in any order. A segment seals when every slot in it is filled, and its +manifest records the cursor of its highest block. No reorder buffer, no per-row locking. + +### 2.3 Segment layout + +``` +// + download-cursor.json # cursor + block of the last sealed segment + seg-%016d-%s/ # first block num + random suffix + manifest.json # written last; its presence = sealed + .pgcopy # PGCOPY binary, one file per table +``` + +`manifest.json` holds `first_block`, `last_block`, `cursor`, and per table the file name, +row count, byte count and the resolved column list. A segment without a valid sealed +manifest is torn and is deleted on recovery. + +### 2.4 Why PGCOPY binary on disk + +`COPY ... FROM STDIN (FORMAT BINARY)` is a length-prefixed tuple stream. Writing that +exact format to disk makes the flush an `io.Copy` from file to socket: no re-encoding, no +escaping, no parsing at flush time. That is what lets the COPY workers run independently +of the decode workers, and it makes the buffer directory a first-class artifact that can +be loaded into any PostgreSQL later (§7). + +Format: 19-byte header (`"PGCOPY\n\377\r\n\0"`, int32 flags, int32 header extension +length), then per tuple an int16 field count and per field an int32 length (`-1` for +NULL) plus the raw binary value, then an int16 `-1` trailer. Implemented in +`sink/sql/db_proto/sql/postgres/pgcopy`. + +## 3. Column OIDs — the main correctness hazard + +Binary COPY performs **zero type coercion**. The bytes for column *i* must match that +column's actual type OID exactly or PostgreSQL aborts the COPY, sometimes with a +confusing error. Text COPY would coerce; binary will not. + +Do not derive OIDs from `MapFieldType`'s declared type names. Read them from the live +catalog once per table (`pgcopy.LoadColumns` does this against `pg_attribute`). + +Conversions that need care, all handled by `pgcopy.Normalize`: + +| Go value from the walker | Column type | Encoding | +|---|---|---| +| `uint64` | `NUMERIC` | `pgtype.Numeric` from `big.Int`; **never** send an int8 | +| `string` with an int128/uint256/decimalN `ConvertTo` | `NUMERIC`/`DECIMAL(p,s)` | parse to `pgtype.Numeric`; empty string stays 0, as `RowInserter` does today | +| `[]byte` | `BYTEA` / `TEXT` | raw bytes, or `bytesEncoding.EncodeBytes` when `IsStringType()` | +| `*timestamppb.Timestamp`, `time.Time` | `TIMESTAMP` | µs since 2000-01-01, not RFC3339 | +| `[]any` | `[]` | typed slice, then the pgtype array codec | +| inline message | `JSONB` | `protojson.Marshal`, with the JSONB `0x01` version byte | + +`TestPgCopyBinaryRoundTrip` loads 1000 rows across all 13 benchmark column types through +both a parameterised INSERT and binary COPY and asserts the results are byte-identical. +That test is the safety net; extend it whenever a new column type becomes reachable. + +## 4. Auto-tuning: what the knobs are, and what they are not + +Three flags, and nothing else user-facing: + +``` +--local-buffer enable the decoupled path and say where the buffer lives +--local-buffer-max-size disk quota (default: min(10% of free space, 16GiB)) +--local-buffer-only download only, never connect to PostgreSQL (see §7) +``` + +Everything else is derived at runtime: + +| Quantity | How it is chosen | +|---|---| +| decode workers | `max(1, NumCPU-1)`; blocks are independent so this is free parallelism | +| COPY workers | start at 1; add one whenever the segment queue is non-empty **and** the previous addition improved segment throughput; drop back when it does not. Cap at 8 | +| segment size | target a 3s COPY; after each segment `next = clamp(current * clamp(3s/observed, 0.5, 2), 8MiB, 512MiB)`. Seal on size or a 30s age bound, never mid-block | +| direct vs buffered | `isLive` — live blocks go through the existing accumulator, backfill goes through the buffer (§8) | +| backpressure | block the receive loop when the queue is full or the disk quota is reached; count the wait | + +Sizing segments by **bytes, not blocks or rows**: block payload size varies by orders of +magnitude across chains and modules, so a block count gives wildly unstable flush +durations. Multiplicative control with a 2x per-step clamp avoids oscillation. + +## 5. Concurrency and transaction boundaries + +The COPY workers apply whole segments. With more than one worker, segments can commit out +of order, so "applied through block N" stops being expressible as a single number. + +Add a small `_segments_` table: + +```sql +CREATE TABLE ._segments_ ( + first_block BIGINT NOT NULL, + last_block BIGINT NOT NULL, + cursor TEXT NOT NULL, + applied_at TIMESTAMP NOT NULL DEFAULT now(), + PRIMARY KEY (first_block) +); +``` + +Each COPY worker runs one transaction per segment: COPY every table file, insert the +`_segments_` row, commit. The apply cursor in `_cursor_` is advanced by whichever worker +completes a segment that extends the contiguous prefix. Recovery replays any local +segment with no `_segments_` row. + +This keeps exact atomicity per segment — a crash never leaves a partially applied +segment, so there is nothing to clean up and no `DELETE ... WHERE _block_number_ > x` +sweep is needed. Within one segment the table files are COPYed sequentially, because a +transaction is bound to one connection; parallelism comes from running several segments +at once, not several tables. + +`_segments_` is also directly queryable by the user, which is half of goal 2. + +## 6. Observability: naming the bottleneck + +The sink must be able to say which stage is the limiter. All three states are real, and +the measurements in §1 show the third is the common one today. + +| Signal | Verdict | +|---|---| +| block queue empty, decode workers idle | **Substreams is the limiter** — the sink is keeping up, the stream is as fast as it gets | +| block queue full, decode workers saturated, segment queue empty | **The sink's CPU is the limiter** — raise decode workers, or the entity is expensive to walk | +| segment queue growing, disk usage climbing, backpressure engaged | **PostgreSQL is the limiter** — the download is still running at full speed and buffering to disk | + +Emit this as a single periodic log line plus Prometheus gauges, in the vocabulary the +user actually cares about: + +``` +downloaded through #20,431,200 (3.2 GiB on disk, 41k blocks ahead) +applied through #20,390,118 +limiter: postgres copy 118 MiB/s, 4 workers, backpressure 62% of the last 30s +``` + +Metrics to add to `db_proto/stats.Stats`: `DecodeDuration`, `EncodeDuration`, +`CopyDuration`, `CopyBytes`, `BlockQueueDepth`, `SegmentQueueDepth`, `DiskBytesInUse`, +`BackpressureWaitDuration`, `DownloadedBlock`, `AppliedBlock`. `BackpressureWaitDuration` +near zero versus dominant is the single number that separates state 2 from state 3. + +## 7. The buffer as an artifact + +Once the buffer directory holds PGCOPY binary plus manifests, three things become +possible for free, and they serve goal 1 better than any tuning: + +- **`--local-buffer-only`** — stream and encode at full speed, never open a database + connection. The user pays Substreams once, at the maximum rate their CPU allows. +- **`substreams sink postgres buffer-load `** — load a buffer directory into any + PostgreSQL, with as many parallel COPY workers as that server can take. Offline, + resumable, and repeatable into several databases. +- **`substreams sink postgres buffer-status `** — report what has been downloaded + without touching PostgreSQL at all. + +`buffer-load` overlaps heavily with the existing `inject-csv` command +(`cmd/substreams/sink_postgres_inject_csv.go`), which already streams CSV into +`PgConn().CopyFrom`. Same shape, binary format, manifest-driven. + +## 8. Live blocks, undo, and constraints + +- **Live blocks bypass the buffer.** When `isLive`, use the existing accumulator path: + latency matters, volume is trivial, undo signals arrive, and round-tripping through + disk is pure overhead. Backfill uses the buffer. The transition backfill→live must + **drain the segment queue and wait for the last COPY to commit** before the first + direct insert, or a direct row could land ahead of an older buffered one. +- **Undo signals** only occur near head, where the queue is empty. Guard explicitly: + assert an empty queue and no in-flight segment, then run the existing + `HandleBlocksUndo`. Error loudly rather than silently mis-ordering. +- **The buffered path requires `--no-constraints`.** `useConstraints` already selects + `RowInserter`, a different path entirely, and FK checks are immediate per row unless + declared `DEFERRABLE INITIALLY DEFERRED`, which per-table COPY files would violate. + Fail fast at startup if both are set. + +## 9. Crash recovery + +On startup, before running the stream: + +1. Read the apply cursor from PostgreSQL and the applied ranges from `_segments_`. +2. Scan the buffer for segment directories, sorted by `first_block`. +3. Drop segments without a valid sealed manifest (torn write). +4. Drop segments already recorded in `_segments_` (crash between COMMIT and `rm`). +5. Drop segments beyond a hole — if an earlier segment was lost, nothing after it is + usable. +6. Replay the contiguous remainder through the COPY workers. +7. Resume streaming from the download cursor, which is now consistent with what is on + disk. + +**fsync policy:** fsync only `manifest.json` on seal, not the data files. A process crash +is fully covered. A machine crash may leave a sealed segment short, so validate each data +file's trailer and byte length against the manifest at recovery; on mismatch discard from +that segment onward and fall back to the apply cursor. The cost of being wrong is +re-downloading a few segments — correct but billable, never corrupt. + +## 10. Implementation order + +1. `pgcopy` package: binary writer, OID resolution from `pg_attribute`, `Normalize`, + round-trip test per column type. **Done** — `sink/sql/db_proto/sql/postgres/pgcopy/`. +2. Benchmarks establishing the server and client ceilings. **Done** — + `sink/sql/db_proto/benchmarks/`. +3. **Parallel decode workers.** The largest single lever (§1). **Done** — + `sink/sql/db_proto/decoder.go`. Measures 4.13x at eight workers on a wide entity + (129k → 534k rows/s); scaling flattens past eight, the work being allocator-bound + before it runs out of cores, so the default is capped there. +4. Buffer writer: segments, manifests, download cursor, sealing. Synchronous COPY on seal + at first, so the file format and recovery are exercised before concurrency is added. +5. COPY workers, `_segments_` table, bounded queues, disk-quota backpressure. +6. Auto-tuning of segment size and COPY worker count (§4). +7. Recovery and replay (§9), with a test that SIGKILLs mid-backfill and asserts no gaps + and no duplicates. +8. Bottleneck verdict logging and metrics (§6). +9. `--local-buffer-only`, `buffer-load`, `buffer-status` (§7). + +Step 3 is worth doing first and on its own: it needs no new file format, no recovery +story, and on the measured numbers it is where the throughput is. + +## 11. Pre-existing issues in the way + +These interact with adding worker goroutines and must be fixed or consciously accepted +before step 3: + +- ~~`holding` is a package-level global, `Database.Clone()` does not clone, and + `appengine.MultiError` is appended to from several goroutines without a lock.~~ Fixed: + the unreachable `--parallel` path that depended on all three is removed. +- ~~`postgres.ValueToString` renders raw bytes as `E''::BYTEA`, storing twice the + intended bytes.~~ Fixed on its own branch, `fix/sink-sql-bytea-encoding` — unrelated + to this work. +- Blocks still held when a bounded run reaches its stop block are never flushed: nothing + drains `holding` at stream end. Not touched here, but it bounds what any test of the + flush path can assert. +- ~~`WalkMessageDescriptorAndInsertWithDialect` builds `zap.Any` debug fields once per + message whether or not debug is enabled.~~ Fixed: both that and the per-row equivalent + in `RowInserter` are behind `Check` now. Worth ~6% on a narrow entity. + +## 12. Server-side settings worth trying first + +None of this removes WAL amplification, index maintenance, checkpoint I/O or constraint +checks. With two secondary indexes on the benchmark table, binary COPY's advantage drops +from 7.29x to 3.84x. Before assuming the ratios above transfer: + +```sql +SELECT query, calls, total_exec_time, mean_exec_time +FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 20; + +SELECT wait_event_type, wait_event, count(*) +FROM pg_stat_activity WHERE state='active' GROUP BY 1,2; +``` + +Cheap wins that may reduce how much of this is needed: + +- `synchronous_commit = off` on the sink's session — safe here, because a lost commit is + recovered by the buffer replay logic. +- Raise `max_wal_size` and `checkpoint_timeout` for the duration of a backfill. +- Drop secondary indexes for the backfill and build them at the end. Usually the single + largest factor at tens of GB. diff --git a/plans/2026-08-12-coherent-sink-sql-flags.md b/plans/2026-08-12-coherent-sink-sql-flags.md new file mode 100644 index 000000000..1316490f5 --- /dev/null +++ b/plans/2026-08-12-coherent-sink-sql-flags.md @@ -0,0 +1,482 @@ +# Coherent flags for the from-proto SQL sink + +Status: implemented +Date: 2026-08-12 +Follows: the local buffer / binary COPY work on `feature/sink-sql-local-cache` + +## Why + +The spool arrived as a second way of writing to the database without retiring the first +one's vocabulary. Before this plan, the flag surface said different things depending on +the driver, on whether a directory flag was set, and on where the stream happened to be: + +- `--block-batch-size` used to size the database transaction. With the local buffer on — + the default on PostgreSQL — it sizes nothing but the in-memory decode batch: transactions + are a no-op (`postgres/database.go:366`), and the write to the database is sized by a + hardcoded 3s target inside the buffer (`postgres/buffer/buffer.go:386`). Same flag, same + help text, three behaviours across postgres+buffer, postgres+`--local-buffer ""`, and + ClickHouse. +- The knobs that actually decide database load — `TargetFlushDuration` 3s, + `SegmentMinBytes` 8MiB, `SegmentMaxBytes` 512MiB, `QueueDepth` 2 — are unreachable. + Only `Dir` and `MaxBytes` come from flags (`cmd/substreams/sink_sql_common.go:974`). +- `QueueDepth: 2` binds long before `--local-buffer-max-size` 8GiB can: at most 3 sealed + segments of ≤512MiB are ever in flight, so the disk high-water is ~2GiB and `awaitQuota` + never fires at defaults. The advertised backpressure is not the one that runs; the real + limit is the blocking channel send in `Seal`. +- `addFromProtoModeRunFlags` is registered on `setup` and on `apply-constraints` + (`cmd/substreams/sink_postgres.go:71,75`), so two commands that never decode a block or + write a row carry `--block-batch-size`, `--local-buffer` and `--local-buffer-max-size` — + and `apply-constraints` carries `--apply-constraints`. +- Which write path runs is never stated by the user. It is derived from driver + + whether `--local-buffer` is non-empty + whether the schema's foreign key graph happens + to have a cycle (`postgres/database.go:107-115`, `:150-181`), with a `logger.Warn` on + the downgrade. + +## Principles + +1. **One unit.** Everything the operator sizes is expressed against *one durable commit + to the database*. That unit exists in all three write paths, so every knob applies + everywhere. +2. **Three prefixes, three concerns.** `--decode-*` is CPU. `--db-write-*` is the commit + unit. `--spool-*` is how far ahead of the database the stream may run. A flag's prefix + says which resource it spends. +3. **Every `--decode-*`, `--db-write-*` and `--spool-*` flag has an effect in every write + mode.** No mode-conditional flags. +4. **Backfill knobs are named as such.** At chain HEAD the sink inserts directly and + self-bundles against the block interval; the help text says so rather than leaving the + operator to discover it. +5. **Silence becomes a message.** A mode that cannot be honoured is an error; a mode + chosen for the operator is logged with its reason. + +## Design + +### The commit unit + +One commit is **one spool segment**, in every mode: its files plus the cursor, applied in +a single transaction. The mode changes only how the segment's bytes are pushed. + +| write mode | how a segment is applied | +|---|---| +| `copy` | one `COPY ... FROM STDIN (FORMAT BINARY)` per table | +| `batch-insert` | multi-row INSERTs per table, ordered by foreign key | +| `row-insert` | single-row prepared INSERTs, in walk order | + +So the sizer has **one dial in all three modes: segment bytes**. There is no per-mode dial +and no rows-based flag — row count falls out of the segment. +`--db-write-target-duration` runs one control loop: measure the last commit, scale the +segment target by `target/actual` (clamped 0.5–2.0), clamp to an internal floor and to +`--db-write-max-size`. This is `Buffer.resize` generalised out of the buffer package, +unchanged in shape. + +The loop absorbs the cost difference between modes for free. `row-insert` is roughly an +order of magnitude more expensive per row than the multi-row path, so at the same 3s +target it simply converges on a segment about that much smaller. The operator states the +intent once — "no commit should hold my database longer than this" — and it means the same +thing whichever path is running. Exposing a rows knob would instead make them re-derive +the equivalent number per mode. + +Sizing by measured duration rather than by a block count is what keeps this stable across +chains, where block payloads differ by orders of magnitude. + +Two things stay internal, being implementation facts rather than operator preferences: + +- **Statement chunking.** `batch-insert` splits a table's rows across statements to stay + under PostgreSQL's 65535 bind parameters and ClickHouse's `max_query_size`. +- **The lower clamp**, a constant, not a flag. It exists only to stop a death spiral: a + database stalled on lock contention returns a long elapsed time, the loop halves the + segment each round, and without a floor it converges on segments whose cost is entirely + per-segment overhead — manifest write, fsync, transaction, one statement or COPY setup + per table. That is a correctness guard on the controller, not a policy the operator + should have to hold an opinion about, and it is not honestly explainable as a knob: + in `row-insert` the floor can exceed what the mode pushes in the target duration, so a + flag named for the sizer would sometimes silently override the sizer. 8MiB, as today's + `SegmentMinBytes`. + +### Spool becomes the transport, write-mode becomes the drain + +Today the on-disk spool exists only for COPY, which is what makes `--spool-*` a +copy-only concept. Invert it: the sinker always writes decoded rows to a spool segment, +and the applier goroutine drains a sealed segment using whichever write mode is selected. + +``` +stream → decode workers → spool segment (disk) → applier → database + --decode-* --spool-* --db-write-*, --write-mode +``` + +Consequences: + +- `--spool-dir` and `--spool-max-size` mean the same thing in every mode, on both drivers. +- `--write-mode` no longer decides *whether* rows are buffered, only *how* a sealed + segment reaches the database. It stops being entangled with `--local-buffer`'s + empty-string-means-off convention. +- The stream stops waiting on the database in all three modes, not just COPY. +- Blocks already downloaded survive a restart in all three modes. + +### Spool format: per-mode, always pre-rendered + +The spool always holds bytes that are ready to send. Rendering happens on the sinker's +side, where it is off the database's critical path and rides the decode workers; the +applier concatenates and sends. + +| format | used by | contents | +|---|---|---| +| PGCOPY binary | `copy` | PostgreSQL's binary COPY wire format, one file per table | +| rendered tuples | `batch-insert`, `row-insert` | the dialect's own SQL literals, one value tuple per row, one file per table | + +Two formats, three modes, both driver-aware. Pre-encoding to the target format is worth +~8% on its own for COPY (`sink/sql/db_proto/benchmarks`), and the same argument applies to +the INSERT modes, where `AccumulatorInserter` already renders values to strings before +building its statement (`postgres/accumulator_inserter.go:13-16`) — that rendering simply +moves to spool time. + +ClickHouse uses the rendered-tuple format with its own dialect, so it needs no new write +mode: `--write-mode=batch-insert` is what `auto` already picks there. A future native bulk +path (ClickHouse `RowBinary` over the native protocol) would be a third format and a +fourth mode, `clickhouse-native`; out of scope here, but the applier interface should not +foreclose it. + +The flags are universal; the bytes on disk are an implementation detail. + +### One limit on how far ahead of the database the stream may run + +`QueueDepth: 2` goes away, and no flag replaces it. A cap on the *number* of pending +segments was never the operator's concern — the disk budget is — and having two ceilings +meant the one nobody could see always won: at most 3 sealed segments of ≤512MiB are ever +in flight, so the real high-water was ~2GiB and `awaitQuota` could not fire at the 8GiB +default. The advertised backpressure was not the one that ran. + +The bounded channel becomes an unbounded queue (slice plus condition variable) drained by +the applier. Queue entries are pointers, so the queue itself costs nothing; what bounds +the system is `--spool-max-size` and only that. It also drops the decoupling window from a +fixed ~9s of database stall to whatever the operator's disk budget buys, which is what the +flag implied all along. + +Two accounting fixes go with it, both of which let the quota mean what it says: + +- `awaitQuota` runs after `seal()` has already written the files. Check the quota before + writing, not after. +- `bytesOnDisk` excludes the open segment (`BytesOnDisk()` adds it for reporting only), so + the real high-water is `MaxBytes` plus up to one segment. Count the open segment. + +### Sealing on idle + +A segment is written when it is big enough — or when the stream goes quiet. This is the +ordinary size-or-idle batching rule (Kafka's `linger.ms`, group commit, Nagle): the size +trigger buys throughput, the idle trigger bounds what is lost when the producer stops. + +Today the triggers are size, `Close`, range completion (`sinker.go:192`) and the live +switch (`postgres/database.go:203`) — nothing covers a stream that simply stalls +mid-backfill. The open segment then sits unsealed indefinitely, the cursor never advances, +and those blocks are streamed and paid for a second time on restart. +`--spool-max-idle` fills exactly that hole and no other. + +**Idle, not a deadline.** An alternative would be a maximum interval between commits, but +that fires hardest on the case it is not for: a slow-but-progressing stream would have its +segments chopped short precisely when it can least afford small commits. Idle is +self-disabling under load — it can only fire when there is nothing to lose by sealing +early — which is what makes a 10s default safe. + +**Trigger is rows, not blocks.** "No new row written to the open segment", so a chain +producing blocks whose module output is empty still counts as idle. + +Two things this deliberately does not solve: + +- **A trickle is not idle.** Rows arriving steadily but slowly never trip the timer, so + the segment fills over a long stretch and a crash re-streams all of it. That is the case + a deadline would catch. Skipped: backfill throughput is high by definition, a sparse + module produces rows in bursts far enough apart that idle does fire, and the residual + blast radius is one segment. Revisit if it bites. +- **Rows arriving just slower than the window** — every 11s against a 10s idle — make every + row its own segment and its own transaction. At that arrival rate the database is doing + one tiny transaction every 11s. Not worth guarding. + +**It needs a mutex, and that is its real cost.** `b.current` is touched only from the +sinker's goroutine today and is unsynchronized. Idle sealing has to come from a timer +goroutine — when the stream stalls the sinker is blocked in a read and has no next +opportunity to check anything — so the open segment needs a lock shared with +`Insert`, `RecordBlock`, `RecordCursor` and `MaybeSeal`. + +## Command surface + +No `run` subcommand — the root `sink postgres [ []]` stays the run +action. + +``` +sink postgres [ []] run +sink postgres setup create the schema +sink postgres constraints apply create constraints on a loaded database +sink postgres constraints drop drop them again +sink postgres tools ... unchanged +``` + +`apply-constraints` becomes `constraints apply`. `constraints drop` is new: the escape +hatch after `--apply-constraints=always`, and what makes a stalled backfill fast again +without a re-`setup`. + +Persistent flags on `sink postgres` are unchanged: `--dsn`, operator flags. Everything +below is registered non-persistently with `.Flags()`, so no subcommand inherits a knob it +cannot act on — and `addFromProtoModeRunFlags` stops being called from `setup` and from +the constraints commands. + +### Root command (run) — added + +| flag | type | default | description | +|---|---|---|---| +| `--write-mode` | string | `auto` | How a sealed spool segment reaches the database: `copy` (binary COPY), `batch-insert` (one multi-row INSERT per table), `row-insert` (one prepared INSERT per row). `auto` picks copy on PostgreSQL, batch-insert on ClickHouse, and row-insert for a schema whose foreign keys cannot be ordered. An explicit mode the driver or the schema cannot support is an error, not a downgrade. Ignored once the stream reaches chain HEAD. | +| `--decode-workers` | int | `0` | Blocks unmarshalled and walked concurrently. Zero takes one per core less one for the goroutine draining the stream, capped at 8: measured at 4.13x on eight workers and only 4.24x on fifteen, the work being allocator-bound well before it runs out of cores. CPU only — does not change what the database sees. | +| `--decode-batch-size` | int | `0` | Blocks held in memory and decoded together. Zero takes four per decode worker. Larger keeps the workers fed; smaller costs less memory, since every held block keeps its payload and its decoded rows. Sizes the CPU stage, not the database write. | +| `--db-write-target-duration` | duration | `3s` | How long one commit to the database should take. A commit is one spooled segment, whichever write mode applies it; each is measured and the next segment sized toward this. Raise it for fewer, larger commits; lower it to keep the sink from occupying a database shared with something else. Backfill only. | +| `--db-write-max-size` | size | `512MiB` | Ceiling for the segment size the sizer may choose, whatever the target duration would allow. Backfill only. | +| `--spool-dir` | string | `./localdata/spool` | Directory pending segments are written to. The stream never waits on the database, and blocks already downloaded survive a restart instead of being streamed, and paid for, twice. Backfill only. | +| `--spool-max-size` | size | `8GiB` | Disk budget for `--spool-dir`, and the only bound on how far ahead of the database the stream may run. The stream is held once pending segments reach it, which is what turns a slow database into backpressure rather than a full disk. Backfill only. | +| `--spool-max-idle` | duration | `10s` | Write the open segment to the database once no new row has reached it for this long, short of its size target. A stream that stalls would otherwise sit on those rows indefinitely, leaving the cursor where it was and the blocks to be streamed, and paid for, again on restart. Zero disables it. Backfill only. | + +### Root command (run) — changed + +| flag | change | +|---|---| +| `--apply-constraints` | values become `auto` (default), `manual`, `always`. `auto` has the sink create the constraints itself once the backfill reaches chain HEAD, or at the end of a bounded run. `manual` leaves it to `sink postgres constraints apply`. `always` creates them before the first row and loads with them in place — measured through binary COPY, 27x slower than loading without, where building the same constraints afterwards costs 3.3x. | + +Old values map: `head` → `auto`, `manual` → `manual`, `upfront` → `always`. They are +renamed outright with no alias, `--apply-constraints` never having shipped. + +**The default changes from `manual` to `auto`, deliberately.** A sink that reaches HEAD +with no primary keys and no foreign keys has produced a database nobody should query, and +leaving it that way until the operator remembers to run a second command is the worse +failure — it is silent, and it looks like success. `auto` accepts a stop-the-world pass at +the end of the backfill as the price of a database that is correct when the sink says it +is done. Operators who need that pass inside a maintenance window ask for `manual`, which +is exactly what the flag is for. + +What `auto` owes the operator, since the stall is real: + +- A log line **before** it starts, at Info, naming what is about to happen, that the + tables are locked while it runs, and that `--apply-constraints=manual` is how to take + the pass into a maintenance window instead. +- Per-constraint progress as they are built, so a long pass is distinguishable from a + hung one. +- The elapsed time when it finishes. + +Unchanged on the root command: `--disable-foreign-keys`, `--disable-primary-keys`, +`--disable-unique-constraints`, `--proto-file-override`, `--bytes-encoding`. + +### Root command — deprecated + +A deprecation alias is owed only to a flag that has actually shipped. Everything the local +buffer work added is unreleased as of `v1.21.0` — `--apply-constraints`, +`--disable-foreign-keys`, `--disable-primary-keys`, `--disable-unique-constraints`, +`--local-buffer`, `--local-buffer-max-size` all arrived on +`feature/sink-sql-local-cache` and have never been in a tag. Those are renamed outright, +with no alias and no warning: nobody can have a script using them. + +Only these two are in `v1.21.0` and get an alias, honoured for one release with a warning +naming the replacement: + +| old | maps to | +|---|---| +| `--block-batch-size` | `--decode-batch-size` | +| `--no-constraints` | `--disable-foreign-keys --disable-primary-keys=all --disable-unique-constraints=all` | + +`--no-constraints` needs the alias for a second reason: **the local buffer branch deletes +it without a replacement path.** It is registered and read on `develop` +(`sink_sql_common.go:156`, `:294`, `:496`) and is gone entirely on the branch. It maps +exactly onto `sql.DisableAllConstraints()`, so restoring it as an alias is mechanical, and +it must land whether or not the rest of this plan does. + +Renamed with no alias (unreleased): `--local-buffer` → `--spool-dir`, +`--local-buffer-max-size` → `--spool-max-size`, `--apply-constraints` values +`head`/`upfront` → `auto`/`always`. The old `--local-buffer ""` convention for turning +buffering off disappears with no equivalent — the spool is always on. + +### `setup ` — added: none + +Subtractive. Keeps `--disable-foreign-keys`, `--disable-primary-keys`, +`--disable-unique-constraints` (they shape the DDL this command writes), +`--bytes-encoding` (decides column types), `--proto-file-override` (resolves the schema), +plus its existing DatabaseChanges-mode flags. + +Drops `--apply-constraints` (timing is a run-time decision), `--block-batch-size`, +`--local-buffer`, `--local-buffer-max-size`. Does not register any `--write-mode`, +`--decode-*`, `--db-write-*` or `--spool-*`. + +### `constraints apply|drop ` — added: none + +Keeps `--disable-foreign-keys`, `--disable-primary-keys`, `--disable-unique-constraints`, +`--bytes-encoding` and `--proto-file-override`. + +They have to be passed again here, and deliberately so: `setup` is optional — the run +path creates the schema itself when it finds none — so anything it recorded would be +present or absent depending on a step nobody is required to take, and `constraints apply` +would quietly mean different things on two otherwise identical deployments. + +Drops everything else. The common invocation becomes +`sink postgres constraints apply --dsn ...`. + +`constraints drop` takes the same flags and removes the primary keys, unique constraints +and foreign keys the policy describes, skipping the ones already absent. Both are +idempotent. + +## DatabaseChanges mode owns its schema + +A Substreams whose output module is `sf.substreams.sink.database.v1.DatabaseChanges` (or +the older `sf.substreams.database.v1.DatabaseChanges`) does not use the from-proto model +at all. The operator owns the SQL schema entirely: it comes from the `schema.sql` bundled +in the manifest, the sink never derives tables from a proto descriptor, and there is no +constraint policy for the sink to have an opinion about. Mode is detected from the output +module type at run time (`sink_sql_common.go:242`, `:467`), not from a flag. + +Everything in this plan is from-proto only. Today the from-proto flags are silently +ignored in DatabaseChanges mode; that becomes an error. + +- **`constraints apply` and `constraints drop` fail outright** when the resolved output + module is DatabaseChanges: + + > this Substreams outputs DatabaseChanges, where the SQL schema is yours: it is created + > from the `schema.sql` bundled in the manifest, and the sink neither derives it nor + > manages its constraints. `sink postgres constraints` only applies to from-proto + > Substreams. + +- **From-proto flags explicitly set in DatabaseChanges mode fail**, on the root command + and on `setup`: `--write-mode`, `--decode-*`, `--db-write-*`, `--spool-*`, + `--apply-constraints`, `--disable-*`, `--proto-file-override`. The check is on + `cmd.Flags().Changed(name)`, so a default value never trips it — only a flag the + operator typed. + + > `--spool-max-size` only applies to from-proto Substreams. This one outputs + > DatabaseChanges, where rows are written from the module's own database changes and + > the schema is the `schema.sql` in the manifest. + +- **The reverse fails too.** `warnIgnoredDatabaseChangesSetupFlags` + (`sink_sql_common.go:532`) already covers DatabaseChanges-only setup flags passed to a + from-proto setup, but it only warns. Promote it to the same error, on the same + `Changed()` test, and extend it from `setup` to the run command: + `--batch-block-flush-interval`, `--batch-row-flush-interval`, + `--live-block-flush-interval`, `--flush-retry-count`, `--flush-retry-delay`, + `--undo-buffer-size`, `--cursors-table`, `--history-table`. + + Both directions error for the same reason: a flag typed for the other mode means the + operator expects the sink to be doing something it will not do, and a warning in a log + that scrolls past is not how they find that out. + +The detection runs after the package and module are resolved, so the guard belongs in +`newSinkRunE` and `newSinkSetupE` at the point they branch, and at the head of the +constraints commands' `RunE`. + +### Both vocabularies live on one command + +Mode is detected from the module, not chosen by a flag, so a flag cannot be registered +conditionally — the command has not read the package yet when `init()` runs. +`addSinkRunFlags` (`sink_sql_common.go:204`) therefore registers both +`addDatabaseChangesModeRunFlags` and `addFromProtoModeRunFlags` on the same root command, +and that is not fixable. + +The result is that `substreams sink postgres --help` lists both batching vocabularies +side by side — `--batch-block-flush-interval`, `--batch-row-flush-interval`, +`--live-block-flush-interval` next to `--decode-batch-size`, `--db-write-*` and +`--spool-*` — with a `[mode]` prefix per line as the only thing separating them. Removing +the from-proto flags from `setup` and the constraints commands does not help here; the run +command still shows every flag, half of them inert for any given Substreams. + +So partition the help output instead: a custom usage template rendering +`From-proto mode flags:`, `DatabaseChanges mode flags:` and `Common flags:` as separate +sections. That turns a per-line tag the operator has to scan into a heading they can skip. +The `[from-proto mode]` / `[DatabaseChanges mode]` prefixes then come out of the +individual help strings, which shortens every one of them. + +Grouping and the symmetric error are complementary: the sections say which half applies, +and typing from the wrong half fails rather than being ignored. + +## Messages that replace silence + +- At startup: one line with the resolved write mode, why it was resolved that way, and the + effective value of every knob that applies. +- `--write-mode=copy` (or `batch-insert`) with an unorderable foreign key graph: **error**, + naming the fix (`--disable-foreign-keys`). `auto` keeps the fallback but says which mode + it picked and why. +- `--apply-constraints=always` with any write mode: **warn**, that being the measured 27x + path — chosen deliberately or by accident, and the log line is the only thing that tells + them apart. +- At the live switch: the existing message (`postgres/database.go:209`) gains the second + half — which flags stop applying from here on. + +## Decisions worth keeping + +- **`--apply-constraints=auto` is the default**, stop-the-world pass and all. A backfill + that ends with no constraints is a silent wrong result; a stall is a visible one. The + pass is at least visible. +- **Spool format is per-mode and pre-rendered**: binary COPY for `copy`, rendered tuples + for `batch-insert`, an interleaved log for `row-insert`, typed values for ClickHouse. +- **One release of deprecation, and only for flags that have shipped** — `--block-batch-size` + and `--no-constraints`. Everything this branch added is renamed outright, none of it + having been in a tag. +- **One sizer dial: segment bytes, in every mode.** No rows-based flag, so + `--db-write-max-size` keeps one meaning. `row-insert` needs no special case — being + slower per row, it just converges on a smaller segment at the same target duration. +- **No `--db-write-min-size`.** The lower clamp stays as an internal constant. It guards + the controller against a death spiral; it is not a policy, and as a flag it would + sometimes silently override the sizer it appears to configure. +- **No `--spool-max-chunks`.** `--spool-max-size` is the only ceiling on how far ahead of + the database the stream may run. +- **`--spool-max-idle`, 10s, sealing on idle rather than on a deadline.** A spool concern, + not a `--db-write-*` one: it does not size a commit, it stops the sink sitting on rows + nobody is adding to. Rejected alternatives: `--db-write-max-interval` (fires hardest on + the slow-but-progressing streams it is not for) and `--db-write-max-age` (reads as + retention, and names the data's staleness rather than the guarantee). + +### ClickHouse: the spool asks nothing new of it + +ClickHouse has no transactions and keeps its cursor in a file, which looks like it should +force a new recovery model on it. It does not. What ClickHouse does today is: + +1. accumulate rows in memory, in typed columnar builders; +2. send one columnar INSERT per table, with no transaction (`accumulator_inserter.go:562`); +3. write the cursor to `--cursor-file-path` afterwards (`click_house/database.go:363`). + +So a crash between 2 and 3 already re-streams those blocks and re-inserts those rows. +At-least-once is the guarantee ClickHouse ships with, and the spool does not have to +improve on it — it only has to not make it worse. + +Which it does not. The spool is a list of preprocessed blocks, nothing more: + +- **Applying a segment is exactly step 2 followed by step 3.** Same INSERTs, same cursor + file, same order, same window. +- **Recovery needs no bookkeeping table.** The cursor file already says what landed. + Segments on disk whose cursor is at or behind it are dropped; the rest are replayed. A + segment that was applied but whose cursor write was lost gets replayed and duplicates + exactly the rows re-streaming it would have duplicated — parity, and it saves paying + Substreams for them a second time. +- **No `_segments_` table, no deduplication setting, no cursor migration.** + +One honest cost, and it is the same one PostgreSQL pays: the cursor now advances per +segment rather than per flush, so a crash re-does up to one segment's worth of work +instead of one flush's. `--db-write-target-duration` and `--spool-max-idle` are what bound +it. + +What it needed was a **format**, not a recovery model. ClickHouse's inserts are typed and +columnar rather than SQL text, so `FormatTuples` is the wrong fit — rendering to literals +would change the insert path and its type handling. `FormatValues` stores the row values +tag-prefixed instead, and the applier decodes them straight back into the same column +builders the accumulator has always used, so what reaches the server is byte for byte what +an unspooled flush would have sent. + +A `*timestamppb.Timestamp` is normalised to `time.Time` on the way in, the accumulator +taking either. A value of a type the accumulator cannot consume fails at spool time rather +than being silently mangled at apply time. + +## Outcome + +Every flag above is registered, the three PostgreSQL write modes and ClickHouse all spool, +and `constraints apply|drop` takes the same schema switches as the run. Principle 3 holds +on both drivers. + +Two things the first draft of this plan got wrong, kept here because the reasoning is +worth having: + +- **`row-insert` cannot spool by table.** The spool groups rows into one file per table, + and a cyclic foreign key graph — the only reason that mode exists — has no table order + that keeps a parent ahead of its children. Hence `FormatRowLog`, an interleaved file + replayed in the walk's own order. +- **ClickHouse needed no new guarantees.** An earlier draft gave it + `insert_deduplication_token` and a cursor table, which solved a problem the sink does + not have: it already re-streams and re-inserts on a crash between the inserts and the + cursor write. See the section above. diff --git a/schema_proto_embed.go b/schema_proto_embed.go new file mode 100644 index 000000000..fab94fd9b --- /dev/null +++ b/schema_proto_embed.go @@ -0,0 +1,13 @@ +package substreams + +import _ "embed" + +// SQLSchemaProto is the annotations file a from-proto SQL sink's schema is declared with. +// +// It is embedded so `substreams tools extract-proto --sql` can write it next to the proto +// it annotates: the override is parsed with imports resolved from the working directory, +// so the file has to be on disk, and asking the operator to go and find the right version +// of it is most of the work the command exists to remove. +// +//go:embed proto/sf/substreams/sink/sql/schema/v1/schema.proto +var SQLSchemaProto string diff --git a/sink/sql/db_proto/benchmarks/.gitignore b/sink/sql/db_proto/benchmarks/.gitignore new file mode 100644 index 000000000..093b9a398 --- /dev/null +++ b/sink/sql/db_proto/benchmarks/.gitignore @@ -0,0 +1 @@ +.sinkbench/ diff --git a/sink/sql/db_proto/benchmarks/README.md b/sink/sql/db_proto/benchmarks/README.md new file mode 100644 index 000000000..faf979aba --- /dev/null +++ b/sink/sql/db_proto/benchmarks/README.md @@ -0,0 +1,410 @@ +# Relational Mappings Mode Postgres sink: ingestion benchmarks + +Two suites, answering the two questions that decide the design in +`plans/2026-08-07-fast-local-buffer-copy.md`: + +- **`TestCopyVsInsert`** — how fast can PostgreSQL absorb rows, per strategy? Runs against + a real server in a testcontainer. Every artifact is materialised on disk *before* any + timer starts, so a measured duration is transport plus server work, never row + generation — unless the variant's real implementation would do that work at flush time + too. +- **`TestClientEncodeCeiling`** — how fast can the sink *produce* rows, on one core, with + no database at all? Runs the production path: `proto.Unmarshal` into `dynamicpb`, then + `WalkMessageDescriptorAndInsertWithDialect` with the real Postgres dialect. Only the + `Inserter` changes between variants. No docker needed. + +The second one turned out to matter more than the first. See *What the numbers say*. + +## Running + +The correctness tests always run; only the measurements are gated, so plain +`go test ./...` stays fast. It does need a container runtime: + +| gate | covers | | +|---|---|---| +| *(none)* | `TestPgCopyBinaryRoundTrip` | correctness; always runs, and needs a container runtime | +| `SF_SINK_SQL_BENCHMARKS=true` | everything else | measurements; they assert nothing and cost minutes | + +`TestCopyVsInsert` still needs `SF_SINK_SQL_BENCHMARKS=true`; its container now comes on +its own. + +```bash +# correctness first: the binary encoder must be byte-identical to a parameterised INSERT +go test ./sink/sql/db_proto/benchmarks/ -run TestPgCopyBinaryRoundTrip -v + +# server side +PGBENCH_ROWS=250000 PGBENCH_REPEAT=2 \ + go test ./sink/sql/db_proto/benchmarks/ -run TestCopyVsInsert -v -timeout 30m + +# client side, no docker +go test ./sink/sql/db_proto/benchmarks/ -run TestClientEncodeCeiling -v +``` + +| Variable | Default | Meaning | +|---|---|---| +| `PGBENCH_ROWS` | `250000` | rows in the dataset | +| `PGBENCH_REPEAT` | `1` | passes over the variant set; best duration is reported | +| `PGBENCH_WITH_INDEX` | unset | add btrees on `id` and `_block_number_` before loading | +| `PGBENCH_PG_IMAGE` | `postgres:17-alpine` | server image | +| `PGBENCH_KEEP_DATA_DIR` | unset | reuse artifacts across runs instead of a temp dir | + + +## Variants + +| Variant | What it stands for | +|---|---| +| `insert-1row-prepared` | `RowInserter` — one prepared `INSERT` per row, in one tx | +| `insert-multirow-built-at-flush` | `AccumulatorInserter` — build the giant `VALUES` text statement at flush, exec it. **The current baseline.** | +| `insert-multirow-built-at-flush-libpq` | same through `database/sql` + `lib/pq`, to size the driver's share | +| `insert-multirow-prebuilt-from-disk` | the same statements, already built on disk — isolates pure server-side cost | +| `copy-csv-from-disk` | `COPY FROM STDIN (FORMAT CSV)`, `io.Copy` from file | +| `copy-binary-from-disk` | `COPY FROM STDIN (FORMAT BINARY)`, `io.Copy` from file — **the proposed spill path** | +| `copy-binary-encoded-at-flush` | `pgx.CopyFrom` over in-memory rows — binary COPY without pre-encoding | + +Correctness is not optional here: after every variant the table is fingerprinted in SQL +and compared against the same fingerprint computed in Go. A variant that loads different +data fails, however fast it was. + +## Measured + +250,000 rows, 13 columns (`BIGINT`, `TIMESTAMP`, `VARCHAR`, `BYTEA`, `INTEGER`, +`NUMERIC`, `BOOLEAN`, `DOUBLE PRECISION`, `TEXT[]`, `JSONB`), ~88 MiB of data. +`postgres:17-alpine` under OrbStack on an M-series laptop, best of 2 passes. Ratios, +not absolutes, are the point. + +### No indexes (the default, the backfill case) + +| variant | duration | rows/s | MiB/s | vs current | +|---|--:|--:|--:|--:| +| insert-1row-prepared | 22.199s | 11k | 4.1 | 0.10x | +| insert-multirow-built-at-flush | 2.281s | 110k | 40.1 | 1.00x | +| insert-multirow-built-at-flush-libpq | 2.296s | 109k | 39.8 | 0.99x | +| insert-multirow-prebuilt-from-disk | 1.989s | 126k | 46.0 | 1.15x | +| copy-csv-from-disk | 724ms | 345k | 123.1 | 3.15x | +| **copy-binary-from-disk** | **313ms** | **799k** | **280.3** | **7.29x** | +| copy-binary-encoded-at-flush | 339ms | 737k | 258.8 | 6.73x | + +### With two secondary indexes + +| variant | duration | vs current | +|---|--:|--:| +| insert-multirow-built-at-flush | 2.812s | 1.00x | +| copy-csv-from-disk | 1.135s | 2.48x | +| **copy-binary-from-disk** | **733ms** | **3.84x** | + +## What constraints cost, and what COPY is still worth under them (`TestConstraintCost`) + +500,000 rows through a relational shape — `blocks <- parents <- children`, children also +carrying a unique column — loaded by binary COPY and by multi-row INSERT, under each set +of constraints. `postgres:17-alpine`, M-series laptop. + +| load runs with | COPY | vs bare | INSERT | vs bare | COPY gain | +|---|--:|--:|--:|--:|--:| +| no constraints | 127ms | 1.00x | 610ms | 1.00x | **4.79x** | +| primary keys | 552ms | 0.23x | 1.079s | 0.56x | 1.96x | +| primary keys + unique | 748ms | 0.17x | 1.255s | 0.49x | 1.68x | +| primary keys + unique + foreign keys | 4.063s | 0.03x | 4.639s | 0.13x | **1.14x** | +| loaded bare, constraints created after | 453ms | 0.28x | 940ms | 0.65x | 2.07x | + +Single run, and the foreign-key variant is the noisiest of them: a second sample on the +same machine gave 163ms / 609ms / 847ms / 5.151s / 462ms, so read these as ratios with +about 15% of slack, more like 25% on the foreign-key row. The conclusions below survive +either sample. + +**Foreign keys dominate everything else.** Primary keys cost 4.3x, uniques add little on +top, and foreign keys take the load from 748ms to 4.06s — a per-row referential check that +no write path avoids. + +**Under full constraints the write path stops mattering**: 4.06s against 4.64s, a 1.14x +edge. All the work is server-side by then, so the buffer and binary COPY buy nothing. The +COPY advantage is 4.79x precisely when the load is bare, which is the whole argument for +loading bare and creating the constraints afterwards — 453ms for the same end state, 9x +cheaper than the 4.06s of loading with them in place. + +## End to end against a real substreams (`live-benchmark.sh`) + +The Go suites below isolate one stage each. `live-benchmark.sh` measures what an operator +actually experiences: stream, decode and load together, against a live endpoint. It runs +on Linux or macOS and needs only docker, python3, and either go or a prebuilt binary. + +```bash +export SUBSTREAMS_API_KEY=... + +cd sink/sql/db_proto/benchmarks +./live-benchmark.sh # 10k and 50k blocks +SIZES="10000 50000 200000" ./live-benchmark.sh +ENDPOINT=https://mainnet.eth.ca.streamingfast.io ./live-benchmark.sh +WARM=1 ./live-benchmark.sh # range nobody has streamed yet +``` + +It builds the binary from the checkout, starts its own PostgreSQL container, runs both +variants at each size and prints the table. Everything lands in +`./.sinkbench` (gitignored); results accumulate in `results.tsv` across invocations, so +several endpoints can be compared in one table. + +### Requirements + +On a fresh Ubuntu (24.04 tested), the script itself needs only: + +```bash +sudo apt-get update +sudo apt-get install -y docker.io python3 ca-certificates +sudo usermod -aG docker "$USER" # then log out and back in, or: newgrp docker +``` + +`python3` and `ca-certificates` ship with a normal Ubuntu install but are absent from +minimal and container images; without the certificates every HTTPS call fails with +`x509: certificate signed by unknown authority`. + +Then the binary, either of two ways: + +**Bring a prebuilt binary** — nothing else to install, no repo checkout, no Go: + +```bash +# on any machine with Go: +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o substreams ./cmd/substreams +scp substreams live-benchmark.sh live-report.py ubuntu-host:~/bench/ + +# on the Ubuntu host: +cd ~/bench && SUBSTREAMS_BIN=./substreams ./live-benchmark.sh +``` + +The build is CGO-free, so it cross-compiles cleanly and runs on a bare Ubuntu. + +**Or build on the machine** — needs `git`, a checkout, and **Go 1.26**, which apt does +not carry (24.04 ships 1.22): + +```bash +sudo snap install go --classic # or install the official tarball +sudo apt-get install -y git +git clone && cd substreams/sink/sql/db_proto/benchmarks +./live-benchmark.sh +``` + +Also needed: `SUBSTREAMS_API_KEY`, outbound HTTPS to the endpoint, to `spkg.io` for the +package and to Docker Hub for the PostgreSQL image, and disk for the database. + +**Disk is the one that bites.** At roughly 110 KB per block that is about 5.5 GB for the +default sizes and about 20 GB at 200,000 blocks, plus the `postgres:17-alpine` image +(291 MB) and the local buffer directory, which `BUFFER_MAX` bounds. Note the space has to +be free where *docker* stores its data, which is often a different filesystem from the +working directory. The script prints both before it starts and warns when the estimate +does not fit. + +Running out mid-run is worth recognising, because the sink only reports +`driver: bad connection` -- PostgreSQL dies part-way through a write and the client just +sees the connection go away. On any such failure the script now dumps the database's own +log, which says `No space left on device` plainly. + +| variable | default | | +|---|---|---| +| `ENDPOINT` | `https://mainnet.eth.streamingfast.io` | | +| `SIZES` | `10000 50000` | block counts to measure | +| `START_BLOCK` | `20000000` | | +| `PACKAGE` / `MODULE` / `TABLE` | `erc20-balance-changes` / `map_balance_changes` / `balancechange` | | +| `WARM` | `0` | stream the range once first, see below | +| `SUBSTREAMS_BIN` | *(built from source)* | skip the build | +| `PG_PORT` / `PG_IMAGE` / `PG_CONTAINER` | `55432` / `postgres:17-alpine` / `sinkbench-pg` | | +| `BUFFER_MAX` | `8GiB` | `--local-buffer-max-size` | +| `BLOCK_BATCH` | *(sink default, 25)* | `--block-batch-size`; lower it if a backend is killed for memory | + +**Warming, `WARM=1`, is off by default.** The server-side cache holds for about 30 days, +so a range that has been streamed recently is already warm and warming it again only +costs time. + +It matters on a range nobody has touched. A cold range is dominated by that first pass +rather than by the sink, so whichever variant ran first would pay for the other's stream +as well as its own: cold, a 5,000-block comparison read 9.4x where warm it is 2.9x. +Endpoints also cap how many *uncached* blocks a single request may process — +`mainnet.eth.ca` rejects anything over 10,000 outright — so a cold large range fails +rather than merely running slowly. The script recognises that error and points at +`WARM=1`. + +**These are totals, to durable.** The clock is launch to exit, and the sink only exits +after `HandleBlockRangeCompletion` has flushed what is held and drained the buffer. Every +check on the resulting data runs strictly after the process is gone, so a variant that +deferred work past exit reports fewer rows rather than a better time. The `drain` column +is the part of the run that happened after the stream finished — a path that merely +deferred its work would show a long tail there. + +### Measured + +`erc20-balance-changes@v1.4.0` / `map_balance_changes`, Ethereum mainnet from block +20,000,000, into `postgres:17-alpine`. Every row verified identical between variants: +same rows, same blocks, same fingerprint. + +**On a server** — AWS `c5.2xlarge` in `us-east-2`, EBS `gp3` at 1000 MB/s and 10,000 IOPS: + +| endpoint | blocks | rows | accumulator | buffer | speedup | acc rows/s | buffer rows/s | +|---|--:|--:|--:|--:|--:|--:|--:| +| mainnet.eth | 10,000 | 3,375,211 | 94.9s | 18.3s | 5.19x | 35,566 | 184,438 | +| mainnet.eth | 50,000 | 17,018,743 | 473.8s | 66.6s | 7.11x | 35,919 | 255,537 | +| mainnet.eth.ca | 10,000 | 3,375,211 | 92.4s | 16.5s | 5.60x | 36,528 | 204,558 | +| mainnet.eth.ca | 50,000 | 17,018,743 | 445.9s | 65.7s | 6.79x | 38,167 | 259,037 | +| mainnet.eth.ca | 200,000 | 66,273,320 | 1741.0s | 304.8s | 5.71x | 38,066 | 217,432 | + +Reproducibility on this machine is good: 10,000 blocks on mainnet.eth.ca measured twice +came out 90.6s/16.3s and 92.4s/16.5s, a 2.0% and 1.2% spread. The two endpoints agree to +within a few percent as well. + +**The speedup does not trend with size.** It is 5.6x at 10,000, 6.8x at 50,000 and 5.7x +at 200,000 — a hump, not a slope, and the same shape appears on both endpoints. The two +sides explain it separately: + +- The **accumulator is almost perfectly linear**: 35.6k, 35.9k, 36.5k, 38.2k, 38.1k rows/s + across every size and endpoint. It is bound by one core building a multi-megabyte + statement and one backend parsing it, and neither cares how large the table has grown. +- The **buffer is not**: 184k to 259k rows/s. Fixed startup — schema setup, connection, + the segment sizer ramping from its 8 MiB floor — is amortised over less data at 10,000, + and by 200,000 the table is around 20 GB and PostgreSQL's own ingest has slowed a + little. The best case sits in between. + +So the honest headline for server hardware is a range, 5.6x to 7.1x, rather than a trend. + +**On a laptop** — M-series, `postgres:17-alpine` in Docker Desktop: + +| endpoint | blocks | rows | accumulator | buffer | speedup | +|---|--:|--:|--:|--:|--:| +| mainnet.eth | 10,000 | 3,375,211 | 40.0s | 12.2s | 3.28x | +| mainnet.eth | 50,000 | 17,018,743 | 220.7s | 70.2s | 3.14x | +| mainnet.eth | 200,000 | 66,273,320 | 749.6s | 270.2s | 2.77x | +| mainnet.eth.ca | 10,000 | 3,375,211 | 72.9s | 55.2s | 1.32x | +| mainnet.eth.ca | 50,000 | 17,018,743 | 353.8s | 311.3s | 1.14x | +| mainnet.eth.ca | 200,000 | 66,273,320 | 1297.8s | 939.9s | 1.38x | + +Repeat measurement, laptop mainnet.eth.ca 200,000 accumulator: 1297.8s and 1364.8s, +5.2% spread. + +### What the two machines say + +**The server is where the accumulator hurts most.** The buffer lands near the laptop's +figures (18.3s against 12.2s at 10,000) while the accumulator is more than twice as slow +(94.9s against 40.0s). That is the shape to expect: the accumulator is bound by one core +building and one backend parsing a multi-megabyte statement, and the c5's single-core +throughput is well below an M-series laptop's, whereas the buffer spends its time in COPY +and in the stream. Laptop figures are the conservative case, not the flattering one. + +The laptop's mainnet.eth column shows the ratio falling with size (3.28x, 3.14x, 2.77x) +where the server shows a hump. Neither is a law; both are the same two curves — a flat +accumulator and a buffer whose throughput depends on startup amortisation and on how large +the table has grown — sampled on different hardware. + +**The laptop's `.ca` rows are a network artifact, not an endpoint property.** They were +taken over wifi to a geographically distant endpoint, and the same endpoint from a server +behaves like any other. What they do still illustrate is the regime rather than the +cause: when blocks arrive slowly for *any* reason, the sink is stream-bound, there is +little sink cost left to remove, and the ratio collapses toward 1x — 1.1x to 1.4x here, +at every size including 10,000. Measuring the sink over a slow or distant link measures +the link. + +**Throughput is flat in data volume.** On the laptop against mainnet.eth the buffer holds +276k, 242k and 245k rows/s across 10k, 50k and 200k, and the accumulator 84k, 77k and +88k. Neither degrades from 3.4M to 66M rows. + +## Parallel decode scaling (`TestClientDecodeScaling`) + +The per-block work the sinker's decoder parallelises — unmarshal, walk, buffer the +inserts — on a wide entity, 16 cores available: + +| workers | rows/s | speedup | +|--:|--:|--:| +| 1 | 129k | 1.00x | +| 2 | 220k | 1.70x | +| 4 | 355k | 2.75x | +| 8 | 534k | 4.13x | +| 15 | 548k | 4.24x | + +Flat past eight: the work is allocator-bound before it runs out of cores. That is why +the decoder defaults to one worker per core less one, capped at eight. + +## What the numbers say + +**For wide entities the sink's own CPU is the bottleneck, not PostgreSQL.** One core +produces 96k rows/s; binary COPY absorbs 799k. PostgreSQL has ~8x of headroom the current +single-threaded handler cannot reach. For narrow entities the two are within 10%. The +biggest available lever is therefore **parallelising decode + walk + encode across +cores** — blocks are independent — not anything on the database side. + +**Binary COPY is worth ~7x over the current multi-row INSERT**, and 2.3x over text/CSV +COPY. It is also the only strategy that can absorb a parallelised client. + +**Pre-encoding to disk buys only ~8%** (313ms vs 339ms). So spilling PGCOPY bytes to disk +does not justify itself on throughput. It justifies itself by moving encoding off the +flush path so the stream keeps being consumed during a COPY, and by making downloaded +blocks survive a restart. A spill design that still blocks the stream would be strictly +worse than `pgx.CopyFrom` over in-memory rows. + +**Building the SQL string is not the bottleneck.** Prebuilt-from-disk is only 1.15x +better than building at flush time, so `ValueToString` plus `strings.Builder` are ~13% of +the current cost. The other ~87% is the server parsing a multi-megabyte statement. + +**The driver is irrelevant.** `lib/pq` and pgx in simple-protocol mode are within noise +(2.296s vs 2.281s). No reason to switch drivers for the INSERT path. + +**Per-row prepared INSERT is 10x worse than the default.** It is now only the fallback +for a schema whose foreign keys form a cycle and therefore cannot be ordered; constraints +alone no longer select it, since the tables are ordered by their foreign keys instead. + +**Indexes eat over half the COPY advantage** — 7.29x drops to 3.84x with two btrees. +COPY removes client and parse cost, not index maintenance or WAL amplification. + +### Hypotheses tested and rejected + +- **NUMERIC encoding via `big.Int` is the hot spot.** Mapping every unsigned integer to + `BIGINT` instead of `NUMERIC` moves wide-entity throughput from 96k to 100k rows/s — + 4%. Not worth a schema change or a hand-rolled numeric encoder. (`uint32` → `NUMERIC` + at `sql/postgres/types.go:75` is still arguably wrong, since `uint32` fits `BIGINT` + exactly, but not for performance reasons.) +- **Text encoding is slower than binary encoding client-side.** It is the opposite: text + literals are ~1 µs/row *cheaper* to produce than binary COPY tuples. Binary wins on the + round trip because the server side is 7x faster, not because the client side is. + +One optimisation was found and kept: caching the pgtype encode plan per column in +`pgcopy.Writer` instead of letting `Map.Encode` resolve one per value. +8% on wide +entities, +3% on narrow, no downside. + +## Findings + +`postgres.ValueToString` renders raw bytes as `E''::BYTEA`, which casts the hex +*text* to bytea — 32 binary bytes are stored as the 64 ASCII characters of their hex +representation. `TestValueToStringRawBytesAreDoubleEncoded` pins the current behaviour +against a live server; the correct literal is `'\x'::bytea`. This affects the +`db_proto` accumulator path with `--bytes-encoding=raw` today, independently of any of +the work above. + + +## What the block number index costs, and buys + +`TestBlockNumberIndexCost`, 10GiB of rows shaped like `erc20-balance-changes`' +`map_balance_changes` — 32.7M rows over 500,000 blocks — into a containerised +PostgreSQL 17 with `maintenance_work_mem=1GB`: + +| variant | load | index build | table | index | undo 1k blocks | plan | +|---|--:|--:|--:|--:|--:|---| +| load bare, index after (the default) | 44.31s | 2.62s | 10.0GiB | 217.8MiB | 15ms | Index Scan | +| same data, undo without the index | | | | | 1.296s | Seq Scan | +| index in place during the load | 42.21s | | 10.0GiB | 217.8MiB | 25ms | Index Scan | + +Building it after the load costs **2.6s on a 44s load, around 6%**, and **218MiB against +10GiB, around 2%** of the table. Having it in place while the rows arrive costs nothing +measurable: `_block_number_` only ever increases during a backfill, so every insert lands +on the rightmost page of the btree and splits almost nothing. + +What it buys is the reorg path, which deletes from **every** table by that column: 1.296s +of sequential scan against 15ms of index scan, **86x**, for one table. Those are warm +numbers — the same sequential scan measured 5.9s on first touch after the load, against +the index's 15ms either way, since it reads a handful of pages rather than 10GiB. + +`ANALYZE` matters to reproducing this. A `COPY` leaves no statistics behind, and without +them the planner will sequentially scan a predicate matching 0.2% of the table — the first +version of this measurement reported 4.7s for an indexed undo for exactly that reason. The +test analyses before measuring and reports the chosen plan alongside every number. + +```bash +SF_SINK_SQL_BENCHMARKS=true go test ./sink/sql/db_proto/benchmarks/ \ + -run TestBlockNumberIndexCost -count=1 -v -timeout 120m + +# smaller, for a quick check +SF_SINK_SQL_BENCHMARKS=true PGBENCH_TARGET_BYTES=$((200*1024*1024)) PGBENCH_BLOCKS=20000 \ + go test ./sink/sql/db_proto/benchmarks/ -run TestBlockNumberIndexCost -count=1 -v +``` diff --git a/sink/sql/db_proto/benchmarks/block_number_index_test.go b/sink/sql/db_proto/benchmarks/block_number_index_test.go new file mode 100644 index 000000000..407373e9d --- /dev/null +++ b/sink/sql/db_proto/benchmarks/block_number_index_test.go @@ -0,0 +1,340 @@ +package benchmarks + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +// TestBlockNumberIndexCost measures what the index on _block_number_ costs and what it +// buys, at a size where the answer is not noise. +// +// The sink creates it on every table because every table carries that column and every +// reorg deletes from every table by it — a foreign key indexes its referenced side only. +// The question that decides whether it should be on by default is what the load pays for +// it, and that has two shapes: built after the load, which is when the constraint pass +// runs, or already in place while the rows arrive, which is --apply-constraints=always. +// +// The row is shaped like erc20-balance-changes' map_balance_changes output: an id, two +// addresses, two balances as text, a transaction hash and an ordinal, plus the two columns +// the sink adds to every table. +// +// PGBENCH_TARGET_BYTES sets the heap size to aim for, 10GiB by default. Each variant drops +// what came before it, so the peak on disk is one table plus its index rather than all of +// them at once. +func TestBlockNumberIndexCost(t *testing.T) { + requireBenchmark(t) + + ctx := context.Background() + + targetBytes := int64(envInt(t, "PGBENCH_TARGET_BYTES", 10*1024*1024*1024)) + blockSpan := envInt(t, "PGBENCH_BLOCKS", 500_000) + + // Measured at ~328 bytes of heap per row for this shape, tuple header and alignment + // included; the table size actually reached is reported rather than assumed. + rowCount := targetBytes / 328 + + pool, dsn := startBenchmarkPostgres(t, ctx) + _ = dsn + + t.Logf("target %s, %d rows over %d blocks", humanBytes(targetBytes), rowCount, blockSpan) + + type result struct { + name string + load time.Duration + indexBuild time.Duration + tableBytes int64 + indexBytes int64 + undo time.Duration + plan string + } + var results []result + + // Variant one: load bare, then build the index, which is what the sink does. + { + createBalanceSchema(t, ctx, pool) + + load := copyBalanceRows(t, ctx, pool, rowCount, blockSpan) + tableBytes := relationBytes(t, ctx, pool, "idx.balance_changes") + + undoBare, planBare := timeUndo(t, ctx, pool, blockSpan) + + startAt := time.Now() + _, err := pool.Exec(ctx, `CREATE INDEX balance_changes_block_number_idx ON idx.balance_changes (_block_number_)`) + require.NoError(t, err) + indexBuild := time.Since(startAt) + + undoIndexed, planIndexed := timeUndo(t, ctx, pool, blockSpan) + + results = append(results, + result{name: "load bare, index after (the default)", load: load, indexBuild: indexBuild, + tableBytes: tableBytes, indexBytes: relationBytes(t, ctx, pool, "balance_changes_block_number_idx"), undo: undoIndexed, plan: planIndexed}, + result{name: " same data, undo without the index", undo: undoBare, plan: planBare}, + ) + } + + // Variant two: the index already in place while the rows arrive. + { + createBalanceSchema(t, ctx, pool) + _, err := pool.Exec(ctx, `CREATE INDEX balance_changes_block_number_idx ON idx.balance_changes (_block_number_)`) + require.NoError(t, err) + + load := copyBalanceRows(t, ctx, pool, rowCount, blockSpan) + undo, plan := timeUndo(t, ctx, pool, blockSpan) + + results = append(results, result{ + name: "index in place during the load", + load: load, + tableBytes: relationBytes(t, ctx, pool, "idx.balance_changes"), + indexBytes: relationBytes(t, ctx, pool, "balance_changes_block_number_idx"), + undo: undo, + plan: plan, + }) + } + + fmt.Printf("\n%-38s %10s %12s %10s %10s %12s %s\n", "variant", "load", "index build", "table", "index", "undo 1k blk", "plan") + for _, r := range results { + fmt.Printf("%-38s %10s %12s %10s %10s %12s %s\n", + r.name, + durationOrDash(r.load), + durationOrDash(r.indexBuild), + bytesOrDash(r.tableBytes), + bytesOrDash(r.indexBytes), + durationOrDash(r.undo), + r.plan) + } + fmt.Println() +} + +func createBalanceSchema(t *testing.T, ctx context.Context, pool *pgxpool.Pool) { + t.Helper() + + _, err := pool.Exec(ctx, `DROP SCHEMA IF EXISTS idx CASCADE; CREATE SCHEMA idx`) + require.NoError(t, err) + + _, err = pool.Exec(ctx, ` + CREATE TABLE idx.balance_changes ( + _block_number_ INTEGER NOT NULL, + _block_timestamp_ TIMESTAMP NOT NULL, + id TEXT NOT NULL, + contract TEXT NOT NULL, + owner TEXT NOT NULL, + old_balance TEXT NOT NULL, + new_balance TEXT NOT NULL, + transaction_id TEXT NOT NULL, + ordinal BIGINT NOT NULL + )`) + require.NoError(t, err) +} + +// copyBalanceRows streams the rows straight into COPY rather than materialising them: at +// this size the slice alone would not fit in memory. +func copyBalanceRows(t *testing.T, ctx context.Context, pool *pgxpool.Pool, rowCount int64, blockSpan int) time.Duration { + t.Helper() + + source := &balanceRowSource{ + remaining: rowCount, + total: rowCount, + blockSpan: int64(blockSpan), + random: rand.New(rand.NewSource(1)), + baseTime: time.Unix(1_600_000_000, 0).UTC(), + } + + startAt := time.Now() + copied, err := pool.CopyFrom(ctx, + pgx.Identifier{"idx", "balance_changes"}, + []string{"_block_number_", "_block_timestamp_", "id", "contract", "owner", "old_balance", "new_balance", "transaction_id", "ordinal"}, + source) + require.NoError(t, err) + require.Equal(t, rowCount, copied) + + return time.Since(startAt) +} + +// balanceRowSource generates the rows lazily, one at a time. +type balanceRowSource struct { + remaining int64 + total int64 + blockSpan int64 + random *rand.Rand + baseTime time.Time + current []any +} + +func (s *balanceRowSource) Next() bool { + if s.remaining == 0 { + return false + } + s.remaining-- + + // Blocks in order, as a backfill produces them, so the heap is clustered by block the + // way a real load leaves it. + index := s.total - s.remaining - 1 + blockNumber := index * s.blockSpan / s.total + + s.current = []any{ + int32(blockNumber), + s.baseTime.Add(time.Duration(blockNumber) * 12 * time.Second), + hexOf(s.random, 32), + hexOf(s.random, 20), + hexOf(s.random, 20), + strconv.FormatUint(s.random.Uint64(), 10), + strconv.FormatUint(s.random.Uint64(), 10), + hexOf(s.random, 32), + int64(index % 64), + } + + return true +} + +func (s *balanceRowSource) Values() ([]any, error) { return s.current, nil } +func (s *balanceRowSource) Err() error { return nil } + +const hexDigits = "0123456789abcdef" + +func hexOf(random *rand.Rand, bytes int) string { + out := make([]byte, 2+bytes*2) + out[0], out[1] = '0', 'x' + for i := 2; i < len(out); i++ { + out[i] = hexDigits[random.Intn(16)] + } + + return string(out) +} + +// timeUndo measures the reorg path: the sink deletes from every table by _block_number_, +// so this is what one table costs. It rolls back, the point being the scan rather than the +// write, and the next measurement needing the same rows. +// +// ANALYZE first, or the planner is choosing without statistics — a COPY leaves none behind +// and autovacuum has not necessarily caught up, which is enough to make it seq scan a +// predicate that matches a fraction of a percent. The plan is reported alongside the +// duration so a number that looks wrong can be explained rather than guessed at. +// +// Twice, reporting the second: the first pass pays for whatever the load evicted, and a +// variant measured cold against another measured warm compares the cache, not the index. +func timeUndo(t *testing.T, ctx context.Context, pool *pgxpool.Pool, blockSpan int) (time.Duration, string) { + t.Helper() + + _, err := pool.Exec(ctx, `ANALYZE idx.balance_changes`) + require.NoError(t, err) + + var duration time.Duration + for range 2 { + tx, err := pool.Begin(ctx) + require.NoError(t, err) + + startAt := time.Now() + _, err = tx.Exec(ctx, `DELETE FROM idx.balance_changes WHERE _block_number_ > $1`, blockSpan-1_000) + require.NoError(t, err) + duration = time.Since(startAt) + + require.NoError(t, tx.Rollback(ctx)) + } + + return duration, undoPlan(t, ctx, pool, blockSpan) +} + +// undoPlan names the scan the planner chose for that delete. +func undoPlan(t *testing.T, ctx context.Context, pool *pgxpool.Pool, blockSpan int) string { + t.Helper() + + rows, err := pool.Query(ctx, `EXPLAIN DELETE FROM idx.balance_changes WHERE _block_number_ > $1`, blockSpan-1_000) + require.NoError(t, err) + defer rows.Close() + + for rows.Next() { + var line string + require.NoError(t, rows.Scan(&line)) + + trimmed := strings.TrimSpace(line) + for _, node := range []string{"Seq Scan", "Index Scan", "Bitmap Heap Scan", "Bitmap Index Scan"} { + if strings.HasPrefix(trimmed, "-> "+node) || strings.HasPrefix(trimmed, node) { + return node + } + } + } + + return "?" +} + +func relationBytes(t *testing.T, ctx context.Context, pool *pgxpool.Pool, relation string) int64 { + t.Helper() + + qualified := relation + if !containsDot(relation) { + qualified = "idx." + relation + } + + var size int64 + require.NoError(t, pool.QueryRow(ctx, `SELECT pg_relation_size($1)`, qualified).Scan(&size)) + + return size +} + +func containsDot(s string) bool { + for i := range s { + if s[i] == '.' { + return true + } + } + + return false +} + +func startBenchmarkPostgres(t *testing.T, ctx context.Context) (*pgxpool.Pool, string) { + t.Helper() + + container, err := tcpostgres.Run(ctx, envString("PGBENCH_PG_IMAGE", "postgres:17-alpine"), + tcpostgres.WithDatabase("bench"), + tcpostgres.WithUsername("bench"), + tcpostgres.WithPassword("bench"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(120*time.Second)), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = testcontainers.TerminateContainer(container) }) + + dsn := container.MustConnectionString(ctx, "sslmode=disable") + pool, err := pgxpool.New(ctx, dsn) + require.NoError(t, err) + t.Cleanup(pool.Close) + + // An index build is bounded by maintenance_work_mem, and the default 64MB says more + // about the default than about the index. + _, err = pool.Exec(ctx, `ALTER SYSTEM SET maintenance_work_mem = '1GB'`) + require.NoError(t, err) + _, err = pool.Exec(ctx, `SELECT pg_reload_conf()`) + require.NoError(t, err) + + return pool, dsn +} + +func durationOrDash(d time.Duration) string { + if d == 0 { + return "-" + } + + return d.Round(time.Millisecond).String() +} + +func bytesOrDash(n int64) string { + if n == 0 { + return "-" + } + + return humanBytes(n) +} diff --git a/sink/sql/db_proto/benchmarks/client_ceiling_test.go b/sink/sql/db_proto/benchmarks/client_ceiling_test.go new file mode 100644 index 000000000..97f6eed03 --- /dev/null +++ b/sink/sql/db_proto/benchmarks/client_ceiling_test.go @@ -0,0 +1,496 @@ +package benchmarks + +import ( + "fmt" + "io" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + sqlbytes "github.com/streamingfast/substreams/sink/sql/bytes" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + sqlpostgres "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + protoschema "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/dynamicpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestClientEncodeCeiling measures how fast the sink can turn a substreams payload into +// something loadable, with no database involved at all. +// +// This is the number that decides whether decoupling the stream from PostgreSQL is +// worth anything. COPY BINARY absorbs ~800k rows/s (see TestCopyVsInsert); if the +// client cannot produce rows at that rate then PostgreSQL was never the bottleneck and +// the work belongs on this side instead. +// +// It runs the production path: proto.Unmarshal into dynamicpb, then +// BaseDatabase.WalkMessageDescriptorAndInsertWithDialect with the real Postgres +// dialect. Only the Inserter changes between variants. +// +// go test ./sink/sql/db_proto/benchmarks/ -run TestClientEncodeCeiling -v +func TestClientEncodeCeiling(t *testing.T) { + requireBenchmark(t) + + shapes := []struct { + name string + entities int + build func(count int) *pbrelations.Output + rowsPerItem int + }{ + { + name: "narrow-entity (2 columns)", + entities: 200, + build: buildCustomerOutput, + rowsPerItem: 1, + }, + { + name: "wide-entity (60+ columns, arrays, inline JSONB)", + entities: 200, + build: buildTypesTestOutput, + rowsPerItem: 1, + }, + } + + logger := zap.NewNop() + descriptor := pbrelations.File_test_relations_relations_proto.Messages().ByName("Output") + require.NotNil(t, descriptor) + + schema, err := protoschema.NewSchema("bench", descriptor, true, logger) + require.NoError(t, err) + + dialect, err := sqlpostgres.NewDialectPostgres(schema, sqlbytes.EncodingRaw, logger) + require.NoError(t, err) + + base, err := protosql.NewBaseDatabase(string(descriptor.FullName()), descriptor, true, logger) + require.NoError(t, err) + + for _, shape := range shapes { + payload, err := proto.Marshal(shape.build(shape.entities)) + require.NoError(t, err) + + rowsPerPayload := shape.entities * shape.rowsPerItem + + variants := []struct { + name string + notes string + inserter func() protosql.Inserter + decode bool + }{ + { + name: "unmarshal-only", + notes: "proto.Unmarshal into dynamicpb, no walk", + }, + { + name: "unmarshal+walk-discard", + notes: "walk the message, throw the values away: the floor for any strategy", + inserter: func() protosql.Inserter { return &discardInserter{} }, + }, + { + name: "unmarshal+walk+pgcopy-binary", + notes: "walk, then encode straight to the binary COPY format", + inserter: func() protosql.Inserter { return newPgcopyInserter(inferOID) }, + }, + { + name: "unmarshal+walk+pgcopy-binary uint32->BIGINT", + notes: "same, but uint32/fixed32 map to BIGINT instead of NUMERIC", + inserter: func() protosql.Inserter { + return newPgcopyInserter(func(v any) uint32 { + switch v.(type) { + case uint32: + return pgtype.Int8OID + default: + return inferOID(v) + } + }) + }, + }, + { + name: "unmarshal+walk+pgcopy-binary no-NUMERIC", + notes: "upper bound: every unsigned int as BIGINT, isolating the numeric codec's cost", + inserter: func() protosql.Inserter { + return newPgcopyInserter(func(v any) uint32 { + switch v.(type) { + case uint32, uint64, uint: + return pgtype.Int8OID + default: + return inferOID(v) + } + }) + }, + }, + { + name: "unmarshal+walk+text-literal", + notes: "walk, then ValueToString into a VALUES buffer (what happens today)", + inserter: func() protosql.Inserter { return newTextInserter() }, + }, + } + + results := make([]clientResult, 0, len(variants)) + for _, v := range variants { + var inserter protosql.Inserter + if v.inserter != nil { + inserter = v.inserter() + } + + iterations, elapsed := runUntil(2*time.Second, func() { + message := dynamicpb.NewMessage(descriptor) + if err := proto.Unmarshal(payload, message); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if inserter == nil { + return + } + if _, err := base.WalkMessageDescriptorAndInsertWithDialect( + message, 20_000_000, time.Unix(1700000000, 0).UTC(), nil, dialect, inserter, + ); err != nil { + t.Fatalf("walk: %v", err) + } + }) + + results = append(results, clientResult{ + name: v.name, + notes: v.notes, + rowsPerS: float64(iterations*rowsPerPayload) / elapsed.Seconds(), + mibPerS: float64(iterations*len(payload)) / elapsed.Seconds() / (1024 * 1024), + }) + } + + reportClient(t, shape.name, len(payload), rowsPerPayload, results) + } +} + +type clientResult struct { + name string + notes string + rowsPerS float64 + mibPerS float64 +} + +func runUntil(budget time.Duration, fn func()) (iterations int, elapsed time.Duration) { + start := time.Now() + for elapsed < budget { + fn() + iterations++ + if iterations%16 == 0 { + elapsed = time.Since(start) + } + } + + return iterations, time.Since(start) +} + +func reportClient(t *testing.T, shape string, payloadBytes, rowsPerPayload int, results []clientResult) { + t.Helper() + + var b strings.Builder + fmt.Fprintf(&b, "\n\n%s -- %d rows per payload, %s of protobuf\n\n", shape, rowsPerPayload, humanBytes(int64(payloadBytes))) + fmt.Fprintf(&b, "%-32s %12s %10s %s\n", "variant", "rows/s", "MiB/s", "notes") + for _, r := range results { + fmt.Fprintf(&b, "%-32s %12s %10.1f %s\n", r.name, humanCount(r.rowsPerS), r.mibPerS, r.notes) + } + + t.Log(b.String()) +} + +// -- inserters --------------------------------------------------------------------- + +// discardInserter measures the walk itself: dynamicpb reflection, the per-message +// zap.Any debug fields, and the []any building. +type discardInserter struct{ rows int64 } + +func (i *discardInserter) Insert(table string, values []any) error { + i.rows++ + return nil +} + +// pgcopyInserter encodes into the binary COPY format and throws the bytes away. +// +// Column OIDs are inferred from the first row's Go types rather than read from +// pg_attribute, since there is no server here. The encoding work per value is +// identical; only the one-time OID resolution differs from production. +type pgcopyInserter struct { + targets map[string]*pgcopyTarget + oidOf func(any) uint32 +} + +type pgcopyTarget struct { + writer *pgcopy.Writer + columns []pgcopy.Column +} + +func newPgcopyInserter(oidOf func(any) uint32) *pgcopyInserter { + return &pgcopyInserter{targets: map[string]*pgcopyTarget{}, oidOf: oidOf} +} + +func (i *pgcopyInserter) Insert(table string, values []any) error { + target, ok := i.targets[table] + if !ok { + columns := make([]pgcopy.Column, len(values)) + for j, v := range values { + columns[j] = pgcopy.Column{Name: fmt.Sprintf("c%d", j), OID: i.oidOf(v)} + } + + writer, err := pgcopy.NewWriter(io.Discard, columns) + if err != nil { + return err + } + target = &pgcopyTarget{writer: writer, columns: columns} + i.targets[table] = target + } + + if len(values) != len(target.columns) { + return fmt.Errorf("table %s: expected %d values, got %d", table, len(target.columns), len(values)) + } + if err := pgcopy.NormalizeRow(target.columns, values); err != nil { + return fmt.Errorf("normalizing %s: %w", table, err) + } + + return target.writer.WriteRow(values) +} + +// textInserter is the current AccumulatorInserter client-side work: stringify every +// value and append it to a VALUES buffer. +type textInserter struct { + buffers map[string]*strings.Builder +} + +func newTextInserter() *textInserter { + return &textInserter{buffers: map[string]*strings.Builder{}} +} + +func (i *textInserter) Insert(table string, values []any) error { + buffer, ok := i.buffers[table] + if !ok { + buffer = &strings.Builder{} + i.buffers[table] = buffer + } + + buffer.WriteByte('(') + for j, v := range values { + if j > 0 { + buffer.WriteByte(',') + } + buffer.WriteString(sqlpostgres.ValueToString(v, sqlbytes.EncodingRaw)) + } + buffer.WriteString("),") + + // A flush would hand this off and reset; keep the buffer bounded so the benchmark + // measures encoding rather than the allocator growing a multi-gigabyte string. + if buffer.Len() > 8<<20 { + buffer.Reset() + } + + return nil +} + +func inferOID(value any) uint32 { + switch v := value.(type) { + case nil: + return pgtype.TextOID + case bool: + return pgtype.BoolOID + case int32: + return pgtype.Int4OID + case int64: + return pgtype.Int8OID + case uint32, uint64, uint: + return pgtype.NumericOID + case float32: + return pgtype.Float4OID + case float64: + return pgtype.Float8OID + case string: + return pgtype.TextOID + case []byte: + return pgtype.ByteaOID + case time.Time, *timestamppb.Timestamp: + return pgtype.TimestampOID + case []any: + if len(v) == 0 { + return pgtype.TextArrayOID + } + return arrayOIDFor(inferOID(v[0])) + default: + return pgtype.JSONBOID + } +} + +func arrayOIDFor(element uint32) uint32 { + switch element { + case pgtype.BoolOID: + return pgtype.BoolArrayOID + case pgtype.Int4OID: + return pgtype.Int4ArrayOID + case pgtype.Int8OID: + return pgtype.Int8ArrayOID + case pgtype.NumericOID: + return pgtype.NumericArrayOID + case pgtype.Float4OID: + return pgtype.Float4ArrayOID + case pgtype.Float8OID: + return pgtype.Float8ArrayOID + case pgtype.ByteaOID: + return pgtype.ByteaArrayOID + default: + return pgtype.TextArrayOID + } +} + +// -- payload builders -------------------------------------------------------------- + +func buildCustomerOutput(count int) *pbrelations.Output { + out := &pbrelations.Output{Entities: make([]*pbrelations.Entity, count)} + for i := range out.Entities { + out.Entities[i] = &pbrelations.Entity{ + Entity: &pbrelations.Entity_Customer{Customer: &pbrelations.Customer{ + CustomerId: fmt.Sprintf("cust-%08d", i), + Name: fmt.Sprintf("Customer Number %d", i), + }}, + } + } + + return out +} + +func buildTypesTestOutput(count int) *pbrelations.Output { + out := &pbrelations.Output{Entities: make([]*pbrelations.Entity, count)} + optionalString := "set" + optionalInt := int32(42) + + for i := range out.Entities { + out.Entities[i] = &pbrelations.Entity{ + Entity: &pbrelations.Entity_TypesTest{TypesTest: &pbrelations.TypesTest{ + Id: uint64(i), + DoubleField: float64(i) * 1.5, + FloatField: float32(i) * 2.5, + Int32Field: int32(i), + Int64Field: int64(i) * 1000, + Uint32Field: uint32(i), + Uint64Field: uint64(i) * 7, + Sint32Field: int32(-i), + Sint64Field: int64(-i) * 3, + Fixed32Field: uint32(i), + Fixed64Field: uint64(i), + Sfixed32Field: int32(i), + Sfixed64Field: int64(i), + BoolField: i%2 == 0, + StringField: fmt.Sprintf("string value %d", i), + BytesField: []byte(fmt.Sprintf("bytes-%d", i)), + OptionalStringSet: &optionalString, + OptionalInt32FieldSet: &optionalInt, + TimestampField: timestamppb.New(time.Unix(1700000000+int64(i), 0)), + RepeatedInt32Field: []int32{1, 2, 3}, + RepeatedInt64Field: []int64{4, 5, 6}, + RepeatedUint32Field: []uint32{7, 8}, + RepeatedUint64Field: []uint64{9, 10}, + RepeatedStringField: []string{"alpha", "beta", "gamma"}, + RepeatedBoolField: []bool{true, false}, + RepeatedDoubleField: []float64{1.1, 2.2}, + Str_2Int128: "-170141183460469231731687303715884105728", + Str_2Uint128: "340282366920938463463374607431768211455", + Str_2Int256: "-57896044618658097711785492504343953926634992332820282019728792003956564819968", + Str_2Uint256: "115792089237316195423570985008687907853269984665640564039457584007913129639935", + Str_2Decimal128: "1234.5678", + Str_2Decimal256: "8765.4321", + // The inline nested fields (level1, list_of_level1) exist in the .proto + // but not in the checked-in relations.pb.go, so the inline-JSONB path is + // not covered here. + }}, + } + } + + return out +} + +// TestClientDecodeScaling measures how the per-block work the decoder parallelises — +// unmarshal, walk, and buffering the inserts — scales across cores. +// +// It is the same workload sink/sql/db_proto's decoder runs per block, so it bounds what +// the worker pool can achieve. The replay of the buffered inserts is deliberately not +// included: that part stays serial on the flush goroutine. +func TestClientDecodeScaling(t *testing.T) { + requireBenchmark(t) + + logger := zap.NewNop() + descriptor := pbrelations.File_test_relations_relations_proto.Messages().ByName("Output") + require.NotNil(t, descriptor) + + schema, err := protoschema.NewSchema("bench", descriptor, true, logger) + require.NoError(t, err) + + dialect, err := sqlpostgres.NewDialectPostgres(schema, sqlbytes.EncodingRaw, logger) + require.NoError(t, err) + + base, err := protosql.NewBaseDatabase(string(descriptor.FullName()), descriptor, true, logger) + require.NoError(t, err) + + const entities = 200 + payload, err := proto.Marshal(buildTypesTestOutput(entities)) + require.NoError(t, err) + + decodeOne := func() { + message := dynamicpb.NewMessage(descriptor) + if err := proto.Unmarshal(payload, message); err != nil { + panic(err) + } + buffer := protosql.NewBufferedInserter(entities) + if _, err := base.WalkMessageDescriptorAndInsertWithDialect( + message, 20_000_000, time.Unix(1700000000, 0).UTC(), nil, dialect, buffer, + ); err != nil { + panic(err) + } + } + + var b strings.Builder + fmt.Fprintf(&b, "\n\nwide entity, %d rows per block, decode work only (%d cores available)\n\n", entities, runtime.NumCPU()) + fmt.Fprintf(&b, "%-10s %12s %10s\n", "workers", "rows/s", "speedup") + + var serial float64 + for _, workers := range []int{1, 2, 4, 8, max(1, runtime.NumCPU()-1)} { + blocks, elapsed := runUntilParallel(2*time.Second, workers, decodeOne) + rowsPerSecond := float64(blocks*entities) / elapsed.Seconds() + if workers == 1 { + serial = rowsPerSecond + } + fmt.Fprintf(&b, "%-10d %12s %9.2fx\n", workers, humanCount(rowsPerSecond), rowsPerSecond/serial) + } + + t.Log(b.String()) +} + +// runUntilParallel runs fn on the given number of goroutines until the budget elapses, +// returning the total number of calls completed. +func runUntilParallel(budget time.Duration, workers int, fn func()) (calls int, elapsed time.Duration) { + var ( + wg sync.WaitGroup + total atomic.Int64 + start = time.Now() + ) + + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + local := 0 + for time.Since(start) < budget { + for range 16 { + fn() + } + local += 16 + } + total.Add(int64(local)) + }() + } + wg.Wait() + + return int(total.Load()), time.Since(start) +} diff --git a/sink/sql/db_proto/benchmarks/constraint_cost_test.go b/sink/sql/db_proto/benchmarks/constraint_cost_test.go new file mode 100644 index 000000000..3e9a277bc --- /dev/null +++ b/sink/sql/db_proto/benchmarks/constraint_cost_test.go @@ -0,0 +1,300 @@ +package benchmarks + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +// TestConstraintCost measures what running the from-proto sink with database constraints +// costs on the binary COPY path, which is the question that decides whether they can be +// on by default. +// +// Constraints used to force the row-at-a-time inserter, so "with constraints" meant a +// tenth of the throughput. Ordering the tables by their foreign keys removed that, and +// what is left is index maintenance and referential checks during the COPY itself. This +// separates the two: primary keys and uniques build indexes, foreign keys check a lookup +// per row against an index that has to exist anyway. +// +// A relational shape rather than one flat table, since foreign keys are the point: +// blocks <- parents <- children, with children also carrying a unique column. +func TestConstraintCost(t *testing.T) { + requireBenchmark(t) + + ctx := context.Background() + rowCount := envInt(t, "PGBENCH_ROWS", 250_000) + + variants := []struct { + name string + // constraints are applied before the load; afterConstraints once it is done, + // which is the "backfill bare, then turn them on" workflow. + constraints []string + afterConstraints []string + }{ + {name: "no constraints"}, + { + name: "primary keys only", + constraints: []string{ + `ALTER TABLE cost.blocks ADD CONSTRAINT blocks_pk PRIMARY KEY (number)`, + `ALTER TABLE cost.parents ADD CONSTRAINT parents_pk PRIMARY KEY (id)`, + `ALTER TABLE cost.children ADD CONSTRAINT children_pk PRIMARY KEY (id)`, + }, + }, + { + name: "primary keys and unique", + constraints: []string{ + `ALTER TABLE cost.blocks ADD CONSTRAINT blocks_pk PRIMARY KEY (number)`, + `ALTER TABLE cost.parents ADD CONSTRAINT parents_pk PRIMARY KEY (id)`, + `ALTER TABLE cost.children ADD CONSTRAINT children_pk PRIMARY KEY (id)`, + `ALTER TABLE cost.children ADD CONSTRAINT children_ref_unique UNIQUE (ref)`, + }, + }, + { + name: "primary keys, unique and foreign keys", + constraints: []string{ + `ALTER TABLE cost.blocks ADD CONSTRAINT blocks_pk PRIMARY KEY (number)`, + `ALTER TABLE cost.parents ADD CONSTRAINT parents_pk PRIMARY KEY (id)`, + `ALTER TABLE cost.children ADD CONSTRAINT children_pk PRIMARY KEY (id)`, + `ALTER TABLE cost.children ADD CONSTRAINT children_ref_unique UNIQUE (ref)`, + `ALTER TABLE cost.parents ADD CONSTRAINT parents_fk_block FOREIGN KEY (_block_number_) REFERENCES cost.blocks(number) ON DELETE CASCADE`, + `ALTER TABLE cost.children ADD CONSTRAINT children_fk_block FOREIGN KEY (_block_number_) REFERENCES cost.blocks(number) ON DELETE CASCADE`, + `ALTER TABLE cost.children ADD CONSTRAINT children_fk_parent FOREIGN KEY (parent_id) REFERENCES cost.parents(id)`, + }, + }, + } + + full := variants[len(variants)-1].constraints + variants = append(variants, struct { + name string + constraints []string + afterConstraints []string + }{name: "loaded bare, constraints added after", afterConstraints: full}) + + image := envString("PGBENCH_PG_IMAGE", "postgres:17-alpine") + container, err := tcpostgres.Run(ctx, image, + tcpostgres.WithDatabase("bench"), + tcpostgres.WithUsername("bench"), + tcpostgres.WithPassword("bench"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second)), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = testcontainers.TerminateContainer(container) }) + + dsn := container.MustConnectionString(ctx, "sslmode=disable") + pool, err := pgxpool.New(ctx, dsn) + require.NoError(t, err) + t.Cleanup(pool.Close) + + dataDir := t.TempDir() + + type measurement struct { + name string + duration time.Duration + insertDuration time.Duration + } + var measurements []measurement + + for _, variant := range variants { + // A fresh schema per variant: an index built before the load is not the same + // thing as one built after it. + _, err := pool.Exec(ctx, `DROP SCHEMA IF EXISTS cost CASCADE; CREATE SCHEMA cost`) + require.NoError(t, err) + + _, err = pool.Exec(ctx, createCostTablesSQL) + require.NoError(t, err) + + for _, statement := range variant.constraints { + _, err := pool.Exec(ctx, statement) + require.NoError(t, err, statement) + } + + // Materialise every file before the clock starts, so what is measured is the + // transport plus the server's own work, never row generation. + blocksFile := filepath.Join(dataDir, "blocks.pgcopy") + parentsFile := filepath.Join(dataDir, "parents.pgcopy") + childrenFile := filepath.Join(dataDir, "children.pgcopy") + + blockCount := rowCount / 100 + if blockCount < 1 { + blockCount = 1 + } + + writeCopyFile(t, ctx, pool, "blocks", blocksFile, func(write func(...any)) { + for i := 0; i < blockCount; i++ { + write(int32(i), fmt.Sprintf("hash-%d", i)) + } + }) + writeCopyFile(t, ctx, pool, "parents", parentsFile, func(write func(...any)) { + for i := 0; i < rowCount; i++ { + write(int32(i%blockCount), fmt.Sprintf("parent-%d", i), fmt.Sprintf("name-%d", i)) + } + }) + writeCopyFile(t, ctx, pool, "children", childrenFile, func(write func(...any)) { + for i := 0; i < rowCount; i++ { + write(int32(i%blockCount), fmt.Sprintf("child-%d", i), fmt.Sprintf("parent-%d", i), fmt.Sprintf("ref-%d", i), int64(i)) + } + }) + + // Topological order, exactly as the applier loads a segment. + startAt := time.Now() + copyFile(t, ctx, pool, "blocks", blocksFile) + copyFile(t, ctx, pool, "parents", parentsFile) + copyFile(t, ctx, pool, "children", childrenFile) + + for _, statement := range variant.afterConstraints { + _, err := pool.Exec(ctx, statement) + require.NoError(t, err, statement) + } + elapsed := time.Since(startAt) + + var children int + require.NoError(t, pool.QueryRow(ctx, `SELECT count(*) FROM cost.children`).Scan(&children)) + require.Equal(t, rowCount, children, "every variant must load the same data") + + // The same rows through the other write path: multi-row INSERT statements, built + // before the clock starts, in the same table order. + _, err = pool.Exec(ctx, `DROP SCHEMA IF EXISTS cost CASCADE; CREATE SCHEMA cost`) + require.NoError(t, err) + _, err = pool.Exec(ctx, createCostTablesSQL) + require.NoError(t, err) + for _, statement := range variant.constraints { + _, err := pool.Exec(ctx, statement) + require.NoError(t, err, statement) + } + + statements := buildInsertStatements(rowCount, blockCount) + + insertStart := time.Now() + for _, statement := range statements { + _, err := pool.Exec(ctx, statement) + require.NoError(t, err) + } + for _, statement := range variant.afterConstraints { + _, err := pool.Exec(ctx, statement) + require.NoError(t, err, statement) + } + insertElapsed := time.Since(insertStart) + + require.NoError(t, pool.QueryRow(ctx, `SELECT count(*) FROM cost.children`).Scan(&children)) + require.Equal(t, rowCount, children, "both paths must load the same data") + + measurements = append(measurements, measurement{name: variant.name, duration: elapsed, insertDuration: insertElapsed}) + } + + baseline := measurements[0].duration + insertBaseline := measurements[0].insertDuration + t.Log("") + t.Logf("%d blocks, %d parents, %d children per variant", rowCount/100, rowCount, rowCount) + t.Logf("%-38s %10s %9s %10s %9s %11s", "variant", "COPY", "vs bare", "INSERT", "vs bare", "COPY gain") + for _, m := range measurements { + t.Logf("%-38s %10s %8.2fx %10s %8.2fx %10.2fx", + m.name, + m.duration.Round(time.Millisecond), baseline.Seconds()/m.duration.Seconds(), + m.insertDuration.Round(time.Millisecond), insertBaseline.Seconds()/m.insertDuration.Seconds(), + m.insertDuration.Seconds()/m.duration.Seconds()) + } +} + +// writeCopyFile resolves the table's real column layout and writes the rows in PGCOPY +// binary format, the same way the buffer does. +func writeCopyFile(t *testing.T, ctx context.Context, pool *pgxpool.Pool, table, path string, rows func(write func(...any))) { + t.Helper() + + columns, err := pgcopy.LoadColumns(ctx, pool, "cost", table) + require.NoError(t, err) + + file, err := os.Create(path) + require.NoError(t, err) + defer file.Close() + + writer, err := pgcopy.NewWriter(file, columns) + require.NoError(t, err) + + var writeErr error + rows(func(values ...any) { + if writeErr != nil { + return + } + if err := pgcopy.NormalizeRow(columns, values); err != nil { + writeErr = err + return + } + writeErr = writer.WriteRow(values) + }) + require.NoError(t, writeErr) + require.NoError(t, writer.Close()) +} + +func copyFile(t *testing.T, ctx context.Context, pool *pgxpool.Pool, table, path string) { + t.Helper() + + file, err := os.Open(path) + require.NoError(t, err) + defer file.Close() + + conn, err := pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + + _, err = conn.Conn().PgConn().CopyFrom(ctx, file, fmt.Sprintf(`COPY cost.%s FROM STDIN (FORMAT BINARY)`, table)) + require.NoError(t, err) +} + +const createCostTablesSQL = ` + CREATE TABLE cost.blocks (number INTEGER NOT NULL, hash TEXT NOT NULL); + CREATE TABLE cost.parents (_block_number_ INTEGER NOT NULL, id TEXT NOT NULL, name TEXT NOT NULL); + CREATE TABLE cost.children (_block_number_ INTEGER NOT NULL, id TEXT NOT NULL, parent_id TEXT NOT NULL, ref TEXT NOT NULL, quantity BIGINT NOT NULL); +` + +// buildInsertStatements renders the same rows as multi-row INSERT text, the way the +// accumulator does at flush, in the order the tables have to be loaded. +func buildInsertStatements(rowCount, blockCount int) []string { + const perStatement = 1000 + + var statements []string + + appendRows := func(prefix string, total int, row func(i int) string) { + for start := 0; start < total; start += perStatement { + end := start + perStatement + if end > total { + end = total + } + + var b strings.Builder + b.WriteString(prefix) + for i := start; i < end; i++ { + if i > start { + b.WriteString(",") + } + b.WriteString(row(i)) + } + statements = append(statements, b.String()) + } + } + + appendRows("INSERT INTO cost.blocks (number, hash) VALUES ", blockCount, func(i int) string { + return fmt.Sprintf("(%d,'hash-%d')", i, i) + }) + appendRows("INSERT INTO cost.parents (_block_number_, id, name) VALUES ", rowCount, func(i int) string { + return fmt.Sprintf("(%d,'parent-%d','name-%d')", i%blockCount, i, i) + }) + appendRows("INSERT INTO cost.children (_block_number_, id, parent_id, ref, quantity) VALUES ", rowCount, func(i int) string { + return fmt.Sprintf("(%d,'child-%d','parent-%d','ref-%d',%d)", i%blockCount, i, i, i, i) + }) + + return statements +} diff --git a/sink/sql/db_proto/benchmarks/copy_vs_insert_test.go b/sink/sql/db_proto/benchmarks/copy_vs_insert_test.go new file mode 100644 index 000000000..fa67d878d --- /dev/null +++ b/sink/sql/db_proto/benchmarks/copy_vs_insert_test.go @@ -0,0 +1,560 @@ +// Package benchmarks compares the strategies available to the from-proto SQL sink for +// getting rows into PostgreSQL, against a real server in a container. +// +// The question it answers is narrow on purpose: given rows that are already prepared, +// how much of the wall clock is the *strategy* rather than the data? Every artifact is +// materialised on disk before any timer starts, so nothing here measures row +// generation, protobuf decoding or SQL string building unless a variant's real +// implementation would do that work at flush time. +// +// Run it with: +// +// go test ./sink/sql/db_proto/benchmarks/ -run TestCopyVsInsert -v -timeout 30m +// +// Environment: +// +// PGBENCH_ROWS=250000 rows in the dataset +// PGBENCH_REPEAT=1 passes over the variant set, best duration is reported +// PGBENCH_WITH_INDEX=1 add a btree on id and on _block_number_ before loading +// PGBENCH_PG_IMAGE=... postgres image (default postgres:17-alpine) +// PGBENCH_KEEP_DATA_DIR=... reuse artifacts across runs instead of a temp dir +package benchmarks + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "testing" + "text/tabwriter" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + _ "github.com/lib/pq" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +// multiRowBatchSize is how many rows go into one INSERT statement. It matches the +// order of magnitude the sink produces today: blockBatchSize of 25 blocks times a few +// entities per block. +const multiRowBatchSize = 500 + +func TestCopyVsInsert(t *testing.T) { + requireBenchmark(t) + + ctx := context.Background() + rowCount := envInt(t, "PGBENCH_ROWS", 250_000) + repeat := envInt(t, "PGBENCH_REPEAT", 1) + + h := newHarness(t, ctx) + + t.Logf("generating %d rows and materialising on-disk artifacts in %s", rowCount, h.dataDir) + generateStart := time.Now() + rows := generateRows(rowCount, 1) + art := h.materialise(t, ctx, rows) + t.Logf("artifacts ready in %s: binary=%s csv=%s multirow-sql=%s", + time.Since(generateStart).Round(time.Millisecond), + humanBytes(art.binaryBytes), humanBytes(art.csvBytes), humanBytes(art.multiRowBytes)) + + expected := expectedChecksum(rows) + + variants := []variant{ + { + name: "insert-1row-prepared", + notes: "per-row prepared INSERT in one tx (what RowInserter does)", + bytes: art.multiRowBytes, + run: func(ctx context.Context) error { return h.insertPerRow(ctx, rows) }, + }, + { + name: "insert-multirow-built-at-flush", + notes: "build the giant VALUES statement then exec (what AccumulatorInserter does)", + bytes: art.multiRowBytes, + run: func(ctx context.Context) error { return h.insertMultiRowBuilt(ctx, rows) }, + }, + { + name: "insert-multirow-built-at-flush-libpq", + notes: "same, through database/sql + lib/pq, to size the driver's share", + bytes: art.multiRowBytes, + run: func(ctx context.Context) error { return h.insertMultiRowBuiltLibpq(ctx, rows) }, + }, + { + name: "insert-multirow-prebuilt-from-disk", + notes: "statements already built on disk: isolates pure server-side cost", + bytes: art.multiRowBytes, + run: func(ctx context.Context) error { return h.insertMultiRowPrebuilt(ctx, art.multiRowPath) }, + }, + { + name: "copy-csv-from-disk", + notes: "COPY FROM STDIN (FORMAT CSV), io.Copy from file", + bytes: art.csvBytes, + run: func(ctx context.Context) error { return h.copyFromFile(ctx, art.csvPath, h.csvCopySQL) }, + }, + { + name: "copy-binary-from-disk", + notes: "COPY FROM STDIN (FORMAT BINARY), io.Copy from file -- the proposed spill path", + bytes: art.binaryBytes, + run: func(ctx context.Context) error { return h.copyFromFile(ctx, art.binaryPath, h.binaryCopySQL) }, + }, + { + name: "copy-binary-encoded-at-flush", + notes: "pgx.CopyFrom over in-memory rows: binary COPY without pre-encoding", + bytes: art.binaryBytes, + run: func(ctx context.Context) error { return h.copyFromValues(ctx, rows) }, + }, + } + + best := map[string]time.Duration{} + for pass := range repeat { + for _, v := range variants { + h.truncate(t, ctx) + + start := time.Now() + err := v.run(ctx) + elapsed := time.Since(start) + require.NoError(t, err, "variant %q", v.name) + + actual := h.checksum(t, ctx) + require.Equal(t, expected.String(), actual.String(), + "variant %q loaded different data than the dataset", v.name) + + if prev, ok := best[v.name]; !ok || elapsed < prev { + best[v.name] = elapsed + } + t.Logf("pass %d %-36s %10s", pass+1, v.name, elapsed.Round(time.Millisecond)) + } + } + + report(t, variants, best, rowCount) +} + +type variant struct { + name string + notes string + bytes int64 + run func(ctx context.Context) error +} + +func report(t *testing.T, variants []variant, best map[string]time.Duration, rowCount int) { + t.Helper() + + baseline := best["insert-multirow-built-at-flush"] + + out := &tabwriter.Writer{} + buf := &lineBuffer{} + out.Init(buf, 0, 8, 2, ' ', 0) + + fmt.Fprintln(out, "variant\tduration\trows/s\tMiB/s\tvs current\tnotes") + for _, v := range variants { + d := best[v.name] + rowsPerSec := float64(rowCount) / d.Seconds() + mibPerSec := float64(v.bytes) / d.Seconds() / (1024 * 1024) + speedup := baseline.Seconds() / d.Seconds() + + fmt.Fprintf(out, "%s\t%s\t%s\t%.1f\t%.2fx\t%s\n", + v.name, d.Round(time.Millisecond), humanCount(rowsPerSec), mibPerSec, speedup, v.notes) + } + require.NoError(t, out.Flush()) + + t.Logf("\n\n%d rows into %s.%s\n\n%s", rowCount, benchSchema, benchTable, buf.String()) +} + +// -- harness ---------------------------------------------------------------------- + +type harness struct { + pool *pgxpool.Pool // extended protocol, used for prepared inserts and COPY + simplePool *pgxpool.Pool // simple protocol, used for prebuilt literal SQL + libpq *sql.DB + columns []pgcopy.Column + binaryCopySQL string + csvCopySQL string + dataDir string +} + +func newHarness(t *testing.T, ctx context.Context) *harness { + t.Helper() + + image := envString("PGBENCH_PG_IMAGE", "postgres:17-alpine") + container, err := tcpostgres.Run(ctx, image, + tcpostgres.WithDatabase("bench"), + tcpostgres.WithUsername("bench"), + tcpostgres.WithPassword("bench"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second)), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = testcontainers.TerminateContainer(container) }) + + dsn := container.MustConnectionString(ctx, "sslmode=disable") + + pool, err := pgxpool.New(ctx, dsn) + require.NoError(t, err) + t.Cleanup(pool.Close) + + simpleConfig, err := pgxpool.ParseConfig(dsn) + require.NoError(t, err) + simpleConfig.ConnConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol + simplePool, err := pgxpool.NewWithConfig(ctx, simpleConfig) + require.NoError(t, err) + t.Cleanup(simplePool.Close) + + libpqDB, err := sql.Open("postgres", dsn) + require.NoError(t, err) + t.Cleanup(func() { _ = libpqDB.Close() }) + + _, err = pool.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS "+benchSchema) + require.NoError(t, err) + _, err = pool.Exec(ctx, createBenchTableSQL) + require.NoError(t, err) + + if os.Getenv("PGBENCH_WITH_INDEX") != "" { + t.Log("creating secondary indexes: COPY's advantage shrinks when index maintenance dominates") + _, err = pool.Exec(ctx, `CREATE INDEX IF NOT EXISTS transfers_id_idx ON bench.transfers (id)`) + require.NoError(t, err) + _, err = pool.Exec(ctx, `CREATE INDEX IF NOT EXISTS transfers_block_idx ON bench.transfers (_block_number_)`) + require.NoError(t, err) + } + + // Resolve the type OIDs from the live catalog. Binary COPY does no coercion, so + // these must be the server's own, never derived from the declared type names. + all, err := pgcopy.LoadColumns(ctx, pool, benchSchema, benchTable) + require.NoError(t, err) + require.Len(t, all, len(benchColumnNames)) + for i, col := range all { + require.Equal(t, benchColumnNames[i], col.Name, "column %d out of order", i) + } + + dataDir := os.Getenv("PGBENCH_KEEP_DATA_DIR") + if dataDir == "" { + dataDir = t.TempDir() + } else { + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + } + + return &harness{ + pool: pool, + simplePool: simplePool, + libpq: libpqDB, + columns: all, + binaryCopySQL: pgcopy.CopySQL(benchSchema, benchTable, all), + csvCopySQL: copyCSVSQL(all), + dataDir: dataDir, + } +} + +type artifacts struct { + binaryPath string + csvPath string + multiRowPath string + + binaryBytes int64 + csvBytes int64 + multiRowBytes int64 +} + +// materialise writes every on-disk artifact in full. Nothing below this point should +// touch the dataset except to hand already-encoded bytes to the server. +func (h *harness) materialise(t *testing.T, ctx context.Context, rows []*row) artifacts { + t.Helper() + + art := artifacts{ + binaryPath: filepath.Join(h.dataDir, "data.pgcopy"), + csvPath: filepath.Join(h.dataDir, "data.csv"), + multiRowPath: filepath.Join(h.dataDir, "data.multirow.frames"), + } + + require.NoError(t, h.writeBinaryCopy(art.binaryPath, rows)) + require.NoError(t, writeCSV(art.csvPath, rows)) + require.NoError(t, writeMultiRowSQL(art.multiRowPath, rows, multiRowBatchSize)) + + art.binaryBytes = fileSize(t, art.binaryPath) + art.csvBytes = fileSize(t, art.csvPath) + art.multiRowBytes = fileSize(t, art.multiRowPath) + + return art +} + +func (h *harness) writeBinaryCopy(path string, rows []*row) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + defer file.Close() + + writer, err := pgcopy.NewWriter(file, h.columns) + if err != nil { + return fmt.Errorf("creating pgcopy writer: %w", err) + } + + for i, r := range rows { + values := r.values() + if err := pgcopy.NormalizeRow(h.columns, values); err != nil { + return fmt.Errorf("normalizing row %d: %w", i, err) + } + if err := writer.WriteRow(values); err != nil { + return fmt.Errorf("writing row %d: %w", i, err) + } + } + + if err := writer.Close(); err != nil { + return err + } + + return file.Sync() +} + +// -- variants --------------------------------------------------------------------- + +func (h *harness) insertPerRow(ctx context.Context, rows []*row) error { + statement := insertPlaceholderSQL() + + return h.inTx(ctx, h.pool, func(tx pgx.Tx) error { + for i, r := range rows { + values := r.values() + if err := pgcopy.NormalizeRow(h.columns, values); err != nil { + return fmt.Errorf("normalizing row %d: %w", i, err) + } + if _, err := tx.Exec(ctx, statement, values...); err != nil { + return fmt.Errorf("inserting row %d: %w", i, err) + } + } + + return nil + }) +} + +func (h *harness) insertMultiRowBuilt(ctx context.Context, rows []*row) error { + return h.inTx(ctx, h.simplePool, func(tx pgx.Tx) error { + for start := 0; start < len(rows); start += multiRowBatchSize { + end := min(start+multiRowBatchSize, len(rows)) + if _, err := tx.Exec(ctx, buildMultiRowInsert(rows[start:end])); err != nil { + return fmt.Errorf("inserting rows %d..%d: %w", start, end, err) + } + } + + return nil + }) +} + +func (h *harness) insertMultiRowBuiltLibpq(ctx context.Context, rows []*row) error { + tx, err := h.libpq.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin: %w", err) + } + defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op + + for start := 0; start < len(rows); start += multiRowBatchSize { + end := min(start+multiRowBatchSize, len(rows)) + if _, err := tx.ExecContext(ctx, buildMultiRowInsert(rows[start:end])); err != nil { + return fmt.Errorf("inserting rows %d..%d: %w", start, end, err) + } + } + + return tx.Commit() +} + +func (h *harness) insertMultiRowPrebuilt(ctx context.Context, path string) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer file.Close() + + frames := newFrameReader(file) + + return h.inTx(ctx, h.simplePool, func(tx pgx.Tx) error { + for { + statement, err := frames.next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("reading statement: %w", err) + } + if _, err := tx.Exec(ctx, string(statement)); err != nil { + return fmt.Errorf("executing prebuilt statement: %w", err) + } + } + }) +} + +// copyFromFile is the shape the spill design uses at flush time: hand the file to the +// connection and let it shovel bytes. No encoding, no escaping, no parsing client-side. +func (h *harness) copyFromFile(ctx context.Context, path, statement string) error { + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer file.Close() + + conn, err := h.pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquiring connection: %w", err) + } + defer conn.Release() + + if _, err := conn.Conn().PgConn().CopyFrom(ctx, file, statement); err != nil { + return fmt.Errorf("copy from %s: %w", filepath.Base(path), err) + } + + return nil +} + +func (h *harness) copyFromValues(ctx context.Context, rows []*row) error { + names := make([]string, len(h.columns)) + for i, col := range h.columns { + names[i] = col.Name + } + + var encodeErr error + source := pgx.CopyFromSlice(len(rows), func(i int) ([]any, error) { + values := rows[i].values() + if err := pgcopy.NormalizeRow(h.columns, values); err != nil { + encodeErr = err + return nil, err + } + return values, nil + }) + + if _, err := h.pool.CopyFrom(ctx, pgx.Identifier{benchSchema, benchTable}, names, source); err != nil { + return fmt.Errorf("pgx copy from: %w (encode: %v)", err, encodeErr) + } + + return nil +} + +// -- helpers ---------------------------------------------------------------------- + +func (h *harness) inTx(ctx context.Context, pool *pgxpool.Pool, fn func(tx pgx.Tx) error) error { + tx, err := pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op + + if err := fn(tx); err != nil { + return err + } + + return tx.Commit(ctx) +} + +func (h *harness) truncate(t *testing.T, ctx context.Context) { + t.Helper() + + _, err := h.pool.Exec(ctx, fmt.Sprintf("TRUNCATE %s.%s", benchSchema, benchTable)) + require.NoError(t, err) +} + +func (h *harness) checksum(t *testing.T, ctx context.Context) checksum { + t.Helper() + + var out checksum + err := h.pool.QueryRow(ctx, checksumSQL).Scan( + &out.Count, &out.BlockSum, &out.LogIndexSum, &out.AmountSum, + &out.TopicsSum, &out.HashLenSum, &out.HashByte0Sum, &out.HashByteLastSum, + &out.SuccessCount, &out.FeeAbove500, &out.MetaKindCount, + ) + require.NoError(t, err) + + return out +} + +func insertPlaceholderSQL() string { + placeholders := "" + quoted := "" + for i, name := range benchColumnNames { + if i > 0 { + placeholders += ", " + quoted += ", " + } + placeholders += "$" + strconv.Itoa(i+1) + quoted += `"` + name + `"` + } + + return fmt.Sprintf(`INSERT INTO %s.%s (%s) VALUES (%s)`, benchSchema, benchTable, quoted, placeholders) +} + +func copyCSVSQL(cols []pgcopy.Column) string { + quoted := "" + for i, col := range cols { + if i > 0 { + quoted += ", " + } + quoted += pgx.Identifier{col.Name}.Sanitize() + } + + return fmt.Sprintf("COPY %s (%s) FROM STDIN (FORMAT CSV)", + pgx.Identifier{benchSchema, benchTable}.Sanitize(), quoted) +} + +func fileSize(t *testing.T, path string) int64 { + t.Helper() + + info, err := os.Stat(path) + require.NoError(t, err) + + return info.Size() +} + +func envInt(t *testing.T, name string, fallback int) int { + t.Helper() + + raw := os.Getenv(name) + if raw == "" { + return fallback + } + + value, err := strconv.Atoi(raw) + require.NoError(t, err, "invalid %s", name) + + return value +} + +func envString(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + + return fallback +} + +func humanBytes(n int64) string { + f := float64(n) + for _, unit := range []string{"B", "KiB", "MiB", "GiB"} { + if f < 1024 { + return fmt.Sprintf("%.1f%s", f, unit) + } + f /= 1024 + } + + return fmt.Sprintf("%.1fTiB", f) +} + +func humanCount(f float64) string { + switch { + case f >= 1_000_000: + return fmt.Sprintf("%.2fM", f/1_000_000) + case f >= 1_000: + return fmt.Sprintf("%.0fk", f/1_000) + default: + return fmt.Sprintf("%.0f", f) + } +} + +// lineBuffer is a tiny io.Writer so the tabwriter output can go into one t.Logf call +// and stay together in the test output. +type lineBuffer struct{ b []byte } + +func (l *lineBuffer) Write(p []byte) (int, error) { l.b = append(l.b, p...); return len(p), nil } +func (l *lineBuffer) String() string { return string(l.b) } diff --git a/sink/sql/db_proto/benchmarks/correctness_test.go b/sink/sql/db_proto/benchmarks/correctness_test.go new file mode 100644 index 000000000..e873288ef --- /dev/null +++ b/sink/sql/db_proto/benchmarks/correctness_test.go @@ -0,0 +1,147 @@ +package benchmarks + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +// TestPgCopyBinaryRoundTrip is the safety net for the binary encoder: a value written +// through pgcopy must come back out of the server byte-identical to what a +// parameterised INSERT of the same value produces. Binary COPY does no coercion, so an +// encoder bug shows up as either a hard COPY failure or, worse, silently wrong data. +func TestPgCopyBinaryRoundTrip(t *testing.T) { + + ctx := context.Background() + pool := startPostgres(t, ctx) + + _, err := pool.Exec(ctx, "CREATE SCHEMA IF NOT EXISTS "+benchSchema) + require.NoError(t, err) + _, err = pool.Exec(ctx, createBenchTableSQL) + require.NoError(t, err) + + columns, err := pgcopy.LoadColumns(ctx, pool, benchSchema, benchTable) + require.NoError(t, err) + + rows := generateRows(1000, 7) + + // Reference load through parameterised INSERT, which does coerce. + _, err = pool.Exec(ctx, "TRUNCATE bench.transfers") + require.NoError(t, err) + statement := insertPlaceholderSQL() + for i, r := range rows { + values := r.values() + require.NoError(t, pgcopy.NormalizeRow(columns, values)) + _, err := pool.Exec(ctx, statement, values...) + require.NoError(t, err, "row %d", i) + } + viaInsert := dumpRows(t, ctx, pool) + + // Same dataset through the binary COPY path. + _, err = pool.Exec(ctx, "TRUNCATE bench.transfers") + require.NoError(t, err) + + path := t.TempDir() + "/roundtrip.pgcopy" + file, err := os.Create(path) + require.NoError(t, err) + writer, err := pgcopy.NewWriter(file, columns) + require.NoError(t, err) + for i, r := range rows { + values := r.values() + require.NoError(t, pgcopy.NormalizeRow(columns, values)) + require.NoError(t, writer.WriteRow(values), "row %d", i) + } + require.NoError(t, writer.Close()) + require.NoError(t, file.Close()) + + reader, err := os.Open(path) + require.NoError(t, err) + defer reader.Close() + + conn, err := pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + + tag, err := conn.Conn().PgConn().CopyFrom(ctx, reader, pgcopy.CopySQL(benchSchema, benchTable, columns)) + require.NoError(t, err) + require.Equal(t, int64(len(rows)), tag.RowsAffected()) + + viaCopy := dumpRows(t, ctx, pool) + + require.Equal(t, len(viaInsert), len(viaCopy)) + for i := range viaInsert { + require.Equal(t, viaInsert[i], viaCopy[i], "row %d differs between INSERT and binary COPY", i) + } +} + +// dumpRows reads every column back in a stable order, as Go values, so two load paths +// can be compared field by field. +func dumpRows(t *testing.T, ctx context.Context, pool *pgxpool.Pool) []string { + t.Helper() + + const query = ` + SELECT _block_number_, _block_timestamp_, id, encode(tx_hash, 'hex'), log_index, + "from", "to", amount::text, gas_used::text, success, fee, topics, meta::text + FROM bench.transfers + ORDER BY id, log_index` + + rows, err := pool.Query(ctx, query) + require.NoError(t, err) + defer rows.Close() + + var out []string + for rows.Next() { + var ( + blockNumber int64 + timestamp time.Time + id, hash string + logIndex int32 + from, to string + amount, gas string + success bool + fee float64 + topics []string + meta string + ) + require.NoError(t, rows.Scan(&blockNumber, ×tamp, &id, &hash, &logIndex, + &from, &to, &amount, &gas, &success, &fee, &topics, &meta)) + + out = append(out, fmt.Sprintf("%d|%s|%s|%s|%d|%s|%s|%s|%s|%t|%v|%v|%s", + blockNumber, timestamp.UTC().Format(time.RFC3339Nano), id, hash, logIndex, + from, to, amount, gas, success, fee, topics, meta)) + } + require.NoError(t, rows.Err()) + + return out +} + +func startPostgres(t *testing.T, ctx context.Context) *pgxpool.Pool { + t.Helper() + + container, err := tcpostgres.Run(ctx, envString("PGBENCH_PG_IMAGE", "postgres:17-alpine"), + tcpostgres.WithDatabase("bench"), + tcpostgres.WithUsername("bench"), + tcpostgres.WithPassword("bench"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second)), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = testcontainers.TerminateContainer(container) }) + + pool, err := pgxpool.New(ctx, container.MustConnectionString(ctx, "sslmode=disable")) + require.NoError(t, err) + t.Cleanup(pool.Close) + + return pool +} diff --git a/sink/sql/db_proto/benchmarks/dataset_test.go b/sink/sql/db_proto/benchmarks/dataset_test.go new file mode 100644 index 000000000..5e0a6af4e --- /dev/null +++ b/sink/sql/db_proto/benchmarks/dataset_test.go @@ -0,0 +1,355 @@ +package benchmarks + +import ( + "encoding/csv" + "encoding/hex" + "fmt" + "math/big" + "math/rand" + "os" + "strconv" + "strings" + "time" +) + +// The benchmark table mirrors the shape the from-proto sink produces for a typical +// entity: the two synthetic block columns, a string primary key, and a spread of the +// column types whose binary encodings are non-trivial (NUMERIC from uint64, BYTEA, +// TIMESTAMP, TEXT[], JSONB). +const ( + benchSchema = "bench" + benchTable = "transfers" +) + +const createBenchTableSQL = ` +CREATE TABLE IF NOT EXISTS bench.transfers ( + _block_number_ BIGINT NOT NULL, + _block_timestamp_ TIMESTAMP NOT NULL, + id VARCHAR(255) NOT NULL, + tx_hash BYTEA NOT NULL, + log_index INTEGER NOT NULL, + "from" VARCHAR(255) NOT NULL, + "to" VARCHAR(255) NOT NULL, + amount NUMERIC NOT NULL, + gas_used NUMERIC NOT NULL, + success BOOLEAN NOT NULL, + fee DOUBLE PRECISION NOT NULL, + topics TEXT[] NOT NULL, + meta JSONB NOT NULL +)` + +// benchColumnNames is the column order used by every variant, on disk and on the wire. +var benchColumnNames = []string{ + "_block_number_", "_block_timestamp_", "id", "tx_hash", "log_index", + "from", "to", "amount", "gas_used", "success", "fee", "topics", "meta", +} + +// metaColumnIndex is the JSONB column, which the text paths must quote as json rather +// than let the []byte fall through to a bytea literal. +const metaColumnIndex = 12 + +// row is one generated entity. Field order matches benchColumnNames. +type row struct { + BlockNumber int64 + BlockTimestamp time.Time + ID string + TxHash []byte + LogIndex int32 + From string + To string + Amount uint64 + GasUsed uint64 + Success bool + Fee float64 + Topics []any + Meta []byte +} + +// values returns the row exactly as the protobuf walker hands it to Inserter.Insert: +// raw Go types, uint64 for the NUMERIC columns, []any for the array. +func (r *row) values() []any { + return []any{ + r.BlockNumber, r.BlockTimestamp, r.ID, r.TxHash, r.LogIndex, + r.From, r.To, r.Amount, r.GasUsed, r.Success, r.Fee, r.Topics, r.Meta, + } +} + +// generateRows builds a deterministic dataset. Same seed and count always yields the +// same bytes, so cached on-disk artifacts stay valid across runs. +func generateRows(count int, seed int64) []*row { + rng := rand.New(rand.NewSource(seed)) + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + rows := make([]*row, count) + for i := range rows { + blockNum := int64(20_000_000 + i/12) // ~12 entities per block + + txHash := make([]byte, 32) + rng.Read(txHash) + + topicCount := 1 + rng.Intn(4) + topics := make([]any, topicCount) + for j := range topics { + topics[j] = fmt.Sprintf("0x%016x", rng.Uint64()) + } + + rows[i] = &row{ + BlockNumber: blockNum, + BlockTimestamp: base.Add(time.Duration(blockNum-20_000_000) * 12 * time.Second), + ID: fmt.Sprintf("%016x-%04d", rng.Uint64(), i%10000), + TxHash: txHash, + LogIndex: int32(i % 256), + From: fmt.Sprintf("0x%040x", rng.Uint64()), + To: fmt.Sprintf("0x%040x", rng.Uint64()), + // Deliberately spread across the whole uint64 range: above 2^63 is exactly + // where sending a uint64 as an int8 to a NUMERIC column goes wrong. + Amount: rng.Uint64(), + GasUsed: uint64(21000 + rng.Intn(3_000_000)), + Success: rng.Intn(100) != 0, + Fee: rng.Float64() * 1000, + Topics: topics, + Meta: fmt.Appendf(nil, `{"kind":"transfer","seq":%d,"ok":%t}`, i, rng.Intn(2) == 0), + } + } + + return rows +} + +// -- verification ----------------------------------------------------------------- + +// checksum is a cheap fingerprint of a loaded table, computed identically in Go and in +// SQL so every variant can be proven to have loaded the same data. A fast-but-wrong +// load is worth nothing, and binary COPY is exactly the kind of change that can be +// fast and wrong. +type checksum struct { + Count int64 + BlockSum int64 + LogIndexSum int64 + AmountSum string // exact sum of the uint64 amounts, as a decimal string + TopicsSum int64 + HashLenSum int64 + HashByte0Sum int64 + HashByteLastSum int64 + SuccessCount int64 + FeeAbove500 int64 + MetaKindCount int64 +} + +func (c checksum) String() string { + return fmt.Sprintf("count=%d blocks=%d logidx=%d amount=%s topics=%d hashlen=%d hash0=%d hashN=%d ok=%d fee500=%d meta=%d", + c.Count, c.BlockSum, c.LogIndexSum, c.AmountSum, c.TopicsSum, + c.HashLenSum, c.HashByte0Sum, c.HashByteLastSum, c.SuccessCount, c.FeeAbove500, c.MetaKindCount) +} + +func expectedChecksum(rows []*row) checksum { + out := checksum{Count: int64(len(rows))} + amount := new(big.Int) + + for _, r := range rows { + out.BlockSum += r.BlockNumber + out.LogIndexSum += int64(r.LogIndex) + out.TopicsSum += int64(len(r.Topics)) + amount.Add(amount, new(big.Int).SetUint64(r.Amount)) + + out.HashLenSum += int64(len(r.TxHash)) + out.HashByte0Sum += int64(r.TxHash[0]) + out.HashByteLastSum += int64(r.TxHash[len(r.TxHash)-1]) + + if r.Success { + out.SuccessCount++ + } + if r.Fee > 500 { + out.FeeAbove500++ + } + out.MetaKindCount++ // every generated row has "kind":"transfer" + } + out.AmountSum = amount.String() + + return out +} + +// checksumSQL recomputes the same fingerprint server-side. +// +// sum(length(tx_hash)) alone catches the classic bytea double-encoding failure (64 +// bytes instead of 32); the two byte probes catch content corruption at the same +// length. meta->>'kind' proves the JSONB actually parsed rather than landing as text. +// Every aggregate is cast explicitly because sum() over an integer type returns +// numeric, which does not scan into an int64. +const checksumSQL = ` +SELECT + count(*)::bigint, + coalesce(sum(_block_number_), 0)::bigint, + coalesce(sum(log_index), 0)::bigint, + coalesce(sum(amount), 0)::text, + coalesce(sum(array_length(topics, 1)), 0)::bigint, + coalesce(sum(length(tx_hash)), 0)::bigint, + coalesce(sum(get_byte(tx_hash, 0)), 0)::bigint, + coalesce(sum(get_byte(tx_hash, length(tx_hash) - 1)), 0)::bigint, + count(*) FILTER (WHERE success)::bigint, + count(*) FILTER (WHERE fee > 500)::bigint, + count(*) FILTER (WHERE meta->>'kind' = 'transfer')::bigint +FROM bench.transfers` + +// -- on-disk artifacts ------------------------------------------------------------ +// +// Every file below is materialised in full before the comparison starts, so a measured +// duration is transport plus server work and never generation. + +// writeCSV materialises the dataset for COPY ... FROM STDIN (FORMAT CSV). +func writeCSV(path string, rows []*row) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + defer file.Close() + + writer := csv.NewWriter(file) + record := make([]string, len(benchColumnNames)) + + for _, r := range rows { + record[0] = strconv.FormatInt(r.BlockNumber, 10) + record[1] = r.BlockTimestamp.Format("2006-01-02 15:04:05.999999") + record[2] = r.ID + record[3] = `\x` + hex.EncodeToString(r.TxHash) + record[4] = strconv.FormatInt(int64(r.LogIndex), 10) + record[5] = r.From + record[6] = r.To + record[7] = strconv.FormatUint(r.Amount, 10) + record[8] = strconv.FormatUint(r.GasUsed, 10) + record[9] = strconv.FormatBool(r.Success) + record[10] = strconv.FormatFloat(r.Fee, 'g', 17, 64) + record[11] = pgArrayLiteral(r.Topics) + record[12] = string(r.Meta) + + if err := writer.Write(record); err != nil { + return fmt.Errorf("writing csv record: %w", err) + } + } + + writer.Flush() + if err := writer.Error(); err != nil { + return fmt.Errorf("flushing csv: %w", err) + } + + return file.Sync() +} + +// pgArrayLiteral renders {"a","b"} as Postgres array input expects. The csv writer +// quotes the resulting field itself. +func pgArrayLiteral(values []any) string { + var b strings.Builder + b.WriteByte('{') + for i, v := range values { + if i > 0 { + b.WriteByte(',') + } + escaped := strings.ReplaceAll(fmt.Sprint(v), `\`, `\\`) + escaped = strings.ReplaceAll(escaped, `"`, `\"`) + b.WriteByte('"') + b.WriteString(escaped) + b.WriteByte('"') + } + b.WriteByte('}') + + return b.String() +} + +// writeMultiRowSQL materialises complete multi-row INSERT statements, length-prefixed +// so they stream back without re-parsing. This isolates the server-side cost of the +// current accumulator strategy from the client-side cost of building its SQL. +func writeMultiRowSQL(path string, rows []*row, batchSize int) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + defer file.Close() + + frames := newFrameWriter(file) + for start := 0; start < len(rows); start += batchSize { + end := min(start+batchSize, len(rows)) + + if err := frames.write([]byte(buildMultiRowInsert(rows[start:end]))); err != nil { + return fmt.Errorf("writing frame: %w", err) + } + } + + if err := frames.flush(); err != nil { + return err + } + + return file.Sync() +} + +// buildMultiRowInsert is the current AccumulatorInserter.flush strategy: one text +// statement whose length grows with the batch. +func buildMultiRowInsert(rows []*row) string { + var b strings.Builder + b.Grow(len(rows) * 400) + + b.WriteString(insertPrefix()) + for i, r := range rows { + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('(') + for j, v := range r.values() { + if j > 0 { + b.WriteByte(',') + } + b.WriteString(sqlLiteral(j, v)) + } + b.WriteByte(')') + } + + return b.String() +} + +func insertPrefix() string { + quoted := make([]string, len(benchColumnNames)) + for i, name := range benchColumnNames { + quoted[i] = `"` + name + `"` + } + + return fmt.Sprintf(`INSERT INTO %s.%s (%s) VALUES `, benchSchema, benchTable, strings.Join(quoted, ", ")) +} + +// sqlLiteral mirrors postgres.ValueToString, which is what AccumulatorInserter uses +// today, restricted to the types this dataset produces. +func sqlLiteral(columnIndex int, value any) string { + if columnIndex == metaColumnIndex { + return quoteString(string(value.([]byte))) + } + + switch v := value.(type) { + case nil: + return "NULL" + case string: + return quoteString(v) + case int32: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case uint64: + return strconv.FormatUint(v, 10) + case float64: + return strconv.FormatFloat(v, 'g', 17, 64) + case bool: + return strconv.FormatBool(v) + case time.Time: + return "'" + v.UTC().Format("2006-01-02 15:04:05.999999") + "'" + case []byte: + return `'\x` + hex.EncodeToString(v) + `'::bytea` + case []any: + elements := make([]string, len(v)) + for i, e := range v { + elements[i] = sqlLiteral(-1, e) + } + return "array[" + strings.Join(elements, ",") + "]" + default: + panic(fmt.Sprintf("sqlLiteral: unsupported type %T", v)) + } +} + +func quoteString(v string) string { + return "'" + strings.ReplaceAll(v, "'", "''") + "'" +} diff --git a/sink/sql/db_proto/benchmarks/frames_test.go b/sink/sql/db_proto/benchmarks/frames_test.go new file mode 100644 index 000000000..0adc394f4 --- /dev/null +++ b/sink/sql/db_proto/benchmarks/frames_test.go @@ -0,0 +1,63 @@ +package benchmarks + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" +) + +// A frame file is a sequence of uint32-length-prefixed byte blobs. It lets prebuilt +// SQL statements be stored on disk and streamed back without any parsing, so a variant +// reading them pays no cost the real implementation would not. + +type frameWriter struct { + out *bufio.Writer + header []byte +} + +func newFrameWriter(w io.Writer) *frameWriter { + return &frameWriter{out: bufio.NewWriterSize(w, 1024*1024), header: make([]byte, 4)} +} + +func (w *frameWriter) write(payload []byte) error { + binary.BigEndian.PutUint32(w.header, uint32(len(payload))) + if _, err := w.out.Write(w.header); err != nil { + return err + } + _, err := w.out.Write(payload) + + return err +} + +func (w *frameWriter) flush() error { return w.out.Flush() } + +type frameReader struct { + in *bufio.Reader + header []byte + buf []byte +} + +func newFrameReader(r io.Reader) *frameReader { + return &frameReader{in: bufio.NewReaderSize(r, 1024*1024), header: make([]byte, 4)} +} + +// next returns the next frame, or io.EOF. The returned slice is only valid until the +// following call. +func (r *frameReader) next() ([]byte, error) { + if _, err := io.ReadFull(r.in, r.header); err != nil { + return nil, err + } + + size := int(binary.BigEndian.Uint32(r.header)) + if cap(r.buf) < size { + r.buf = make([]byte, size) + } + r.buf = r.buf[:size] + + if _, err := io.ReadFull(r.in, r.buf); err != nil { + return nil, fmt.Errorf("reading frame payload of %d bytes: %w", size, err) + } + + return r.buf, nil +} diff --git a/sink/sql/db_proto/benchmarks/gate_test.go b/sink/sql/db_proto/benchmarks/gate_test.go new file mode 100644 index 000000000..cc94c22e0 --- /dev/null +++ b/sink/sql/db_proto/benchmarks/gate_test.go @@ -0,0 +1,25 @@ +package benchmarks + +import ( + "os" + "testing" +) + +// This package holds two different kinds of test. +// +// A correctness test asserts something and always runs, container and all: the suite +// needs a container runtime anyway, so a test that quietly skipped itself without one +// would only hide a broken environment. +// +// A measurement asserts nothing — it prints a table for a human — and costs a minute or +// two, so it stays opt-in even on a machine that could run it. + +// requireBenchmark skips a measurement. These assert nothing, so running them in CI +// spends minutes producing a table nobody reads. +func requireBenchmark(t *testing.T) { + t.Helper() + + if os.Getenv("SF_SINK_SQL_BENCHMARKS") == "" { + t.Skip("measurement only; set SF_SINK_SQL_BENCHMARKS=true to run") + } +} diff --git a/sink/sql/db_proto/benchmarks/live-benchmark.sh b/sink/sql/db_proto/benchmarks/live-benchmark.sh new file mode 100755 index 000000000..85c129abe --- /dev/null +++ b/sink/sql/db_proto/benchmarks/live-benchmark.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# +# End-to-end comparison of the from-proto PostgreSQL sink's two ingestion paths, against +# a live Substreams endpoint. +# +# accumulator the current path: rows are buffered in memory as SQL literals and +# flushed as one large multi-row INSERT per table, synchronously +# buffer --local-buffer: rows are written to disk in the binary COPY wire format +# and loaded by a background goroutine, one transaction per segment +# +# Both variants run the same binary over the same block range; only the flag differs. +# The wall clock is launch to exit, and every check on the resulting data runs strictly +# after the process has exited, so a variant that deferred work past exit would show up +# short rather than fast. +# +# Usage: +# export SUBSTREAMS_API_KEY=... +# ./live-benchmark.sh +# SIZES="10000 50000 200000" ./live-benchmark.sh +# ENDPOINT=https://mainnet.eth.ca.streamingfast.io ./live-benchmark.sh +# WARM=1 ./live-benchmark.sh # first run over a range nobody has streamed +# +# Requirements: docker, python3, and either go or a prebuilt binary in SUBSTREAMS_BIN. + +set -uo pipefail + +ENDPOINT=${ENDPOINT:-https://mainnet.eth.streamingfast.io} +PACKAGE=${PACKAGE:-erc20-balance-changes} +MODULE=${MODULE:-map_balance_changes} +TABLE=${TABLE:-balancechange} +START_BLOCK=${START_BLOCK:-20000000} +SIZES=${SIZES:-"10000 50000"} +WARM=${WARM:-0} +WARM_CHUNK=${WARM_CHUNK:-10000} +BUFFER_MAX=${BUFFER_MAX:-8GiB} +BLOCK_BATCH=${BLOCK_BATCH:-} +# Rough on-disk cost of one block for the default package, measured at ~324 bytes per row +# and ~338 rows per block. Only used to size the preflight check. +BYTES_PER_BLOCK=${BYTES_PER_BLOCK:-110000} + +PG_CONTAINER=${PG_CONTAINER:-sinkbench-pg} +PG_PORT=${PG_PORT:-55432} +PG_IMAGE=${PG_IMAGE:-postgres:17-alpine} +WORKDIR=${WORKDIR:-./.sinkbench} +RESULTS="$WORKDIR/results.tsv" + +log() { printf '\n=== %s ===\n' "$*"; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +human_bytes() { python3 -c " +n = float($1) +for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): + if n < 1024: + print('%.1f%s' % (n, unit)); break + n /= 1024 +"; } + +docker_root() { docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo /var/lib/docker; } + +# free_space_bytes -- bytes available on the filesystem holding path. +# +# Walks up until it finds something that exists: the docker root is often a path inside a +# VM that the host cannot see, in which case there is nothing useful to report. +free_space_bytes() { + local path=$1 + while [ -n "$path" ] && [ "$path" != "/" ] && [ ! -d "$path" ]; do path=$(dirname "$path"); done + [ -d "$path" ] || return 0 + df -Pk "$path" 2>/dev/null | awk 'NR==2 {print $4*1024}' +} + +free_space() { + local bytes + bytes=$(free_space_bytes "$1") + if [ -z "$bytes" ]; then echo "unknown"; else human_bytes "$bytes"; fi +} + +# --- prerequisites ----------------------------------------------------------------- + +command -v docker >/dev/null || die "docker is required" +command -v python3 >/dev/null || die "python3 is required" +[ -n "${SUBSTREAMS_API_KEY:-}" ] || [ -n "${SUBSTREAMS_API_TOKEN:-}" ] || + die "set SUBSTREAMS_API_KEY (or SUBSTREAMS_API_TOKEN)" + +mkdir -p "$WORKDIR" + +BIN=${SUBSTREAMS_BIN:-} +if [ -z "$BIN" ]; then + command -v go >/dev/null || die "set SUBSTREAMS_BIN or install go to build from source" + BIN="$(cd "$WORKDIR" && pwd)/substreams" + root=$(git rev-parse --show-toplevel 2>/dev/null) || die "run inside the repo, or set SUBSTREAMS_BIN" + log "building $BIN" + ( cd "$root" && go build -o "$BIN" ./cmd/substreams ) || die "build failed" +fi +[ -x "$BIN" ] || die "$BIN is not executable" + +# --- database ---------------------------------------------------------------------- + +if ! docker ps --format '{{.Names}}' | grep -qx "$PG_CONTAINER"; then + log "starting $PG_CONTAINER ($PG_IMAGE) on port $PG_PORT" + docker rm -f "$PG_CONTAINER" >/dev/null 2>&1 + docker run -d --name "$PG_CONTAINER" \ + -e POSTGRES_PASSWORD=bench -e POSTGRES_USER=bench -e POSTGRES_DB=bench \ + -p "${PG_PORT}:5432" "$PG_IMAGE" >/dev/null || die "could not start postgres" + + # Docker can take a moment to accept exec into a freshly started container, and the + # first boot also runs initdb, so give it room rather than racing it. + ready=0 + for attempt in $(seq 1 120); do + if docker exec "$PG_CONTAINER" pg_isready -U bench >/dev/null 2>&1; then + ready=1 + break + fi + [ $(( attempt % 15 )) -eq 0 ] && printf ' still waiting for postgres (%ss)\n' "$attempt" + sleep 1 + done + + if [ "$ready" -ne 1 ]; then + printf 'container status: %s\n' "$(docker ps -a --filter "name=${PG_CONTAINER}" --format '{{.Status}}')" >&2 + docker logs --tail 20 "$PG_CONTAINER" >&2 2>&1 + die "postgres never became ready" + fi +fi + +psql_q() { docker exec -i "$PG_CONTAINER" psql -U bench -d bench -t -A -c "$1"; } + +DSN_BASE="psql://bench:bench@localhost:${PG_PORT}/bench?sslmode=disable" + +# --- warming ----------------------------------------------------------------------- +# +# Off by default: the server-side cache stays valid for about 30 days, so a range that +# has been streamed recently is already warm and warming it again just costs time. +# +# It matters on a range nobody has touched. A cold range is dominated by the first pass +# rather than by the sink, so whichever variant ran first would pay for the other's +# stream as well as its own: cold, a 5,000-block comparison reads 9.4x where warm it is +# 2.9x. Set WARM=1 for a range you have not streamed before. + +warm() { + local total=$1 off s + log "warming ${total} blocks from ${START_BLOCK} in chunks of ${WARM_CHUNK}" + for (( off=0; off "$WORKDIR/warm_${s}.log" 2>&1 + if ! grep -q "Completed successfully" "$WORKDIR/warm_${s}.log"; then + printf ' chunk %s FAILED, see %s\n' "$s" "$WORKDIR/warm_${s}.log" + tail -3 "$WORKDIR/warm_${s}.log" | sed 's/^/ /' + return 1 + fi + printf ' warmed %s..+%s\n' "$s" "$WARM_CHUNK" + done +} + +# --- one measured run -------------------------------------------------------------- + +measure() { # measure + local size=$1 variant=$2 + local schema="sb_${variant}_${size}" + local bufferdir="$WORKDIR/buffer_${variant}_${size}" + local log="$WORKDIR/log_${variant}_${size}.txt" + + psql_q "DROP SCHEMA IF EXISTS ${schema} CASCADE;" >/dev/null 2>&1 + rm -rf "$bufferdir" + + local extra=() + # The buffer is on by default now, so the accumulator variant is the one that has to + # say so: an empty --local-buffer turns it off. + if [ "$variant" = buffer ]; then + extra=(--local-buffer "$bufferdir" --local-buffer-max-size "$BUFFER_MAX") + else + extra=(--local-buffer "") + fi + [ -n "$BLOCK_BATCH" ] && extra+=(--block-batch-size "$BLOCK_BATCH") + + local t0 t1 rc + t0=$(python3 -c 'import time;print(time.monotonic())') + + "$BIN" sink postgres "$PACKAGE" "$MODULE" \ + --dsn "${DSN_BASE}&schemaName=${schema}" -e "$ENDPOINT" \ + -s "$START_BLOCK" -t "+${size}" "${extra[@]}" > "$log" 2>&1 + rc=$? + + t1=$(python3 -c 'import time;print(time.monotonic())') + + if [ "$rc" -ne 0 ]; then + printf ' %s FAILED (rc=%s), see %s\n' "$variant" "$rc" "$log" + tail -3 "$log" | sed 's/^/ /' + + # A range nobody has streamed before trips the endpoint's cap on uncached blocks. + if grep -q "limit-processed-blocks" "$log"; then + printf ' this range is cold: re-run with WARM=1 to stream it once first\n' + fi + + # "bad connection" means the server went away mid-statement rather than refusing the + # work, so the reason is on the PostgreSQL side and not in this log. + if grep -qE "bad connection|connection reset|EOF" "$log"; then + printf '\n the database dropped the connection, so the reason is in its log:\n' + docker logs --tail 25 "$PG_CONTAINER" 2>&1 | sed 's/^/ /' + + local pglog + pglog=$(docker logs --tail 300 "$PG_CONTAINER" 2>&1) + + if grep -qiE "no space left on device|could not write to file|could not extend file" <<< "$pglog"; then + printf '\n the database ran out of disk. this benchmark writes about %s per\n' "$(human_bytes "$BYTES_PER_BLOCK")" + printf ' block, so %s blocks needs roughly %s, plus the local buffer directory.\n' \ + "$size" "$(human_bytes $(( size * BYTES_PER_BLOCK )))" + printf ' free space now: docker %s, workdir %s\n' \ + "$(free_space "$(docker_root)")" "$(free_space "$WORKDIR")" + elif grep -qiE "out of memory|terminated by signal 9" <<< "$pglog"; then + printf '\n a backend was killed for memory. the accumulator sends one INSERT per\n' + printf ' batch, so its statement grows with --block-batch-size (default 25).\n' + printf ' retry with a smaller batch: BLOCK_BATCH=5 %s\n' "$0" + fi + fi + fi + + # Strictly after the process exits. A variant still flushing in the background would + # report fewer rows here, not a better time. + local stats + stats=$(psql_q "SELECT count(*) || '|' || (SELECT count(*) FROM ${schema}._blocks_) || '|' || + sum(_block_number_)::text + FROM ${schema}.${TABLE};" 2>/dev/null) + + python3 - "$ENDPOINT" "$size" "$variant" "$t0" "$t1" "$stats" "$rc" "$RESULTS" \ + "$log" <<'PYEOF' +import datetime +import json +import sys + +endpoint, size, variant = sys.argv[1:4] +t0, t1 = float(sys.argv[4]), float(sys.argv[5]) +stats, rc, results, logpath = sys.argv[6], sys.argv[7], sys.argv[8], sys.argv[9] + +rows, blocks, fp = (stats.split('|') + ['', '', ''])[:3] +seconds = t1 - t0 + + +def timestamp(line): + """The sink logs as JSON when stderr is not a terminal and as console text when it + is, and which one you get varies by platform. Handle both rather than silently + losing the drain figure to a format difference.""" + line = line.strip() + if line.startswith('{'): + try: + return datetime.datetime.fromisoformat(json.loads(line)['timestamp']) + except (ValueError, KeyError, json.JSONDecodeError): + return None + try: + return datetime.datetime.fromisoformat(line.split(' ', 1)[0]) + except (ValueError, IndexError): + return None + + +# How much of the run happened after the stream was already done, i.e. flushing what was +# still buffered. A path that merely deferred its work would show a long tail here rather +# than a genuinely shorter run. +drain = '' +try: + stream_end = run_end = None + with open(logpath, errors='replace') as fh: + for line in fh: + when = timestamp(line) + if when is None: + continue + run_end = when + if 'reached your stop block' in line and stream_end is None: + stream_end = when + if stream_end and run_end: + drain = '%.1f' % (run_end - stream_end).total_seconds() +except OSError: + pass + +with open(results, 'a') as fh: + fh.write('\t'.join([endpoint, size, variant, '%.1f' % seconds, + rows, blocks, fp, rc, drain]) + '\n') + +tail = (' drain=%ss' % drain) if drain else '' +print(' %-11s %8.1fs rows=%s rc=%s%s' % (variant, seconds, rows or '?', rc, tail), flush=True) +PYEOF + + psql_q "DROP SCHEMA IF EXISTS ${schema} CASCADE;" >/dev/null 2>&1 + rm -rf "$bufferdir" +} + +# --- run --------------------------------------------------------------------------- + +[ -f "$RESULTS" ] || + printf 'endpoint\tsize\tvariant\tseconds\trows\tblocks\tfingerprint\trc\tdrain\n' > "$RESULTS" + +log "endpoint $ENDPOINT" +printf 'package %s / %s, from block %s, sizes: %s\n' "$PACKAGE" "$MODULE" "$START_BLOCK" "$SIZES" + +largest=0 +for size in $SIZES; do [ "$size" -gt "$largest" ] && largest=$size; done + +needed=$(( largest * BYTES_PER_BLOCK )) +dockerfree=$(free_space_bytes "$(docker_root)") +printf 'disk: about %s needed for %s blocks; docker has %s free, workdir %s\n' \ + "$(human_bytes "$needed")" "$largest" "$(free_space "$(docker_root)")" "$(free_space "$WORKDIR")" +if [ -n "$dockerfree" ] && [ "$dockerfree" -lt "$needed" ]; then + printf '\nwarning: the database will very likely run out of disk. PostgreSQL dies mid-write\n' + printf ' when that happens and the sink reports only "driver: bad connection".\n' + printf ' free some space, point WORKDIR elsewhere, or use a smaller SIZES.\n\n' +fi + +if [ "$WARM" = 1 ]; then + warm "$largest" || die "warming failed; a measured run over cold data would be dominated by it" +else + printf 'warming skipped (WARM=1 to enable); the server-side cache holds for ~30 days\n' +fi + +for size in $SIZES; do + log "${size} blocks" + measure "$size" accumulator + measure "$size" buffer +done + +log "results" +python3 "$(dirname "$0")/live-report.py" "$RESULTS" diff --git a/sink/sql/db_proto/benchmarks/live-report.py b/sink/sql/db_proto/benchmarks/live-report.py new file mode 100755 index 000000000..196d16e69 --- /dev/null +++ b/sink/sql/db_proto/benchmarks/live-report.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Turn live-benchmark.sh's results.tsv into a comparison table. + +Correctness is checked before any speedup is printed: for a given endpoint and size the +two variants must have produced the same rows, the same blocks and the same fingerprint. +A speedup over data that does not match is meaningless, so it is reported as a mismatch +rather than as a number. +""" + +import collections +import pathlib +import sys + +Row = collections.namedtuple("Row", "endpoint size variant seconds rows blocks fp rc drain") + + +def load(path): + lines = pathlib.Path(path).read_text().splitlines() + out = [] + for line in lines[1:]: + parts = line.split("\t") + if len(parts) < 8: + continue + drain = float(parts[8]) if len(parts) > 8 and parts[8] else None + out.append(Row(parts[0], int(parts[1]), parts[2], float(parts[3]), + int(parts[4] or 0), int(parts[5] or 0), parts[6], int(parts[7]), drain)) + return out + + +def short(endpoint): + return endpoint.replace("https://", "").replace(".streamingfast.io", "") + + +def main(argv): + path = argv[1] if len(argv) > 1 else ".sinkbench/results.tsv" + rows = load(path) + + failed = [r for r in rows if r.rc != 0] + good = [r for r in rows if r.rc == 0] + if failed: + print("failed runs, excluded:") + for r in failed: + print(f" {short(r.endpoint)} {r.size} {r.variant} rc={r.rc}") + print() + + pairs = collections.defaultdict(dict) + for r in good: + pairs[(r.endpoint, r.size)][r.variant] = r + + print(f"{'endpoint':22s} {'blocks':>8s} {'rows':>12s} " + f"{'accumulator':>12s} {'buffer':>10s} {'speedup':>8s} {'identical':>10s} {'drain a/c':>14s}") + for (endpoint, size), variants in sorted(pairs.items(), key=lambda kv: (kv[0][0], kv[0][1])): + acc, buffer = variants.get("accumulator"), variants.get("buffer") + if not (acc and buffer): + only = next(iter(variants.values())) + print(f"{short(endpoint):22s} {size:>8,} {only.rows:>12,} " + f"{'(only ' + only.variant + ')':>32s}") + continue + + identical = acc.rows == buffer.rows and acc.blocks == buffer.blocks and acc.fp == buffer.fp + drain = "-" + if acc.drain is not None and buffer.drain is not None: + drain = f"{acc.drain:.1f}s/{buffer.drain:.1f}s" + print(f"{short(endpoint):22s} {size:>8,} {acc.rows:>12,} " + f"{acc.seconds:>11.1f}s {buffer.seconds:>9.1f}s " + f"{acc.seconds / buffer.seconds:>7.2f}x " + f"{'yes' if identical else 'NO -- MISMATCH':>10s} {drain:>14s}") + + repeats = collections.defaultdict(list) + for r in good: + repeats[(r.endpoint, r.size, r.variant)].append(r.seconds) + dupes = {k: v for k, v in repeats.items() if len(v) > 1} + if dupes: + print("\nrepeat measurements:") + for (endpoint, size, variant), seconds in sorted(dupes.items()): + spread = (max(seconds) - min(seconds)) / min(seconds) * 100 + print(f" {short(endpoint)} {size:,} {variant}: " + f"{', '.join(f'{s:.1f}s' for s in seconds)} (spread {spread:.1f}%)") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/sink/sql/db_proto/decoder.go b/sink/sql/db_proto/decoder.go index b9850176c..99c2d18dd 100644 --- a/sink/sql/db_proto/decoder.go +++ b/sink/sql/db_proto/decoder.go @@ -107,15 +107,23 @@ type decoded struct { walkDuration time.Duration } -func newDecoder(rootMessageDescriptor protoreflect.MessageDescriptor, workers int) *decoder { - if workers <= 0 { - // One per core, less one for the goroutine draining the gRPC stream, capped at - // 8: TestClientDecodeScaling measures 4.52x at eight workers and 5.06x at - // fifteen, so the seven extra cores together buy 11%. Taking them would cost the - // rest of the machine for almost nothing. - workers = min(8, max(1, runtime.NumCPU()-1)) +// ResolveDecodeWorkers turns a requested worker count into the one that will be used. +// +// One per core, less one for the goroutine draining the gRPC stream, capped at 8: +// TestClientDecodeScaling measures 4.52x at eight workers and 5.06x at fifteen, so the +// seven extra cores together buy 11%. Taking them would cost the rest of the machine for +// almost nothing. +func ResolveDecodeWorkers(workers int) int { + if workers > 0 { + return workers } + return min(8, max(1, runtime.NumCPU()-1)) +} + +func newDecoder(rootMessageDescriptor protoreflect.MessageDescriptor, workers int) *decoder { + workers = ResolveDecodeWorkers(workers) + return &decoder{ messageType: hyperpb.CompileMessageDescriptor(rootMessageDescriptor), workers: workers, diff --git a/sink/sql/db_proto/sinker.go b/sink/sql/db_proto/sinker.go index 0eaa99620..4888d769c 100644 --- a/sink/sql/db_proto/sinker.go +++ b/sink/sql/db_proto/sinker.go @@ -22,10 +22,18 @@ type Sinker struct { stats *stats.Stats logger *zap.Logger rootMessageDescriptor protoreflect.MessageDescriptor - useConstraints bool lastAppliedBlockNum uint64 lastAppliedBlockTime time.Time + // directInserts records that the stream reached the chain head and the database was + // switched off any buffered write path. It only ever goes from false to true. + directInserts bool + + // constraints decides what is created and when. Anything not applied upfront is + // applied once the backfill is over, which is the point of the default. + constraints sql.ConstraintPolicy + constraintsApplied bool + // holding buffers the blocks received since the last flush. It is only ever touched // from the sinker's callbacks, which are called sequentially. holding []*Holder @@ -36,11 +44,12 @@ type Sinker struct { // NewSinker builds the from-proto sinker. decodeWorkers bounds how many blocks are // unmarshalled and walked concurrently at flush time; zero picks one per core, less one, // capped at eight. -func NewSinker(rootMessageDescriptor protoreflect.MessageDescriptor, sink *sink.Sinker, db sql.Database, useTransaction bool, useConstraints bool, blockBatchSize int, decodeWorkers int, stats *stats.Stats, logger *zap.Logger) *Sinker { +func NewSinker(rootMessageDescriptor protoreflect.MessageDescriptor, sink *sink.Sinker, db sql.Database, useTransaction bool, constraints sql.ConstraintPolicy, blockBatchSize int, decodeWorkers int, stats *stats.Stats, logger *zap.Logger) *Sinker { return &Sinker{ db: db, rootMessageDescriptor: rootMessageDescriptor, useTransaction: useTransaction, + constraints: constraints, blockBatchSize: uint64(blockBatchSize), stats: stats, Sinker: sink, @@ -52,12 +61,25 @@ func NewSinker(rootMessageDescriptor protoreflect.MessageDescriptor, sink *sink. func (s *Sinker) Run(ctx context.Context) error { // Show stats one last time before exiting run defer s.LogStats() + defer s.closeDatabase() cursor, err := s.db.FetchCursor() if err != nil { return fmt.Errorf("fetch cursor: %w", err) } + // The step of the stored cursor deliberately decides nothing here. It says where the + // previous run stopped, not where the chain is now: a sink that was live, went down for + // an hour and comes back has a STEP_NEW — or, if its block forked out meanwhile, a + // STEP_UNDO — cursor with a full backfill ahead of it, which is precisely the run the + // spool exists for. Only the step of the blocks now arriving says whether we are at the + // head, and HandleBlockScopedData already reads it per block. + // + // A run that resumes on a forked-out block gets the undo signal as its first message, + // before any block. That is handled where it lands: HandleBlockUndoSignal drains the + // spool, which is empty, deletes the rows and records the cursor on the open segment. + // Whatever follows — irreversible blocks to catch up on, or live ones — then decides + // what happens to the spool, as it would on any other run. //clean up the mess from running without a transaction if cursor != nil { err = s.db.HandleBlocksUndo(cursor.Block().Num()) @@ -67,6 +89,11 @@ func (s *Sinker) Run(ctx context.Context) error { } //panic("Testing 12 12") s.logger.Info("fetched cursor", zap.Stringer("block", cursor.Block())) + if cursor != nil { + // Seed the applied mark, otherwise the first downloaded block would be measured + // against zero and reported as a chain-height-sized backlog. + s.stats.Progress.SetResumeBlock(cursor.Block().Num()) + } s.stats.LastBlockProcessAt = time.Now() s.Sinker.Run(ctx, cursor, s) @@ -78,6 +105,31 @@ func (s *Sinker) LogStats() { s.stats.Log() } +// closeTimeout bounds the shutdown seal. Writing the open segment's manifest and fsyncing +// it is local work of a few milliseconds; anything past this is a disk that is not coming +// back, and holding the process there helps nobody. +const closeTimeout = 30 * time.Second + +// closeDatabase seals whatever is still held and releases the connections, on every way +// out of the run rather than only on the one that ends a bounded range. +// +// A backfill is normally interrupted rather than completed — Ctrl-C, a killed pod, a +// stream that errors — and not sealing the open segment on the way out throws away every +// block in it, up to --db-write-max-size: recovery discards an unsealed segment, so those +// blocks are streamed, and paid for, a second time. That is the outcome the spool exists +// to prevent. +// +// It closes on a fresh context because the run's is already cancelled by the time an +// interrupt reaches here, and the seal it has to perform is a write. +func (s *Sinker) closeDatabase() { + ctx, cancel := context.WithTimeout(context.Background(), closeTimeout) + defer cancel() + + if err := s.db.Close(ctx); err != nil { + s.logger.Warn("closing the database", zap.Error(err)) + } +} + type Holder struct { output *pbsubstreamsrpc.MapModuleOutput data *pbsubstreamsrpc.BlockScopedData @@ -96,10 +148,6 @@ func (s *Sinker) HandleBlockScopedData(ctx context.Context, data *pbsubstreamsrp return fmt.Errorf("received data from wrong output module, expected to received from %q but got module's output for %q", s.OutputModuleName(), output.Name) } - if (isLive != nil && *isLive) && s.useConstraints { - return fmt.Errorf("live mode is not supported without constraints") - } - startAt := time.Now() defer func() { s.stats.LastBlockProcessAt = time.Now() @@ -113,6 +161,36 @@ func (s *Sinker) HandleBlockScopedData(ctx context.Context, data *pbsubstreamsrp } s.stats.BlockCount++ + // The switch happens before this block is held, which is what makes the spool safe + // against reorgs: `isLive` here comes from the cursor-based liveness checker the sink + // installs itself (see runFromProtoSink; --live-block-time-delta is deliberately not + // registered on this command), so it turns true on the first cursor at STEP_NEW — + // the first block that can ever be undone. Everything spooled up to here was delivered + // at STEP_NEW_IRREVERSIBLE and cannot be undone, and from here on there is no spool. + // An undo therefore never reaches a spool holding undoable blocks. It can still reach a + // spool holding nothing — a run resuming on a block that forked out while it was down + // gets the undo before any block — which is what the drain in HandleBlocksUndo is for. + if isLive != nil && *isLive && !s.directInserts { + // Write what is held before the switch, with the cursor of the last held block: + // those blocks belong to the buffered path, and this block's cursor covers a + // block that has not been applied yet. + if len(s.holding) > 0 { + if err := s.flushHolding(s.holding[len(s.holding)-1].cursor); err != nil { + return fmt.Errorf("flushing held blocks before switching to direct inserts: %w", err) + } + } + + if err := s.db.SwitchToDirectInserts(ctx, "stream reached the chain head", true); err != nil { + return fmt.Errorf("switching to direct inserts: %w", err) + } + + if err := s.applyConstraintsOnce(); err != nil { + return err + } + + s.directInserts = true + } + holder := &Holder{ output: output, data: data, @@ -120,6 +198,9 @@ func (s *Sinker) HandleBlockScopedData(ctx context.Context, data *pbsubstreamsrp cursor: cursor, } s.holding = append(s.holding, holder) + s.stats.Progress.RecordDownloaded(data.Clock.Number) + s.recordBuffered() + if data.Clock.Number > (s.lastAppliedBlockNum+s.blockBatchSize) || s.blockBatchSize == 1 || (isLive != nil && *isLive) { if isLive != nil && *isLive && s.stats.FlushDuration.Average() > data.Clock.Timestamp.AsTime().Sub(s.lastAppliedBlockTime) { s.logger.Debug("skipping a flush because we are LIVE and flush average duration is above time between blocks", zapx.HumanDuration("flush_duration_average", s.stats.FlushDuration.Average()), zap.Time("last_block_time", s.lastAppliedBlockTime), zap.Time("block_time", data.Clock.Timestamp.AsTime())) @@ -140,13 +221,44 @@ func (s *Sinker) HandleBlockScopedData(ctx context.Context, data *pbsubstreamsrp // --block-batch-size larger than the range, that meant an empty database and a run that // looked successful. func (s *Sinker) HandleBlockRangeCompletion(ctx context.Context, cursor *sink.Cursor) error { - if len(s.holding) == 0 { - return nil + + if len(s.holding) > 0 { + s.logger.Info("flushing blocks held at the end of the requested range", zap.Int("block_count", len(s.holding))) + + if err := s.flushHolding(cursor); err != nil { + return err + } + } + + // The spool still holds whatever has not reached its segment size, and it owns the + // transactions while it is open. Draining it first is what keeps those blocks from + // being streamed twice. + if err := s.db.SwitchToDirectInserts(ctx, "stream reached the end of the requested range", false); err != nil { + return fmt.Errorf("draining before the end of the range: %w", err) } - s.logger.Info("flushing blocks held at the end of the requested range", zap.Int("block_count", len(s.holding))) + // The constraints are deliberately left alone. A stop block says this run is over, not + // that the backfill is: a range is routinely one chunk of several, and building the + // constraints here would make every chunk after it load into a constrained schema, + // which is measured at 27.7x. Only reaching chain HEAD says there is nothing left. + s.reportConstraintsLeftToApply() - return s.flushHolding(cursor) + return s.db.Close(ctx) +} + +// reportConstraintsLeftToApply says what the schema is still missing when a bounded run +// ends short of the chain head. +// +// Saying nothing is the worse failure: a database with no primary keys and no foreign keys +// answers queries, slowly and without rejecting anything, and looks exactly like a database +// that is fine. +func (s *Sinker) reportConstraintsLeftToApply() { + if s.constraintsApplied || s.constraints.SkipsEverything() || !s.constraints.ApplyAtHead() { + return + } + + s.logger.Info("the run reached its stop block without reaching chain HEAD, so the schema's constraints are left alone: a stop block ends the run, it does not say the backfill is done. " + + "Run `substreams sink postgres constraints apply --dsn ...` once there is nothing more to load") } // flushHolding applies every held block, stores the cursor and commits. @@ -205,9 +317,30 @@ func (s *Sinker) flushHolding(cursor *sink.Cursor) (err error) { } s.holding = s.holding[:0] + // With a local buffer the rows are only queued at this point, not durable, so the + // applied mark has to come from what the buffer actually committed. + if _, _, applied, buffering := s.db.BufferStats(); !buffering { + s.stats.Progress.RecordApplied(lastClock.Number) + } else if applied > 0 { + s.stats.Progress.RecordApplied(applied) + } + s.recordBuffered() + return nil } +// recordBuffered reports what sits between the stream and the database: blocks held in +// memory for the next flush, plus whatever a local buffer has queued on disk. +func (s *Sinker) recordBuffered() { + bufferedBlocks, bufferedBytes, _, buffering := s.db.BufferStats() + if !buffering { + s.stats.Progress.RecordBuffered(len(s.holding), 0) + return + } + + s.stats.Progress.RecordBuffered(len(s.holding)+int(bufferedBlocks), bufferedBytes) +} + // recordDecodeStats folds the per-block timings the workers measured back into the // shared stats, on this goroutine: Average.Add is not safe for concurrent use. func (s *Sinker) recordDecodeStats(results []*decoded, insertDuration time.Duration) { @@ -226,6 +359,13 @@ func (s *Sinker) HandleBlockUndoSignal(ctx context.Context, undoSignal *pbsubstr s.logger.Info("Handling undo block signal", zap.Stringer("block", cursor.Block()), zap.Stringer("cursor", cursor)) + // Blocks are held in memory until the batch fills, so the undone ones may not have + // reached the database yet. Writing them first and deleting after is what keeps a + // later flush from putting back exactly what the undo removed. + if err := s.flushHolding(cursor); err != nil { + return fmt.Errorf("flushing held blocks before an undo: %w", err) + } + err = s.db.HandleBlocksUndo(lastValidBlockNum) if err != nil { return fmt.Errorf("handle blocks undo from %d : %w", lastValidBlockNum, err) @@ -238,3 +378,36 @@ func (s *Sinker) HandleBlockUndoSignal(ctx context.Context, undoSignal *pbsubstr return nil } + +// applyConstraintsOnce creates the schema's constraints now that the bulk of the loading +// is behind us, which is where they belong: measured through binary COPY, loading with +// foreign keys in place costs 27.7x against 3.3x for building them afterwards. +// +// It runs when the stream reaches the chain head, and only then. A stop block ends the +// run, which is not the same thing as the backfill being over: a range is routinely one +// chunk of several, and creating the constraints at the end of one would leave every chunk +// after it loading into a constrained schema — the case this whole arrangement exists to +// avoid. Reaching the head is the one signal that says there is nothing left to load. +func (s *Sinker) applyConstraintsOnce() error { + if s.constraintsApplied || !s.constraints.ApplyAtHead() { + if !s.constraintsApplied && s.constraints.Timing == sql.ConstraintsManual && !s.constraints.SkipsEverything() { + s.constraintsApplied = true + s.logger.Info("the backfill is done and the schema has no constraints yet. Creating them locks every table while indexes are built and foreign keys validated, " + + "so it is left to you: run `substreams sink postgres constraints apply --dsn ...` when a maintenance window suits") + } + + return nil + } + s.constraintsApplied = true + + s.logger.Info("the stream reached chain HEAD, creating the schema's constraints as --apply-constraints=auto asked. This locks every table while it runs") + + startAt := time.Now() + if err := applyConstraints(s.db, s.logger); err != nil { + return err + } + + s.logger.Debug("constraints created", zap.Duration("duration", time.Since(startAt))) + + return nil +} diff --git a/sink/sql/db_proto/sinker_factory.go b/sink/sql/db_proto/sinker_factory.go index d7c002d13..ad891aa47 100644 --- a/sink/sql/db_proto/sinker_factory.go +++ b/sink/sql/db_proto/sinker_factory.go @@ -13,6 +13,7 @@ import ( clickhouse "github.com/streamingfast/substreams/sink/sql/db_proto/sql/click_house" "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres" schema2 "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" stats2 "github.com/streamingfast/substreams/sink/sql/db_proto/stats" "go.uber.org/zap" "google.golang.org/protobuf/reflect/protoreflect" @@ -21,17 +22,30 @@ import ( type SinkerFactoryFunc func(ctx context.Context, dsnString, schemaName string, logger *zap.Logger, tracer logging.Tracer) (*Sinker, error) type SinkerFactoryOptions struct { - UseProtoOption bool - UseConstraints bool + UseProtoOption bool + // Constraints says which constraints the schema gets and when; the zero value means + // all of them, created once the backfill is done. + Constraints protosql.ConstraintPolicy UseTransactions bool - BlockBatchSize int + // WriteMode says how a sealed spool segment reaches the database. The zero value + // resolves per driver and schema. + WriteMode protosql.WriteMode // DecodeWorkers bounds how many blocks are unmarshalled and walked concurrently at - // flush time. Zero picks one per core, less one. + // flush time. Zero picks one per core, less one, capped at 8. DecodeWorkers int - Encoding bytes.Encoding - Clickhouse SinkerFactoryClickhouse + // DecodeBatchSize is how many blocks are held in memory and decoded together. Zero + // picks four per decode worker. It sizes the CPU stage, not the database write. + DecodeBatchSize int + Encoding bytes.Encoding + // Spool, when set, holds rows on disk and applies them from a background goroutine. + Spool *spool.Options + Clickhouse SinkerFactoryClickhouse } +// defaultSpoolMaxBytes mirrors the spool's own default, for the one place that has to know +// the quota before the spool is built. +const defaultSpoolMaxBytes int64 = 8 << 30 + type SinkerFactoryClickhouse struct { SinkInfoFolder string CursorFilePath string @@ -40,8 +54,12 @@ type SinkerFactoryClickhouse struct { } func (o SinkerFactoryOptions) Defaults() SinkerFactoryOptions { - if o.BlockBatchSize <= 0 { - o.BlockBatchSize = 25 + o.DecodeWorkers = ResolveDecodeWorkers(o.DecodeWorkers) + if o.DecodeBatchSize <= 0 { + // Four per worker: enough that the slowest block in a batch does not leave the + // others idle, small enough that the held payloads and their decoded rows stay a + // bounded amount of memory. + o.DecodeBatchSize = 4 * o.DecodeWorkers } o.UseTransactions = true if o.Encoding == 0 { @@ -56,6 +74,8 @@ func SinkerFactory( rootMessageDescriptor protoreflect.MessageDescriptor, options SinkerFactoryOptions, ) SinkerFactoryFunc { + options = options.Defaults() + return func(ctx context.Context, dsnString string, schemaName string, logger *zap.Logger, tracer logging.Tracer) (*Sinker, error) { database, err := SetupDatabaseSchema(ctx, dsnString, schemaName, outputModuleName, rootMessageDescriptor, options, logger, tracer) if err != nil { @@ -67,15 +87,35 @@ func SinkerFactory( return nil, fmt.Errorf("opening database: %w", err) } + // Before anything is streamed, and concurrently, so a restart onto a loaded table + // neither waits for a lock nor takes one. + if err := database.EnsureBlockNumberIndexes(ctx); err != nil { + return nil, fmt.Errorf("creating the block number indexes: %w", err) + } + + warnAboutMissingConstraints(database, options.Constraints, logger) + + stats := stats2.NewStats(logger, options.DecodeBatchSize) + if options.Spool != nil { + // What "falling behind" is measured against once rows go to disk. A block count + // cannot say it: a sparse backfill spans millions of blocks holding almost + // nothing, and would report itself as behind from the first minute. + quota := options.Spool.MaxBytes + if quota <= 0 { + quota = defaultSpoolMaxBytes + } + stats.Progress.SetBufferQuota(quota) + } + return NewSinker( rootMessageDescriptor, baseSink, database, options.UseTransactions, - options.UseConstraints, - options.BlockBatchSize, + options.Constraints, + options.DecodeBatchSize, options.DecodeWorkers, - stats2.NewStats(logger), + stats, logger, ), nil } @@ -112,13 +152,25 @@ func SetupDatabaseSchema( switch dsn.Driver() { case "postgres": - database, err = postgres.NewDatabase(schema, dsn, outputModuleName, rootMessageDescriptor, options.UseProtoOption, options.UseConstraints, options.Encoding, logger) - if err != nil { - return nil, fmt.Errorf("creating postgres database: %w", err) + pgDatabase, pgErr := postgres.NewDatabase(schema, dsn, outputModuleName, rootMessageDescriptor, options.UseProtoOption, options.Constraints, options.Encoding, logger) + if pgErr != nil { + return nil, fmt.Errorf("creating postgres database: %w", pgErr) + } + if options.Spool != nil { + pgDatabase.WithSpool(*options.Spool) } + if err := pgDatabase.WithWriteMode(options.WriteMode); err != nil { + return nil, err + } + database = pgDatabase case "clickhouse": - database, err = clickhouse.NewDatabase( + if options.WriteMode != "" && options.WriteMode != protosql.WriteModeAuto && options.WriteMode != protosql.WriteModeBatchInsert { + return nil, fmt.Errorf("--write-mode=%s is not available on ClickHouse, whose inserts are columnar and typed rather than SQL text; use %q or %q", + options.WriteMode, protosql.WriteModeAuto, protosql.WriteModeBatchInsert) + } + + chDatabase, err := clickhouse.NewDatabase( ctx, schema, dsn, @@ -126,7 +178,7 @@ func SetupDatabaseSchema( rootMessageDescriptor, options.Clickhouse.SinkInfoFolder, options.Clickhouse.CursorFilePath, - true, + options.UseProtoOption, options.Encoding, logger, tracer, @@ -136,12 +188,26 @@ func SetupDatabaseSchema( if err != nil { return nil, fmt.Errorf("creating clickhouse database: %w", err) } + if options.Spool != nil { + chDatabase.WithSpool(*options.Spool) + } + database = chDatabase default: panic(fmt.Sprintf("unsupported driver: %s", dsn.Driver())) } + // Before anything is created or written: a database set up from a different revision + // of the same package can hold tables this run would keep using as they are. + if verifier, ok := database.(interface { + VerifySchemaCompatibility(ctx context.Context) error + }); ok { + if err := verifier.VerifySchemaCompatibility(ctx); err != nil { + return nil, err + } + } + sinkInfo, err := database.FetchSinkInfo(schema.Name) if err != nil { return nil, fmt.Errorf("fetching sink info: %w", err) @@ -153,7 +219,7 @@ func SetupDatabaseSchema( if err != nil { return nil, fmt.Errorf("begin transaction: %w", err) } - err = database.CreateDatabase(options.UseConstraints) + err = database.CreateDatabase(options.Constraints.ApplyUpfront()) if err != nil { database.RollbackTransaction() return nil, fmt.Errorf("creating database: %w", err) @@ -171,6 +237,17 @@ func SetupDatabaseSchema( } } else { + if options.Constraints.ApplyUpfront() { + // The schema exists, but nothing says it carries constraints: a run without + // them creates the very same tables, and the sink info hash is computed over + // the DDL the dialect would emit either way. Adding the missing ones is the + // only way to get there, and doing it here means constraints on a + // database synced without them behaves the same as one created with them. + if err := applyConstraints(database, logger); err != nil { + return nil, err + } + } + migrationNeeded := sinkInfo.SchemaHash != database.GetDialect().SchemaHash() if migrationNeeded { @@ -221,3 +298,61 @@ func SetupDatabaseSchema( return database, nil } + +// applyConstraints adds the constraints an earlier run left out, in one transaction. +// +// On a populated database this can run for a long time: every index has to be built and +// every foreign key validated, with the table locked while it happens. Hence the warning +// rather than a silent wait. +func applyConstraints(database protosql.Database, logger *zap.Logger) error { + logger.Warn("adding the SQL constraints to the existing schema, this can take a long time and locks the tables while it runs") + + startAt := time.Now() + + // The pass owns its own transactions, committing as it goes so that a run killed + // part-way keeps what it finished. + if err := database.ApplyConstraints(); err != nil { + return fmt.Errorf("applying constraints: %w", err) + } + + logger.Info("constraints applied", zap.Duration("duration", time.Since(startAt))) + + return nil +} + +// warnAboutMissingConstraints says on every start whether the schema carries the +// constraints the flags describe. +// +// A run that was interrupted before the backfill ended, or whose constraint pass was +// killed part-way, leaves a database with no primary keys and no foreign keys — which +// answers queries, slowly and without rejecting anything, and looks exactly like a +// database that is fine. The check is one indexed catalog query, so it costs nothing next +// to what it reports on. +// +// It only reports. Creating them is still the constraint timing's business: auto does it +// when the backfill ends, manual leaves it to `sink postgres constraints apply`. +func warnAboutMissingConstraints(database protosql.Database, constraints protosql.ConstraintPolicy, logger *zap.Logger) { + if constraints.SkipsEverything() { + return + } + + missing, err := database.MissingConstraints() + if err != nil { + logger.Debug("could not check which constraints the schema carries", zap.Error(err)) + return + } + if len(missing) == 0 { + return + } + + next := "run `sink postgres constraints apply ` when a maintenance window suits" + if constraints.ApplyAtHead() { + // Only reaching the head creates them, and a run given a stop block may well never + // get there — saying they "will be created" would then be a promise nothing keeps. + next = "they are created once the stream reaches chain HEAD; a run that stops before that leaves them to `sink postgres constraints apply `" + } + + logger.Warn("the schema is missing constraints it is meant to have, so its tables have no indexes to match and reject nothing. "+next, + zap.Int("missing_count", len(missing)), + zap.Strings("missing", missing)) +} diff --git a/sink/sql/db_proto/sql/click_house/accumulator_inserter.go b/sink/sql/db_proto/sql/click_house/accumulator_inserter.go index 659959f73..010128a34 100644 --- a/sink/sql/db_proto/sql/click_house/accumulator_inserter.go +++ b/sink/sql/db_proto/sql/click_house/accumulator_inserter.go @@ -93,13 +93,18 @@ func createAccumulators(dialect *DialectClickHouse) (map[string]*accumulator, er input[sql2.DialectFieldDeleted] = &proto.ColBool{} columns[3] = &schema.Column{Name: sql2.DialectFieldDeleted} + if dialect.UseRowIDField(table.Name) { + input[sql2.DialectFieldRowID] = &proto.ColUInt32{} + columns[len(columns)] = &schema.Column{Name: sql2.DialectFieldRowID} + } + primaryName := "" if table.PrimaryKey != nil { pk := table.PrimaryKey primaryName = pk.Name input[pk.Name] = ColInputForColumn(pk.FieldDescriptor, dialect.bytesEncoding, table.Columns[pk.Index]) - columns[4] = &schema.Column{Name: pk.Name} + columns[len(columns)] = &schema.Column{Name: pk.Name} } offset := len(columns) diff --git a/sink/sql/db_proto/sql/click_house/chapplier.go b/sink/sql/db_proto/sql/click_house/chapplier.go new file mode 100644 index 000000000..35a97b3b4 --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/chapplier.go @@ -0,0 +1,131 @@ +package clickhouse + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "sort" + + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "go.uber.org/zap" +) + +// chApplier sends a sealed segment to ClickHouse and then advances the cursor, which is +// exactly the order the sink has always written in. +// +// ClickHouse has no transactions and keeps its cursor in a file, so a crash between the +// inserts and the cursor write re-streams those blocks and re-inserts those rows. That is +// the guarantee the sink already ships with; the spool does not improve on it and does not +// have to. What it must not do is make it worse, and it does not: applying a segment is +// the same two steps in the same order. +type chApplier struct { + database *Database + inserter *AccumulatorInserter + logger *zap.Logger +} + +func newCHApplier(database *Database, inserter *AccumulatorInserter, logger *zap.Logger) *chApplier { + return &chApplier{database: database, inserter: inserter, logger: logger.Named("spool_applier")} +} + +// EnsureSchema has nothing to create: recovery reads the cursor the sink already stores. +func (a *chApplier) EnsureSchema(context.Context) error { return nil } + +// AlreadyApplied answers from how far the stored cursor got. +// +// Without transactions there is no exact answer, and none is needed: replaying a segment +// the database had in fact taken duplicates precisely the rows re-streaming it would have +// duplicated. Skipping what the cursor already covers is what keeps a restart from redoing +// the whole spool. +func (a *chApplier) AlreadyApplied(_ context.Context, manifest *spool.Manifest) (bool, error) { + cursor, err := a.database.FetchCursor() + if err != nil || cursor == nil || cursor.IsBlank() { + //nolint:nilerr // a missing or unreadable cursor means nothing has been applied + return false, nil + } + + return manifest.LastBlock != 0 && manifest.LastBlock <= cursor.Block().Num(), nil +} + +// Apply replays one segment's rows into the column builders, sends them, then stores the +// cursor. +func (a *chApplier) Apply(_ context.Context, dir string, manifest *spool.Manifest) error { + // Tables go in the order the dialect assigns, which is the order the accumulator's own + // flush uses, so a segment reaches the server the same way an unspooled flush would. + tables := make([]spool.TableRecord, len(manifest.Tables)) + copy(tables, manifest.Tables) + sort.SliceStable(tables, func(i, j int) bool { + return a.ordinal(tables[i].Name) < a.ordinal(tables[j].Name) + }) + + for _, table := range tables { + if err := a.replayTable(dir, table); err != nil { + return err + } + } + + if err := a.inserter.flush(a.database); err != nil { + return fmt.Errorf("flushing a spooled segment: %w", err) + } + + if manifest.Cursor == "" { + return nil + } + + cursor, err := sink.NewCursor(manifest.Cursor) + if err != nil { + return fmt.Errorf("parsing the cursor of a spooled segment: %w", err) + } + + // Straight to the file: StoreCursor would route it back into the spool, which is where + // this cursor came from. + return a.database.storeCursorFile(cursor) +} + +func (a *chApplier) ordinal(table string) int { + if accumulator, found := a.inserter.accumulators[table]; found { + return accumulator.ordinal + } + + return len(a.inserter.accumulators) +} + +func (a *chApplier) replayTable(dir string, table spool.TableRecord) error { + path := filepath.Join(dir, table.File) + + reader, err := spool.OpenFrameReader(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer reader.Close() + + var replayed int64 + for { + encoded, err := reader.ReadField() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + values, err := decodeValues(encoded) + if err != nil { + return fmt.Errorf("decoding a row of %q: %w", table.Name, err) + } + + if err := a.inserter.insert(table.Name, values); err != nil { + return fmt.Errorf("replaying a row of %q: %w", table.Name, err) + } + replayed++ + } + + if replayed != table.Rows { + return fmt.Errorf("replayed %d rows of %q but the manifest recorded %d", replayed, table.Name, table.Rows) + } + + return nil +} diff --git a/sink/sql/db_proto/sql/click_house/chcodec.go b/sink/sql/db_proto/sql/click_house/chcodec.go new file mode 100644 index 000000000..f7777f561 --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/chcodec.go @@ -0,0 +1,320 @@ +package clickhouse + +import ( + "encoding/binary" + "fmt" + "math" + "os" + "path/filepath" + "time" + + sql2 "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// chCodec spools rows as typed values, one file per table. +// +// ClickHouse inserts columnar and typed rather than as SQL text, so rendering to literals +// the way the PostgreSQL formats do would change both the insert path and how types are +// handled. Keeping the values means a spooled row is appended back into exactly the same +// column builders the accumulator has always used. +type chCodec struct{} + +func newCHCodec() *chCodec { return &chCodec{} } + +func (c *chCodec) Format() spool.Format { return spool.FormatValues } + +func (c *chCodec) OpenSegment(dir string) (spool.SegmentWriter, error) { + return &chSegment{dir: dir, tables: map[string]*chTableFile{}}, nil +} + +// Verify checks each file's length against the manifest. The format is length-framed, so +// that is the whole of it: a torn write leaves the file short of what was recorded. +func (c *chCodec) Verify(dir string, manifest *spool.Manifest) error { + for _, table := range manifest.Tables { + path := filepath.Join(dir, table.File) + + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("%s is missing: %w", table.File, err) + } + if info.Size() != table.Bytes { + return fmt.Errorf("%s is %d bytes, the manifest recorded %d", table.File, info.Size(), table.Bytes) + } + } + + return nil +} + +type chTableFile struct { + path string + file *os.File + writer *spool.FrameWriter + rows int64 +} + +type chSegment struct { + dir string + tables map[string]*chTableFile +} + +func (s *chSegment) WriteRow(table string, values []any) error { + target, ok := s.tables[table] + if !ok { + path := filepath.Join(s.dir, spool.SanitizeFileName(table)+".values") + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + + target = &chTableFile{path: path, file: file, writer: spool.NewFrameWriter(file)} + s.tables[table] = target + } + + encoded, err := encodeValues(values) + if err != nil { + return fmt.Errorf("encoding a row of %q: %w", table, err) + } + target.rows++ + + return target.writer.WriteRecord(encoded) +} + +func (s *chSegment) PendingBytes() int64 { + var total int64 + for _, target := range s.tables { + total += target.writer.Bytes() + } + + return total +} + +func (s *chSegment) Seal(manifest *spool.Manifest) error { + for name, target := range s.tables { + if err := target.writer.Close(); err != nil { + return fmt.Errorf("closing the stream of %q: %w", name, err) + } + + info, err := os.Stat(target.path) + if err != nil { + return fmt.Errorf("sizing %s: %w", target.path, err) + } + + manifest.Tables = append(manifest.Tables, spool.TableRecord{ + Name: name, + File: filepath.Base(target.path), + Rows: target.rows, + Bytes: info.Size(), + }) + } + + return nil +} + +func (s *chSegment) Discard() { + for _, target := range s.tables { + target.file.Close() + } + os.RemoveAll(s.dir) +} + +// Value tags. Every type the accumulator's column switch consumes has one, and anything +// else is an error at spool time rather than a silent mangling at apply time. +const ( + tagNil byte = iota + tagBool + tagInt32 + tagInt64 + tagUint32 + tagUint64 + tagFloat32 + tagFloat64 + tagString + tagBytes + tagTime + tagList +) + +// encodeValues writes one row as tag-prefixed values. +// +// A *timestamppb.Timestamp is normalised to time.Time on the way in: the accumulator +// accepts either, and carrying one shape rather than two keeps the decoder honest. +func encodeValues(values []any) (string, error) { + out := make([]byte, 0, 64) + + var err error + for _, value := range values { + out, err = appendValue(out, value) + if err != nil { + return "", err + } + } + + return string(out), nil +} + +func appendValue(out []byte, value any) ([]byte, error) { + switch v := value.(type) { + case nil: + return append(out, tagNil), nil + case bool: + if v { + return append(out, tagBool, 1), nil + } + return append(out, tagBool, 0), nil + case int32: + return binary.BigEndian.AppendUint32(append(out, tagInt32), uint32(v)), nil + case sql2.EnumValue: + return binary.BigEndian.AppendUint32(append(out, tagInt32), uint32(v.Number)), nil + case protoreflect.EnumNumber: + return binary.BigEndian.AppendUint32(append(out, tagInt32), uint32(v)), nil + case int: + return binary.BigEndian.AppendUint64(append(out, tagInt64), uint64(int64(v))), nil + case int64: + return binary.BigEndian.AppendUint64(append(out, tagInt64), uint64(v)), nil + case uint32: + return binary.BigEndian.AppendUint32(append(out, tagUint32), v), nil + case uint: + return binary.BigEndian.AppendUint64(append(out, tagUint64), uint64(v)), nil + case uint64: + return binary.BigEndian.AppendUint64(append(out, tagUint64), v), nil + case float32: + return binary.BigEndian.AppendUint32(append(out, tagFloat32), math.Float32bits(v)), nil + case float64: + return binary.BigEndian.AppendUint64(append(out, tagFloat64), math.Float64bits(v)), nil + case string: + return appendBlob(append(out, tagString), []byte(v)), nil + case []byte: + return appendBlob(append(out, tagBytes), v), nil + case time.Time: + return binary.BigEndian.AppendUint64(append(out, tagTime), uint64(v.UnixNano())), nil + case *timestamppb.Timestamp: + return binary.BigEndian.AppendUint64(append(out, tagTime), uint64(v.AsTime().UnixNano())), nil + case []any: + out = binary.BigEndian.AppendUint32(append(out, tagList), uint32(len(v))) + var err error + for _, element := range v { + out, err = appendValue(out, element) + if err != nil { + return nil, err + } + } + + return out, nil + } + + return nil, fmt.Errorf("cannot spool a value of type %T", value) +} + +func appendBlob(out []byte, blob []byte) []byte { + return append(binary.BigEndian.AppendUint32(out, uint32(len(blob))), blob...) +} + +// decodeValues reads back one row. +func decodeValues(encoded string) ([]any, error) { + data := []byte(encoded) + + var values []any + for len(data) > 0 { + value, rest, err := decodeValue(data) + if err != nil { + return nil, err + } + values = append(values, value) + data = rest + } + + return values, nil +} + +func decodeValue(data []byte) (any, []byte, error) { + if len(data) == 0 { + return nil, nil, fmt.Errorf("a row ended mid-value") + } + + tag, rest := data[0], data[1:] + switch tag { + case tagNil: + return nil, rest, nil + case tagBool: + if len(rest) < 1 { + return nil, nil, fmt.Errorf("a bool ended mid-value") + } + return rest[0] == 1, rest[1:], nil + case tagInt32: + v, rest, err := take4(rest) + return int32(v), rest, err + case tagInt64: + v, rest, err := take8(rest) + return int64(v), rest, err + case tagUint32: + v, rest, err := take4(rest) + return v, rest, err + case tagUint64: + v, rest, err := take8(rest) + return v, rest, err + case tagFloat32: + v, rest, err := take4(rest) + return math.Float32frombits(v), rest, err + case tagFloat64: + v, rest, err := take8(rest) + return math.Float64frombits(v), rest, err + case tagString: + blob, rest, err := takeBlob(rest) + return string(blob), rest, err + case tagBytes: + blob, rest, err := takeBlob(rest) + return blob, rest, err + case tagTime: + v, rest, err := take8(rest) + return time.Unix(0, int64(v)).UTC(), rest, err + case tagList: + count, rest, err := take4(rest) + if err != nil { + return nil, nil, err + } + list := make([]any, 0, count) + for range count { + var element any + element, rest, err = decodeValue(rest) + if err != nil { + return nil, nil, err + } + list = append(list, element) + } + + return list, rest, nil + } + + return nil, nil, fmt.Errorf("unknown value tag %d", tag) +} + +func take4(data []byte) (uint32, []byte, error) { + if len(data) < 4 { + return 0, nil, fmt.Errorf("a 4 byte value ended after %d", len(data)) + } + + return binary.BigEndian.Uint32(data), data[4:], nil +} + +func take8(data []byte) (uint64, []byte, error) { + if len(data) < 8 { + return 0, nil, fmt.Errorf("an 8 byte value ended after %d", len(data)) + } + + return binary.BigEndian.Uint64(data), data[8:], nil +} + +func takeBlob(data []byte) ([]byte, []byte, error) { + size, rest, err := take4(data) + if err != nil { + return nil, nil, err + } + if uint32(len(rest)) < size { + return nil, nil, fmt.Errorf("a %d byte blob ended after %d", size, len(rest)) + } + + return rest[:size], rest[size:], nil +} diff --git a/sink/sql/db_proto/sql/click_house/chcodec_block_test.go b/sink/sql/db_proto/sql/click_house/chcodec_block_test.go new file mode 100644 index 000000000..a70262e2a --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/chcodec_block_test.go @@ -0,0 +1,23 @@ +package clickhouse + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestEncodeValuesRoundTripsABlockRow covers the row shape InsertBlock produces, now that +// it goes through the spool like every other one. A tag missing here would fail the run at +// spool time rather than silently, but only once a block is written — which is every block. +func TestEncodeValuesRoundTripsABlockRow(t *testing.T) { + timestamp := time.Unix(1700000000, 0).UTC() + row := []any{uint64(42), "0xabc", timestamp, int64(7), false} + + encoded, err := encodeValues(row) + require.NoError(t, err) + + decoded, err := decodeValues(encoded) + require.NoError(t, err) + require.Equal(t, row, decoded) +} diff --git a/sink/sql/db_proto/sql/click_house/chcodec_enum_test.go b/sink/sql/db_proto/sql/click_house/chcodec_enum_test.go new file mode 100644 index 000000000..4d929b91e --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/chcodec_enum_test.go @@ -0,0 +1,17 @@ +package clickhouse + +import ( + "testing" + + sql2 "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/stretchr/testify/require" +) + +func TestEncodeValuesSupportsEnumValue(t *testing.T) { + encoded, err := encodeValues([]any{sql2.EnumValue{Number: 2, Name: "ACTIVE"}}) + require.NoError(t, err) + + decoded, err := decodeValues(encoded) + require.NoError(t, err) + require.Equal(t, []any{int32(2)}, decoded) +} diff --git a/sink/sql/db_proto/sql/click_house/cursor_test.go b/sink/sql/db_proto/sql/click_house/cursor_test.go new file mode 100644 index 000000000..6bfce745f --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/cursor_test.go @@ -0,0 +1,35 @@ +package clickhouse + +import ( + "path/filepath" + "testing" + + "github.com/streamingfast/bstream" + sink "github.com/streamingfast/substreams/sink" + "github.com/stretchr/testify/require" +) + +// TestStoreCursorFileWritesWhereFetchCursorReads pins the path the spool applier uses. +// +// StoreCursor records onto the open segment whenever a spool is active, so this is the +// only thing that makes a spooled ClickHouse backfill resumable: routing the applier +// through StoreCursor instead leaves the file untouched for the whole run. +func TestStoreCursorFileWritesWhereFetchCursorReads(t *testing.T) { + database := &Database{cursorFilePath: filepath.Join(t.TempDir(), "cursor.txt")} + + block := bstream.NewBlockRef("10a", 10) + opaque := (&bstream.Cursor{ + Step: bstream.StepNewIrreversible, + Block: block, + HeadBlock: block, + LIB: block, + }).ToOpaque() + + cursor, err := sink.NewCursor(opaque) + require.NoError(t, err) + require.NoError(t, database.storeCursorFile(cursor)) + + read, err := database.FetchCursor() + require.NoError(t, err) + require.Equal(t, uint64(10), read.Block().Num()) +} diff --git a/sink/sql/db_proto/sql/click_house/database.go b/sink/sql/db_proto/sql/click_house/database.go index 74a663391..1290691fa 100644 --- a/sink/sql/db_proto/sql/click_house/database.go +++ b/sink/sql/db_proto/sql/click_house/database.go @@ -11,6 +11,7 @@ import ( "time" "github.com/ClickHouse/ch-go" + chproto "github.com/ClickHouse/ch-go/proto" "github.com/streamingfast/logging" "github.com/streamingfast/logging/zapx" sink "github.com/streamingfast/substreams/sink" @@ -18,6 +19,7 @@ import ( "github.com/streamingfast/substreams/sink/sql/db_changes/db" "github.com/streamingfast/substreams/sink/sql/db_proto/sql" "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" "go.uber.org/zap" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -33,6 +35,8 @@ type Database struct { dsn *db.DSN ctx context.Context inserter *AccumulatorInserter + spoolOptions *spool.Options + spool *spool.Spool bytesEncoding bytes.Encoding queryRetryCount int queryRetrySleep time.Duration @@ -90,7 +94,28 @@ func NewDatabase( return database, nil } +// WithSpool turns on the on-disk spool. It must be called before Open. +func (d *Database) WithSpool(options spool.Options) { + d.spoolOptions = &options +} + +// Open starts the spool, if one is configured. +// +// Rows then land on disk and a background goroutine applies whole segments, so the stream +// stops waiting on ClickHouse. What reaches the server is unchanged: a segment is replayed +// into the same column builders and sent by the same flush, followed by the same cursor +// write. func (d *Database) Open() error { + if d.spoolOptions == nil { + return nil + } + + created, err := spool.New(d.ctx, *d.spoolOptions, newCHCodec(), newCHApplier(d, d.inserter, d.logger), d.schema.Name, d.logger) + if err != nil { + return fmt.Errorf("starting the local spool: %w", err) + } + d.spool = created + return nil } @@ -163,6 +188,52 @@ func (d *Database) clientNoCache(dsn *db.DSN) (*ch.Client, error) { return client, nil } +// SwitchToDirectInserts drains the spool and inserts inline from here on. The reason says +// what brought the switch on, since the database cannot tell the chain head from the end +// of a bounded range. +// +// The spool trades freshness for throughput, which is what a backfill wants and the +// opposite of what a sink at the chain head wants, where a block should be queryable when +// it arrives rather than when the segment it lands in is full. +func (d *Database) SwitchToDirectInserts(ctx context.Context, reason string, _ bool) error { + if d.spool == nil { + return nil + } + + d.logger.Info(reason + ", draining the spool and switching to direct inserts. " + + "--db-write-* and --spool-* no longer apply from here on") + + if err := d.spool.Close(ctx); err != nil { + return fmt.Errorf("draining the spool: %w", err) + } + d.spool = nil + d.spoolOptions = nil + + return nil +} + +// ApplyConstraints does nothing: ClickHouse has no primary/foreign key constraints to +// apply, which is also why CreateDatabase ignores its useConstraints argument. +func (d *Database) ApplyConstraints() error { + return nil +} + +// EnsureBlockNumberIndexes does nothing: a ClickHouse table's ORDER BY is its index, and +// the sink has no separate one to create. +func (d *Database) EnsureBlockNumberIndexes(context.Context) error { + return nil +} + +// MissingConstraints reports none: ClickHouse has no constraints to be missing. +func (d *Database) MissingConstraints() ([]string, error) { + return nil, nil +} + +// DropConstraints does nothing, for the same reason as ApplyConstraints. +func (d *Database) DropConstraints() error { + return nil +} + func (d *Database) CreateDatabase(useConstraints bool) error { dsn := d.dsn.Clone() dsn.Database = "default" @@ -217,8 +288,104 @@ func (d *Database) CreateDatabase(useConstraints bool) error { return nil } +// VerifySchemaCompatibility rejects a database whose tables disagree with the schema the +// current package would create, on the one point the sink cannot paper over: the presence +// of _row_id_. +// +// That column is added exactly when the message carries no 'order_by_fields', so +// annotating a message that had none — or dropping the annotation from one that had them — +// changes the sorting key of a table that already holds rows. CREATE TABLE IF NOT EXISTS +// leaves the old table in place, and the inserts that follow would silently write their +// values into the wrong columns. +func (d *Database) VerifySchemaCompatibility(ctx context.Context) error { + existing, err := d.rowIDColumnByTable(ctx) + if err != nil { + return fmt.Errorf("reading the columns of schema %q: %w", d.schema.Name, err) + } + + for _, table := range d.dialect.GetTables() { + found, exists := existing[table.Name] + if !exists { + // A table the next CREATE TABLE will add. + continue + } + + expected := d.dialect.UseRowIDField(table.Name) + if found == expected { + continue + } + + if expected { + return fmt.Errorf("table %q in database %q was created from a schema declaring 'order_by_fields' for it, but the package now carries none, so the sink would sort it on (%s, %s) instead. Sink into a fresh database, or restore the annotation", + table.Name, d.schema.Name, sql.DialectFieldBlockNumber, sql.DialectFieldRowID) + } + + return fmt.Errorf("table %q in database %q was created without 'order_by_fields' and carries the %s column the sink adds in that case, but the package now declares them. Sink into a fresh database, or drop the annotation", + table.Name, d.schema.Name, sql.DialectFieldRowID) + } + + return nil +} + +// rowIDColumnByTable reports, for every table of the schema that exists in ClickHouse, +// whether it carries the _row_id_ column. An absent schema yields an empty map rather +// than an error: nothing is set up yet, so there is nothing to disagree with. +// +// It connects to 'default' rather than to the schema, for the same reason CreateDatabase +// does: this runs before the database exists on a first run, and dialing one that is not +// there never comes back — newClient retries a failed dial forever. system.columns is +// global, so which database the connection names does not change the answer. +func (d *Database) rowIDColumnByTable(ctx context.Context) (map[string]bool, error) { + dsn := d.dsn.Clone() + dsn.Database = "default" + + client, err := d.clientNoCache(dsn) + if err != nil { + return nil, fmt.Errorf("getting clickhouse client: %w", err) + } + defer client.Close() + + var ( + tables chproto.ColStr + hasRowD chproto.ColUInt64 + ) + + out := map[string]bool{} + query := fmt.Sprintf("SELECT table, sum(name = '%s') AS has_row_id FROM system.columns WHERE database = '%s' GROUP BY table", + sql.DialectFieldRowID, d.schema.Name) + + if err := client.Do(ctx, ch.Query{ + Body: query, + Result: chproto.Results{ + {Name: "table", Data: &tables}, + {Name: "has_row_id", Data: &hasRowD}, + }, + OnResult: func(_ context.Context, _ chproto.Block) error { + for i := 0; i < tables.Rows(); i++ { + out[tables.Row(i)] = hasRowD[i] > 0 + } + + return nil + }, + }); err != nil { + return nil, fmt.Errorf("querying system.columns: %w", err) + } + + return out, nil +} + func (d *Database) Insert(table string, values []any) error { - return d.inserter.insert(table, values) + if d.spool == nil { + return d.inserter.insert(table, values) + } + + if table == sql.DialectTableBlock { + if blockNum, ok := values[0].(uint64); ok { + d.spool.RecordBlock(blockNum) + } + } + + return d.spool.Insert(table, values) } func (d *Database) WalkMessageDescriptorAndInsert(dm protoreflect.Message, blockNum uint64, blockTimestamp time.Time, parent *sql.Parent) (time.Duration, error) { @@ -244,6 +411,14 @@ func (d *Database) Flush() (time.Duration, error) { d.logger.Debug("flushing") startFlush := time.Now() + + // With a spool the rows are already on disk and the write to ClickHouse happens later, + // on the applier's goroutine. The segment is sealed when the cursor covering it is + // recorded, which happens after this. + if d.spool != nil { + return time.Since(startFlush), nil + } + err := d.inserter.flush(d) if err != nil { return 0, fmt.Errorf("flushing: %w", err) @@ -255,9 +430,16 @@ func (d *Database) GetDialect() sql.Dialect { return d.dialect } +// InsertBlock writes the block row through the same path as every other row. +// +// It must not reach the accumulator directly: with a spool open that instance belongs to +// the applier's goroutine, so appending to it from here is an unsynchronised write to the +// map the applier swaps out on every flush. Going through Insert is also what tells the +// spool which block the rows now being written belong to, without which a segment records +// no block range at all. func (d *Database) InsertBlock(blockNum uint64, hash string, timestamp time.Time) error { d.logger.Debug("inserting _block_", zap.Uint64("block_num", blockNum), zap.String("block_hash", hash)) - err := d.inserter.insert("_blocks_", []any{blockNum, hash, timestamp, time.Now().UnixNano(), false}) + err := d.Insert(sql.DialectTableBlock, []any{blockNum, hash, timestamp, time.Now().UnixNano(), false}) if err != nil { return fmt.Errorf("inserting block %d: %w", blockNum, err) } @@ -338,6 +520,27 @@ func (d *Database) FetchCursor() (*sink.Cursor, error) { } func (d *Database) StoreCursor(cursor *sink.Cursor) error { + // With a spool the cursor belongs to the segment being written: it is what makes that + // segment resumable, and it must not run ahead of the rows it covers. + if d.spool != nil { + d.spool.RecordCursor(cursor.String()) + + // Sealed here rather than at flush, the cursor being the last thing a flush writes: + // sealing before it would close the segment on the cursor the previous flush stored, + // leaving it to claim a range its own cursor does not cover. + return d.spool.MaybeSeal(d.ctx) + } + + return d.storeCursorFile(cursor) +} + +// storeCursorFile writes the cursor where the next run reads it back. +// +// It is what the applier calls once a segment has reached the server. Going through +// StoreCursor would hand the cursor straight back to the spool it just came out of — +// leaving the file untouched for the whole backfill, and stamping an already-applied +// cursor over the newer one on the segment still being written. +func (d *Database) storeCursorFile(cursor *sink.Cursor) error { if d.cursorFilePath == "" { return fmt.Errorf("cursor file path is not set") } @@ -357,6 +560,14 @@ func (d *Database) StoreCursor(cursor *sink.Cursor) error { } func (d *Database) HandleBlocksUndo(lastValidBlockNum uint64) error { + // Rows still in the spool would otherwise land after the delete that was supposed to + // remove them. + if d.spool != nil { + if err := d.spool.Drain(d.ctx); err != nil { + return fmt.Errorf("draining the spool before an undo: %w", err) + } + } + tables := d.dialect.GetTables() // Sort tables in descending order based on their Ordinal field @@ -427,6 +638,12 @@ func (d *Database) HandleBlocksUndo(lastValidBlockNum uint64) error { tableFullName := d.dialect.FullTableName(table) fields := "" + // The tombstone only collapses onto the row it deletes if it carries the same + // sorting key, and _row_id_ is part of it wherever the schema did not declare one. + if d.dialect.UseRowIDField(table.Name) { + fields += fmt.Sprintf(", %s", sql.DialectFieldRowID) + } + if table.ChildOf != nil { parentTable, parentFound := d.dialect.TableRegistry[table.ChildOf.ParentTable] if !parentFound { @@ -481,6 +698,25 @@ func (d *Database) HandleBlocksUndo(lastValidBlockNum uint64) error { return nil } +// Close drains the spool, so blocks held at shutdown reach the server rather than being +// streamed again. Without one there is nothing held: inserts flush inline. +func (d *Database) Close(ctx context.Context) error { + if d.spool == nil { + return nil + } + + return d.spool.Close(ctx) +} + +// BufferStats reports what sits between the stream and the server. +func (d *Database) BufferStats() (int64, int64, uint64, bool) { + if d.spool == nil { + return 0, 0, 0, false + } + + return d.spool.BlocksBuffered(), d.spool.BytesOnDisk(), d.spool.AppliedBlock(), true +} + func (d *Database) DatabaseHash(schemaName string) (uint64, error) { panic("not implemented") } diff --git a/sink/sql/db_proto/sql/click_house/dialect.go b/sink/sql/db_proto/sql/click_house/dialect.go index 7379c8576..61f35be82 100644 --- a/sink/sql/db_proto/sql/click_house/dialect.go +++ b/sink/sql/db_proto/sql/click_house/dialect.go @@ -35,8 +35,6 @@ const staticSqlCreateBlock = ` allow_experimental_replacing_merge_with_cleanup = 1; ` -const clickhouseTableOptionsErrorMsg = "schema annotation 'clickhouse_table_options' is required in table annotation 'option (schema.table) = { name: %q, ... }' , see: https://github.com/streamingfast/substreams/blob/develop/docs/references/sql/proto-annotations.md#clickhouse-specific-options for configuration details" - type DialectClickHouse struct { *sql2.BaseDialect schemaName string @@ -73,6 +71,15 @@ func (d *DialectClickHouse) UseDeletedField() bool { return true } +func (d *DialectClickHouse) UseRowIDField(tableName string) bool { + table := d.GetTable(tableName) + if table == nil { + return false + } + + return hasDefaultOrderBy(table) +} + func (d *DialectClickHouse) init() error { return nil } @@ -89,6 +96,10 @@ func (d *DialectClickHouse) createTable(table *schema.Table) error { sb.WriteString(fmt.Sprintf(" %s Int64 NOT NULL,", sql2.DialectFieldVersion)) sb.WriteString(fmt.Sprintf(" %s bool NOT NULL,", sql2.DialectFieldDeleted)) + if hasDefaultOrderBy(table) { + sb.WriteString(fmt.Sprintf(" %s UInt32 NOT NULL,", sql2.DialectFieldRowID)) + } + var primaryKeyFieldName string if table.PrimaryKey != nil { pk := table.PrimaryKey @@ -165,6 +176,11 @@ func (d *DialectClickHouse) createTable(table *schema.Table) error { primaryKey := "" if primaryKeyFieldName != "" { primaryKey = fmt.Sprintf("PRIMARY KEY (%s)", primaryKeyFieldName) + } else if hasDefaultOrderBy(table) { + // ClickHouse takes the whole sorting key as the primary key when none is given. + // _row_id_ is only there to keep rows apart, never to look them up, so it is left + // out of the sparse index rather than doubling its size for nothing. + primaryKey = fmt.Sprintf("PRIMARY KEY (%s)", sql2.DialectFieldBlockNumber) } orderBy, err := orderByString(table) @@ -301,16 +317,41 @@ func tableName(schemaName string, tableName string) string { return fmt.Sprintf("%s.%s", schemaName, tableName) } -func orderByString(table *schema.Table) (string, error) { +// hasDefaultOrderBy reports whether the table's sorting key is the one the sink picks on +// its own, which is the case for any message that does not declare 'order_by_fields'. +// Those tables get the extra _row_id_ column; annotated ones are left exactly as they +// were. +func hasDefaultOrderBy(table *schema.Table) bool { + if table.PbTableInfo == nil { + return true + } + info := table.PbTableInfo.ClickhouseTableOptions - if info == nil { - return "", fmt.Errorf(clickhouseTableOptionsErrorMsg, table.Name) + + return info == nil || len(info.OrderByFields) == 0 +} + +// defaultOrderByFields is the sorting key used when the message carries no +// 'order_by_fields': the block number and the per-block row counter that keeps the key +// unique. A declared primary key leads, because ClickHouse requires the primary key to be +// a prefix of the sorting key. +func defaultOrderByFields(table *schema.Table) []string { + var fields []string + if table.PrimaryKey != nil { + fields = append(fields, table.PrimaryKey.Name) } + fields = append(fields, sql2.DialectFieldBlockNumber) + + return append(fields, sql2.DialectFieldRowID) +} - if len(info.OrderByFields) == 0 { - return "", fmt.Errorf("clickhouse table options for table %q don't have any 'order_by_fields'. Require at least 1", table.Name) +func orderByString(table *schema.Table) (string, error) { + if hasDefaultOrderBy(table) { + return fmt.Sprintf("ORDER BY (%s)", strings.Join(defaultOrderByFields(table), ", ")), nil } + info := table.PbTableInfo.ClickhouseTableOptions + out := "" for i, field := range info.OrderByFields { w := wrapWithClickhouseFunction(field.Name, field.Function) @@ -327,9 +368,13 @@ func orderByString(table *schema.Table) (string, error) { } func partitionByString(table *schema.Table) (string, error) { - info := table.PbTableInfo.ClickhouseTableOptions + var info *pbSchmema.ClickhouseTableOptions + if table.PbTableInfo != nil { + info = table.PbTableInfo.ClickhouseTableOptions + } if info == nil { - return "", fmt.Errorf(clickhouseTableOptionsErrorMsg, table.Name) + // Same partitioning an annotated table gets when it declares no partition field. + return fmt.Sprintf("PARTITION BY (%s)", wrapWithClickhouseFunction(sql2.DialectFieldBlockTimestamp, pbSchmema.Function_toYYYYMM)), nil } var parts []string diff --git a/sink/sql/db_proto/sql/click_house/dialect_row_id_test.go b/sink/sql/db_proto/sql/click_house/dialect_row_id_test.go new file mode 100644 index 000000000..bd6054cac --- /dev/null +++ b/sink/sql/db_proto/sql/click_house/dialect_row_id_test.go @@ -0,0 +1,49 @@ +package clickhouse + +import ( + "testing" + + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "github.com/streamingfast/substreams/sink/sql/bytes" + schema2 "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// TestUnannotatedTableGetsDefaultSortingKey covers the case a package with no +// schema.proto annotations lands in: the setup used to fail outright for want of +// 'order_by_fields', where it now sorts on the block number and a per-block row counter. +func TestUnannotatedTableGetsDefaultSortingKey(t *testing.T) { + dialect := dialectFor(t, (&pbsubstreams.Clock{}).ProtoReflect().Descriptor()) + + create := dialect.GetCreateTableSql("Clock") + require.NotEmpty(t, create) + + assert.Contains(t, create, "_row_id_ UInt32 NOT NULL") + assert.Contains(t, create, "PRIMARY KEY (_block_number_)") + assert.Contains(t, create, "ORDER BY (_block_number_, _row_id_)") + assert.Contains(t, create, "PARTITION BY (toYYYYMM(_block_timestamp_))") + assert.True(t, dialect.UseRowIDField("Clock")) +} + +// TestUseRowIDFieldIgnoresUnknownTable guards the walk, which asks about every message it +// meets, including the ones that never became a table. +func TestUseRowIDFieldIgnoresUnknownTable(t *testing.T) { + dialect := dialectFor(t, (&pbsubstreams.Clock{}).ProtoReflect().Descriptor()) + + assert.False(t, dialect.UseRowIDField("not_a_table")) +} + +func dialectFor(t *testing.T, descriptor protoreflect.MessageDescriptor) *DialectClickHouse { + t.Helper() + + schema, err := schema2.NewSchema("test", descriptor, false, zap.NewNop()) + require.NoError(t, err) + + dialect, err := NewDialectClickHouse(schema, bytes.EncodingRaw, zap.NewNop()) + require.NoError(t, err) + + return dialect +} diff --git a/sink/sql/db_proto/sql/constraint.go b/sink/sql/db_proto/sql/constraint.go index f3ebfbf1e..7383a7298 100644 --- a/sink/sql/db_proto/sql/constraint.go +++ b/sink/sql/db_proto/sql/constraint.go @@ -13,6 +13,11 @@ type ForeignKey struct { type Constraint struct { Table string Sql string + + // ReferencedTable is the logical name of the table a foreign key points at, empty for + // any other kind of constraint. It is what the apply order is computed from: the + // SQL carries schema-qualified names, which is not what the registry is keyed by. + ReferencedTable string } func (f *ForeignKey) String() string { diff --git a/sink/sql/db_proto/sql/constraint_policy.go b/sink/sql/db_proto/sql/constraint_policy.go new file mode 100644 index 000000000..376675a98 --- /dev/null +++ b/sink/sql/db_proto/sql/constraint_policy.go @@ -0,0 +1,200 @@ +package sql + +import ( + "fmt" + "strings" +) + +// AllTables is what a per-table constraint switch takes to mean every table at once. +const AllTables = "all" + +// ConstraintPolicy says which constraints the schema gets and when. +// +// Constraints are created after the load rather than during it — measured on 500k rows +// through binary COPY, loading with foreign keys in place costs 27.7x, while building the +// same constraints after the load costs 3.3x, for an identical schema. See +// TestConstraintCost in sink/sql/db_proto/benchmarks. + +// ConstraintTiming says when the constraints are created. +type ConstraintTiming string + +const ( + // ConstraintsAuto has the sink create them itself at the first live block, and only + // there: a stop block ends a run without saying the backfill is over, and a range is + // routinely one chunk of several. It is the default, stop-the-world pass and all. A backfill that ends with no primary keys and no foreign keys has + // produced a database nobody should query, and leaving it that way until the operator + // remembers a second command is the worse failure — it is silent, and it looks like + // success. + ConstraintsAuto ConstraintTiming = "auto" + + // ConstraintsManual leaves it to the operator, through `sink postgres constraints + // apply`. Building them locks every table while indexes are built and every foreign + // key is validated, so on a large database this is how the pass goes into a + // maintenance window instead. + ConstraintsManual ConstraintTiming = "manual" + + // ConstraintsAlways creates them before the first row is written, so the database + // rejects bad data from the start and the load pays for it throughout. + ConstraintsAlways ConstraintTiming = "always" +) + +type ConstraintPolicy struct { + // Timing decides when they are created; the zero value is ConstraintsAuto. + Timing ConstraintTiming + + // DisableForeignKeys leaves out every foreign key, including the one to the block + // table. Those are what a load pays most for. + DisableForeignKeys bool + + // DisablePrimaryKeys and DisableUniques name the tables that go without, or AllTables. + DisablePrimaryKeys []string + DisableUniques []string + + // DisableBlockNumberIndex leaves out the index on _block_number_. Every table carries + // that column and the reorg path deletes by it on every table, so without the index + // each undo is a sequential scan per table. It is only dead weight on a run that can + // never reorg. + // + // Unlike the rest of this policy it is not about what the schema declares, and not + // governed by Timing: the index is created when the sink starts. + DisableBlockNumberIndex bool + + // Parallelism is how many constraints are created or dropped at once. Zero means one. + // + // They go on independent relations, so the only ordering that matters is between the + // waves — a foreign key needs the key it references to exist — and within a wave the + // server is free to build them side by side. Each statement still commits on its own, + // which is what keeps the pass restartable: a run that is killed keeps what it + // finished. It is an execution knob rather than a property of the schema. + Parallelism int + + // WorkMem is what maintenance_work_mem is set to for the duration of each statement, + // empty leaving the server's own setting alone. The default is 64MB on most servers, + // at which an index build over a large table spills to an external merge sort; raising + // it for the pass alone is the cheapest thing that makes it faster. + // + // It multiplies with Parallelism, each concurrent build taking its own. + WorkMem string +} + +// ConstraintsParallelism is how many statements the pass runs at once. +func (p ConstraintPolicy) ConstraintsParallelism() int { + if p.Parallelism <= 0 { + return 1 + } + + return p.Parallelism +} + +// SkipPrimaryKey reports whether the given table is meant to go without its primary key. +func (p ConstraintPolicy) SkipPrimaryKey(table string) bool { + return matchesTable(p.DisablePrimaryKeys, table) +} + +// SkipUnique reports whether the given table's unique constraints are meant to be left out. +func (p ConstraintPolicy) SkipUnique(table string) bool { + return matchesTable(p.DisableUniques, table) +} + +// SkipForeignKey reports whether the given table's foreign keys are meant to be left out. +func (p ConstraintPolicy) SkipForeignKey(string) bool { + return p.DisableForeignKeys +} + +// SkipsEverything reports a policy that declares no constraints, in which case there is +// nothing to apply and nothing to wait for. The block number index is not one of them: it +// is created when the sink starts, whatever the constraints say. +func (p ConstraintPolicy) SkipsEverything() bool { + return p.DisableForeignKeys && matchesTable(p.DisablePrimaryKeys, AllTables) && matchesTable(p.DisableUniques, AllTables) +} + +// DisableAllConstraints returns the policy that declares none of them, which is what an +// output with no schema annotations leaves the sink with. The block number index survives +// it: nothing in the annotations asks for that one, the reorg path does. +func DisableAllConstraints() ConstraintPolicy { + return ConstraintPolicy{ + DisableForeignKeys: true, + DisablePrimaryKeys: []string{AllTables}, + DisableUniques: []string{AllTables}, + } +} + +// WithBlockNumberIndex carries the index switch over from another policy, the index being +// the one thing that survives an output having no annotations to declare anything. +func (p ConstraintPolicy) WithBlockNumberIndex(from ConstraintPolicy) ConstraintPolicy { + p.Timing = from.Timing + p.Parallelism = from.Parallelism + p.WorkMem = from.WorkMem + p.DisableBlockNumberIndex = from.DisableBlockNumberIndex + + return p +} + +// Describe renders the policy for a log line. +func (p ConstraintPolicy) Describe() string { + if p.SkipsEverything() { + return "none" + } + + var parts []string + if p.DisableBlockNumberIndex { + parts = append(parts, "no block number index") + } + if p.DisableForeignKeys { + parts = append(parts, "no foreign keys") + } + if len(p.DisablePrimaryKeys) > 0 { + parts = append(parts, fmt.Sprintf("no primary key on %s", strings.Join(p.DisablePrimaryKeys, ","))) + } + if len(p.DisableUniques) > 0 { + parts = append(parts, fmt.Sprintf("no unique constraint on %s", strings.Join(p.DisableUniques, ","))) + } + if len(parts) == 0 { + parts = append(parts, "all") + } + + when := "created once the backfill is done" + switch p.Timing { + case ConstraintsAlways: + when = "created before the load" + case ConstraintsManual: + when = "created by `sink postgres constraints apply`" + } + + return strings.Join(parts, ", ") + ", " + when +} + +func matchesTable(list []string, table string) bool { + for _, entry := range list { + entry = strings.TrimSpace(entry) + if strings.EqualFold(entry, AllTables) || strings.EqualFold(entry, table) { + return true + } + } + + return false +} + +// ApplyUpfront reports whether the constraints go in before the first row. +func (p ConstraintPolicy) ApplyUpfront() bool { + return p.Timing == ConstraintsAlways && !p.SkipsEverything() +} + +// ApplyAtHead reports whether the sink creates them itself once the backfill is over. +func (p ConstraintPolicy) ApplyAtHead() bool { + return (p.Timing == ConstraintsAuto || p.Timing == "") && !p.SkipsEverything() +} + +// ParseConstraintTiming validates the flag value. +func ParseConstraintTiming(in string) (ConstraintTiming, error) { + switch ConstraintTiming(in) { + case "", ConstraintsAuto: + return ConstraintsAuto, nil + case ConstraintsManual: + return ConstraintsManual, nil + case ConstraintsAlways: + return ConstraintsAlways, nil + } + + return "", fmt.Errorf("invalid constraint timing %q, expected one of %q, %q or %q", in, ConstraintsAuto, ConstraintsManual, ConstraintsAlways) +} diff --git a/sink/sql/db_proto/sql/database.go b/sink/sql/db_proto/sql/database.go index c17ed2049..4426da60c 100644 --- a/sink/sql/db_proto/sql/database.go +++ b/sink/sql/db_proto/sql/database.go @@ -1,6 +1,7 @@ package sql import ( + "context" "database/sql" "fmt" "strings" @@ -24,6 +25,37 @@ type Database interface { StoreSinkInfo(schemaName string, schemaHash string) error CreateDatabase(useConstraints bool) error + // ApplyConstraints adds the schema's constraints to a database that already exists, + // skipping the ones already in place. A schema first synced without constraints has + // none of them, and only this puts them there. + ApplyConstraints() error + + // EnsureBlockNumberIndexes creates the index the sink needs for its own reorg path, + // when it starts. It is not part of the constraint pass: --apply-constraints describes + // the schema and is the operator's to schedule, where this one the sink depends on to + // undo a reorg without sequentially scanning every table. + EnsureBlockNumberIndexes(ctx context.Context) error + + // MissingConstraints names the constraints the policy says the schema should carry and + // the database does not have. It is what turns "this schema has no indexes" from + // something you find out by querying it into something the sink says on every start. + MissingConstraints() ([]string, error) + + // DropConstraints removes the constraints this schema's DDL would create, leaving + // anything the sink did not put there alone. It is the escape hatch after + // --apply-constraints=always, and what makes a stalled backfill fast again without a + // second setup. + DropConstraints() error + // SwitchToDirectInserts leaves any buffered write path behind and inserts straight + // into the database from now on. It is a one-way switch, and a no-op for a backend + // that never buffered. The reason is what the caller knows and the database does not + // — reaching the chain head, or reaching the end of a bounded range — and it is what + // the switch reports, so the log does not claim a head the stream never saw. + // atChainHead says the sink carries on from here rather than exiting, which is what + // makes the spool's own bookkeeping worth clearing: a run that stays live seals a + // segment or two on every restart and would otherwise accumulate their records for as + // long as it lives. + SwitchToDirectInserts(ctx context.Context, reason string, atChainHead bool) error // WalkMessageDescriptorAndInsert reads the message through protoreflect only, so it // does not care which dynamic implementation produced it — dynamicpb and hyperpb are // both accepted, and the decoder picks. @@ -49,6 +81,16 @@ type Database interface { GetDialect() Dialect Open() error + + // Close releases anything the database buffered locally, so blocks held at shutdown + // reach the server rather than being streamed again. + Close(ctx context.Context) error + + // BufferStats reports what is buffered between the stream and the server: how many + // blocks, how many bytes on disk, and the last block actually committed. enabled is + // false when nothing buffers locally, in which case the caller knows a flush means + // the rows are stored. + BufferStats() (blocks int64, bytes int64, appliedBlock uint64, enabled bool) } type BaseDatabase struct { @@ -87,10 +129,25 @@ type Parent struct { // bytes point into its arena — so an inserter that keeps a []any past the walk keeps the // message alive too. See decoder.arenas for where that lifetime is managed. func (d *BaseDatabase) WalkMessageDescriptorAndInsertWithDialect(dm protoreflect.Message, blockNum uint64, blockTimestamp time.Time, parent *Parent, dialect Dialect, inserter Inserter) (time.Duration, error) { + // The row counters belong to this block alone. They cannot live on the database: the + // decoder walks several blocks at once, on its own goroutines. + return d.walkMessageDescriptorAndInsert(dm, blockNum, blockTimestamp, parent, dialect, inserter, map[string]uint32{}) +} + +func (d *BaseDatabase) walkMessageDescriptorAndInsert(dm protoreflect.Message, blockNum uint64, blockTimestamp time.Time, parent *Parent, dialect Dialect, inserter Inserter, rowIDs map[string]uint32) (time.Duration, error) { if dm == nil { return 0, fmt.Errorf("received a nil message") } + md := dm.Descriptor() + tableInfo := proto.TableInfo(md) + + if tableInfo == nil && !d.useProtoOptions { + tableInfo = &pbSchema.Table{ + Name: string(md.Name()), + } + } + var fieldValues []any fieldValues = append(fieldValues, blockNum) fieldValues = append(fieldValues, blockTimestamp) @@ -106,13 +163,14 @@ func (d *BaseDatabase) WalkMessageDescriptorAndInsertWithDialect(dm protoreflect primaryKeyOffset += 1 } - md := dm.Descriptor() - tableInfo := proto.TableInfo(md) - - if tableInfo == nil && !d.useProtoOptions { - tableInfo = &pbSchema.Table{ - Name: string(md.Name()), - } + // The count of rows this block already wrote to this table. The walk is a + // deterministic function of the message — fields in descriptor order, children after + // their parent — so replaying the same block hands every row the same number, which is + // what makes the sorting key stable across a reprocess. + if tableInfo != nil && dialect.UseRowIDField(tableInfo.Name) { + fieldValues = append(fieldValues, rowIDs[tableInfo.Name]) + rowIDs[tableInfo.Name] += 1 + primaryKeyOffset += 1 } // Guarded: this runs once per message, and zap.Any on the table info allocates @@ -175,7 +233,7 @@ func (d *BaseDatabase) WalkMessageDescriptorAndInsertWithDialect(dm protoreflect // Array of native values - add as a single field value (the array itself) var values []interface{} for j := 0; j < list.Len(); j++ { - values = append(values, ScalarFieldValue(fd,list.Get(j))) + values = append(values, ScalarFieldValue(fd, list.Get(j))) } fieldValues = append(fieldValues, values) } else { @@ -208,7 +266,7 @@ func (d *BaseDatabase) WalkMessageDescriptorAndInsertWithDialect(dm protoreflect childs = append(childs, fm) //need to be handled after current message inserted } } else { - fieldValues = append(fieldValues, ScalarFieldValue(fd,fv)) + fieldValues = append(fieldValues, ScalarFieldValue(fd, fv)) } } @@ -242,7 +300,7 @@ func (d *BaseDatabase) WalkMessageDescriptorAndInsertWithDialect(dm protoreflect } for _, fm := range childs { - sqlDuration, err := d.WalkMessageDescriptorAndInsertWithDialect(fm, blockNum, blockTimestamp, p, dialect, inserter) + sqlDuration, err := d.walkMessageDescriptorAndInsert(fm, blockNum, blockTimestamp, p, dialect, inserter, rowIDs) if err != nil { return 0, fmt.Errorf("processing child %q: %w", string(fm.Descriptor().FullName()), err) } diff --git a/sink/sql/db_proto/sql/dialect.go b/sink/sql/db_proto/sql/dialect.go index 72e2debbd..597ca99d5 100644 --- a/sink/sql/db_proto/sql/dialect.go +++ b/sink/sql/db_proto/sql/dialect.go @@ -1,6 +1,10 @@ package sql import ( + "fmt" + "sort" + "strings" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" "go.uber.org/zap" "golang.org/x/exp/maps" @@ -15,6 +19,13 @@ const DialectFieldBlockTimestamp = "_block_timestamp_" const DialectFieldVersion = "_version_" const DialectFieldDeleted = "_deleted_" +// DialectFieldRowID numbers the rows a single block writes to a single table, starting at +// zero. It only exists where the sorting key would otherwise not be unique, which today +// means a ClickHouse table whose message carries no 'order_by_fields' annotation: the +// sink then sorts on (_block_number_, _row_id_), and without the second column the +// ReplacingMergeTree would collapse every row of a block into one. +const DialectFieldRowID = "_row_id_" + type Dialect interface { SchemaHash() string FullTableName(table *schema.Table) string @@ -22,6 +33,9 @@ type Dialect interface { GetTables() []*schema.Table UseVersionField() bool UseDeletedField() bool + // UseRowIDField reports whether the given table carries the DialectFieldRowID column. + // It is per-table because a schema can annotate some of its messages and not others. + UseRowIDField(table string) bool AppendInlineFieldValues(fieldValues []any, fd protoreflect.FieldDescriptor, fv protoreflect.Value, dm protoreflect.Message) ([]any, error) } @@ -30,8 +44,12 @@ type BaseDialect struct { PrimaryKeySql []*Constraint ForeignKeySql []*Constraint UniqueConstraintSql []*Constraint - TableRegistry map[string]*schema.Table - Logger *zap.Logger + // IndexSql holds the indexes the sink creates for itself rather than because the + // schema asked for one. They are built in the same pass as the constraints, being the + // same kind of expensive. + IndexSql []*Constraint + TableRegistry map[string]*schema.Table + Logger *zap.Logger } func NewBaseDialect(registry map[string]*schema.Table, logger *zap.Logger) *BaseDialect { @@ -42,6 +60,12 @@ func NewBaseDialect(registry map[string]*schema.Table, logger *zap.Logger) *Base } } +// UseRowIDField defaults to false: only the ClickHouse dialect needs a tie-breaker in its +// sorting key. +func (d *BaseDialect) UseRowIDField(table string) bool { + return false +} + func (d *BaseDialect) AddCreateTableSql(table string, sql string) { d.CreateTableSql[table] = sql } @@ -58,6 +82,120 @@ func (d *BaseDialect) AddForeignKeySql(table string, sql string) { d.ForeignKeySql = append(d.ForeignKeySql, &Constraint{Table: table, Sql: sql}) } +// AddForeignKeyReferencing records the same statement along with the logical table it +// points at, which is what TableApplyOrder needs. +func (d *BaseDialect) AddForeignKeyReferencing(table string, referencedTable string, sql string) { + d.ForeignKeySql = append(d.ForeignKeySql, &Constraint{Table: table, Sql: sql, ReferencedTable: referencedTable}) +} + +// TableApplyOrder returns every table ordered so that a referenced table always comes +// before the tables referencing it. +// +// Rows have to reach the server in that order whenever foreign keys are enforced, and +// the write paths group rows by table: a multi-row INSERT per table at flush, and one +// binary COPY per table in the buffer. Insertion order within a block is not enough, +// because the grouping loses it. +// +// Nesting depth is not the answer even though it looks like it: a table can point at a +// sibling it has no ancestry with, so this is a topological sort over the foreign keys +// themselves. +// +// A cycle has no valid order and is reported as an error rather than silently ordered +// wrong. A table referencing itself is not a cycle for these purposes — it constrains the +// order of rows within one table, which no table-level ordering can address — so those +// edges are skipped. +func (d *BaseDialect) TableApplyOrder() ([]string, error) { + names := d.TableNames() + + known := make(map[string]bool, len(names)) + for _, name := range names { + known[name] = true + } + + // referencedBy[x] lists the tables that must come after x; remaining[x] counts what x + // still waits on. + referencedBy := map[string][]string{} + remaining := map[string]int{} + for _, constraint := range d.ForeignKeySql { + referenced := constraint.ReferencedTable + if referenced == "" || referenced == constraint.Table || !known[referenced] || !known[constraint.Table] { + continue + } + + referencedBy[referenced] = append(referencedBy[referenced], constraint.Table) + remaining[constraint.Table]++ + } + + var ready []string + for _, name := range names { + if remaining[name] == 0 { + ready = append(ready, name) + } + } + + ordered := make([]string, 0, len(names)) + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + ordered = append(ordered, name) + + for _, referencing := range referencedBy[name] { + remaining[referencing]-- + if remaining[referencing] == 0 { + ready = append(ready, referencing) + } + } + } + + if len(ordered) != len(names) { + var cycle []string + for _, name := range names { + if remaining[name] > 0 { + cycle = append(cycle, name) + } + } + + return nil, fmt.Errorf("the foreign keys between %s form a cycle, so no order of tables can satisfy them; the schema needs one of those references dropped", strings.Join(cycle, ", ")) + } + + return ordered, nil +} + +// TableNames lists every table of the schema in a stable order: the block table first, +// being referenced by every other one and referencing none, then the rest sorted. +// +// It is what TableApplyOrder starts from, and what the reorg path falls back to when no +// foreign key order exists. +func (d *BaseDialect) TableNames() []string { + names := []string{DialectTableBlock} + for name := range d.TableRegistry { + names = append(names, name) + } + sort.Strings(names[1:]) + + return names +} + +// TableApplyRanks is TableApplyOrder as a lookup, for sorting a set of tables that is not +// the whole schema. A table the dialect does not know about ranks last. +func (d *BaseDialect) TableApplyRanks() (map[string]int, error) { + ordered, err := d.TableApplyOrder() + if err != nil { + return nil, err + } + + ranks := make(map[string]int, len(ordered)) + for rank, name := range ordered { + ranks[name] = rank + } + + return ranks, nil +} + +func (d *BaseDialect) AddIndexSql(table string, sql string) { + d.IndexSql = append(d.IndexSql, &Constraint{Table: table, Sql: sql}) +} + func (d *BaseDialect) AddUniqueConstraintSql(table string, sql string) { d.UniqueConstraintSql = append(d.UniqueConstraintSql, &Constraint{Table: table, Sql: sql}) } diff --git a/sink/sql/db_proto/sql/dialect_order_test.go b/sink/sql/db_proto/sql/dialect_order_test.go new file mode 100644 index 000000000..104e1eadc --- /dev/null +++ b/sink/sql/db_proto/sql/dialect_order_test.go @@ -0,0 +1,174 @@ +package sql + +import ( + "testing" + + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func dialectWith(tables []string, foreignKeys map[string]string) *BaseDialect { + registry := map[string]*schema.Table{} + for _, table := range tables { + registry[table] = &schema.Table{Name: table} + } + + dialect := NewBaseDialect(registry, nil) + for referencing, referenced := range foreignKeys { + dialect.AddForeignKeyReferencing(referencing, referenced, "ALTER TABLE "+referencing+" ADD CONSTRAINT fk FOREIGN KEY (x) REFERENCES "+referenced+"(y)") + } + + return dialect +} + +// before asserts that one table is applied ahead of another, which is the whole point of +// the order: rows of the referenced table must reach the server first. +func before(t *testing.T, ordered []string, first, second string) { + t.Helper() + + positions := map[string]int{} + for i, name := range ordered { + positions[name] = i + } + + require.Contains(t, positions, first) + require.Contains(t, positions, second) + assert.Less(t, positions[first], positions[second], "%q must be applied before %q, got %v", first, second, ordered) +} + +func TestTableApplyOrder(t *testing.T) { + t.Run("the block table always leads", func(t *testing.T) { + dialect := dialectWith([]string{"customers"}, map[string]string{"customers": DialectTableBlock}) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + before(t, ordered, DialectTableBlock, "customers") + }) + + t.Run("deep nesting", func(t *testing.T) { + // orders -> order_items -> order_item_options -> order_item_option_extras + dialect := dialectWith( + []string{"orders", "order_items", "order_item_options", "order_item_option_extras"}, + map[string]string{ + "order_items": "orders", + "order_item_options": "order_items", + "order_item_option_extras": "order_item_options", + }, + ) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + before(t, ordered, "orders", "order_items") + before(t, ordered, "order_items", "order_item_options") + before(t, ordered, "order_item_options", "order_item_option_extras") + }) + + t.Run("a sibling reference, which nesting depth cannot order", func(t *testing.T) { + // Both are top-level tables, so they sit at the same depth; only the foreign key + // says which has to be loaded first. + dialect := dialectWith( + []string{"orders", "customers"}, + map[string]string{"orders": "customers"}, + ) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + before(t, ordered, "customers", "orders") + }) + + t.Run("a sibling reference pointing the other way", func(t *testing.T) { + dialect := dialectWith( + []string{"orders", "customers"}, + map[string]string{"customers": "orders"}, + ) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + before(t, ordered, "orders", "customers") + }) + + t.Run("nesting and siblings together", func(t *testing.T) { + dialect := dialectWith( + []string{"customers", "orders", "order_items", "products"}, + map[string]string{ + "orders": "customers", + "order_items": "orders", + }, + ) + dialect.AddForeignKeyReferencing("order_items", "products", "ALTER TABLE order_items ADD CONSTRAINT fk_product FOREIGN KEY (p) REFERENCES products(id)") + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + before(t, ordered, "customers", "orders") + before(t, ordered, "orders", "order_items") + before(t, ordered, "products", "order_items") + assert.Len(t, ordered, 5, "every table plus the block table") + }) + + t.Run("a table referencing itself is not a cycle", func(t *testing.T) { + // It constrains the order of rows within one table, which no table-level order + // can address, so it is left to the write order of the rows themselves. + dialect := dialectWith([]string{"employees"}, map[string]string{"employees": "employees"}) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + assert.Contains(t, ordered, "employees") + }) + + t.Run("a cycle is reported rather than ordered wrong", func(t *testing.T) { + dialect := dialectWith([]string{"a", "b"}, map[string]string{"a": "b", "b": "a"}) + + _, err := dialect.TableApplyOrder() + require.Error(t, err) + assert.Contains(t, err.Error(), "cycle") + assert.Contains(t, err.Error(), "a") + assert.Contains(t, err.Error(), "b") + }) + + t.Run("a longer cycle", func(t *testing.T) { + dialect := dialectWith([]string{"a", "b", "c"}, map[string]string{"a": "b", "b": "c", "c": "a"}) + + _, err := dialect.TableApplyOrder() + require.Error(t, err) + assert.Contains(t, err.Error(), "cycle") + }) + + t.Run("a table nobody references keeps its place", func(t *testing.T) { + dialect := dialectWith([]string{"alone", "customers", "orders"}, map[string]string{"orders": "customers"}) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + assert.Contains(t, ordered, "alone") + assert.Len(t, ordered, 4) + }) + + t.Run("ranks match the order", func(t *testing.T) { + dialect := dialectWith([]string{"customers", "orders"}, map[string]string{"orders": "customers"}) + + ordered, err := dialect.TableApplyOrder() + require.NoError(t, err) + + ranks, err := dialect.TableApplyRanks() + require.NoError(t, err) + require.Len(t, ranks, len(ordered)) + for i, name := range ordered { + assert.Equal(t, i, ranks[name]) + } + }) +} + +// TestTableNamesCoversEverySchemaTable is what the reorg path falls back to when the +// foreign keys cannot be ordered. It has to name every table, or an undo would leave rows +// of the ones it skipped behind. +func TestTableNamesCoversEverySchemaTable(t *testing.T) { + dialect := dialectWith( + []string{"orders", "customers"}, + map[string]string{"orders": "customers", "customers": "orders"}, + ) + + _, err := dialect.TableApplyOrder() + require.Error(t, err, "the schema is cyclic on purpose") + + assert.Equal(t, []string{DialectTableBlock, "customers", "orders"}, dialect.TableNames()) +} diff --git a/sink/sql/db_proto/sql/postgres/accumulator_inserter.go b/sink/sql/db_proto/sql/postgres/accumulator_inserter.go index c9d589a36..bf7da52be 100644 --- a/sink/sql/db_proto/sql/postgres/accumulator_inserter.go +++ b/sink/sql/db_proto/sql/postgres/accumulator_inserter.go @@ -17,8 +17,13 @@ type accumulator struct { type AccumulatorInserter struct { accumulators map[string]*accumulator - cursorStmt *sql.Stmt - logger *zap.Logger + // flushOrder lists the tables in the order their statements must reach the server: + // a referenced table before the tables referencing it. Flushing groups rows by table, + // which loses the order the walk produced them in, so a foreign key would otherwise + // see a child before its parent. + flushOrder []string + cursorStmt *sql.Stmt + logger *zap.Logger } func NewAccumulatorInserter(logger *zap.Logger) (*AccumulatorInserter, error) { @@ -46,6 +51,11 @@ func (i *AccumulatorInserter) init(database *Database) error { query: fmt.Sprintf("INSERT INTO %s (number, hash, timestamp) VALUES ", tableName(database.schema.Name, "_blocks_")), } + flushOrder, err := database.dialect.TableApplyOrder() + if err != nil { + return err + } + cursorQuery := fmt.Sprintf("INSERT INTO %s (name, cursor) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET cursor = $2", tableName(database.schema.Name, "_cursor_")) cs, err := database.db.Prepare(cursorQuery) if err != nil { @@ -53,6 +63,7 @@ func (i *AccumulatorInserter) init(database *Database) error { } i.accumulators = accumulators + i.flushOrder = flushOrder i.cursorStmt = cs return nil @@ -122,8 +133,9 @@ func (i *AccumulatorInserter) insert(table string, values []any, database *Datab } func (i *AccumulatorInserter) flush(database *Database) error { - for _, acc := range i.accumulators { - if len(acc.rowValues) == 0 { + for _, table := range i.flushOrder { + acc, found := i.accumulators[table] + if !found || len(acc.rowValues) == 0 { continue } var b strings.Builder diff --git a/sink/sql/db_proto/sql/postgres/database.go b/sink/sql/db_proto/sql/postgres/database.go index efe36222c..4f9519930 100644 --- a/sink/sql/db_proto/sql/postgres/database.go +++ b/sink/sql/db_proto/sql/postgres/database.go @@ -5,31 +5,159 @@ import ( pgsql "database/sql" "fmt" "hash/fnv" + "regexp" + "strings" + "sync" "time" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/lib/pq" "github.com/streamingfast/logging/zapx" sink "github.com/streamingfast/substreams/sink" "github.com/streamingfast/substreams/sink/sql/bytes" "github.com/streamingfast/substreams/sink/sql/db_changes/db" "github.com/streamingfast/substreams/sink/sql/db_proto/sql" "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" "go.uber.org/zap" "google.golang.org/protobuf/reflect/protoreflect" ) +const ( + // maxOpenConnections bounds the sink's own pool: inserts are serialised on one + // goroutine, and the queries around them are occasional. + maxOpenConnections = 8 + maxIdleConnections = 2 + + // maxCopyConnections bounds the binary COPY pool, driven by a single applier. + maxCopyConnections = 2 +) + type Database struct { *sql.BaseDatabase - db *pgsql.DB - tx *pgsql.Tx - schema *schema.Schema - logger *zap.Logger - dialect *DialectPostgres - inserter pgInserter - flusher pgFlusher - useConstraints bool + db *pgsql.DB + // pool is only created when the spool is enabled: binary COPY needs pgx, the rest of + // the sink talks to PostgreSQL through database/sql and lib/pq. + pool *pgxpool.Pool + spoolOptions *spool.Options + tx *pgsql.Tx + dsn *db.DSN + schema *schema.Schema + logger *zap.Logger + dialect *DialectPostgres + inserter pgInserter + flusher pgFlusher + constraints sql.ConstraintPolicy + + // writeMode is what the operator asked for; resolvedWriteMode is what Open settled on + // and is what the startup log reports. + writeMode sql.WriteMode + resolvedWriteMode sql.WriteMode +} + +// bufferActive reports whether rows are currently being routed to the spool. +// +// This is deliberately not "was a spool configured": schema creation runs before Open +// and does need real transactions, so the distinction is what keeps DDL working. +func (d *Database) bufferActive() bool { + _, ok := d.inserter.(*localBufferInserter) + + return ok +} + +// WithSpool turns on the on-disk spool. It must be called before Open. +func (d *Database) WithSpool(options spool.Options) { + d.spoolOptions = &options +} + +// WithWriteMode records how sealed segments should reach the database. It must be called +// before Open, which is where the mode is resolved and validated against the schema. +func (d *Database) WithWriteMode(mode sql.WriteMode) error { + parsed, err := sql.ParseWriteMode(string(mode)) + if err != nil { + return err + } + d.writeMode = parsed + + return nil +} + +// WriteMode reports the mode Open settled on, for the startup log. +func (d *Database) WriteMode() sql.WriteMode { return d.resolvedWriteMode } + +// tablesOrderable reports whether the schema's tables can be put in an order that keeps a +// referenced table ahead of the tables referencing it. +// +// Grouping rows by table — which both the spool and the multi-row INSERT path do — only +// works under foreign keys if such an order exists. A cycle has none, and only the +// row-at-a-time inserter can cope, because it follows the walk, which produces a parent +// before its children. +func (d *Database) tablesOrderable() (bool, error) { + if d.constraints.DisableForeignKeys { + return true, nil + } + + if _, err := d.dialect.TableApplyOrder(); err != nil { + return false, err + } + + return true, nil +} + +// spoolFormat is the on-disk layout each write mode needs. +// +// The spool always holds bytes that are ready to send, so the format follows the mode +// rather than the other way round: pre-rendering is worth ~8% for COPY and the same +// argument carries to the INSERT modes, where the rendering the accumulator used to do at +// flush time simply happens earlier. +func spoolFormat(mode sql.WriteMode) spool.Format { + switch mode { + case sql.WriteModeBatchInsert: + return spool.FormatTuples + case sql.WriteModeRowInsert: + return spool.FormatRowLog + default: + return spool.FormatPGCopy + } +} + +// resolveWriteMode turns the requested mode into the one Open will use. +// +// An explicit mode the schema cannot support is an error rather than a downgrade: a +// silent fall back to row-at-a-time inserts is an order of magnitude slower, and finding +// that out from a log line three days into a backfill is not finding it out. +func (d *Database) resolveWriteMode() (sql.WriteMode, error) { + orderable, orderErr := d.tablesOrderable() + + switch d.writeMode { + case sql.WriteModeRowInsert: + return sql.WriteModeRowInsert, nil + + case sql.WriteModeCopy, sql.WriteModeBatchInsert: + if !orderable { + return "", fmt.Errorf("--write-mode=%s groups rows by table, which needs the schema's tables to be orderable by their foreign keys, and this schema's cannot be (%w). "+ + "Use --disable-foreign-keys, or --write-mode=%s, which replays the walk instead and so always keeps a parent ahead of its children", + d.writeMode, orderErr, sql.WriteModeRowInsert) + } + + return d.writeMode, nil + + default: + if !orderable { + d.logger.Info("the schema's foreign keys cannot be ordered, replaying the walk one row at a time", + zap.String("write_mode", string(sql.WriteModeRowInsert)), zap.Error(orderErr)) + + return sql.WriteModeRowInsert, nil + } + if d.spoolOptions == nil { + return sql.WriteModeBatchInsert, nil + } + + return sql.WriteModeCopy, nil + } } -func NewDatabase(schema *schema.Schema, dsn *db.DSN, moduleOutputType string, rootMessageDescriptor protoreflect.MessageDescriptor, useProtoOptions bool, useConstraints bool, bytesEncoding bytes.Encoding, logger *zap.Logger) (*Database, error) { +func NewDatabase(schema *schema.Schema, dsn *db.DSN, moduleOutputType string, rootMessageDescriptor protoreflect.MessageDescriptor, useProtoOptions bool, constraints sql.ConstraintPolicy, bytesEncoding bytes.Encoding, logger *zap.Logger) (*Database, error) { logger = logger.Named("postgres") logger.Info("connecting to db", zap.String("host", dsn.Host), zap.Int64("port", dsn.Port), zap.String("database", dsn.Database)) @@ -38,6 +166,11 @@ func NewDatabase(schema *schema.Schema, dsn *db.DSN, moduleOutputType string, ro return nil, fmt.Errorf("open db connection: %w", err) } + // The sink writes from one goroutine and queries occasionally around it, so an + // unbounded pool only ever buys a way to exhaust the server's connection slots. + sqlDB.SetMaxOpenConns(maxOpenConnections) + sqlDB.SetMaxIdleConns(maxIdleConnections) + if reachable, err := isDatabaseReachable(sqlDB); !reachable { return nil, fmt.Errorf("database not reachable: %w", err) } @@ -52,19 +185,66 @@ func NewDatabase(schema *schema.Schema, dsn *db.DSN, moduleOutputType string, ro return nil, fmt.Errorf("failed to create base database: %w", err) } database := &Database{ - db: sqlDB, - schema: schema, - useConstraints: useConstraints, - BaseDatabase: baseDB, - dialect: dialect, - logger: logger, + db: sqlDB, + dsn: dsn, + schema: schema, + constraints: constraints, + BaseDatabase: baseDB, + dialect: dialect, + logger: logger, } return database, nil } func (d *Database) Open() error { - if d.useConstraints { + mode, err := d.resolveWriteMode() + if err != nil { + return err + } + d.resolvedWriteMode = mode + + if d.spoolOptions == nil { + return d.openDirectInserter() + } + ctx := context.Background() + + poolConfig, err := pgxpool.ParseConfig(d.dsn.ConnString()) + if err != nil { + return fmt.Errorf("parsing the connection string for binary COPY: %w", err) + } + // One applier goroutine COPYs one segment at a time, so the pgx default of four + // connections per core is a fleet of idle connections against the server. + poolConfig.MaxConns = maxCopyConnections + + pool, err := pgxpool.NewWithConfig(ctx, poolConfig) + if err != nil { + return fmt.Errorf("connecting to the database for binary COPY: %w", err) + } + d.pool = pool + + inserter, err := newLocalBufferInserter(ctx, d, spoolFormat(mode), *d.spoolOptions, d.logger) + if err != nil { + return fmt.Errorf("starting the local spool: %w", err) + } + d.inserter = inserter + d.flusher = inserter + + return nil +} + +// openDirectInserter installs the inserter that writes to the database itself: one +// prepared INSERT per row in row-insert mode, a multi-row INSERT built at flush otherwise. +// +// Constraints used to mean one prepared INSERT per row, which the benchmarks put at a +// tenth of the multi-row path. That was only ever needed for ordering: the walk produces a +// parent before its children, and inserting row by row preserved it. Ordering the +// multi-row statements by the foreign keys themselves preserves it too, so row-at-a-time +// is now reserved for a schema whose references cannot be ordered at all. +func (d *Database) openDirectInserter() error { + // The chain-head switch reopens this after the spool is gone, and copy is not a + // direct mode, so it lands on the multi-row path. + if d.resolvedWriteMode == sql.WriteModeRowInsert { inserter, err := NewRowInserter(d.logger) if err != nil { return fmt.Errorf("creating row inserter: %w", err) @@ -74,17 +254,79 @@ func (d *Database) Open() error { } d.inserter = inserter d.flusher = inserter - } else { - inserter, err := NewAccumulatorInserter(d.logger) - if err != nil { - return fmt.Errorf("creating accumulator inserter: %w", err) - } - if err := inserter.init(d); err != nil { - return fmt.Errorf("initializing row inserter: %w", err) + + return nil + } + + inserter, err := NewAccumulatorInserter(d.logger) + if err != nil { + return fmt.Errorf("creating accumulator inserter: %w", err) + } + if err := inserter.init(d); err != nil { + return fmt.Errorf("initializing accumulator inserter: %w", err) + } + d.inserter = inserter + d.flusher = inserter + + return nil +} + +// SwitchToDirectInserts drains the spool and inserts straight into the database from here +// on. The reason says what brought the switch on, since the database cannot tell the chain +// head from the end of a bounded range. +// +// The spool trades freshness for throughput: rows sit on disk until a segment fills, +// which is what a backfill wants and the opposite of what a sink at the chain head wants, +// where a block should be queryable when it arrives rather than when the segment it +// happens to land in is full. Reorgs are also cheaper to undo out of a table than out of +// a segment that has not been applied yet. +// +// Everything spooled is applied before the switch, so no row is left behind, and the +// spool is closed for good — a stream that has reached the head does not go back. +func (d *Database) SwitchToDirectInserts(ctx context.Context, reason string, atChainHead bool) error { + inserter, ok := d.inserter.(*localBufferInserter) + if !ok { + return nil + } + + d.logger.Info(reason+", draining the spool and switching to direct inserts. "+ + "--write-mode, --db-write-* and --spool-* no longer apply from here on", + zap.String("write_mode_was", string(d.resolvedWriteMode))) + + if err := inserter.close(ctx); err != nil { + return fmt.Errorf("draining the spool: %w", err) + } + + if atChainHead { + // The drain above applied every segment and removed its directory, so there is + // nothing left on disk for a record to answer for. Only a run that carries on past + // here bothers: it seals a segment or two on each restart before reaching the head + // again, and those records would otherwise pile up for the life of the sink. + if err := inserter.applier.clearSegments(ctx); err != nil { + // Bookkeeping, and the spool is already drained: the rows are safe either way, + // so this is not worth failing the switch over. + d.logger.Warn("could not clear the applied-segment records", zap.Error(err)) } - d.inserter = inserter - d.flusher = inserter } + + // The spool is what copy mode is; without it the direct path is the multi-row one. + if d.resolvedWriteMode == sql.WriteModeCopy { + d.resolvedWriteMode = sql.WriteModeBatchInsert + } + + if err := d.openDirectInserter(); err != nil { + return fmt.Errorf("switching to direct inserts: %w", err) + } + + // bufferActive() reads the inserter, so transactions resume from here; the options + // go with it to keep the two from disagreeing. + d.spoolOptions = nil + + if d.pool != nil { + d.pool.Close() + d.pool = nil + } + return nil } @@ -92,13 +334,13 @@ func (d *Database) GetDialect() sql.Dialect { return d.dialect } -func (d *Database) CreateDatabase(useConstraints bool) error { +func (d *Database) CreateDatabase(applyConstraints bool) error { err := d.createDatabase() if err != nil { return fmt.Errorf("creating database: %w", err) } - if useConstraints { + if applyConstraints { err = d.applyConstraints() if err != nil { return fmt.Errorf("applying constraints: %w", err) @@ -125,34 +367,499 @@ func (d *Database) createDatabase() error { return nil } +// ApplyConstraints puts the dialect's constraints on a schema that already exists, +// leaving the ones already there alone. +// +// It is what `sink postgres apply-constraints` does to a database synced without them: the sink +// info hash cannot tell those two apart, since it is computed over the DDL the dialect +// would emit, constraints included, either way. +// +// The caller owns the transaction. On a populated database this is not quick — every +// index has to be built and every foreign key validated, with the table locked meanwhile. +func (d *Database) ApplyConstraints() error { + return d.applyConstraints() +} + +// applyConstraints creates what the policy asks for and the catalog does not have. +// +// It owns its transactions rather than running inside the caller's, and commits every +// --constraints-per-transaction statements. Building an index and validating a foreign key +// are the two most memory-hungry things the sink ever asks of the server, and holding them +// all open at once is what turns a large schema into an OOM that loses the entire pass. +// Committing as it goes bounds that, and leaves a killed run's finished work in place for +// the next one to carry on from. func (d *Database) applyConstraints() error { startAt := time.Now() - for _, constraint := range d.dialect.PrimaryKeySql { - d.logger.Info("executing pk statement", zap.String("sql", constraint.Sql)) - _, err := d.tx.Exec(constraint.Sql) - if err != nil { - return fmt.Errorf("executing pk statement: %w %s", err, constraint.Sql) + + existing, err := d.existingConstraints(d.querier()) + if err != nil { + return err + } + + var pending []statement + collect := func(kind string, constraints []*sql.Constraint, skip func(string) bool) { + for _, constraint := range constraints { + if skip(constraint.Table) { + d.logger.Debug("constraint disabled by the policy, skipping", zap.String("kind", kind), zap.String("table", constraint.Table)) + continue + } + + if key, ok := constraintTarget(constraint.Sql); ok && existing[key] { + d.logger.Debug("constraint already in place, skipping", zap.String("constraint", key.name), zap.String("relation", key.relation)) + continue + } + + pending = append(pending, statement{kind: kind, sql: constraint.Sql}) } } - for _, constraint := range d.dialect.UniqueConstraintSql { - d.logger.Info("executing unique statement", zap.String("sql", constraint.Sql)) - _, err := d.tx.Exec(constraint.Sql) - if err != nil { - return fmt.Errorf("executing unique statement: %w %s", err, constraint.Sql) + + collect("pk", d.dialect.PrimaryKeySql, d.constraints.SkipPrimaryKey) + collect("unique", d.dialect.UniqueConstraintSql, d.constraints.SkipUnique) + keys := pending + + pending = nil + collect("fk constraint", d.dialect.ForeignKeySql, d.constraints.SkipForeignKey) + foreignKeys := pending + + // Two waves: every key first, then the foreign keys, which need the key they point at + // to exist. Inside a wave the relations are independent. + if err := d.runStatements([][]statement{keys, foreignKeys}); err != nil { + return err + } + + d.logger.Info("applying constraints", + zap.Int("created", len(keys)+len(foreignKeys)), + zap.Int("parallelism", d.constraints.ConstraintsParallelism()), + zap.String("work_mem", d.constraints.WorkMem), + zapx.HumanDuration("duration", time.Since(startAt))) + + return nil +} + +// EnsureBlockNumberIndexes creates the index the sink needs for its own reorg path, on +// every table, when the sink starts. +// +// It is not part of the constraint pass and not governed by --apply-constraints. Those +// describe the schema and are the operator's to schedule; this one the sink depends on to +// undo a reorg without sequentially scanning every table, so waiting for a maintenance +// window would mean running without it for as long as the operator likes. +// +// Concurrently, and therefore outside any transaction: a restart onto an already-loaded +// table must not lock out the writers. On a schema that already has them this is one +// catalog query and nothing else. +func (d *Database) EnsureBlockNumberIndexes(ctx context.Context) error { + if d.constraints.DisableBlockNumberIndex { + return nil + } + + existing, err := d.existingIndexes(d.db) + if err != nil { + return err + } + + invalid, err := d.invalidIndexes() + if err != nil { + return err + } + + startAt := time.Now() + created := 0 + + for _, index := range d.dialect.IndexSql { + key, ok := indexTarget(index.Sql) + if !ok { + continue } + + if invalid[key.name] { + // A concurrent build that was interrupted leaves an index behind that no query + // will ever use, and IF NOT EXISTS would happily keep it forever. + d.logger.Info("dropping an index a previous concurrent build left unusable", zap.String("index", key.name)) + if _, err := d.db.ExecContext(ctx, fmt.Sprintf("DROP INDEX CONCURRENTLY IF EXISTS %s.%s", + pq.QuoteIdentifier(d.schema.Name), pq.QuoteIdentifier(key.name))); err != nil { + return fmt.Errorf("dropping the invalid index %q: %w", key.name, err) + } + } else if existing[key] { + continue + } + + d.logger.Info("creating the block number index", zap.String("index", key.name), zap.String("sql", index.Sql)) + if _, err := d.db.ExecContext(ctx, index.Sql); err != nil { + return fmt.Errorf("creating the index %q: %w", key.name, err) + } + created++ } - for _, constraint := range d.dialect.ForeignKeySql { - d.logger.Info("executing fk constraint statement", zap.String("sql", constraint.Sql)) - _, err := d.tx.Exec(constraint.Sql) - if err != nil { - return fmt.Errorf("executing fk constraint statement: %w %s", err, constraint.Sql) + + if created > 0 { + d.logger.Info("block number indexes created", zap.Int("created", created), zapx.HumanDuration("duration", time.Since(startAt))) + } + + return nil +} + +// invalidIndexes names the indexes an interrupted concurrent build left behind. They exist +// as far as IF NOT EXISTS is concerned and are used by nothing. +func (d *Database) invalidIndexes() (map[string]bool, error) { + rows, err := d.db.Query(` + SELECT cl.relname + FROM pg_index i + JOIN pg_class cl ON cl.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = cl.relnamespace + WHERE n.nspname = $1 AND NOT i.indisvalid`, d.schema.Name) + if err != nil { + return nil, fmt.Errorf("listing the invalid indexes of schema %q: %w", d.schema.Name, err) + } + defer rows.Close() + + invalid := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scanning an invalid index name: %w", err) + } + invalid[name] = true + } + + return invalid, rows.Err() +} + +// existingIndexes reads which indexes the schema carries. They live in pg_indexes rather +// than pg_constraint, a plain index not being a constraint. +func (d *Database) existingIndexes(from rowQuerier) (map[constraintKey]bool, error) { + rows, err := from.Query(` + SELECT indexname, schemaname || '.' || tablename + FROM pg_indexes + WHERE schemaname = $1`, d.schema.Name) + if err != nil { + return nil, fmt.Errorf("listing the existing indexes of schema %q: %w", d.schema.Name, err) + } + defer rows.Close() + + existing := map[constraintKey]bool{} + for rows.Next() { + var name, relation string + if err := rows.Scan(&name, &relation); err != nil { + return nil, fmt.Errorf("scanning index name: %w", err) } + existing[constraintKey{relation: normalizeRelation(relation), name: name}] = true } - d.logger.Info("applying constraints", zapx.HumanDuration("duration", time.Since(startAt))) + + return existing, rows.Err() +} + +// indexTargetPattern pulls the name and the relation out of the dialect's own CREATE INDEX. +var indexTargetPattern = regexp.MustCompile(`(?i)create\s+index\s+(?:concurrently\s+)?(?:if\s+not\s+exists\s+)?"?([a-z0-9_]+)"?\s+on\s+(\S+)`) + +func indexTarget(statement string) (constraintKey, bool) { + match := indexTargetPattern.FindStringSubmatch(statement) + if len(match) != 3 { + return constraintKey{}, false + } + + return constraintKey{relation: normalizeRelation(match[2]), name: match[1]}, true +} + +// statement is one piece of DDL the constraint pass has to run. +type statement struct { + kind string + sql string +} + +// workMemPattern is what a maintenance_work_mem value may look like. SET takes no bind +// parameters, so the value is interpolated and has to be checked rather than escaped. +var workMemPattern = regexp.MustCompile(`^[0-9]+(kB|MB|GB|TB)?$`) + +// runStatements executes the DDL one wave at a time. +// +// Statements inside a wave touch independent relations, so they are handed to the server +// together: ten primary keys on ten tables have no reason to queue behind each other, and +// the small ones would otherwise wait out the largest. What cannot overlap is a wave +// boundary — a foreign key needs the key it references to be there — so the caller orders +// the waves and this only spreads what is inside one. +// +// Each statement still commits on its own, which is what keeps the pass restartable: a run +// that is killed keeps what it finished and the next one carries on. +// +// When the caller already has a transaction open — creating the schema does — the +// statements join it instead, sequentially: the tables they constrain are not committed +// yet, so nothing else could see them. +func (d *Database) runStatements(waves [][]statement) error { + if d.tx != nil { + for _, wave := range waves { + for _, s := range wave { + d.logger.Info("executing "+s.kind+" statement", zap.String("sql", s.sql)) + if _, err := d.tx.Exec(s.sql); err != nil { + return fmt.Errorf("executing %s statement: %w %s", s.kind, err, s.sql) + } + } + } + + return nil + } + + parallelism := d.constraints.ConstraintsParallelism() + + // The pool is sized for a sink that writes from one goroutine; the pass wants one + // connection per statement in flight, and gives them back when it is done. + if parallelism > maxOpenConnections { + d.db.SetMaxOpenConns(parallelism + 1) + defer d.db.SetMaxOpenConns(maxOpenConnections) + } + + for _, wave := range waves { + if err := d.runStatementWave(wave, parallelism); err != nil { + return err + } + } + + return nil +} + +// runStatementWave runs one wave, at most parallelism statements at a time, and reports +// the first failure once the wave has drained. +func (d *Database) runStatementWave(statements []statement, parallelism int) error { + if len(statements) == 0 { + return nil + } + + var ( + waitGroup sync.WaitGroup + mutex sync.Mutex + failure error + ) + slots := make(chan struct{}, parallelism) + + for _, s := range statements { + waitGroup.Add(1) + + go func(s statement) { + defer waitGroup.Done() + + slots <- struct{}{} + defer func() { <-slots }() + + // One statement failing does not stop the ones already running, but there is no + // point starting the rest: the pass reports the first failure and the operator + // runs it again, which skips whatever did land. + mutex.Lock() + stop := failure != nil + mutex.Unlock() + if stop { + return + } + + if err := d.runStatement(s); err != nil { + mutex.Lock() + if failure == nil { + failure = err + } + mutex.Unlock() + } + }(s) + } + + waitGroup.Wait() + + return failure +} + +// runStatement executes one piece of DDL in a transaction of its own. +func (d *Database) runStatement(s statement) error { + startAt := time.Now() + d.logger.Info("executing "+s.kind+" statement", zap.String("sql", s.sql)) + + tx, err := d.db.Begin() + if err != nil { + return fmt.Errorf("beginning a constraint transaction: %w", err) + } + + if mem := d.constraints.WorkMem; mem != "" { + if !workMemPattern.MatchString(mem) { + tx.Rollback() + return fmt.Errorf("invalid maintenance_work_mem %q, expected a size such as 512MB", mem) + } + // SET LOCAL, so it lasts exactly as long as this statement's transaction. + if _, err := tx.Exec(fmt.Sprintf("SET LOCAL maintenance_work_mem = '%s'", mem)); err != nil { + tx.Rollback() + return fmt.Errorf("setting maintenance_work_mem: %w", err) + } + } + + if _, err := tx.Exec(s.sql); err != nil { + tx.Rollback() + return fmt.Errorf("executing %s statement: %w %s", s.kind, err, s.sql) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("committing a constraint transaction: %w", err) + } + + d.logger.Debug("statement done", zap.String("sql", s.sql), zapx.HumanDuration("duration", time.Since(startAt))) + return nil } +// MissingConstraints names the constraints the policy says this schema should carry and +// the catalog does not have. +// +// It is one query against pg_constraint filtered by namespace — indexed, and nothing like +// the cost of building the constraints it reports on — so it is cheap enough to run on +// every start. +func (d *Database) MissingConstraints() ([]string, error) { + existing, err := d.existingConstraints(d.querier()) + if err != nil { + return nil, err + } + + var missing []string + collect := func(constraints []*sql.Constraint, skip func(string) bool) { + for _, constraint := range constraints { + if skip(constraint.Table) { + continue + } + if key, ok := constraintTarget(constraint.Sql); ok && !existing[key] { + missing = append(missing, key.relation+"."+key.name) + } + } + } + + collect(d.dialect.PrimaryKeySql, d.constraints.SkipPrimaryKey) + collect(d.dialect.UniqueConstraintSql, d.constraints.SkipUnique) + collect(d.dialect.ForeignKeySql, d.constraints.SkipForeignKey) + + return missing, nil +} + +// DropConstraints removes the constraints this schema's DDL would create, leaving +// anything the sink did not put there alone. +// +// Foreign keys go first, then unique constraints, then primary keys: a primary key still +// referenced by a foreign key cannot be dropped. Constraints already absent are skipped, +// so this is idempotent and safe to run against a schema that never had them. +func (d *Database) DropConstraints() error { + existing, err := d.existingConstraints(d.querier()) + if err != nil { + return err + } + + var pending []statement + collect := func(kind string, constraints []*sql.Constraint) { + for _, constraint := range constraints { + key, ok := constraintTarget(constraint.Sql) + if !ok || !existing[key] { + d.logger.Debug("constraint already absent, skipping", zap.String("constraint", key.name)) + continue + } + + pending = append(pending, statement{ + kind: kind, + sql: fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT IF EXISTS %s", key.relation, pq.QuoteIdentifier(key.name)), + }) + } + } + + // Foreign keys first, then uniques, then primary keys: a primary key still referenced + // by a foreign key cannot be dropped, so each of the three is its own wave. + collect("fk constraint", d.dialect.ForeignKeySql) + foreignKeys := pending + + pending = nil + collect("unique constraint", d.dialect.UniqueConstraintSql) + uniques := pending + + pending = nil + collect("pk", d.dialect.PrimaryKeySql) + primaryKeys := pending + + return d.runStatements([][]statement{foreignKeys, uniques, primaryKeys}) +} + +// existingConstraintRelations maps every constraint name in the schema to the relation it +// belongs to, already quoted. Reading the relation from the catalog rather than deriving +// it from the logical table name is what keeps this working under the server's own +// identifier folding. +// rowQuerier is what both a transaction and the pool itself satisfy. +type rowQuerier interface { + Query(query string, args ...any) (*pgsql.Rows, error) +} + +// querier is the transaction if one is open, the pool otherwise. +// +// Creating the schema runs the constraint pass inside its own transaction, against tables +// that are not committed yet — so the pass has to read and write through that transaction +// or it cannot see them. On a database that is already loaded there is no open +// transaction, and the pass owns its own. +func (d *Database) querier() rowQuerier { + if d.tx != nil { + return d.tx + } + + return d.db +} + +// constraintKey identifies a constraint the way PostgreSQL does: by relation and name. +// +// Names are only unique per table, not per schema — every table carries a foreign key +// called fk_block — so keying by name alone both drops the wrong number of them and +// reports a constraint present on one table as present on all of them. +type constraintKey struct { + relation string + name string +} + +// existingConstraints reads which of them the schema actually carries. +func (d *Database) existingConstraints(from rowQuerier) (map[constraintKey]bool, error) { + rows, err := from.Query(` + SELECT c.conname, n.nspname || '.' || cl.relname + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + JOIN pg_class cl ON cl.oid = c.conrelid + WHERE n.nspname = $1`, d.schema.Name) + if err != nil { + return nil, fmt.Errorf("listing the existing constraints of schema %q: %w", d.schema.Name, err) + } + defer rows.Close() + + existing := map[constraintKey]bool{} + for rows.Next() { + var name, relation string + if err := rows.Scan(&name, &relation); err != nil { + return nil, fmt.Errorf("scanning constraint name: %w", err) + } + existing[constraintKey{relation: normalizeRelation(relation), name: name}] = true + } + + return existing, rows.Err() +} + +// constraintTargetPattern pulls the relation and the name out of the dialect's own DDL. +var constraintTargetPattern = regexp.MustCompile(`(?i)alter\s+table\s+(\S+)\s+add\s+constraint\s+"?([a-z0-9_]+)"?`) + +// constraintTarget says which constraint a statement of ours creates. It reports false for +// anything it does not recognise, which is then left to the server to accept or reject. +func constraintTarget(statement string) (constraintKey, bool) { + match := constraintTargetPattern.FindStringSubmatch(statement) + if len(match) != 3 { + return constraintKey{}, false + } + + return constraintKey{relation: normalizeRelation(match[1]), name: match[2]}, true +} + +// normalizeRelation puts a schema-qualified name in the shape the catalog reports, the +// dialect writing its DDL unquoted and the server folding it. +func normalizeRelation(relation string) string { + return strings.ToLower(strings.ReplaceAll(strings.TrimSpace(relation), `"`, "")) +} + func (d *Database) BeginTransaction() (err error) { + if d.bufferActive() { + // The buffer owns its transactions: one per segment, in the applier goroutine. + // Holding one here would pin a connection for the whole buffering window and + // would not cover the writes anyway. + return nil + } + d.tx, err = d.db.Begin() if err != nil { return fmt.Errorf("beginning transaction: %w", err) @@ -161,6 +868,10 @@ func (d *Database) BeginTransaction() (err error) { } func (d *Database) CommitTransaction() (err error) { + if d.bufferActive() { + return nil + } + err = d.tx.Commit() if err != nil { return fmt.Errorf("committing transaction: %w", err) @@ -170,6 +881,10 @@ func (d *Database) CommitTransaction() (err error) { } func (d *Database) RollbackTransaction() { + if d.bufferActive() { + return + } + err := d.tx.Rollback() if err != nil { panic("RollbackTransaction failed: " + err.Error()) @@ -205,6 +920,39 @@ func (d *Database) InsertBlock(blockNum uint64, hash string, timestamp time.Time return nil } +// Close drains the local buffer, if one is in use, so the blocks buffered at shutdown +// reach the database rather than being streamed again on the next run. +// Close drains a local buffer, if one is in use, and then releases the connections. It +// is called once the stream is done with the database, so holding the pools open past it +// only occupies connection slots on the server. +func (d *Database) Close(ctx context.Context) error { + var err error + if inserter, ok := d.inserter.(*localBufferInserter); ok { + err = inserter.close(ctx) + } + + if d.pool != nil { + d.pool.Close() + d.pool = nil + } + + if closeErr := d.db.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("closing the database connections: %w", closeErr) + } + + return err +} + +// BufferStats reports what the local buffer is holding, for the progress line. +func (d *Database) BufferStats() (blocks int64, bytes int64, appliedBlock uint64, enabled bool) { + inserter, ok := d.inserter.(*localBufferInserter) + if !ok { + return 0, 0, 0, false + } + + return inserter.buffer.BlocksBuffered(), inserter.buffer.BytesOnDisk(), inserter.buffer.AppliedBlock(), true +} + func (d *Database) Flush() (time.Duration, error) { startFlush := time.Now() err := d.flusher.flush(d) @@ -232,6 +980,7 @@ func (d *Database) FetchSinkInfo(schemaName string) (*sql.SinkInfo, error) { if err != nil { return nil, fmt.Errorf("fetching sync info: %w", err) } + return out, nil } @@ -279,7 +1028,38 @@ func (d *Database) StoreCursor(cursor *sink.Cursor) error { return err } +// HandleBlocksUndo removes everything a reorg invalidated: every entity row above the +// last valid block, then the block rows themselves. +// +// The deletes are explicit rather than left to `fk_block ... ON DELETE CASCADE`, because +// that foreign key only exists once the constraints have been created. Deleting just the block rows, as +// this used to, silently orphaned every entity row of the undone blocks on a schema +// without constraints — and that is now the default. Every table carries +// `_block_number_`, so the same delete works either way, and the descending Ordinal +// visits children before parents so a foreign key never blocks its own cleanup. func (d *Database) HandleBlocksUndo(lastValidBlockNum uint64) (err error) { + // Rows for the undone blocks may still be sitting in the buffer, on their way to a + // COPY; applying them after the delete would resurrect exactly what it removed. + // + // The spool holds no undoable block whenever this runs: an undo only ever concerns a + // block delivered at STEP_NEW, everything spooled arrived at STEP_NEW_IRREVERSIBLE, and + // the first STEP_NEW block closes the spool before it is held — see the live switch in + // db_proto.Sinker's HandleBlockScopedData. The drain is not therefore dead code: a run + // that resumes on a block which forked out while it was down gets the undo signal as its + // first message, with the spool open and empty. It also stands as a guard, since the + // invariant it rests on lives in another package and losing it silently would mean + // deleted rows coming back. + // + // Note that the delete below does not prune `_segments_`. It does not have to while that + // invariant holds: every recorded segment ends at a block that was irreversible when it + // was written, so none can cover a block above lastValidBlockNum. Prune it here if a + // spool is ever allowed to hold undoable blocks. + if inserter, ok := d.inserter.(*localBufferInserter); ok { + if err := inserter.buffer.Drain(context.Background()); err != nil { + return fmt.Errorf("draining the local buffer before an undo: %w", err) + } + } + tx, err := d.db.Begin() if err != nil { return fmt.Errorf("HandleBlocksUndo beginning transaction: %w", err) @@ -298,16 +1078,59 @@ func (d *Database) HandleBlocksUndo(lastValidBlockNum uint64) (err error) { }() d.logger.Info("undoing blocks", zap.Uint64("last_valid_block_num", lastValidBlockNum)) + startAt := time.Now() + + // Reverse apply order: a referencing table is emptied before the table it points at, + // so a foreign key never blocks its own cleanup. Nesting depth would order the + // parent-child keys and quietly get sibling references wrong. + ordered, err := d.dialect.TableApplyOrder() + if err != nil { + // A cyclic foreign key graph has no such order, and refusing here would leave the + // sink unable to undo a reorg — or even to start, since Run undoes from the stored + // cursor — on exactly the schema row-insert mode exists to support. The deletes all + // run in one transaction and key on _block_number_ alone, so a stable arbitrary + // order is right wherever the references are absent, and where a cycle really is + // enforced no order would have worked either. + d.logger.Debug("the schema's tables cannot be ordered by their foreign keys, undoing in schema order", + zap.Error(err)) + ordered = d.dialect.TableNames() + } + + var rowsAffected int64 + for i := len(ordered) - 1; i >= 0; i-- { + name := ordered[i] + if name == sql.DialectTableBlock { + // Deleted last, below, and counted separately. + continue + } + + query := fmt.Sprintf(`DELETE FROM %s WHERE %s > $1`, tableName(d.schema.Name, name), sql.DialectFieldBlockNumber) + result, err := tx.Exec(query, lastValidBlockNum) + if err != nil { + return fmt.Errorf("deleting rows of %q from %d: %w", name, lastValidBlockNum, err) + } + + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("fetching rows affected: %w", err) + } + rowsAffected += affected + } + query := fmt.Sprintf(`DELETE FROM %s._blocks_ WHERE "number" > $1`, d.schema.Name) result, err := tx.Exec(query, lastValidBlockNum) if err != nil { return fmt.Errorf("deleting block from %d: %w", lastValidBlockNum, err) } - rowsAffected, err := result.RowsAffected() + blocksAffected, err := result.RowsAffected() if err != nil { return fmt.Errorf("fetching rows affected: %w", err) } - d.logger.Info("undo completed", zap.Int64("row_affected", rowsAffected)) + + d.logger.Info("undo completed", + zap.Int64("row_affected", rowsAffected), + zap.Int64("block_affected", blocksAffected), + zapx.HumanDuration("duration", time.Since(startAt))) return nil } diff --git a/sink/sql/db_proto/sql/postgres/database_test.go b/sink/sql/db_proto/sql/postgres/database_test.go new file mode 100644 index 000000000..2e3a6ba43 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/database_test.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestConstraintTarget pins how a constraint is identified out of the dialect's own DDL. +// +// Relation as well as name, because names are only unique per table: every table carries +// a foreign key called fk_block, so a name on its own says a constraint present on one +// table is present on all of them — which would skip creating the rest and drop only one. +func TestConstraintTarget(t *testing.T) { + tests := []struct { + name string + statement string + expectFound bool + expectRelate string + expectName string + }{ + { + "primary key, as the dialect writes it", + `alter table myschema.balancechange add constraint balancechange_pk primary key ("id");`, + true, "myschema.balancechange", "balancechange_pk", + }, + { + "the block table's primary key", + `alter table myschema._blocks_ add constraint block_pk primary key (number);`, + true, "myschema._blocks_", "block_pk", + }, + { + "unique constraint", + `alter table myschema.transfer add constraint transfer_hash_unique unique ("hash");`, + true, "myschema.transfer", "transfer_hash_unique", + }, + { + // ForeignKey.String() puts two spaces after the name, and upper-cases the DDL. + "foreign key", + `ALTER TABLE myschema.transfer ADD CONSTRAINT transfer_block_fk FOREIGN KEY (_block_number_) REFERENCES myschema._blocks_(number)`, + true, "myschema.transfer", "transfer_block_fk", + }, + { + "quoted name and relation are folded the way the server folds them", + `ALTER TABLE "MySchema"."Transfer" ADD CONSTRAINT "fk_block" FOREIGN KEY (a) REFERENCES b(c)`, + true, "myschema.transfer", "fk_block", + }, + { + // Nothing to skip on: the statement gets executed and the server decides. + "not a named constraint", + `CREATE TABLE IF NOT EXISTS myschema.transfer (id text)`, + false, "", "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + key, found := constraintTarget(test.statement) + + assert.Equal(t, test.expectFound, found) + assert.Equal(t, test.expectRelate, key.relation) + assert.Equal(t, test.expectName, key.name) + }) + } +} + +// TestConstraintTargetDistinguishesTables is the case that made keying by name alone a +// bug: the same constraint name on two tables has to be two different constraints. +func TestConstraintTargetDistinguishesTables(t *testing.T) { + customers, _ := constraintTarget(`ALTER TABLE myschema.customers ADD CONSTRAINT fk_block FOREIGN KEY (a) REFERENCES b(c)`) + orders, _ := constraintTarget(`ALTER TABLE myschema.orders ADD CONSTRAINT fk_block FOREIGN KEY (a) REFERENCES b(c)`) + + assert.NotEqual(t, customers, orders) + assert.Equal(t, customers.name, orders.name) +} diff --git a/sink/sql/db_proto/sql/postgres/dialect.go b/sink/sql/db_proto/sql/postgres/dialect.go index 736f1a74d..ee6cd4b18 100644 --- a/sink/sql/db_proto/sql/postgres/dialect.go +++ b/sink/sql/db_proto/sql/postgres/dialect.go @@ -116,7 +116,7 @@ func (d *DialectPostgres) createTable(table *schema.Table) error { ForeignField: parentField.Name, } - d.AddForeignKeySql(table.Name, foreignKey.String()) + d.AddForeignKeyReferencing(table.Name, parentTable.Name, foreignKey.String()) fieldFound = true break @@ -155,7 +155,7 @@ func (d *DialectPostgres) createTable(table *schema.Table) error { ForeignTable: d.FullTableName(childTable), ForeignField: childTable.PrimaryKey.Name, } - d.AddForeignKeySql(table.Name, foreignKey.String()) + d.AddForeignKeyReferencing(table.Name, childTable.Name, foreignKey.String()) case f.ForeignKey != nil: foreignTable, found := d.TableRegistry[f.ForeignKey.Table] @@ -181,7 +181,7 @@ func (d *DialectPostgres) createTable(table *schema.Table) error { ForeignTable: d.FullTableName(foreignTable), ForeignField: foreignField.Name, } - d.AddForeignKeySql(table.Name, foreignKey.String()) + d.AddForeignKeyReferencing(table.Name, foreignTable.Name, foreignKey.String()) } fieldType := MapFieldType(f.FieldDescriptor, d.bytesEncoding, f) if f.IsUnique { @@ -200,7 +200,18 @@ func (d *DialectPostgres) createTable(table *schema.Table) error { sb.WriteString(");\n") - d.AddForeignKeySql(tableName, fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT fk_block FOREIGN KEY (%s) REFERENCES %s.%s(number) ON DELETE CASCADE", tableName, sql2.DialectFieldBlockNumber, d.schemaName, sql2.DialectTableBlock)) + d.AddForeignKeyReferencing(table.Name, sql2.DialectTableBlock, fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT fk_block FOREIGN KEY (%s) REFERENCES %s.%s(number) ON DELETE CASCADE", tableName, sql2.DialectFieldBlockNumber, d.schemaName, sql2.DialectTableBlock)) + + // A foreign key indexes its referenced side only, so _block_number_ would carry none. + // Every undo deletes from every table by that column, which without this is a + // sequential scan per table, and a cascade would be worse still: PostgreSQL looks the + // child rows up once per deleted parent row. + // + // CONCURRENTLY because this is built when the sink starts rather than when the load is + // over: a restart onto an already-loaded table must not lock out the writers. It is + // also why the index cannot be part of the constraint pass, which runs in transactions + // and a concurrent build cannot. + d.AddIndexSql(table.Name, fmt.Sprintf("CREATE INDEX CONCURRENTLY IF NOT EXISTS %s_block_number_idx ON %s (%s)", table.Name, tableName, sql2.DialectFieldBlockNumber)) d.AddCreateTableSql(table.Name, sb.String()) return nil diff --git a/sink/sql/db_proto/sql/postgres/local_buffer_inserter.go b/sink/sql/db_proto/sql/postgres/local_buffer_inserter.go new file mode 100644 index 000000000..1822ea0d2 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/local_buffer_inserter.go @@ -0,0 +1,120 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "go.uber.org/zap" +) + +// spoolInserter routes rows into the on-disk spool instead of into the database. It is +// unrelated to sql.BufferedInserter, which only records a walk's inserts in memory for +// replay. +// +// It sits behind the same pgInserter/pgFlusher interfaces as the accumulator, so the +// sinker is unchanged: what differs is that a "flush" only seals a segment, and the +// database is written to later, by the spool's own goroutine. That is the whole point — +// the stream stops waiting on PostgreSQL. +type localBufferInserter struct { + buffer *spool.Spool + // applier is kept so the switch to direct inserts can clear the bookkeeping it owns + // once the spool it belongs to has been drained. + applier *pgApplier + logger *zap.Logger +} + +func newLocalBufferInserter(ctx context.Context, database *Database, format spool.Format, options spool.Options, logger *zap.Logger) (*localBufferInserter, error) { + copyRanks, err := database.dialect.TableApplyRanks() + if err != nil { + // A cyclic foreign key graph has no table order, which is exactly the schema + // row-insert mode exists for: its segments are an interleaved log replayed in walk + // order, so nothing needs ordering by table. + if format != spool.FormatRowLog { + return nil, err + } + copyRanks = nil + } + + applier := newPGApplier(database.pool, database.schema.Name, copyRanks, logger) + if err := applier.EnsureSchema(ctx); err != nil { + return nil, err + } + + tables, err := loadColumnLayouts(ctx, database) + if err != nil { + return nil, err + } + + codec := newPGCodec(format, tables, database.dialect.bytesEncoding) + + buf, err := spool.New(ctx, options, codec, applier, database.schema.Name, logger) + if err != nil { + return nil, err + } + + return &localBufferInserter{buffer: buf, applier: applier, logger: logger.Named("spool_inserter")}, nil +} + +// loadColumnLayouts resolves every table's column order and type OIDs from the live +// catalog. Binary COPY does no coercion, so these must be the server's own rather than +// anything derived from the declared type names. +func loadColumnLayouts(ctx context.Context, database *Database) (map[string]*pgcopy.Table, error) { + layouts := map[string]*pgcopy.Table{} + + names := []string{"_blocks_"} + for _, table := range database.dialect.GetTables() { + names = append(names, table.Name) + } + + for _, name := range names { + // Resolve through the same reference the dialect writes into its DDL, so the + // server applies its own identifier folding rather than us guessing at it. + resolved, err := pgcopy.ResolveTable(ctx, database.pool, tableName(database.schema.Name, name)) + if err != nil { + return nil, fmt.Errorf("resolving the column layout of %q: %w", name, err) + } + layouts[name] = resolved + } + + return layouts, nil +} + +func (i *localBufferInserter) insert(table string, values []any, database *Database) error { + switch table { + case "_cursor_": + // The cursor is not a row to buffer: it is what makes a segment resumable, and + // it is written by the applier in the same transaction as the segment. + cursor, ok := values[1].(string) + if !ok { + return fmt.Errorf("expected a string cursor, got %T", values[1]) + } + i.buffer.RecordCursor(cursor) + + // Sealing happens here rather than at flush, because the cursor is the last thing a + // flush writes: sealing before it would close the segment on the cursor the previous + // flush stored, leaving it to claim a range its own cursor does not cover. A restart + // then resumes behind rows the segment record says are applied. + return i.buffer.MaybeSeal(context.Background()) + + case "_blocks_": + blockNum, ok := values[0].(uint64) + if !ok { + return fmt.Errorf("expected a uint64 block number, got %T", values[0]) + } + i.buffer.RecordBlock(blockNum) + } + + return i.buffer.Insert(table, values) +} + +// flush has nothing to do: a spooled row is already on disk, and the segment is sealed +// when the cursor covering it is recorded, which happens after this. +func (i *localBufferInserter) flush(database *Database) error { + return nil +} + +func (i *localBufferInserter) close(ctx context.Context) error { + return i.buffer.Close(ctx) +} diff --git a/sink/sql/db_proto/sql/postgres/pgapplier.go b/sink/sql/db_proto/sql/postgres/pgapplier.go new file mode 100644 index 000000000..c0076371d --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgapplier.go @@ -0,0 +1,270 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "go.uber.org/zap" +) + +// Applier loads sealed segments into PostgreSQL. +// +// One segment is one transaction: every table file is COPYed, the segment is recorded, +// and the cursor is advanced, all together. Nothing can therefore be half-applied, which +// is what lets recovery decide a segment's fate by looking it up in one table. +type pgApplier struct { + pool *pgxpool.Pool + schema string + logger *zap.Logger + + // copyRanks orders the table files within a segment: a referenced table has to be + // COPYed before the tables referencing it, or a foreign key rejects a child row that + // reaches the server ahead of its parent. It comes from the dialect rather than the + // manifest so that segments written by an earlier build still apply correctly. + copyRanks map[string]int + + // applied is the segments table read once, on the first recovery question: first block + // to last block of every segment the database already holds. + applied map[uint64]uint64 +} + +func newPGApplier(pool *pgxpool.Pool, schema string, copyRanks map[string]int, logger *zap.Logger) *pgApplier { + return &pgApplier{pool: pool, schema: schema, copyRanks: copyRanks, logger: logger.Named("spool_applier")} +} + +// copyOrder returns the segment's tables in the order they must be loaded. A table the +// dialect does not know about keeps its manifest position, after the ones it does. +func (a *pgApplier) copyOrder(tables []spool.TableRecord) []spool.TableRecord { + ordered := make([]spool.TableRecord, len(tables)) + copy(ordered, tables) + + rank := func(table spool.TableRecord) int { + if at, found := a.copyRanks[table.Name]; found { + return at + } + + return len(a.copyRanks) + } + + sort.SliceStable(ordered, func(i, j int) bool { + return rank(ordered[i]) < rank(ordered[j]) + }) + + return ordered +} + +// EnsureSchema creates the bookkeeping table recovery relies on. +func (a *pgApplier) EnsureSchema(ctx context.Context) error { + statement := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + first_block BIGINT NOT NULL, + last_block BIGINT NOT NULL, + cursor TEXT NOT NULL, + applied_at TIMESTAMP NOT NULL DEFAULT now(), + PRIMARY KEY (first_block) + )`, a.segmentsTable()) + + if _, err := a.pool.Exec(ctx, statement); err != nil { + return fmt.Errorf("creating the applied-segments table: %w", err) + } + + return a.dropSegmentsPastCursor(ctx) +} + +// dropSegmentsPastCursor removes the segment records the stored cursor does not cover. +// +// The cursor is what a restart resumes from, and Run undoes every row above it before a +// block arrives — so a record reaching past it describes rows that are no longer there. +// Left in place it answers AlreadyApplied for the segment carrying exactly those rows, +// which recovery then discards while replaying the segments behind it, moving the cursor +// over a gap it just created. +// +// Segments sealed before the cursor covering them was recorded are how such a record came +// about, which no longer happens; this also clears the ones a database synced by an earlier +// build is still carrying. +func (a *pgApplier) dropSegmentsPastCursor(ctx context.Context) error { + var stored string + err := a.pool.QueryRow(ctx, fmt.Sprintf(`SELECT cursor FROM %s WHERE name = 'cursor'`, + pgx.Identifier{a.schema, "_cursor_"}.Sanitize())).Scan(&stored) + if err != nil { + // No cursor yet means nothing has been applied, so there is nothing to contradict. + return nil + } + + cursor, err := sink.NewCursor(stored) + if err != nil || cursor.IsBlank() { + //nolint:nilerr // an unreadable cursor is not this pass's to report + return nil + } + + tag, err := a.pool.Exec(ctx, fmt.Sprintf(`DELETE FROM %s WHERE last_block > $1`, a.segmentsTable()), + cursor.Block().Num()) + if err != nil { + return fmt.Errorf("dropping the segment records past the stored cursor: %w", err) + } + + if tag.RowsAffected() > 0 { + a.logger.Info("dropped segment records the stored cursor does not cover", + zap.Int64("dropped", tag.RowsAffected()), zap.Uint64("cursor_block", cursor.Block().Num())) + } + + return nil +} + +func (a *pgApplier) segmentsTable() string { + return pgx.Identifier{a.schema, "_segments_"}.Sanitize() +} + +// Apply loads one segment. The COPYs run sequentially because a transaction is bound to +// one connection; parallelism, if it is ever needed, belongs between segments. +func (a *pgApplier) Apply(ctx context.Context, dir string, manifest *spool.Manifest) error { + tx, err := a.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck // a rollback after commit is a no-op + + switch manifest.Format { + case spool.FormatTuples: + if err := a.applyTuples(ctx, tx, dir, manifest); err != nil { + return err + } + + case spool.FormatRowLog: + if err := a.applyRowLog(ctx, tx, dir, manifest); err != nil { + return err + } + + default: + for _, table := range a.copyOrder(manifest.Tables) { + if err := a.copyTable(ctx, tx, dir, table); err != nil { + return err + } + } + } + + // A cursor-only segment has no rows to replay, so recovery has nothing to decide about + // it and re-applying it would only store the same cursor again. Recording it would + // also collide on first_block, which every such segment leaves at zero. + if !manifest.CursorOnly() { + if _, err := tx.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (first_block, last_block, cursor) VALUES ($1, $2, $3) + ON CONFLICT (first_block) DO UPDATE SET last_block = $2, cursor = $3, applied_at = now()`, + a.segmentsTable(), + ), manifest.FirstBlock, manifest.LastBlock, manifest.Cursor); err != nil { + return fmt.Errorf("recording the applied segment: %w", err) + } + } + + if _, err := tx.Exec(ctx, fmt.Sprintf( + `INSERT INTO %s (name, cursor) VALUES ('cursor', $1) + ON CONFLICT (name) DO UPDATE SET cursor = $1`, + pgx.Identifier{a.schema, "_cursor_"}.Sanitize(), + ), manifest.Cursor); err != nil { + return fmt.Errorf("storing the cursor: %w", err) + } + + return tx.Commit(ctx) +} + +// copyTable streams one pre-encoded file straight into the server. Because the bytes on +// disk are already in the binary COPY wire format, this is a copy from file to socket: +// no encoding, no escaping, no parsing. +func (a *pgApplier) copyTable(ctx context.Context, tx pgx.Tx, dir string, table spool.TableRecord) error { + path := filepath.Join(dir, table.File) + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer file.Close() + + columns := make([]pgcopy.Column, len(table.Columns)) + for i, name := range table.Columns { + columns[i] = pgcopy.Column{Name: name} + } + + statement := pgcopy.CopySQL(table.Schema, table.Relation, columns) + + tag, err := tx.Conn().PgConn().CopyFrom(ctx, file, statement) + if err != nil { + return fmt.Errorf("copying %s into %q: %w", table.File, table.Name, err) + } + if tag.RowsAffected() != table.Rows { + return fmt.Errorf("copied %d rows into %q but the manifest recorded %d", + tag.RowsAffected(), table.Name, table.Rows) + } + + return nil +} + +// AlreadyApplied answers from the segments table, which the applier writes in the same +// transaction as the rows: with transactions available, a segment either landed whole or +// not at all, so this is exact rather than a best guess. +func (a *pgApplier) AlreadyApplied(ctx context.Context, manifest *spool.Manifest) (bool, error) { + if manifest.CursorOnly() { + // Nothing to replay and nothing recorded, so replaying only stores the same + // cursor again. + return false, nil + } + + if a.applied == nil { + applied, err := a.appliedSegments(ctx) + if err != nil { + return false, err + } + a.applied = applied + } + + // Both ends have to match. Segments are keyed by their first block, but the block a + // segment ends on is whatever the sizer settled on at the time, so a range re-streamed + // after its segment was discarded comes back starting at the same block and ending + // somewhere else. Answering from the first block alone would call that one applied and + // drop every row it carries. + lastBlock, found := a.applied[manifest.FirstBlock] + + return found && lastBlock == manifest.LastBlock, nil +} + +// appliedSegments returns the block range of every segment already in the database, so +// recovery can tell what still needs replaying. +func (a *pgApplier) appliedSegments(ctx context.Context) (map[uint64]uint64, error) { + rows, err := a.pool.Query(ctx, fmt.Sprintf(`SELECT first_block, last_block FROM %s`, a.segmentsTable())) + if err != nil { + return nil, fmt.Errorf("listing applied segments: %w", err) + } + defer rows.Close() + + applied := map[uint64]uint64{} + for rows.Next() { + var firstBlock, lastBlock uint64 + if err := rows.Scan(&firstBlock, &lastBlock); err != nil { + return nil, fmt.Errorf("scanning an applied segment: %w", err) + } + applied[firstBlock] = lastBlock + } + + return applied, rows.Err() +} + +// clearSegments empties the bookkeeping table. +// +// It is only ever called once the spool has been drained and closed, which leaves no +// segment on disk for a record to answer for: the records exist to tell recovery whether a +// directory it found was already applied, and there are no directories left. +func (a *pgApplier) clearSegments(ctx context.Context) error { + if _, err := a.pool.Exec(ctx, fmt.Sprintf(`DELETE FROM %s`, a.segmentsTable())); err != nil { + return fmt.Errorf("clearing the applied-segments table: %w", err) + } + a.applied = nil + + return nil +} diff --git a/sink/sql/db_proto/sql/postgres/pgapplier_rendered.go b/sink/sql/db_proto/sql/postgres/pgapplier_rendered.go new file mode 100644 index 000000000..fb8d9328c --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgapplier_rendered.go @@ -0,0 +1,173 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" +) + +// maxStatementBytes caps one generated INSERT. The values are rendered literals rather +// than bind parameters, so the 65535 parameter limit does not apply and the real ceiling +// is what the server will parse in one go. A few megabytes keeps the parse cost per row +// negligible without building a statement the server has to hold whole. +const maxStatementBytes = 4 << 20 + +// applyTuples loads a FormatTuples segment: one multi-row INSERT per table, chunked, in +// the same foreign key order the COPY path uses. +func (a *pgApplier) applyTuples(ctx context.Context, tx pgx.Tx, dir string, manifest *spool.Manifest) error { + for _, table := range a.copyOrder(manifest.Tables) { + if table.File == "" { + continue + } + + if err := a.insertTupleFile(ctx, tx, filepath.Join(dir, table.File), table); err != nil { + return err + } + } + + return nil +} + +func (a *pgApplier) insertTupleFile(ctx context.Context, tx pgx.Tx, path string, table spool.TableRecord) error { + reader, err := spool.OpenFrameReader(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer reader.Close() + + prefix := insertPrefix(table) + + var ( + statement strings.Builder + batched int + applied int64 + ) + flush := func() error { + if batched == 0 { + return nil + } + if _, err := tx.Exec(ctx, statement.String()); err != nil { + return fmt.Errorf("inserting %d rows into %q: %w", batched, table.Name, err) + } + statement.Reset() + batched = 0 + + return nil + } + + for { + tuple, err := reader.ReadField() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + if batched > 0 && statement.Len()+len(tuple)+3 > maxStatementBytes { + if err := flush(); err != nil { + return err + } + } + if batched == 0 { + statement.WriteString(prefix) + } else { + statement.WriteString(",") + } + statement.WriteString("(") + statement.WriteString(tuple) + statement.WriteString(")") + + batched++ + applied++ + } + + if err := flush(); err != nil { + return err + } + + if applied != table.Rows { + return fmt.Errorf("inserted %d rows into %q but the manifest recorded %d", applied, table.Name, table.Rows) + } + + return nil +} + +// applyRowLog loads a FormatRowLog segment, replaying the walk's own order. +// +// Rows are not grouped by table here, and that is the point: this format exists for a +// schema whose foreign keys form a cycle, where no table order can keep a parent ahead of +// its children. The walk always does, so its order is what gets replayed — one statement +// per row, which is what makes it the slowest mode and the fallback rather than a choice. +func (a *pgApplier) applyRowLog(ctx context.Context, tx pgx.Tx, dir string, manifest *spool.Manifest) error { + if manifest.LogFile == "" { + return nil + } + + path := filepath.Join(dir, manifest.LogFile) + reader, err := spool.OpenFrameReader(path) + if err != nil { + return fmt.Errorf("opening %s: %w", path, err) + } + defer reader.Close() + + prefixes := make(map[string]string, len(manifest.Tables)) + expected := make(map[string]int64, len(manifest.Tables)) + for _, table := range manifest.Tables { + prefixes[table.Name] = insertPrefix(table) + expected[table.Name] = table.Rows + } + + applied := make(map[string]int64, len(manifest.Tables)) + for { + table, err := reader.ReadField() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + tuple, err := reader.ReadField() + if err != nil { + return fmt.Errorf("reading the row of %q in %s: %w", table, path, err) + } + + prefix, found := prefixes[table] + if !found { + return fmt.Errorf("%s holds a row of %q, which the manifest does not describe", path, table) + } + + if _, err := tx.Exec(ctx, prefix+"("+tuple+")"); err != nil { + return fmt.Errorf("inserting a row into %q: %w", table, err) + } + applied[table]++ + } + + for name, want := range expected { + if applied[name] != want { + return fmt.Errorf("inserted %d rows into %q but the manifest recorded %d", applied[name], name, want) + } + } + + return nil +} + +// insertPrefix builds `INSERT INTO schema.relation (cols) VALUES `, quoting through the +// identifiers the server itself reported so the statement matches what was created. +func insertPrefix(table spool.TableRecord) string { + columns := make([]string, len(table.Columns)) + for i, name := range table.Columns { + columns[i] = pgx.Identifier{name}.Sanitize() + } + + return fmt.Sprintf("INSERT INTO %s (%s) VALUES ", + pgx.Identifier{table.Schema, table.Relation}.Sanitize(), + strings.Join(columns, ", ")) +} diff --git a/sink/sql/db_proto/sql/postgres/pgcodec.go b/sink/sql/db_proto/sql/postgres/pgcodec.go new file mode 100644 index 000000000..feef27d38 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcodec.go @@ -0,0 +1,334 @@ +package postgres + +import ( + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/streamingfast/substreams/sink/sql/bytes" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" +) + +// pgCodec lays a segment out the way the chosen write mode can send it unchanged. +// +// Three formats, one per mode: binary COPY files, rendered SQL tuples per table, and a +// single interleaved log in walk order. The rendering the accumulator used to do at flush +// time happens here instead, off the database's critical path. +type pgCodec struct { + format spool.Format + tables map[string]*pgcopy.Table + encoding bytes.Encoding +} + +func newPGCodec(format spool.Format, tables map[string]*pgcopy.Table, encoding bytes.Encoding) *pgCodec { + return &pgCodec{format: format, tables: tables, encoding: encoding} +} + +func (c *pgCodec) Format() spool.Format { return c.format } + +func (c *pgCodec) OpenSegment(dir string) (spool.SegmentWriter, error) { + return &pgSegment{dir: dir, codec: c, tables: map[string]*pgTableFile{}}, nil +} + +// Verify checks each data file against what the manifest recorded. The rendered formats +// are length-framed, so the size check is the whole of it. Binary COPY is not, which is +// why it also gets its trailer checked. +func (c *pgCodec) Verify(dir string, manifest *spool.Manifest) error { + if manifest.Format == spool.FormatRowLog { + if manifest.LogFile == "" { + return nil + } + + return verifySize(filepath.Join(dir, manifest.LogFile), manifest.LogBytes) + } + + for _, table := range manifest.Tables { + path := filepath.Join(dir, table.File) + + if err := verifySize(path, table.Bytes); err != nil { + return err + } + + if manifest.Format == spool.FormatPGCopy { + if err := verifyTrailer(path); err != nil { + return err + } + } + } + + return nil +} + +func verifySize(path string, expected int64) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("%s is missing: %w", filepath.Base(path), err) + } + if info.Size() != expected { + return fmt.Errorf("%s is %d bytes, the manifest recorded %d", filepath.Base(path), info.Size(), expected) + } + + return nil +} + +// verifyTrailer checks the two bytes that terminate a binary COPY stream. +func verifyTrailer(path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return err + } + if info.Size() < int64(pgcopy.HeaderSize+pgcopy.TrailerSize) { + return fmt.Errorf("%s is too short to be a pgcopy stream", filepath.Base(path)) + } + + trailer := make([]byte, pgcopy.TrailerSize) + if _, err := file.ReadAt(trailer, info.Size()-int64(pgcopy.TrailerSize)); err != nil { + return fmt.Errorf("reading the trailer of %s: %w", filepath.Base(path), err) + } + if trailer[0] != 0xFF || trailer[1] != 0xFF { + return fmt.Errorf("%s does not end with a pgcopy trailer", filepath.Base(path)) + } + + return nil +} + +// rowWriter is one table's open stream, whichever format it is in. Closing one closes the +// file it was writing to: the segment hands the stream its file and never touches it +// again, so nothing else is left to release it. +type rowWriter interface { + WriteRow(values []any) error + Rows() int64 + Bytes() int64 + Close() error +} + +// pgCopyWriter closes the file under a binary COPY stream. +// +// pgcopy.Writer.Close only writes the trailer and flushes, deliberately leaving the +// underlying writer to whoever opened it — which here is the segment, and a segment is +// sealed and forgotten. Without this every sealed segment leaks one descriptor per table, +// and the applier's os.RemoveAll then frees no space while they are held. +type pgCopyWriter struct { + *pgcopy.Writer + file *os.File +} + +func (w *pgCopyWriter) Close() error { + err := w.Writer.Close() + if closeErr := w.file.Close(); err == nil { + err = closeErr + } + + return err +} + +// pgTableFile is one table's stream within a segment. Under FormatRowLog there is no +// per-table stream and writer stays nil: the record carries only the column layout a +// replay needs, and the row counter. +type pgTableFile struct { + target *pgcopy.Table + path string + file *os.File + writer rowWriter + rows int64 +} + +type pgSegment struct { + dir string + codec *pgCodec + tables map[string]*pgTableFile + + // log is the single interleaved file of FormatRowLog, shared by every table. + log *spool.FrameWriter + logPath string + values []string +} + +func (s *pgSegment) WriteRow(table string, values []any) error { + target, ok := s.tables[table] + if !ok { + layout, known := s.codec.tables[table] + if !known { + return fmt.Errorf("no column layout known for table %q", table) + } + + created, err := s.openTable(table, layout) + if err != nil { + return err + } + target = created + } + + if s.codec.format == spool.FormatPGCopy { + // Binary COPY does no coercion, so the values have to match the column types the + // server reported exactly. This is value normalization, including materializing + // the configured protobuf-bytes representation for text columns; it is not SQL + // rendering. The rendered formats go through the dialect instead. + if err := pgcopy.NormalizeRowWithEncoding(target.target.Columns, values, s.codec.encoding); err != nil { + return fmt.Errorf("normalizing a row of %q: %w", table, err) + } + } + + target.rows++ + + if s.codec.format == spool.FormatRowLog { + tuple := s.renderTuple(values) + + return s.log.WriteRecord(table, tuple) + } + + return target.writer.WriteRow(values) +} + +func (s *pgSegment) renderTuple(values []any) string { + s.values = s.values[:0] + for _, value := range values { + s.values = append(s.values, s.codec.render(value)) + } + + return strings.Join(s.values, ",") +} + +func (c *pgCodec) render(value any) string { return ValueToString(value, c.encoding) } + +func (s *pgSegment) openTable(table string, layout *pgcopy.Table) (*pgTableFile, error) { + target := &pgTableFile{target: layout} + + switch s.codec.format { + case spool.FormatRowLog: + if s.log == nil { + s.logPath = filepath.Join(s.dir, "rows.log") + file, err := os.Create(s.logPath) + if err != nil { + return nil, fmt.Errorf("creating %s: %w", s.logPath, err) + } + s.log = spool.NewFrameWriter(file) + } + + case spool.FormatTuples: + path := filepath.Join(s.dir, spool.SanitizeFileName(table)+".tuples") + file, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("creating %s: %w", path, err) + } + target.path, target.file = path, file + target.writer = &pgTupleWriter{frames: spool.NewFrameWriter(file), segment: s} + + default: + path := filepath.Join(s.dir, spool.SanitizeFileName(table)+".pgcopy") + file, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("creating %s: %w", path, err) + } + + writer, err := pgcopy.NewWriter(file, layout.Columns) + if err != nil { + file.Close() + return nil, fmt.Errorf("starting pgcopy stream for %q: %w", table, err) + } + target.path, target.file, target.writer = path, file, &pgCopyWriter{Writer: writer, file: file} + } + + s.tables[table] = target + + return target, nil +} + +func (s *pgSegment) PendingBytes() int64 { + if s.log != nil { + return s.log.Bytes() + } + + var total int64 + for _, target := range s.tables { + if target.writer != nil { + total += target.writer.Bytes() + } + } + + return total +} + +func (s *pgSegment) Seal(manifest *spool.Manifest) error { + if s.log != nil { + if err := s.log.Close(); err != nil { + return fmt.Errorf("closing %s: %w", s.logPath, err) + } + + info, err := os.Stat(s.logPath) + if err != nil { + return fmt.Errorf("sizing %s: %w", s.logPath, err) + } + manifest.LogFile = filepath.Base(s.logPath) + manifest.LogBytes = info.Size() + } + + for _, name := range slices.Sorted(maps.Keys(s.tables)) { + target := s.tables[name] + + record := spool.TableRecord{ + Name: name, + Schema: target.target.Schema, + Relation: target.target.Name, + Rows: target.rows, + } + + record.Columns = make([]string, len(target.target.Columns)) + for i, column := range target.target.Columns { + record.Columns[i] = column.Name + } + + if target.writer != nil { + if err := target.writer.Close(); err != nil { + return fmt.Errorf("closing the stream of %q: %w", name, err) + } + + info, err := os.Stat(target.path) + if err != nil { + return fmt.Errorf("sizing %s: %w", target.path, err) + } + record.File = filepath.Base(target.path) + record.Bytes = info.Size() + } + + manifest.Tables = append(manifest.Tables, record) + } + + return nil +} + +func (s *pgSegment) Discard() { + for _, target := range s.tables { + if target.file != nil { + target.file.Close() + } + } + if s.log != nil { + s.log.Close() + } + os.RemoveAll(s.dir) +} + +// pgTupleWriter renders a row into one framed record. +type pgTupleWriter struct { + frames *spool.FrameWriter + segment *pgSegment +} + +func (w *pgTupleWriter) WriteRow(values []any) error { + return w.frames.WriteRecord(w.segment.renderTuple(values)) +} + +func (w *pgTupleWriter) Rows() int64 { return w.frames.Rows() } +func (w *pgTupleWriter) Bytes() int64 { return w.frames.Bytes() } +func (w *pgTupleWriter) Close() error { return w.frames.Close() } diff --git a/sink/sql/db_proto/sql/postgres/pgcodec_bytes_test.go b/sink/sql/db_proto/sql/postgres/pgcodec_bytes_test.go new file mode 100644 index 000000000..76bba1dc9 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcodec_bytes_test.go @@ -0,0 +1,62 @@ +package postgres + +import ( + "encoding/binary" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + sqlbytes "github.com/streamingfast/substreams/sink/sql/bytes" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "github.com/stretchr/testify/require" +) + +func TestPGSegmentCopyUsesConfiguredBytesEncoding(t *testing.T) { + const tableName = "payloads" + + codec := newPGCodec( + spool.FormatPGCopy, + map[string]*pgcopy.Table{ + tableName: { + Schema: "public", + Name: tableName, + Columns: []pgcopy.Column{ + {Name: "payload", OID: pgtype.TextOID}, + }, + }, + }, + sqlbytes.EncodingHex, + ) + segment := &pgSegment{ + dir: t.TempDir(), + codec: codec, + tables: map[string]*pgTableFile{}, + } + + require.NoError(t, segment.WriteRow(tableName, []any{[]byte{0xde, 0xad, 0xbe, 0xef}})) + + table := segment.tables[tableName] + require.NoError(t, table.writer.Close()) + encoded, err := os.ReadFile(table.path) + require.NoError(t, err) + + fieldCountOffset := pgcopy.HeaderSize + require.Equal(t, int16(1), int16(binary.BigEndian.Uint16(encoded[fieldCountOffset:]))) + + fieldLengthOffset := fieldCountOffset + 2 + fieldLength := int(binary.BigEndian.Uint32(encoded[fieldLengthOffset:])) + fieldOffset := fieldLengthOffset + 4 + require.Equal(t, "deadbeef", string(encoded[fieldOffset:fieldOffset+fieldLength])) +} + +func TestPGCopyNormalizationUsesConfiguredBytesEncodingForArrays(t *testing.T) { + values := []any{[]any{ + []byte{0xde, 0xad, 0xbe, 0xef}, + []byte{0x00, 0xff}, + }} + columns := []pgcopy.Column{{Name: "payloads", OID: pgtype.TextArrayOID}} + + require.NoError(t, pgcopy.NormalizeRowWithEncoding(columns, values, sqlbytes.EncodingHex)) + require.Equal(t, []string{"deadbeef", "00ff"}, values[0]) +} diff --git a/sink/sql/db_proto/sql/postgres/pgcodec_close_test.go b/sink/sql/db_proto/sql/postgres/pgcodec_close_test.go new file mode 100644 index 000000000..5bf321b77 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcodec_close_test.go @@ -0,0 +1,49 @@ +package postgres + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgtype" + sqlbytes "github.com/streamingfast/substreams/sink/sql/bytes" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/postgres/pgcopy" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + "github.com/stretchr/testify/require" +) + +// TestSealClosesEveryStreamFile covers the descriptors a sealed segment must not keep. +// +// A segment is sealed and forgotten, so whatever it opened has to be released by Seal. A +// leak here is invisible until a long backfill runs out of descriptors, and the applier's +// os.RemoveAll frees no space while they are held. +func TestSealClosesEveryStreamFile(t *testing.T) { + const tableName = "payloads" + + for _, format := range []spool.Format{spool.FormatPGCopy, spool.FormatTuples} { + t.Run(string(format), func(t *testing.T) { + codec := newPGCodec( + format, + map[string]*pgcopy.Table{ + tableName: { + Schema: "public", + Name: tableName, + Columns: []pgcopy.Column{ + {Name: "payload", OID: pgtype.TextOID}, + }, + }, + }, + sqlbytes.EncodingHex, + ) + segment := &pgSegment{ + dir: t.TempDir(), + codec: codec, + tables: map[string]*pgTableFile{}, + } + + require.NoError(t, segment.WriteRow(tableName, []any{[]byte{0xde, 0xad}})) + require.NoError(t, segment.Seal(&spool.Manifest{})) + + // Closing an already closed file is what says the seal released it. + require.Error(t, segment.tables[tableName].file.Close()) + }) + } +} diff --git a/sink/sql/db_proto/sql/postgres/pgcopy/normalize.go b/sink/sql/db_proto/sql/postgres/pgcopy/normalize.go new file mode 100644 index 000000000..1ebb05bc8 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcopy/normalize.go @@ -0,0 +1,248 @@ +package pgcopy + +import ( + "fmt" + "math/big" + "time" + + "github.com/jackc/pgx/v5/pgtype" + sqlbytes "github.com/streamingfast/substreams/sink/sql/bytes" + sql2 "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Normalize converts a value produced by the protobuf walker into something the +// pgtype binary encoder can write for the given column OID. +// +// Binary COPY does no coercion, so the conversions that the text path gets for +// free have to happen here. In particular an uint64 must not be sent as an int8 +// to a NUMERIC column: it would be rejected outright, and values above 2^63 would +// be wrong even if it were not. +func Normalize(oid uint32, value any) (any, error) { + return normalize(oid, value, sqlbytes.EncodingRaw) +} + +// NormalizeWithEncoding converts a value for binary COPY while applying the configured +// protobuf-bytes representation. Non-raw bytes columns are text columns, so their payload +// must be encoded before the COPY encoder sees it. +func NormalizeWithEncoding(oid uint32, value any, encoding sqlbytes.Encoding) (any, error) { + return normalize(oid, value, encoding) +} + +func normalize(oid uint32, value any, encoding sqlbytes.Encoding) (any, error) { + switch v := value.(type) { + case nil: + return nil, nil + + case uint64: + if oid == pgtype.NumericOID { + return numericFromBigInt(new(big.Int).SetUint64(v)), nil + } + return v, nil + + case uint32: + if oid == pgtype.NumericOID { + return numericFromBigInt(new(big.Int).SetUint64(uint64(v))), nil + } + return v, nil + + case uint: + if oid == pgtype.NumericOID { + return numericFromBigInt(new(big.Int).SetUint64(uint64(v))), nil + } + return v, nil + + case *big.Int: + return numericFromBigInt(v), nil + + case *timestamppb.Timestamp: + if v == nil { + return nil, nil + } + return v.AsTime().UTC(), nil + + case time.Time: + return v.UTC(), nil + + case string: + // A NUMERIC column fed from a proto string field (the int128/uint256/decimal + // conversions) arrives here as text. Empty means "no value" upstream, which the + // row inserter turns into 0; keep that behaviour rather than writing NULL. + if oid == pgtype.NumericOID { + var n pgtype.Numeric + if v == "" { + v = "0" + } + if err := n.Scan(v); err != nil { + return nil, fmt.Errorf("parsing %q as numeric: %w", v, err) + } + return n, nil + } + return v, nil + + case []byte: + encoded, err := encoding.EncodeBytes(v) + if err != nil { + return nil, fmt.Errorf("encoding bytes: %w", err) + } + return encoded, nil + + case []any: + return normalizeSlice(oid, v, encoding) + + default: + return value, nil + } +} + +// NormalizeRow applies [Normalize] to every value of a row, in place. +func NormalizeRow(cols []Column, values []any) error { + return NormalizeRowWithEncoding(cols, values, sqlbytes.EncodingRaw) +} + +// NormalizeRowWithEncoding applies NormalizeWithEncoding to every value of a row, in place. +func NormalizeRowWithEncoding(cols []Column, values []any, encoding sqlbytes.Encoding) error { + if len(cols) != len(values) { + return fmt.Errorf("expected %d values, got %d", len(cols), len(values)) + } + + for i := range values { + normalized, err := NormalizeWithEncoding(cols[i].OID, values[i], encoding) + if err != nil { + return fmt.Errorf("column %q: %w", cols[i].Name, err) + } + values[i] = normalized + } + + return nil +} + +// normalizeSlice turns the walker's []any array into a concretely typed slice, which +// the pgtype array codec can then encode against the array's element OID. +func normalizeSlice(oid uint32, in []any, encoding sqlbytes.Encoding) (any, error) { + if len(in) == 0 { + // An empty array carries no element to infer a type from, and pgtype resolves the + // encode plan from the Go type rather than from the column: []string reaches no + // plan at all on a numeric[], bigint[], bytea[], bool[] or timestamp[] column, and + // the row is refused. The walker's own []any encodes against every array OID, so + // the right thing to do with it is nothing. + // + // Nothing is rendered here, unlike the element cases below: with no element there + // is no value whose representation could drift from what the rendered write modes + // produce, only a plan that has to resolve. + return in, nil + } + + switch in[0].(type) { + case string: + if elementOID(oid) == pgtype.NumericOID { + out := make([]any, len(in)) + for i, v := range in { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("mixed element types in array: %T and string", v) + } + normalized, err := normalize(pgtype.NumericOID, s, encoding) + if err != nil { + return nil, fmt.Errorf("normalizing array element %d: %w", i, err) + } + out[i] = normalized + } + return out, nil + } + + out := make([]string, len(in)) + for i, v := range in { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("mixed element types in array: %T and string", v) + } + out[i] = s + } + return out, nil + + case []byte: + if encoding.IsStringType() { + out := make([]string, len(in)) + for i, v := range in { + b, ok := v.([]byte) + if !ok { + return nil, fmt.Errorf("mixed element types in array: %T and []byte", v) + } + encoded, err := encoding.EncodeBytes(b) + if err != nil { + return nil, fmt.Errorf("encoding array element %d: %w", i, err) + } + out[i] = encoded.(string) + } + return out, nil + } + + out := make([][]byte, len(in)) + for i, v := range in { + b, ok := v.([]byte) + if !ok { + return nil, fmt.Errorf("mixed element types in array: %T and []byte", v) + } + out[i] = b + } + return out, nil + + case sql2.EnumValue: + // The dialect declares an enum column TEXT, so a repeated enum is TEXT[]. + // + // Handing the values over as they are would encode identically today — pgtype picks + // the Stringer, which is what ValueToString renders with too. What it would not do + // is keep the two agreeing: pgtype chooses between TextValuer, driver.Valuer and + // fmt.Stringer in that order, so the day EnumValue gains any of the earlier ones, + // binary COPY starts writing something the rendered write modes do not, and nothing + // fails. Rendering here is what keeps that choice ours. + out := make([]string, len(in)) + for i, v := range in { + enum, ok := v.(sql2.EnumValue) + if !ok { + return nil, fmt.Errorf("mixed element types in array: %T and sql.EnumValue", v) + } + out[i] = enum.String() + } + + return out, nil + + case int32, int64, uint32, uint64, float32, float64, bool: + out := make([]any, len(in)) + for i, v := range in { + normalized, err := Normalize(elementOID(oid), v) + if err != nil { + return nil, err + } + out[i] = normalized + } + return out, nil + + default: + return nil, fmt.Errorf("unsupported array element type %T", in[0]) + } +} + +// elementOID maps the well-known array OIDs back to their element OID, so that +// numeric-typed elements get the same treatment as a scalar column would. +func elementOID(arrayOID uint32) uint32 { + switch arrayOID { + case pgtype.NumericArrayOID: + return pgtype.NumericOID + case pgtype.Int8ArrayOID: + return pgtype.Int8OID + case pgtype.Int4ArrayOID: + return pgtype.Int4OID + case pgtype.TextArrayOID: + return pgtype.TextOID + case pgtype.ByteaArrayOID: + return pgtype.ByteaOID + default: + return arrayOID + } +} + +func numericFromBigInt(v *big.Int) pgtype.Numeric { + return pgtype.Numeric{Int: v, Exp: 0, Valid: true} +} diff --git a/sink/sql/db_proto/sql/postgres/pgcopy/normalize_test.go b/sink/sql/db_proto/sql/postgres/pgcopy/normalize_test.go new file mode 100644 index 000000000..dae2ca99f --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcopy/normalize_test.go @@ -0,0 +1,88 @@ +package pgcopy + +import ( + "io" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + sqltypes "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/stretchr/testify/require" +) + +func TestNormalizeNumericStringArrayForCopy(t *testing.T) { + values := []any{[]any{ + "123456789012345678901234567890", + "", + }} + columns := []Column{{Name: "amounts", OID: pgtype.NumericArrayOID}} + + require.NoError(t, NormalizeRow(columns, values)) + + expectedLarge, err := numericFromString("123456789012345678901234567890") + require.NoError(t, err) + expectedZero, err := numericFromString("0") + require.NoError(t, err) + + normalized, ok := values[0].([]any) + require.True(t, ok, "expected numeric array elements, got %T", values[0]) + require.Equal(t, []any{expectedLarge, expectedZero}, normalized) +} + +// TestNormalizeEmptyArrayEncodesForEveryArrayType pins the empty-array shape against the +// real encoder rather than against an expected Go type: what matters is that pgx finds a +// binary plan for it, which is exactly what a wrongly typed empty slice fails to do. +func TestNormalizeEmptyArrayEncodesForEveryArrayType(t *testing.T) { + // Every array type the from-proto dialect can declare for a repeated field. + oids := []uint32{ + pgtype.NumericArrayOID, + pgtype.Int8ArrayOID, + pgtype.Int4ArrayOID, + pgtype.Int2ArrayOID, + pgtype.Float8ArrayOID, + pgtype.Float4ArrayOID, + pgtype.BoolArrayOID, + pgtype.ByteaArrayOID, + pgtype.TextArrayOID, + pgtype.VarcharArrayOID, + pgtype.TimestampArrayOID, + pgtype.TimestamptzArrayOID, + pgtype.DateArrayOID, + pgtype.JSONBArrayOID, + } + + for _, oid := range oids { + values := []any{[]any{}} + columns := []Column{{Name: "values", OID: oid}} + require.NoError(t, NormalizeRow(columns, values), "oid %d", oid) + + writer, err := NewWriter(io.Discard, columns) + require.NoError(t, err) + require.NoError(t, writer.WriteRow(values), "oid %d", oid) + require.NoError(t, writer.Close()) + } +} + +func TestNormalizeEnumArrayForCopy(t *testing.T) { + values := []any{[]any{ + sqltypes.EnumValue{Number: 1, Name: "TRANSFER"}, + sqltypes.EnumValue{Number: 7}, + }} + columns := []Column{{Name: "kinds", OID: pgtype.TextArrayOID}} + + require.NoError(t, NormalizeRow(columns, values)) + require.Equal(t, []string{"TRANSFER", "7"}, values[0]) + + writer, err := NewWriter(io.Discard, columns) + require.NoError(t, err) + require.NoError(t, writer.WriteRow(values)) + require.NoError(t, writer.Close()) +} + +func numericFromString(value string) (pgtype.Numeric, error) { + var numeric pgtype.Numeric + if err := numeric.Scan(value); err != nil { + return pgtype.Numeric{}, err + } + + return numeric, nil +} diff --git a/sink/sql/db_proto/sql/postgres/pgcopy/writer.go b/sink/sql/db_proto/sql/postgres/pgcopy/writer.go new file mode 100644 index 000000000..4960f1702 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/pgcopy/writer.go @@ -0,0 +1,271 @@ +// Package pgcopy writes the PostgreSQL binary COPY format ("PGCOPY"). +// +// The point of writing this format directly is that a file holding it can be +// handed to `COPY ... FROM STDIN (FORMAT BINARY)` as raw bytes: the flush becomes +// an io.Copy from file to socket, with no re-encoding, no escaping and no parsing. +// +// Binary COPY performs no type coercion whatsoever: the bytes for a column must +// match that column's type OID exactly or the server aborts the COPY. Always +// resolve OIDs from the live catalog with [LoadColumns] rather than deriving them +// from a declared type name. +package pgcopy + +import ( + "bufio" + "context" + "encoding/binary" + "fmt" + "io" + "reflect" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// signature is the fixed 11-byte PGCOPY header signature. +var signature = []byte{'P', 'G', 'C', 'O', 'P', 'Y', '\n', 0xFF, '\r', '\n', 0x00} + +// HeaderSize is the size of the file header: 11-byte signature, flags and header +// extension length. +const HeaderSize = 11 + 4 + 4 + +// TrailerSize is the size of the file trailer, an int16 holding -1. +const TrailerSize = 2 + +// Column is a target column with the type OID the server expects for it. +type Column struct { + Name string + OID uint32 +} + +// Querier is the subset of pgx used to read the catalog, satisfied by *pgx.Conn, +// *pgxpool.Pool and pgx.Tx. +type Querier interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) +} + +// LoadColumns returns the columns of schema.table in attribute order, with the +// type OID the server will require during a binary COPY. +func LoadColumns(ctx context.Context, q Querier, schema, table string) ([]Column, error) { + resolved, err := ResolveTable(ctx, q, pgx.Identifier{schema, table}.Sanitize()) + if err != nil { + return nil, err + } + + return resolved.Columns, nil +} + +// Table is a table as the server actually holds it: the identifier it stored, and the +// column layout a binary COPY has to match. +type Table struct { + Schema string + Name string + Columns []Column +} + +// ResolveTable looks a table up by the same reference the dialect writes into its DDL, +// letting the server apply its own parsing rules. +// +// This matters because the from-proto dialect emits table names unquoted, so a message +// called BalanceChange becomes the relation balancechange. Querying pg_class for the +// name as written finds nothing, and a COPY that quotes it as written targets a relation +// that does not exist. Going through to_regclass and reading the stored name back avoids +// having to reimplement identifier folding. +func ResolveTable(ctx context.Context, q Querier, reference string) (*Table, error) { + const query = ` + SELECT n.nspname, c.relname, a.attname, a.atttypid + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE a.attrelid = to_regclass($1) AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum` + + rows, err := q.Query(ctx, query, reference) + if err != nil { + return nil, fmt.Errorf("querying columns of %s: %w", reference, err) + } + defer rows.Close() + + out := &Table{} + for rows.Next() { + var col Column + if err := rows.Scan(&out.Schema, &out.Name, &col.Name, &col.OID); err != nil { + return nil, fmt.Errorf("scanning column of %s: %w", reference, err) + } + out.Columns = append(out.Columns, col) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating columns of %s: %w", reference, err) + } + if len(out.Columns) == 0 { + return nil, fmt.Errorf("table %s has no columns, does it exist?", reference) + } + + return out, nil +} + +// CopySQL builds the statement to feed a stream written by [Writer] into the server. +func CopySQL(schema, table string, cols []Column) string { + names := make([]string, len(cols)) + for i, col := range cols { + names[i] = pgx.Identifier{col.Name}.Sanitize() + } + + return fmt.Sprintf("COPY %s (%s) FROM STDIN (FORMAT BINARY)", + pgx.Identifier{schema, table}.Sanitize(), + joinComma(names), + ) +} + +func joinComma(in []string) string { + out := "" + for i, s := range in { + if i > 0 { + out += ", " + } + out += s + } + return out +} + +// Writer encodes rows into the binary COPY format. It is not safe for concurrent use. +type Writer struct { + out *bufio.Writer + cols []Column + types *pgtype.Map + scratch []byte + rows int64 + bytes int64 + closed bool + + // Map.Encode resolves an encode plan on every call, which dominates the per-value + // cost once the column type is known. Rows in a COPY stream are homogeneous, so the + // plan is resolved once per column and reused, re-resolving only if a later row + // presents a different Go type for that column. + plans []pgtype.EncodePlan + planTypes []reflect.Type +} + +// NewWriter wraps w and writes the file header. The caller must call [Writer.Close] +// to emit the trailer, without which the server rejects the stream. +func NewWriter(w io.Writer, cols []Column) (*Writer, error) { + if len(cols) == 0 { + return nil, fmt.Errorf("at least one column is required") + } + if len(cols) > 1<<15-1 { + return nil, fmt.Errorf("too many columns: %d", len(cols)) + } + + buffered := bufio.NewWriterSize(w, 256*1024) + + header := make([]byte, 0, HeaderSize) + header = append(header, signature...) + header = binary.BigEndian.AppendUint32(header, 0) // flags: no OIDs + header = binary.BigEndian.AppendUint32(header, 0) // header extension area length + if _, err := buffered.Write(header); err != nil { + return nil, fmt.Errorf("writing header: %w", err) + } + + return &Writer{ + out: buffered, + cols: cols, + types: pgtype.NewMap(), + scratch: make([]byte, 0, 4096), + plans: make([]pgtype.EncodePlan, len(cols)), + planTypes: make([]reflect.Type, len(cols)), + }, nil +} + +// WriteRow encodes one tuple. values must be positional against the columns given +// to [NewWriter]; a nil value is written as SQL NULL. +func (w *Writer) WriteRow(values []any) error { + if w.closed { + return fmt.Errorf("writer is closed") + } + if len(values) != len(w.cols) { + return fmt.Errorf("expected %d values, got %d", len(w.cols), len(values)) + } + + w.scratch = binary.BigEndian.AppendUint16(w.scratch[:0], uint16(len(w.cols))) + + for i, value := range values { + lengthAt := len(w.scratch) + w.scratch = binary.BigEndian.AppendUint32(w.scratch, 0) + + if value == nil { + binary.BigEndian.PutUint32(w.scratch[lengthAt:], 0xFFFFFFFF) + continue + } + + plan, err := w.planFor(i, value) + if err != nil { + return err + } + + encoded, err := plan.Encode(value, w.scratch) + if err != nil { + return fmt.Errorf("encoding column %q (oid %d) from %T: %w", w.cols[i].Name, w.cols[i].OID, value, err) + } + + if encoded == nil { + // SQL NULL is a length of -1 and no payload. + binary.BigEndian.PutUint32(w.scratch[lengthAt:], 0xFFFFFFFF) + continue + } + + w.scratch = encoded + binary.BigEndian.PutUint32(w.scratch[lengthAt:], uint32(len(w.scratch)-lengthAt-4)) + } + + if _, err := w.out.Write(w.scratch); err != nil { + return fmt.Errorf("writing tuple: %w", err) + } + w.rows++ + w.bytes += int64(len(w.scratch)) + + return nil +} + +// planFor returns the cached encode plan for a column, resolving it on first use and +// whenever the Go type presented for that column changes. +func (w *Writer) planFor(column int, value any) (pgtype.EncodePlan, error) { + valueType := reflect.TypeOf(value) + if w.plans[column] != nil && w.planTypes[column] == valueType { + return w.plans[column], nil + } + + plan := w.types.PlanEncode(w.cols[column].OID, pgtype.BinaryFormatCode, value) + if plan == nil { + return nil, fmt.Errorf("no binary encoder for column %q (oid %d) from %T", + w.cols[column].Name, w.cols[column].OID, value) + } + + w.plans[column] = plan + w.planTypes[column] = valueType + + return plan, nil +} + +// Rows returns how many tuples have been written so far. +func (w *Writer) Rows() int64 { return w.rows } + +// Bytes returns how many tuple bytes have been written, excluding header and trailer. +// It is a counter rather than a file size so a caller can poll it cheaply. +func (w *Writer) Bytes() int64 { return w.bytes } + +// Close writes the trailer and flushes. It does not close the underlying writer. +func (w *Writer) Close() error { + if w.closed { + return nil + } + w.closed = true + + if _, err := w.out.Write([]byte{0xFF, 0xFF}); err != nil { + return fmt.Errorf("writing trailer: %w", err) + } + if err := w.out.Flush(); err != nil { + return fmt.Errorf("flushing: %w", err) + } + + return nil +} diff --git a/sink/sql/db_proto/sql/postgres/types.go b/sink/sql/db_proto/sql/postgres/types.go index 00d0e002e..36df24532 100644 --- a/sink/sql/db_proto/sql/postgres/types.go +++ b/sink/sql/db_proto/sql/postgres/types.go @@ -122,8 +122,15 @@ func MapFieldType(fd protoreflect.FieldDescriptor, bytesEncoding bytes.Encoding, return baseType } +// quoteLiteral renders a Go string as a PostgreSQL string literal. +// +// Only the quote is doubled. A backslash carries no meaning inside a standard string +// literal, and standard_conforming_strings has been on by default since PostgreSQL 9.1, so +// escaping it stored two backslashes where the value had one. That also made the write +// mode visible in the data: the COPY path hands pgtype the string as it is, and only the +// rendered paths came through here. func quoteLiteral(v string) string { - return "'" + strings.ReplaceAll(strings.ReplaceAll(v, "'", "''"), `\`, `\\`) + "'" + return "'" + strings.ReplaceAll(v, "'", "''") + "'" } func ValueToString(value any, bytesEncoding bytes.Encoding) (s string) { @@ -182,7 +189,7 @@ func ValueToString(value any, bytesEncoding bytes.Encoding) (s string) { if err != nil { panic(fmt.Sprintf("failed to marshal protobuf message to JSON: %v", err)) } - s = "'" + strings.ReplaceAll(strings.ReplaceAll(string(jsonBytes), "'", "''"), "\\", "\\\\") + "'" + s = quoteLiteral(string(jsonBytes)) return default: if msg, ok := v.(protoreflect.ProtoMessage); ok { @@ -190,7 +197,7 @@ func ValueToString(value any, bytesEncoding bytes.Encoding) (s string) { if err != nil { panic(fmt.Sprintf("failed to marshal protobuf message to JSON: %v", err)) } - s = "'" + strings.ReplaceAll(strings.ReplaceAll(string(jsonBytes), "'", "''"), "\\", "\\\\") + "'" + s = quoteLiteral(string(jsonBytes)) return } panic(fmt.Sprintf("unsupported type: %T", v)) diff --git a/sink/sql/db_proto/sql/postgres/types_test.go b/sink/sql/db_proto/sql/postgres/types_test.go new file mode 100644 index 000000000..1871260a2 --- /dev/null +++ b/sink/sql/db_proto/sql/postgres/types_test.go @@ -0,0 +1,41 @@ +package postgres + +import ( + "testing" + + "github.com/streamingfast/substreams/sink/sql/bytes" + sql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/stretchr/testify/require" +) + +// TestValueToStringStoresWhatCopyWouldStore pins the rendered write modes against the +// binary COPY one. --write-mode picks between them for throughput, so a value that +// survives one path and not the other makes the same substream produce different rows +// depending on a performance flag. +// +// The literals below are what the server stores under standard_conforming_strings = on, +// the default since PostgreSQL 9.1, which is also what binary COPY writes verbatim. +func TestValueToStringStoresWhatCopyWouldStore(t *testing.T) { + cases := []struct { + name string + in string + expected string + }{ + {name: "plain", in: "hello", expected: "'hello'"}, + {name: "a quote is doubled", in: "it's", expected: "'it''s'"}, + {name: "a backslash is stored as one", in: `C:\path`, expected: `'C:\path'`}, + {name: "an escape sequence is not one", in: `a\nb`, expected: `'a\nb'`}, + {name: "both at once", in: `a\'b`, expected: `'a\''b'`}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, c.expected, ValueToString(c.in, bytes.EncodingRaw)) + }) + } +} + +func TestValueToStringRendersAnEnumByName(t *testing.T) { + require.Equal(t, "'TRANSFER'", ValueToString(sql.EnumValue{Number: 1, Name: "TRANSFER"}, bytes.EncodingRaw)) + require.Equal(t, "'7'", ValueToString(sql.EnumValue{Number: 7}, bytes.EncodingRaw)) +} diff --git a/sink/sql/db_proto/sql/row_id_test.go b/sink/sql/db_proto/sql/row_id_test.go new file mode 100644 index 000000000..411e22544 --- /dev/null +++ b/sink/sql/db_proto/sql/row_id_test.go @@ -0,0 +1,135 @@ +package sql + +import ( + "testing" + "time" + + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" +) + +// TestWalkNumbersRowsPerBlock checks the counter the ClickHouse sorting key relies on: +// every row a block writes to a table gets the next number, starting over at zero for the +// next block. Without it the ReplacingMergeTree collapses a block's rows into one. +func TestWalkNumbersRowsPerBlock(t *testing.T) { + message := &pbsubstreams.Modules{ + Modules: []*pbsubstreams.Module{ + {Name: "map_one"}, + {Name: "map_two"}, + {Name: "map_three"}, + }, + } + + inserter := &recordingInserter{} + database := databaseFor(t, message.ProtoReflect().Descriptor()) + + dynamic := dynamicFor(t, message) + _, err := database.WalkMessageDescriptorAndInsertWithDialect(dynamic, 100, time.Unix(0, 0), nil, rowIDDialect{}, inserter) + require.NoError(t, err) + + // Row 0 is the Modules message itself, then one per module. + assert.Equal(t, []uint32{0}, inserter.rowIDs["Modules"]) + assert.Equal(t, []uint32{0, 1, 2}, inserter.rowIDs["Module"]) + + // A second block starts its own numbering rather than continuing the first one's. + inserter.rowIDs = nil + _, err = database.WalkMessageDescriptorAndInsertWithDialect(dynamic, 101, time.Unix(0, 0), nil, rowIDDialect{}, inserter) + require.NoError(t, err) + + assert.Equal(t, []uint32{0, 1, 2}, inserter.rowIDs["Module"]) +} + +// TestWalkOmitsRowIDWhenDialectDeclinesIt is the PostgreSQL side: nothing extra is +// appended, so the column count the inserters expect is unchanged. +func TestWalkOmitsRowIDWhenDialectDeclinesIt(t *testing.T) { + message := &pbsubstreams.Modules{Modules: []*pbsubstreams.Module{{Name: "map_one"}}} + + inserter := &recordingInserter{} + database := databaseFor(t, message.ProtoReflect().Descriptor()) + + _, err := database.WalkMessageDescriptorAndInsertWithDialect(dynamicFor(t, message), 100, time.Unix(0, 0), nil, plainDialect{}, inserter) + require.NoError(t, err) + + for table, rows := range inserter.values { + for _, values := range rows { + // blockNum, blockTimestamp and nothing else injected. + require.GreaterOrEqual(t, len(values), 2, "table %q", table) + if len(values) < 3 { + continue + } + _, isRowID := values[2].(uint32) + assert.False(t, isRowID, "table %q got a row id", table) + } + } +} + +func databaseFor(t *testing.T, descriptor protoreflect.MessageDescriptor) *BaseDatabase { + t.Helper() + + database, err := NewBaseDatabase("test", descriptor, false, zap.NewNop()) + require.NoError(t, err) + + return database +} + +func dynamicFor(t *testing.T, message proto.Message) protoreflect.Message { + t.Helper() + + encoded, err := proto.Marshal(message) + require.NoError(t, err) + + dynamic := dynamicpb.NewMessage(message.ProtoReflect().Descriptor()) + require.NoError(t, proto.Unmarshal(encoded, dynamic)) + + return dynamic +} + +type recordingInserter struct { + rowIDs map[string][]uint32 + values map[string][][]any +} + +func (i *recordingInserter) Insert(table string, values []any) error { + if i.values == nil { + i.values = map[string][][]any{} + } + i.values[table] = append(i.values[table], values) + + if len(values) < 3 { + return nil + } + + if rowID, ok := values[2].(uint32); ok { + if i.rowIDs == nil { + i.rowIDs = map[string][]uint32{} + } + i.rowIDs[table] = append(i.rowIDs[table], rowID) + } + + return nil +} + +// rowIDDialect keeps every message it is asked about, with the row id column on, and no +// version or deleted column so the id sits at a fixed index. +type rowIDDialect struct{ plainDialect } + +func (rowIDDialect) UseRowIDField(string) bool { return true } + +type plainDialect struct{} + +func (plainDialect) SchemaHash() string { return "" } +func (plainDialect) FullTableName(table *schema.Table) string { return table.Name } +func (plainDialect) GetTable(table string) *schema.Table { return &schema.Table{Name: table} } +func (plainDialect) GetTables() []*schema.Table { return nil } +func (plainDialect) UseVersionField() bool { return false } +func (plainDialect) UseDeletedField() bool { return false } +func (plainDialect) UseRowIDField(string) bool { return false } +func (plainDialect) AppendInlineFieldValues(fieldValues []any, fd protoreflect.FieldDescriptor, fv protoreflect.Value, dm protoreflect.Message) ([]any, error) { + return fieldValues, nil +} diff --git a/sink/sql/db_proto/sql/spool/codec.go b/sink/sql/db_proto/sql/spool/codec.go new file mode 100644 index 000000000..3061c0978 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/codec.go @@ -0,0 +1,80 @@ +package spool + +import "context" + +// Format names an on-disk layout. It is stored in the manifest so a segment written by an +// earlier run is read back the way it was written. +type Format string + +const ( + // FormatPGCopy is one file per table in PostgreSQL's binary COPY wire format. It is + // also what a segment written before formats existed carries, hence the zero value. + FormatPGCopy Format = "" + + // FormatTuples is one file per table of rendered SQL value tuples, which the applier + // wraps into multi-row INSERTs. Tables are applied in foreign key order. + FormatTuples Format = "tuples" + + // FormatRowLog is a single interleaved file of (table, rendered tuple) in walk order. + // + // It exists because grouping rows by table is exactly what row-insert mode cannot do. + // That mode is the fallback for a schema whose foreign keys form a cycle, and a cycle + // has no table order that keeps a parent ahead of its children — only the walk does, + // so only the walk's own order can be replayed. + FormatRowLog Format = "rowlog" + + // FormatValues is one file per table of typed row values, appended straight back into + // the driver's own column builders at apply time. ClickHouse inserts columnar and + // typed rather than as SQL text, so rendering to literals would change both the insert + // path and how types are handled. + FormatValues Format = "values" +) + +// Codec owns what a segment looks like on disk. +type Codec interface { + // Format is what gets recorded in the manifest. + Format() Format + + // OpenSegment prepares the writers for a new segment in dir, which already exists. + OpenSegment(dir string) (SegmentWriter, error) + + // Verify checks a sealed segment's files against its manifest. Only the manifest is + // fsynced on seal, so a machine crash can leave a data file short and this is what + // catches it before anything reaches the database. + Verify(dir string, manifest *Manifest) error +} + +// SegmentWriter accumulates one segment's rows. +type SegmentWriter interface { + // WriteRow appends one row of the named table. + WriteRow(table string, values []any) error + + // PendingBytes is how much has been written so far, from the writers' own counters + // rather than a stat() per call. It is what the sizer compares against. + PendingBytes() int64 + + // Seal closes every stream and fills in the manifest's file records. + Seal(manifest *Manifest) error + + // Discard closes and abandons a segment that will never be applied. + Discard() +} + +// Applier sends sealed segments to the database. +type Applier interface { + // EnsureSchema creates whatever bookkeeping the applier needs, once, before any + // segment is written. + EnsureSchema(ctx context.Context) error + + // AlreadyApplied reports a segment recovery found on disk that the database already + // holds, so it can be dropped rather than replayed. + // + // A driver with transactions answers this from its own record of applied segments. One + // without answers it from how far the stored cursor got: replaying a segment it had in + // fact applied duplicates exactly the rows re-streaming it would have duplicated, which + // is the guarantee such a driver already ships with. + AlreadyApplied(ctx context.Context, manifest *Manifest) (bool, error) + + // Apply loads one segment and advances the cursor with it. + Apply(ctx context.Context, dir string, manifest *Manifest) error +} diff --git a/sink/sql/db_proto/sql/spool/frame.go b/sink/sql/db_proto/sql/spool/frame.go new file mode 100644 index 000000000..22bec9962 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/frame.go @@ -0,0 +1,118 @@ +package spool + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "math" + "os" +) + +// FrameWriter writes length-prefixed records. +// +// Framing rather than lines because a rendered tuple carries SQL literals, and a text +// column holding a newline would end a line in the middle of a row. The length check that +// recovery does against the manifest then covers a torn write, the same way the binary +// COPY trailer does for FormatPGCopy. +type FrameWriter struct { + file *os.File + writer *bufio.Writer + rows int64 + written int64 + header [4]byte +} + +func NewFrameWriter(file *os.File) *FrameWriter { + return &FrameWriter{file: file, writer: bufio.NewWriterSize(file, 1<<20)} +} + +// WriteRecord appends one record made of the given fields, each length-prefixed. +func (w *FrameWriter) WriteRecord(fields ...string) error { + for _, field := range fields { + // The prefix is four bytes, so a longer field would be written with a truncated + // length and read back as a shorter one — a segment that looks intact and is not. + if int64(len(field)) > math.MaxUint32 { + return fmt.Errorf("a field of %d bytes cannot be framed, the length prefix holds at most %d", len(field), int64(math.MaxUint32)) + } + + binary.BigEndian.PutUint32(w.header[:], uint32(len(field))) + if _, err := w.writer.Write(w.header[:]); err != nil { + return err + } + if _, err := w.writer.WriteString(field); err != nil { + return err + } + w.written += int64(len(w.header)) + int64(len(field)) + } + w.rows++ + + return nil +} + +func (w *FrameWriter) Rows() int64 { return w.rows } +func (w *FrameWriter) Bytes() int64 { return w.written } + +func (w *FrameWriter) Close() error { + if err := w.writer.Flush(); err != nil { + return err + } + + return w.file.Close() +} + +// FrameReader reads back what FrameWriter produced. +type FrameReader struct { + file *os.File + reader *bufio.Reader + header [4]byte + + // remaining is what is left of the file, and is what bounds a record's declared + // length. A fixed ceiling here used to reject rows the writer had accepted, which no + // restart could get past: the segment verified, failed to apply, and was replayed + // again on the next start. + remaining int64 +} + +func OpenFrameReader(path string) (*FrameReader, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + info, err := file.Stat() + if err != nil { + file.Close() + return nil, fmt.Errorf("sizing %s: %w", path, err) + } + + return &FrameReader{file: file, reader: bufio.NewReaderSize(file, 1<<20), remaining: info.Size()}, nil +} + +func (r *FrameReader) Close() error { return r.file.Close() } + +// ReadField returns the next field, or io.EOF once the file is exhausted. +func (r *FrameReader) ReadField() (string, error) { + if _, err := io.ReadFull(r.reader, r.header[:]); err != nil { + return "", err + } + + r.remaining -= int64(len(r.header)) + + // A record cannot be longer than what is left of the file holding it. That is what + // catches a corrupt length before it allocates, and it is the whole of the bound: a + // record the writer produced always fits, and the row it came from was already held + // whole in memory to be written. + size := binary.BigEndian.Uint32(r.header[:]) + if int64(size) > r.remaining { + return "", fmt.Errorf("a record of %d bytes does not fit in the %d bytes left of the file", size, r.remaining) + } + + field := make([]byte, size) + if _, err := io.ReadFull(r.reader, field); err != nil { + return "", fmt.Errorf("reading a %d byte record: %w", size, err) + } + r.remaining -= int64(size) + + return string(field), nil +} diff --git a/sink/sql/db_proto/sql/spool/frame_test.go b/sink/sql/db_proto/sql/spool/frame_test.go new file mode 100644 index 000000000..6c9446069 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/frame_test.go @@ -0,0 +1,80 @@ +package spool + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestFrameRoundTripsARecordLargerThanAnyFixedCeiling covers the row a rendered write mode +// can produce and a fixed read ceiling used to refuse. +// +// The writer accepted it, the manifest matched the bytes on disk so Verify passed, and the +// read failed only at apply — after which recovery replayed the same segment on every +// start and the sink could not come up at all. +func TestFrameRoundTripsARecordLargerThanAnyFixedCeiling(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.tuples") + file, err := os.Create(path) + require.NoError(t, err) + + // Comfortably past the 64MiB ceiling this used to carry. + field := strings.Repeat("x", (64<<20)+1) + + writer := NewFrameWriter(file) + require.NoError(t, writer.WriteRecord(field)) + require.NoError(t, writer.Close()) + + reader, err := OpenFrameReader(path) + require.NoError(t, err) + defer reader.Close() + + read, err := reader.ReadField() + require.NoError(t, err) + require.Equal(t, len(field), len(read)) +} + +// TestFrameRejectsALengthTheFileCannotHold is what replaced the fixed ceiling: a corrupt +// prefix still cannot make the reader allocate, because no record can be longer than what +// is left of the file carrying it. +func TestFrameRejectsALengthTheFileCannotHold(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.tuples") + // A prefix claiming ~4GiB in front of three bytes of payload. + require.NoError(t, os.WriteFile(path, []byte{0xFF, 0xFF, 0xFF, 0xF0, 'a', 'b', 'c'}, 0o600)) + + reader, err := OpenFrameReader(path) + require.NoError(t, err) + defer reader.Close() + + _, err = reader.ReadField() + require.ErrorContains(t, err, "does not fit in the 3 bytes left of the file") +} + +// TestFrameReadsSeveralRecordsBack pins that the running bound does not drift, the reader +// having to subtract both the prefix and the payload of every record it hands out. +func TestFrameReadsSeveralRecordsBack(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.log") + file, err := os.Create(path) + require.NoError(t, err) + + writer := NewFrameWriter(file) + require.NoError(t, writer.WriteRecord("customers", "1,'alpha'")) + require.NoError(t, writer.WriteRecord("orders", "2,'beta'")) + require.NoError(t, writer.Close()) + + reader, err := OpenFrameReader(path) + require.NoError(t, err) + defer reader.Close() + + var got []string + for { + field, err := reader.ReadField() + if err != nil { + break + } + got = append(got, field) + } + require.Equal(t, []string{"customers", "1,'alpha'", "orders", "2,'beta'"}, got) +} diff --git a/sink/sql/db_proto/sql/spool/manifest.go b/sink/sql/db_proto/sql/spool/manifest.go new file mode 100644 index 000000000..269c2ab62 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/manifest.go @@ -0,0 +1,120 @@ +// Package spool holds a from-proto sink's rows on local disk, pre-encoded into whatever +// the target database can take unchanged, and applies whole segments from a separate +// goroutine. +// +// The point is not raw throughput — pre-encoding is worth about 8% on its own, see +// sink/sql/db_proto/benchmarks. It is that the stream stops waiting for the database. +// Substreams throughput is paid for, so a slow or stalled database should cost disk, not +// download progress, and blocks already paid for should survive a restart rather than be +// streamed again. +// +// What the bytes on disk look like, and how a sealed segment reaches the server, are the +// driver's business: this package owns the segment lifecycle, the disk budget and the +// sizing, and delegates the rest through Codec and Applier. +package spool + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +const manifestFileName = "manifest.json" + +// Manifest is a segment's commit record. Its presence, parseable and sealed, is what +// makes a segment eligible to be applied; anything else is a torn write from a crash. +type Manifest struct { + FirstBlock uint64 `json:"first_block"` + LastBlock uint64 `json:"last_block"` + Cursor string `json:"cursor"` + Tables []TableRecord `json:"tables"` + Sealed bool `json:"sealed"` + + // Format says how the rows are laid out. Absent means FormatPGCopy, which is what + // every segment written before formats existed carries. + Format Format `json:"format,omitempty"` + // LogFile and LogBytes describe the single interleaved file of FormatRowLog. The + // table records still carry the column layout, since replaying a row needs it. + LogFile string `json:"log_file,omitempty"` + LogBytes int64 `json:"log_bytes,omitempty"` +} + +// BlockCount is how many blocks the segment covers. A segment carrying nothing but a +// cursor — every block in the flush produced no output — covers none. +func (m *Manifest) BlockCount() int64 { + if m.FirstBlock == 0 || m.LastBlock < m.FirstBlock { + return 0 + } + + return int64(m.LastBlock-m.FirstBlock) + 1 +} + +// CursorOnly reports a segment that carries no rows. It exists so that a stretch of +// blocks whose module output is empty still advances the cursor, rather than being +// streamed, and paid for, again on restart. +func (m *Manifest) CursorOnly() bool { return len(m.Tables) == 0 } + +// TableRecord describes one table's PGCOPY file inside a segment. +// +// Schema and Name are the identifiers the server actually stored, not the logical table +// name the walk uses: the dialect writes DDL unquoted, so a message called BalanceChange +// lives in the relation balancechange, and a COPY has to say so. +type TableRecord struct { + Name string `json:"name"` + Schema string `json:"schema"` + Relation string `json:"relation"` + File string `json:"file"` + Columns []string `json:"columns"` + Rows int64 `json:"rows"` + Bytes int64 `json:"bytes"` +} + +// WriteManifest is exported for a codec that needs to re-seal a segment. +func WriteManifest(dir string, manifest *Manifest) error { + encoded, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("encoding manifest: %w", err) + } + + path := filepath.Join(dir, manifestFileName) + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + defer file.Close() + + if _, err := file.Write(encoded); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + + // The manifest is the commit record, so it is the one thing worth the fsync. + return file.Sync() +} + +func readManifest(dir string) (*Manifest, error) { + encoded, err := os.ReadFile(filepath.Join(dir, manifestFileName)) + if err != nil { + return nil, err + } + + var manifest Manifest + if err := json.Unmarshal(encoded, &manifest); err != nil { + return nil, fmt.Errorf("decoding manifest in %s: %w", dir, err) + } + + return &manifest, nil +} + +// SanitizeFileName keeps a table name usable as a path component. +func SanitizeFileName(name string) string { + return strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '-': + return r + default: + return '_' + } + }, name) +} diff --git a/sink/sql/db_proto/sql/spool/recover.go b/sink/sql/db_proto/sql/spool/recover.go new file mode 100644 index 000000000..959653ec1 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/recover.go @@ -0,0 +1,119 @@ +package spool + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "go.uber.org/zap" +) + +// recover decides the fate of everything already on disk, before a single new row is +// written. +// +// A segment is replayed only if it is sealed, intact, and not already recorded in the +// database. Anything else is removed: a torn write from a crash, a segment applied but +// not yet deleted, or anything sitting past a hole left by a lost segment. Whatever is +// discarded is re-streamed from the cursor, which costs Substreams throughput but can +// never corrupt. +func (b *Spool) recover(ctx context.Context) error { + entries, err := os.ReadDir(b.options.Dir) + if err != nil { + return fmt.Errorf("listing %s: %w", b.options.Dir, err) + } + + dirs := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "seg-") { + dirs = append(dirs, entry.Name()) + } + } + if len(dirs) == 0 { + return nil + } + slices.Sort(dirs) + + var ( + replayed int + discarded int + holeFound bool + ) + + for _, name := range dirs { + dir := filepath.Join(b.options.Dir, name) + + // Segment names carry a sequence number, so the highest one seen tells the next + // run where to continue numbering. + if sequence := parseSequence(name); sequence > b.nextSequence { + b.nextSequence = sequence + } + + if holeFound { + os.RemoveAll(dir) + discarded++ + continue + } + + manifest, err := readManifest(dir) + if err != nil || !manifest.Sealed { + // Never sealed: the process died mid-segment. Its blocks are re-streamed. + b.logger.Info("discarding an unsealed spool segment", zap.String("dir", dir)) + os.RemoveAll(dir) + discarded++ + holeFound = true + continue + } + + already, err := b.applier.AlreadyApplied(ctx, manifest) + if err != nil { + return err + } + if already { + // Already in the database, and the process died before the directory was + // removed. + os.RemoveAll(dir) + continue + } + + if err := b.codec.Verify(dir, manifest); err != nil { + b.logger.Warn("discarding a truncated spool segment, its blocks will be streamed again", + zap.String("dir", dir), zap.Error(err)) + os.RemoveAll(dir) + discarded++ + holeFound = true + continue + } + + if err := b.applier.Apply(ctx, dir, manifest); err != nil { + // Left to fail, because a segment the database refuses says something is wrong + // with the rows or the schema, and dropping it quietly would hide that. It + // does mean the sink cannot start until the cause is dealt with, so the way + // out has to be said out loud rather than worked out from a stack trace. + return fmt.Errorf("replaying segment %s: %w. The sink cannot start while it is there. "+ + "Deleting that directory drops the %d block(s) it holds, which are then streamed again from the stored cursor", + dir, err, manifest.BlockCount()) + } + os.RemoveAll(dir) + replayed++ + } + + if replayed > 0 || discarded > 0 { + b.logger.Info("recovered the local spool", + zap.Int("segments_replayed", replayed), + zap.Int("segments_discarded", discarded)) + } + + return nil +} + +func parseSequence(name string) uint64 { + var sequence uint64 + if _, err := fmt.Sscanf(name, "seg-%d", &sequence); err != nil { + return 0 + } + + return sequence +} diff --git a/sink/sql/db_proto/sql/spool/sizer.go b/sink/sql/db_proto/sql/spool/sizer.go new file mode 100644 index 000000000..35f38f89e --- /dev/null +++ b/sink/sql/db_proto/sql/spool/sizer.go @@ -0,0 +1,71 @@ +package spool + +import ( + "sync" + "time" +) + +// segmentFloorBytes is the smallest segment the sizer may choose. It is a constant rather +// than a flag because it guards the controller, not a policy: a database stalled on lock +// contention returns a long elapsed time, the loop halves the segment each round, and +// without a floor it converges on segments whose cost is entirely per-segment overhead — +// manifest write, fsync, transaction, one statement or COPY setup per table. +// +// It is deliberately not exposed. In the slower write modes the floor can exceed what the +// mode pushes within the target duration, so a flag named for the sizer would sometimes +// silently override the sizer it appears to configure. +const segmentFloorBytes int64 = 8 << 20 + +// sizer steers the segment size toward a target commit duration. +// +// Sizing by measured duration rather than by a block count is what keeps this stable +// across chains, where block payloads differ by orders of magnitude. Sizing by bytes +// rather than by rows is what keeps one dial meaningful in every write mode: the mode +// changes the cost per byte, and the loop absorbs that by converging somewhere else. +// +// It is read on the sinker's goroutine and written on the applier's, so it locks. +type sizer struct { + target time.Duration + maxBytes int64 + + mutex sync.Mutex + current int64 +} + +func newSizer(target time.Duration, maxBytes int64) *sizer { + return &sizer{target: target, maxBytes: maxBytes, current: min(segmentFloorBytes, maxBytes)} +} + +// size reports how large a segment should grow before it is committed. +func (s *sizer) size() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.current +} + +// observe folds one measured commit back into the target. +// +// The size to aim for comes from the throughput the commit actually demonstrated, not +// from scaling the current target by target/elapsed: a segment is routinely sealed well +// short of the target — the idle seal, the drain before an undo, the seal at shutdown — +// and such a segment applies quickly for reasons that say nothing about how fast the +// database is. Scaling the target on that evidence doubles it while the database may in +// fact be slow. Deriving from bytes/elapsed cannot make that mistake, and it errs on the +// conservative side for a short segment, whose elapsed time is mostly per-segment +// overhead and therefore understates throughput. +func (s *sizer) observe(bytes int64, elapsed time.Duration) { + if elapsed <= 0 || bytes <= 0 { + return + } + + s.mutex.Lock() + defer s.mutex.Unlock() + + next := float64(bytes) * (s.target.Seconds() / elapsed.Seconds()) + + // Clamp per step so the sizer converges instead of oscillating. + next = min(max(next, float64(s.current)*0.5), float64(s.current)*2.0) + + s.current = min(max(int64(next), segmentFloorBytes), s.maxBytes) +} diff --git a/sink/sql/db_proto/sql/spool/sizer_test.go b/sink/sql/db_proto/sql/spool/sizer_test.go new file mode 100644 index 000000000..0b2e8ba2c --- /dev/null +++ b/sink/sql/db_proto/sql/spool/sizer_test.go @@ -0,0 +1,75 @@ +package spool + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const mib = int64(1) << 20 + +func TestSizerConvergesOnMeasuredThroughput(t *testing.T) { + const target = 3 * time.Second + const maxBytes = 512 * (1 << 20) + + cases := []struct { + name string + current int64 + bytes int64 + elapsed time.Duration + expected int64 + }{ + { + // The one the controller exists for: 64 MiB in 3 s is exactly the target. + name: "a commit at target holds the size", current: 64 * mib, + bytes: 64 * mib, elapsed: 3 * time.Second, expected: 64 * mib, + }, + { + // A short seal — idle, drain or shutdown — applies fast for reasons that say + // nothing about throughput. The size must not grow on it. + name: "a short segment applied fast does not grow the size", current: 64 * mib, + bytes: 1 * mib, elapsed: 100 * time.Millisecond, expected: 32 * mib, + }, + { + name: "a slow database shrinks the size, one step at a time", current: 64 * mib, + bytes: 64 * mib, elapsed: 30 * time.Second, expected: 32 * mib, + }, + { + name: "a fast database grows the size, one step at a time", current: 64 * mib, + bytes: 64 * mib, elapsed: 500 * time.Millisecond, expected: 128 * mib, + }, + { + name: "the floor holds", current: segmentFloorBytes, + bytes: 1 * mib, elapsed: 30 * time.Second, expected: segmentFloorBytes, + }, + { + name: "the ceiling holds", current: maxBytes, + bytes: maxBytes, elapsed: time.Millisecond, expected: maxBytes, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + sizer := newSizer(target, maxBytes) + sizer.current = c.current + + sizer.observe(c.bytes, c.elapsed) + + require.Equal(t, c.expected, sizer.size()) + }) + } +} + +// TestSizerIgnoresAnEmptyCommit covers the segment a drain or a shutdown seals with +// nothing in it: it carries no measurement, so it must not move the size at all. +func TestSizerIgnoresAnEmptyCommit(t *testing.T) { + sizer := newSizer(3*time.Second, 512*mib) + sizer.current = 64 * mib + + sizer.observe(0, time.Millisecond) + require.Equal(t, 64*mib, sizer.size()) + + sizer.observe(64*mib, 0) + require.Equal(t, 64*mib, sizer.size()) +} diff --git a/sink/sql/db_proto/sql/spool/spool.go b/sink/sql/db_proto/sql/spool/spool.go new file mode 100644 index 000000000..78291c831 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/spool.go @@ -0,0 +1,594 @@ +package spool + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "go.uber.org/zap" +) + +// Options configures the on-disk spool. Everything not set here is derived at runtime. +type Options struct { + // Dir is where segments live. Required. + Dir string + // MaxBytes is the disk quota, and the only bound on how far ahead of the database the + // stream may run. Writes are held once the spool holds this much waiting to be + // applied, which is what turns a slow database into backpressure rather than a full + // disk. Zero picks 8GiB. + MaxBytes int64 + // WriteTargetDuration is how long one commit to the database should take. The sizer + // measures each commit and steers the next segment toward it. Zero picks 3s. + WriteTargetDuration time.Duration + // SegmentMaxBytes is the ceiling the sizer may choose, whatever the target duration + // would allow. Zero picks 512MiB. The floor is segmentFloorBytes and not + // configurable, see sizer.go. + SegmentMaxBytes int64 + // MaxIdle commits the open segment once no new row has reached it for this long, short + // of its size target. Without it a stalled stream sits on those rows indefinitely, + // leaving the cursor where it was. Zero picks 10s; negative disables idle sealing. + MaxIdle time.Duration +} + +func (o Options) withDefaults() Options { + if o.MaxBytes <= 0 { + o.MaxBytes = 8 << 30 + } + if o.WriteTargetDuration <= 0 { + o.WriteTargetDuration = 3 * time.Second + } + if o.SegmentMaxBytes <= 0 { + o.SegmentMaxBytes = 512 << 20 + } + // A normal segment must fit within the total spool budget. A single row can still + // exceed it; that one segment is allowed through once the spool is otherwise empty. + o.SegmentMaxBytes = min(o.SegmentMaxBytes, o.MaxBytes) + if o.MaxIdle == 0 { + o.MaxIdle = 10 * time.Second + } + + return o +} + +// Spool holds rows on disk and applies whole segments in the background. +// +// Rows go in through Insert on the sinker's goroutine; sealed segments leave through a +// queue that one applier goroutine drains. The disk quota is what bounds how far ahead of +// the database the stream may run — there is deliberately no second ceiling on the number +// of queued segments, which would otherwise be the limit that binds while the operator +// watches the one they set. +type Spool struct { + options Options + codec Codec + applier Applier + logger *zap.Logger + schema string + sizer *sizer + + // mutex guards the open segment. It is not only the sinker's goroutine any more: the + // idle timer seals from its own, and when the stream stalls the sinker is blocked in a + // read with no next opportunity to check anything. + mutex sync.Mutex + current *openSegment + lastWriteAt time.Time + nextSequence uint64 + + // sealMutex serializes seal-and-enqueue so segments reach the applier in the order + // they were written, whichever goroutine sealed them. It is deliberately not the same + // lock as mutex: sealing waits on the disk quota, and holding the open segment's lock + // across that wait would stop the sinker for the whole time. + sealMutex sync.Mutex + + queueMutex sync.Mutex + queueCond *sync.Cond + queue []*sealedSegment + queueDone bool + + // bytesOnDisk counts sealed segments waiting plus the one being written, so the + // operator sees the spool, not just the queue length. + bytesOnDisk atomic.Int64 + blocksAhead atomic.Int64 + // appliedBlock is the last block the applier actually committed. It is the only + // honest answer to "what is in the database": with a spool, a block reaching the + // sinker's flush means it was queued, not stored. + appliedBlock atomic.Uint64 + + applyErr atomic.Pointer[error] + waitGroup sync.WaitGroup + closeOnce sync.Once + done chan struct{} +} + +type sealedSegment struct { + dir string + manifest *Manifest + bytes int64 + + // barrier marks a segment that carries no data: the applier closes it and moves on, + // which tells Drain that everything queued ahead of it has reached the database. + barrier chan struct{} +} + +// New prepares the spool directory and starts the applier. +// +// Recovery runs first: anything already on disk is either replayed or discarded before a +// single new row is written, so the applied state and the resume cursor cannot disagree. +func New(ctx context.Context, options Options, codec Codec, applier Applier, schema string, logger *zap.Logger) (*Spool, error) { + options = options.withDefaults() + + root := filepath.Join(options.Dir, schema) + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("creating spool directory %s: %w", root, err) + } + + b := &Spool{ + options: options, + codec: codec, + applier: applier, + logger: logger.Named("spool"), + schema: schema, + sizer: newSizer(options.WriteTargetDuration, options.SegmentMaxBytes), + done: make(chan struct{}), + } + b.queueCond = sync.NewCond(&b.queueMutex) + b.options.Dir = root + + if err := b.recover(ctx); err != nil { + return nil, fmt.Errorf("recovering spool at %s: %w", root, err) + } + + b.waitGroup.Add(1) + go b.applyLoop(ctx) + + if options.MaxIdle > 0 { + b.waitGroup.Add(1) + go b.idleLoop(ctx) + } + + return b, nil +} + +// Insert buffers one row. It blocks when the spool is full, which is the backpressure +// that keeps a slow database from filling the disk. +func (b *Spool) Insert(table string, values []any) error { + if err := b.pendingError(); err != nil { + return err + } + + b.mutex.Lock() + defer b.mutex.Unlock() + + if b.current == nil { + if err := b.startSegmentLocked(); err != nil { + return err + } + } + b.lastWriteAt = time.Now() + + return b.current.writer.WriteRow(table, values) +} + +// RecordBlock notes which block the rows now being written belong to. +func (b *Spool) RecordBlock(blockNum uint64) { + b.mutex.Lock() + defer b.mutex.Unlock() + + if b.current == nil { + if err := b.startSegmentLocked(); err != nil { + b.setError(err) + return + } + } + if b.current.firstBlock == 0 { + b.current.firstBlock = blockNum + } + b.current.lastBlock = blockNum +} + +// RecordCursor notes the cursor covering everything written so far. The segment carries +// it so that applying the segment and advancing the cursor commit together. +// +// A flush that produced no rows still has to advance the cursor, or a long stretch of +// blocks whose module output is empty is streamed, and paid for, again on restart. Such a +// flush opens a segment carrying nothing but the cursor. +func (b *Spool) RecordCursor(cursor string) { + b.mutex.Lock() + defer b.mutex.Unlock() + + if b.current == nil { + if err := b.startSegmentLocked(); err != nil { + b.setError(err) + return + } + } + b.current.cursor = cursor +} + +// MaybeSeal hands the current segment to the applier once it is big enough. It is called +// at every sink flush, so it is also where backpressure is applied. +func (b *Spool) MaybeSeal(ctx context.Context) error { + if err := b.pendingError(); err != nil { + return err + } + + b.mutex.Lock() + big := b.current != nil && b.current.writer.PendingBytes() >= b.sizer.size() + b.mutex.Unlock() + + if !big { + return nil + } + + return b.Seal(ctx) +} + +// Seal closes the current segment and queues it, holding the stream if the disk quota is +// reached. +func (b *Spool) Seal(ctx context.Context) error { + // Serialized so segments reach the applier in the order they were written, whichever + // goroutine sealed them. + b.sealMutex.Lock() + defer b.sealMutex.Unlock() + + b.mutex.Lock() + pending := b.current + if pending == nil || pending.cursor == "" { + // Without a cursor the segment could not be resumed from, so it must not be + // applied on its own. + b.mutex.Unlock() + return nil + } + b.current = nil + b.mutex.Unlock() + + // The quota is checked before the manifest is written rather than after, and against + // this segment's own bytes. An individually oversized segment cannot be split without + // breaking row atomicity, so it is allowed once the spool is otherwise empty. + if err := b.awaitQuota(ctx, pending.writer.PendingBytes()); err != nil { + pending.writer.Discard() + return err + } + + manifest, err := pending.seal(b.codec.Format()) + if err != nil { + pending.writer.Discard() + return fmt.Errorf("sealing segment %s: %w", pending.dir, err) + } + + bytes := segmentBytes(manifest) + + b.bytesOnDisk.Add(bytes) + b.blocksAhead.Add(manifest.BlockCount()) + + b.enqueue(&sealedSegment{dir: pending.dir, manifest: manifest, bytes: bytes}) + + return nil +} + +// awaitQuota blocks until the spool has room for the incoming segment. +func (b *Spool) awaitQuota(ctx context.Context, incoming int64) error { + warned := false + for { + onDisk := b.bytesOnDisk.Load() + if onDisk+incoming <= b.options.MaxBytes { + return nil + } + + // A single row or segment cannot be split without breaking row atomicity. Once + // previously queued data has drained, allow that one oversized segment through + // rather than waiting forever for room it can never fit into. + if onDisk == 0 && incoming > b.options.MaxBytes { + b.logger.Warn("local spool segment exceeds its disk budget, allowing it because the spool is empty", + zap.String("segment", humanBytes(incoming)), + zap.String("quota", humanBytes(b.options.MaxBytes))) + return nil + } + + if err := b.pendingError(); err != nil { + return err + } + if !warned { + b.logger.Warn("local spool is full, holding the stream until the database catches up", + zap.String("on_disk", humanBytes(onDisk)), + zap.String("quota", humanBytes(b.options.MaxBytes))) + warned = true + } + + select { + case <-time.After(100 * time.Millisecond): + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// startSegmentLocked opens a new segment. The caller holds mutex. +func (b *Spool) startSegmentLocked() error { + b.nextSequence++ + dir := filepath.Join(b.options.Dir, fmt.Sprintf("seg-%012d", b.nextSequence)) + + // A directory left over from a previous run under the same name was already handled + // by recovery; removing it keeps a restart from appending to a stale stream. + os.RemoveAll(dir) + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating segment directory %s: %w", dir, err) + } + + writer, err := b.codec.OpenSegment(dir) + if err != nil { + return err + } + b.current = &openSegment{dir: dir, writer: writer} + b.lastWriteAt = time.Now() + + return nil +} + +// BytesOnDisk is how much is buffered waiting for the database, including the segment +// still being written. Counting only sealed segments would badly under-report: a segment +// grows to hundreds of megabytes before it is handed over. +func (b *Spool) BytesOnDisk() int64 { + b.mutex.Lock() + defer b.mutex.Unlock() + + total := b.bytesOnDisk.Load() + if b.current != nil { + total += b.current.writer.PendingBytes() + } + + return total +} + +// BlocksBuffered is how many blocks are waiting, sealed or still being written. +func (b *Spool) BlocksBuffered() int64 { + b.mutex.Lock() + defer b.mutex.Unlock() + + total := b.blocksAhead.Load() + if b.current != nil { + total += b.current.blockCount() + } + + return total +} + +// AppliedBlock is the last block committed to the database, zero before the first one. +func (b *Spool) AppliedBlock() uint64 { return b.appliedBlock.Load() } + +func (b *Spool) pendingError() error { + if err := b.applyErr.Load(); err != nil { + return *err + } + + return nil +} + +func (b *Spool) setError(err error) { + b.applyErr.CompareAndSwap(nil, &err) +} + +// Drain seals what is being written and returns once every segment queued before it has +// been applied, so the caller can act on a database that holds everything the spool has +// accepted so far. An undo needs that: rows still in flight would otherwise land after +// the delete that was supposed to remove them. +func (b *Spool) Drain(ctx context.Context) error { + if err := b.Seal(ctx); err != nil { + return err + } + + barrier := make(chan struct{}) + b.enqueue(&sealedSegment{barrier: barrier}) + + select { + case <-barrier: + return b.pendingError() + case <-ctx.Done(): + return ctx.Err() + } +} + +// Close seals whatever is left, drains the applier and reports the first failure. +func (b *Spool) Close(ctx context.Context) error { + var err error + b.closeOnce.Do(func() { + close(b.done) + err = b.Seal(ctx) + + b.queueMutex.Lock() + b.queueDone = true + b.queueMutex.Unlock() + b.queueCond.Broadcast() + + b.waitGroup.Wait() + }) + + if err != nil { + return err + } + + return b.pendingError() +} + +func (b *Spool) enqueue(pending *sealedSegment) { + b.queueMutex.Lock() + b.queue = append(b.queue, pending) + b.queueMutex.Unlock() + + b.queueCond.Signal() +} + +// dequeue waits for the next segment. It reports false once the queue is closed and +// empty, which is the applier's signal to stop. +func (b *Spool) dequeue() (*sealedSegment, bool) { + b.queueMutex.Lock() + defer b.queueMutex.Unlock() + + for len(b.queue) == 0 && !b.queueDone { + b.queueCond.Wait() + } + + if len(b.queue) == 0 { + return nil, false + } + + pending := b.queue[0] + b.queue = b.queue[1:] + + return pending, true +} + +// idleLoop commits the open segment when the stream goes quiet. +// +// The size trigger buys throughput; this one bounds what is lost when the producer stops. +// A stream that stalls mid-backfill would otherwise leave the open segment unsealed +// indefinitely, so the cursor never advances and those blocks are streamed, and paid for, +// a second time on restart. +func (b *Spool) idleLoop(ctx context.Context) { + defer b.waitGroup.Done() + + // Checking more often than the window keeps the worst-case delay to a fraction of it + // rather than to twice it. + ticker := time.NewTicker(max(b.options.MaxIdle/4, 100*time.Millisecond)) + defer ticker.Stop() + + for { + select { + case <-b.done: + return + case <-ctx.Done(): + return + case <-ticker.C: + b.sealIfIdle(ctx) + } + } +} + +func (b *Spool) sealIfIdle(ctx context.Context) { + if b.pendingError() != nil { + return + } + + b.mutex.Lock() + idle := b.current != nil && b.current.cursor != "" && time.Since(b.lastWriteAt) >= b.options.MaxIdle + b.mutex.Unlock() + + if !idle { + return + } + + if err := b.Seal(ctx); err != nil { + b.setError(fmt.Errorf("sealing an idle segment: %w", err)) + } +} + +func (b *Spool) applyLoop(ctx context.Context) { + defer b.waitGroup.Done() + + for { + pending, ok := b.dequeue() + if !ok { + return + } + + if pending.barrier != nil { + close(pending.barrier) + continue + } + + if b.pendingError() != nil { + // Keep draining so Close does not deadlock, but do not touch the database + // again after a failure. + continue + } + + startAt := time.Now() + if err := b.applier.Apply(ctx, pending.dir, pending.manifest); err != nil { + b.setError(fmt.Errorf("applying segment %s: %w", pending.dir, err)) + continue + } + elapsed := time.Since(startAt) + + b.appliedBlock.Store(pending.manifest.LastBlock) + b.bytesOnDisk.Add(-pending.bytes) + b.blocksAhead.Add(-pending.manifest.BlockCount()) + os.RemoveAll(pending.dir) + + b.sizer.observe(pending.bytes, elapsed) + + b.logger.Debug("applied segment", + zap.Uint64("first_block", pending.manifest.FirstBlock), + zap.Uint64("last_block", pending.manifest.LastBlock), + zap.String("bytes", humanBytes(pending.bytes)), + zap.Duration("elapsed", elapsed)) + } +} + +func segmentBytes(manifest *Manifest) int64 { + bytes := manifest.LogBytes + for _, table := range manifest.Tables { + bytes += table.Bytes + } + + return bytes +} + +func humanBytes(n int64) string { + value := float64(n) + for _, unit := range []string{"B", "KiB", "MiB", "GiB"} { + if value < 1024 { + return fmt.Sprintf("%.1f%s", value, unit) + } + value /= 1024 + } + + return fmt.Sprintf("%.1fTiB", value) +} + +// openSegment is the segment being written: the driver's writers plus the block range and +// cursor the spool tracks itself, since those are the same whatever the bytes look like. +type openSegment struct { + dir string + writer SegmentWriter + firstBlock uint64 + lastBlock uint64 + cursor string +} + +// blockCount is how many blocks this segment covers so far. +func (s *openSegment) blockCount() int64 { + if s.firstBlock == 0 || s.lastBlock < s.firstBlock { + return 0 + } + + return int64(s.lastBlock-s.firstBlock) + 1 +} + +// seal closes the writers and writes the manifest last, so a segment is either fully +// described or visibly incomplete. +// +// Only the manifest is fsynced. A process crash is fully covered by that; a machine crash +// may leave a data file short, which recovery catches through the codec's own check. +func (s *openSegment) seal(format Format) (*Manifest, error) { + manifest := &Manifest{ + FirstBlock: s.firstBlock, + LastBlock: s.lastBlock, + Cursor: s.cursor, + Sealed: true, + Format: format, + } + + if err := s.writer.Seal(manifest); err != nil { + return nil, err + } + + if err := WriteManifest(s.dir, manifest); err != nil { + return nil, err + } + + return manifest, nil +} diff --git a/sink/sql/db_proto/sql/spool/spool_test.go b/sink/sql/db_proto/sql/spool/spool_test.go new file mode 100644 index 000000000..5c3d1cf50 --- /dev/null +++ b/sink/sql/db_proto/sql/spool/spool_test.go @@ -0,0 +1,42 @@ +package spool + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestAwaitQuotaAllowsAnOversizedSegmentWhenSpoolIsOtherwiseEmpty(t *testing.T) { + spool := &Spool{ + options: Options{MaxBytes: 10}, + logger: zap.NewNop(), + } + + require.NoError(t, spool.awaitQuota(context.Background(), 11)) +} + +func TestSizerRespectsMaxBytesBelowSegmentFloor(t *testing.T) { + sizer := newSizer(time.Second, 1) + + require.Equal(t, int64(1), sizer.size()) +} + +func TestOptionsClampSegmentMaxToSpoolMax(t *testing.T) { + options := (Options{MaxBytes: 10, SegmentMaxBytes: 20}).withDefaults() + + require.Equal(t, int64(10), options.SegmentMaxBytes) +} + +func TestSegmentBytesIncludesRowLogBytes(t *testing.T) { + manifest := &Manifest{ + LogBytes: 123, + Tables: []TableRecord{ + {Bytes: 7}, + }, + } + + require.Equal(t, int64(130), segmentBytes(manifest)) +} diff --git a/sink/sql/db_proto/sql/write_mode.go b/sink/sql/db_proto/sql/write_mode.go new file mode 100644 index 000000000..eb9cfc1f3 --- /dev/null +++ b/sink/sql/db_proto/sql/write_mode.go @@ -0,0 +1,47 @@ +package sql + +import "fmt" + +// WriteMode says how a sealed spool segment reaches the database. +// +// It is deliberately the operator's choice rather than something derived from a directory +// flag and the shape of the schema's foreign keys, which is how the sink used to decide. +// A mode that the driver or the schema cannot support is an error, not a silent downgrade +// to something an order of magnitude slower. +type WriteMode string + +const ( + // WriteModeAuto picks copy on PostgreSQL, batch-insert on ClickHouse, and row-insert + // for a schema whose foreign keys cannot be ordered. + WriteModeAuto WriteMode = "auto" + + // WriteModeCopy loads each table file with `COPY ... FROM STDIN (FORMAT BINARY)`. + // Measured at ~7x the multi-row INSERT path, see sink/sql/db_proto/benchmarks. + WriteModeCopy WriteMode = "copy" + + // WriteModeBatchInsert builds one multi-row INSERT per table, split to stay under the + // driver's bind-parameter and statement-size limits. + WriteModeBatchInsert WriteMode = "batch-insert" + + // WriteModeRowInsert issues one prepared INSERT per row, in walk order. It is the + // fallback for a schema whose foreign keys form a cycle: a cycle has no table order, + // so grouping rows by table cannot keep a parent ahead of its children, while the walk + // itself always does. + WriteModeRowInsert WriteMode = "row-insert" +) + +func ParseWriteMode(in string) (WriteMode, error) { + switch WriteMode(in) { + case "", WriteModeAuto: + return WriteModeAuto, nil + case WriteModeCopy: + return WriteModeCopy, nil + case WriteModeBatchInsert: + return WriteModeBatchInsert, nil + case WriteModeRowInsert: + return WriteModeRowInsert, nil + } + + return "", fmt.Errorf("invalid write mode %q, expected one of %q, %q, %q or %q", + in, WriteModeAuto, WriteModeCopy, WriteModeBatchInsert, WriteModeRowInsert) +} diff --git a/sink/sql/db_proto/stats/progress.go b/sink/sql/db_proto/stats/progress.go new file mode 100644 index 000000000..797b1127b --- /dev/null +++ b/sink/sql/db_proto/stats/progress.go @@ -0,0 +1,163 @@ +package stats + +import ( + "fmt" + "sync/atomic" + + "go.uber.org/zap" +) + +// Progress tracks the two ends of the sink pipeline: what has been downloaded from +// Substreams, and what has actually been committed to the database. +// +// The distance between them is the number the operator needs. Substreams throughput is +// paid for, so a run should be limited by the stream, not by the database — and when it +// is not, the gap is where that shows. A gap that sits around one batch is the normal +// working set; a gap that keeps growing means the database cannot keep up and blocks are +// piling up in the buffer waiting for it. +// +// Every field is written from the sinker's goroutine and read from the logging ticker, +// hence the atomics. +type Progress struct { + downloadedBlock atomic.Uint64 + appliedBlock atomic.Uint64 + heldBlocks atomic.Int64 + bufferedBytes atomic.Int64 + peakBlocksAhead atomic.Uint64 + + // warnAboveBlocks is the gap past which the buffer is reported as a problem rather + // than as the normal working set. It only decides anything while nothing spools: a + // block count says how much memory is held when the blocks are held in memory. + warnAboveBlocks uint64 + + // bufferQuota is the spool's disk budget, zero when nothing spools. Once rows go to + // disk the block count stops meaning anything — a sparse backfill spans millions of + // blocks holding a few hundred kilobytes — and how full the spool is against what it + // was given is the thing that says whether the database is keeping up. + bufferQuota int64 +} + +// NewProgress returns a tracker whose "falling behind" threshold is derived from the +// batch size: a few batches in flight is normal, an order of magnitude more is not. +func NewProgress(blockBatchSize int) *Progress { + warnAbove := uint64(blockBatchSize) * 4 + if warnAbove < 100 { + warnAbove = 100 + } + + return &Progress{warnAboveBlocks: warnAbove} +} + +// SetBufferQuota records the spool's disk budget, which is what "falling behind" is +// measured against once rows are buffered on disk. +func (p *Progress) SetBufferQuota(bytes int64) { p.bufferQuota = bytes } + +// fallingBehind reports a buffer that has stopped looking like a working set. +// +// With a spool that is a share of the disk budget it was given, not a number of blocks: +// the budget is the whole point of the spool, and the operator set it. Without one, the +// blocks are held in memory and their count is what matters. +func (p *Progress) fallingBehind(ahead uint64) bool { + if p.bufferQuota > 0 { + return p.bufferedBytes.Load()*2 >= p.bufferQuota + } + + return ahead > p.warnAboveBlocks +} + +// RecordDownloaded notes a block received from the stream. +func (p *Progress) RecordDownloaded(blockNum uint64) { + p.downloadedBlock.Store(blockNum) + p.trackPeak() +} + +// RecordApplied notes a block committed to the database. +func (p *Progress) RecordApplied(blockNum uint64) { + p.appliedBlock.Store(blockNum) + p.trackPeak() +} + +// RecordBuffered notes how much is waiting between the two, in blocks and in bytes on +// disk. Bytes stay zero until blocks are buffered somewhere other than memory. +func (p *Progress) RecordBuffered(blocks int, bytes int64) { + p.heldBlocks.Store(int64(blocks)) + p.bufferedBytes.Store(bytes) + p.trackPeak() +} + +// SetResumeBlock seeds the applied mark from the cursor the run resumes at, so the +// first gap is measured against real progress rather than against zero. +func (p *Progress) SetResumeBlock(blockNum uint64) { + p.appliedBlock.Store(blockNum) +} + +// BlocksAhead is how far the download is in front of the database. +func (p *Progress) BlocksAhead() uint64 { + applied := p.appliedBlock.Load() + if applied == 0 { + // Nothing committed yet in this run and no cursor to resume from, so the + // downloaded block number says nothing about a gap — subtracting zero from it + // would report the whole chain height as backlog. What is buffered is the gap. + if held := p.heldBlocks.Load(); held > 0 { + return uint64(held) + } + return 0 + } + + downloaded := p.downloadedBlock.Load() + if downloaded <= applied { + return 0 + } + + return downloaded - applied +} + +func (p *Progress) trackPeak() { + ahead := p.BlocksAhead() + for { + peak := p.peakBlocksAhead.Load() + if ahead <= peak || p.peakBlocksAhead.CompareAndSwap(peak, ahead) { + return + } + } +} + +// Log reports the gap, at warning level once the buffer stops looking like a working set. +func (p *Progress) Log(logger *zap.Logger) { + downloaded := p.downloadedBlock.Load() + if downloaded == 0 { + return + } + + ahead := p.BlocksAhead() + fields := []zap.Field{ + zap.Uint64("downloaded_through", downloaded), + zap.Uint64("applied_through", p.appliedBlock.Load()), + zap.Uint64("blocks_ahead", ahead), + zap.Int64("blocks_buffered", p.heldBlocks.Load()), + zap.Uint64("peak_blocks_ahead", p.peakBlocksAhead.Load()), + } + + if buffered := p.bufferedBytes.Load(); buffered > 0 { + fields = append(fields, zap.String("buffered_on_disk", humanBytes(buffered))) + } + + if p.fallingBehind(ahead) { + logger.Warn("database is falling behind the stream, the buffer is over half of what it was given", fields...) + return + } + + logger.Info(" Pipeline progress", fields...) +} + +func humanBytes(n int64) string { + value := float64(n) + for _, unit := range []string{"B", "KiB", "MiB", "GiB"} { + if value < 1024 { + return fmt.Sprintf("%.1f%s", value, unit) + } + value /= 1024 + } + + return fmt.Sprintf("%.1fTiB", value) +} diff --git a/sink/sql/db_proto/stats/progress_test.go b/sink/sql/db_proto/stats/progress_test.go new file mode 100644 index 000000000..895993053 --- /dev/null +++ b/sink/sql/db_proto/stats/progress_test.go @@ -0,0 +1,143 @@ +package stats + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestProgressBlocksAhead(t *testing.T) { + progress := NewProgress(25) + + assert.Equal(t, uint64(0), progress.BlocksAhead(), "nothing seen yet") + + // Before the first commit there is no applied mark. Subtracting zero from the block + // number would report the whole chain height as backlog and warn on every startup. + progress.RecordDownloaded(20_000_050) + progress.RecordBuffered(3, 0) + assert.Equal(t, uint64(3), progress.BlocksAhead(), "with nothing applied, the gap is what is buffered") + + progress.RecordDownloaded(20_000_100) + progress.RecordApplied(20_000_075) + assert.Equal(t, uint64(25), progress.BlocksAhead()) + + // The applied mark can momentarily equal the downloaded one, and must never report + // a negative gap as a huge unsigned number. + progress.RecordApplied(20_000_100) + assert.Equal(t, uint64(0), progress.BlocksAhead()) + + progress.RecordApplied(20_000_200) + assert.Equal(t, uint64(0), progress.BlocksAhead(), "applied ahead of downloaded is still no gap") +} + +func TestProgressPeakIsRetained(t *testing.T) { + progress := NewProgress(25) + + progress.RecordApplied(400) + progress.RecordDownloaded(1000) + require.Equal(t, uint64(600), progress.BlocksAhead()) + + progress.RecordApplied(995) + assert.Equal(t, uint64(5), progress.BlocksAhead()) + assert.Equal(t, uint64(600), progress.peakBlocksAhead.Load(), "the peak is what tells the operator it backed up earlier") +} + +func TestProgressWarnsOnlyWhenTheBufferStopsLookingLikeAWorkingSet(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + logger := zap.New(core) + + // Threshold is 4 batches, floored at 100. + progress := NewProgress(500) + + progress.RecordApplied(1500) + progress.RecordDownloaded(3000) // 1500 behind, under 2000 + progress.RecordBuffered(1500, 0) + progress.Log(logger) + + require.Equal(t, 1, logs.Len()) + assert.Equal(t, zapcore.InfoLevel, logs.All()[0].Level, "a few batches in flight is normal") + + progress.RecordApplied(500) // 2500 behind, over 2000 + progress.RecordBuffered(2500, 4<<20) + progress.Log(logger) + + require.Equal(t, 2, logs.Len()) + entry := logs.All()[1] + assert.Equal(t, zapcore.WarnLevel, entry.Level) + assert.Contains(t, entry.Message, "falling behind") + + fields := entry.ContextMap() + assert.Equal(t, uint64(2500), fields["blocks_ahead"]) + assert.Equal(t, uint64(3000), fields["downloaded_through"]) + assert.Equal(t, uint64(500), fields["applied_through"]) + assert.Equal(t, "4.0MiB", fields["buffered_on_disk"]) +} + +func TestProgressStaysQuietBeforeTheFirstBlock(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + + NewProgress(25).Log(zap.New(core)) + + assert.Equal(t, 0, logs.Len(), "nothing useful to say before the first block arrives") +} + +// TestProgressResumeBlockAvoidsStartupBacklog covers the shape of a real restart: the +// cursor puts the sink 20 million blocks in, and the first block downloaded must not be +// reported as 20 million blocks of backlog. +func TestProgressResumeBlockAvoidsStartupBacklog(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + logger := zap.New(core) + + progress := NewProgress(25) + progress.SetResumeBlock(20_000_000) + + progress.RecordDownloaded(20_000_010) + progress.RecordBuffered(10, 0) + progress.Log(logger) + + require.Equal(t, uint64(10), progress.BlocksAhead()) + require.Equal(t, 1, logs.Len()) + assert.Equal(t, zapcore.InfoLevel, logs.All()[0].Level, "resuming mid-chain is not a backlog") +} + +// TestFallingBehindMeasuresTheSpoolAgainstItsQuota covers the warning that fired through +// the whole sparse start of every large backfill. +// +// A block count cannot say whether a spool is coping: the early blocks of a chain carry +// almost no rows, so the spool spans millions of them holding a few hundred kilobytes of +// an eight gigabyte budget. What says it is how full the spool is against what it was +// given. +func TestFallingBehindMeasuresTheSpoolAgainstItsQuota(t *testing.T) { + t.Run("a huge block span holding almost nothing is not behind", func(t *testing.T) { + progress := NewProgress(10) + progress.SetBufferQuota(8 << 30) + progress.RecordApplied(1) + progress.RecordDownloaded(5_000_000) + progress.RecordBuffered(4_999_999, 150<<10) + + require.False(t, progress.fallingBehind(progress.BlocksAhead())) + }) + + t.Run("past half the quota is behind", func(t *testing.T) { + progress := NewProgress(10) + progress.SetBufferQuota(8 << 30) + progress.RecordApplied(1) + progress.RecordDownloaded(1000) + progress.RecordBuffered(999, 5<<30) + + require.True(t, progress.fallingBehind(progress.BlocksAhead())) + }) + + t.Run("without a spool the block count still decides", func(t *testing.T) { + progress := NewProgress(10) + progress.RecordApplied(1) + progress.RecordDownloaded(5000) + progress.RecordBuffered(4999, 0) + + require.True(t, progress.fallingBehind(progress.BlocksAhead())) + }) +} diff --git a/sink/sql/db_proto/stats/stats.go b/sink/sql/db_proto/stats/stats.go index 478df30d4..cc9f67d8b 100644 --- a/sink/sql/db_proto/stats/stats.go +++ b/sink/sql/db_proto/stats/stats.go @@ -72,11 +72,15 @@ type Stats struct { LastBlockProcessAt time.Time TotalProcessingDuration time.Duration TotalDurationBetween time.Duration + + // Progress is how far the download is ahead of the database. + Progress *Progress } -func NewStats(logger *zap.Logger) *Stats { +func NewStats(logger *zap.Logger, blockBatchSize int) *Stats { s := &Stats{ logger: logger, + Progress: NewProgress(blockBatchSize), WaitDurationBetweenBlocks: NewAverage(" Wait Duration Between Blocks", 250_000, 1000), BlockProcessingDuration: NewAverage(" Block Processing Duration", 250_000, 1000), UnmarshallingDuration: NewAverage(" Unmarshalling Duration", 250_000, 1000), @@ -115,6 +119,7 @@ func (s *Stats) Log() { s.BlockInsertDuration.Log(s.logger) s.EntitiesInsertDuration.Log(s.logger) s.FlushDuration.Log(s.logger) + s.Progress.Log(s.logger) } s.logger.Info("-----------------------------------") diff --git a/sink/sql/tests/integration/db_changes_clickhouse_test.go b/sink/sql/tests/integration/db_changes_clickhouse_test.go index ad02f2a16..d9246cca2 100644 --- a/sink/sql/tests/integration/db_changes_clickhouse_test.go +++ b/sink/sql/tests/integration/db_changes_clickhouse_test.go @@ -10,12 +10,12 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/jmoiron/sqlx" "github.com/streamingfast/bstream" - sink "github.com/streamingfast/substreams/sink" pbdatabase "github.com/streamingfast/substreams-sink-database-changes/pb/sf/substreams/sink/database/v1" + "github.com/streamingfast/substreams/manifest" + pbsql "github.com/streamingfast/substreams/pb/sf/substreams/sink/sql/services/v1" + sink "github.com/streamingfast/substreams/sink" db2 "github.com/streamingfast/substreams/sink/sql/db_changes/db" "github.com/streamingfast/substreams/sink/sql/db_changes/sinker" - pbsql "github.com/streamingfast/substreams/pb/sf/substreams/sink/sql/services/v1" - "github.com/streamingfast/substreams/manifest" "github.com/stretchr/testify/require" "go.uber.org/zap" "google.golang.org/protobuf/types/known/anypb" diff --git a/sink/sql/tests/integration/db_changes_postgres_test.go b/sink/sql/tests/integration/db_changes_postgres_test.go index 1c1a36dcd..2366c5011 100644 --- a/sink/sql/tests/integration/db_changes_postgres_test.go +++ b/sink/sql/tests/integration/db_changes_postgres_test.go @@ -14,13 +14,13 @@ import ( _ "github.com/lib/pq" "github.com/streamingfast/bstream" "github.com/streamingfast/cli" - sink "github.com/streamingfast/substreams/sink" pbdatabase "github.com/streamingfast/substreams-sink-database-changes/pb/sf/substreams/sink/database/v1" - db2 "github.com/streamingfast/substreams/sink/sql/db_changes/db" - "github.com/streamingfast/substreams/sink/sql/db_changes/sinker" - pbsql "github.com/streamingfast/substreams/pb/sf/substreams/sink/sql/services/v1" "github.com/streamingfast/substreams/manifest" pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" + pbsql "github.com/streamingfast/substreams/pb/sf/substreams/sink/sql/services/v1" + sink "github.com/streamingfast/substreams/sink" + db2 "github.com/streamingfast/substreams/sink/sql/db_changes/db" + "github.com/streamingfast/substreams/sink/sql/db_changes/sinker" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" ) diff --git a/sink/sql/tests/integration/db_proto_clickhouse_fresh_database_test.go b/sink/sql/tests/integration/db_proto_clickhouse_fresh_database_test.go new file mode 100644 index 000000000..72770d9f3 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_clickhouse_fresh_database_test.go @@ -0,0 +1,60 @@ +package tests + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/modules/clickhouse" +) + +// TestDbProtoClickhouseSetupOnAFreshDatabase covers pointing the sink at a server that +// does not hold the database yet, which is what `substreams sink clickhouse setup` is for +// and the one case every other ClickHouse test skips by creating it first. +// +// The schema check that runs ahead of CreateDatabase used to connect to the database it +// was about to create. ClickHouse refuses that with UNKNOWN_DATABASE, and newClient +// retries a failed dial forever on a context of its own, so setup never came back. +func TestDbProtoClickhouseSetupOnAFreshDatabase(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + + var seededDatabase string + clickhouseDSN, _ := setupClickhouseContainer(t, func(ctx context.Context, user, password, database, dsn string, container *clickhouse.ClickHouseContainer) error { + seededDatabase = database + return nil + }) + + // Deliberately never created: that is the whole point. + schemaName := "fresh_database" + testDSN := strings.Replace(clickhouseDSN, seededDatabase, schemaName, 1) + + stateFolder := t.TempDir() + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.DisableAllConstraints(), + Clickhouse: db_proto.SinkerFactoryClickhouse{ + SinkInfoFolder: stateFolder, + CursorFilePath: filepath.Join(stateFolder, "cursor.txt"), + }, + }.Defaults() + + done := make(chan error, 1) + go func() { + _, err := db_proto.SetupDatabaseSchema(context.Background(), testDSN, schemaName, + defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + done <- err + }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(20 * time.Second): + t.Fatal("setup never returned, the schema check is dialing a database that does not exist yet") + } +} diff --git a/sink/sql/tests/integration/db_proto_clickhouse_no_annotations_test.go b/sink/sql/tests/integration/db_proto_clickhouse_no_annotations_test.go new file mode 100644 index 000000000..d5c1f7bdf --- /dev/null +++ b/sink/sql/tests/integration/db_proto_clickhouse_no_annotations_test.go @@ -0,0 +1,176 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "strings" + "testing" + + _ "github.com/ClickHouse/clickhouse-go/v2" + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/modules/clickhouse" +) + +type moduleRow struct { + BlockNumber uint64 `db:"_block_number_"` + RowID uint32 `db:"_row_id_"` + Name string `db:"name"` +} + +// TestDbProtoClickhouseWithoutAnnotations covers a package whose output proto carries no +// schema.proto annotations at all, which is what a Substreams written without the sink in +// mind looks like. Setup used to fail outright for want of 'order_by_fields'; the sink now +// sorts those tables on (_block_number_, _row_id_). +// +// The rows are what makes this worth running against a real server: the tables are +// ReplacingMergeTree, so a sorting key that does not tell a block's rows apart loses all +// but one of them at merge time. OPTIMIZE FINAL forces that merge rather than waiting for +// ClickHouse to get around to it. +func TestDbProtoClickhouseWithoutAnnotations(t *testing.T) { + dbx, schema, dsn, options := runUnannotatedClickhouseSink(t, "no_annotations", + blockScopedData(t, "1a", modulesOutput("map_one", "map_two", "map_three")), + ) + + var createTable string + require.NoError(t, dbx.Get(&createTable, fmt.Sprintf("SHOW CREATE TABLE %s.Module", schema))) + assert.Contains(t, createTable, "ORDER BY (_block_number_, _row_id_)") + + _, err := dbx.Exec(fmt.Sprintf("OPTIMIZE TABLE %s.Module FINAL", schema)) + require.NoError(t, err) + + var rows []moduleRow + require.NoError(t, dbx.Select(&rows, fmt.Sprintf("SELECT _block_number_, _row_id_, name FROM %s.Module ORDER BY _row_id_", schema))) + + assert.Equal(t, []moduleRow{ + {1, 0, "map_one"}, + {1, 1, "map_two"}, + {1, 2, "map_three"}, + }, rows) + + // The guardrail: a database whose tables no longer agree with what the package would + // create is refused rather than written into. A table without _row_id_ is what an + // earlier run of an annotated package leaves behind, and CREATE TABLE IF NOT EXISTS + // would keep it as it is. + _, err = dbx.Exec(fmt.Sprintf("DROP TABLE %s.Module", schema)) + require.NoError(t, err) + + _, err = dbx.Exec(fmt.Sprintf(`CREATE TABLE %s.Module ( + _block_number_ UInt64, _block_timestamp_ timestamp, _version_ Int64, _deleted_ bool, name VARCHAR + ) ENGINE = ReplacingMergeTree(_version_, _deleted_) ORDER BY (_block_number_)`, schema)) + require.NoError(t, err) + + _, err = db_proto.SetupDatabaseSchema(context.Background(), dsn, schema, defaultOutputModuleName, + (&pbsubstreams.Modules{}).ProtoReflect().Descriptor(), options, logger, tracer) + require.ErrorContains(t, err, "_row_id_") +} + +// TestDbProtoClickhouseWithoutAnnotationsUndo checks the other half of the sorting key: +// an undo writes one tombstone per row, and a tombstone only removes anything if it lands +// on the very same key. _row_id_ is part of that key here, so it has to be carried into +// the tombstone as well. +func TestDbProtoClickhouseWithoutAnnotationsUndo(t *testing.T) { + dbx, schema, _, _ := runUnannotatedClickhouseSink(t, "no_annotations_undo", + blockScopedData(t, "1a", modulesOutput("map_one", "map_two")), + blockScopedData(t, "2a", modulesOutput("map_three", "map_four")), + blockUndo(t, "1a"), + ) + + var rows []moduleRow + require.NoError(t, dbx.Select(&rows, fmt.Sprintf("SELECT _block_number_, _row_id_, name FROM %s.Module FINAL WHERE _deleted_ = 0 ORDER BY _block_number_, _row_id_", schema))) + + assert.Equal(t, []moduleRow{ + {1, 0, "map_one"}, + {1, 1, "map_two"}, + }, rows) +} + +func modulesOutput(names ...string) *pbsubstreams.Modules { + out := &pbsubstreams.Modules{} + for _, name := range names { + out.Modules = append(out.Modules, &pbsubstreams.Module{Name: name}) + } + + return out +} + +// runUnannotatedClickhouseSink streams the given responses into a fresh ClickHouse +// database, with the schema derived from a proto carrying no annotations, and returns a +// handle on that database. +func runUnannotatedClickhouseSink(t *testing.T, schemaName string, responses ...*pbsubstreamsrpc.Response) (*sqlx.DB, string, string, db_proto.SinkerFactoryOptions) { + t.Helper() + + outputMessageDescriptor := (&pbsubstreams.Modules{}).ProtoReflect().Descriptor() + + var dbDatabase string + clickhouseDSN, _ := setupClickhouseContainer(t, func(ctx context.Context, user, password, database, dsn string, container *clickhouse.ClickHouseContainer) error { + dbDatabase = database + return nil + }) + + pattern := make([]interface{}, len(responses)) + for i, response := range responses { + pattern[i] = response + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, pattern...) + substreamsPackage := substreamsTestPackage(pbsubstreams.File_sf_substreams_v1_modules_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-3", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + clickhouseStateFolder := t.TempDir() + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: false, + Constraints: protosql.DisableAllConstraints(), + UseTransactions: true, + DecodeBatchSize: 1, + Clickhouse: db_proto.SinkerFactoryClickhouse{ + SinkInfoFolder: clickhouseStateFolder, + CursorFilePath: filepath.Join(clickhouseStateFolder, "cursor.txt"), + }, + }.Defaults() + + sinkerFactory := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options) + + createTestDatabase(t, clickhouseDSN, schemaName) + testDSN := strings.Replace(clickhouseDSN, dbDatabase, schemaName, 1) + + ctx := context.Background() + dbSinker, err := sinkerFactory(ctx, testDSN, schemaName, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("clickhouse", testDSN) + require.NoError(t, err) + + // Unsafe: the rows read back leave _version_ and _deleted_ out. + dbx := sqlx.NewDb(db, "clickhouse").Unsafe() + t.Cleanup(func() { dbx.Close() }) + + return dbx, schemaName, testDSN, options +} diff --git a/sink/sql/tests/integration/db_proto_clickhouse_spool_test.go b/sink/sql/tests/integration/db_proto_clickhouse_spool_test.go new file mode 100644 index 000000000..03bacb490 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_clickhouse_spool_test.go @@ -0,0 +1,113 @@ +package tests + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + _ "github.com/ClickHouse/clickhouse-go/v2" + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/modules/clickhouse" +) + +// TestDbProtoClickhouseSpool runs the same blocks with and without the spool and requires +// the same rows either way. +// +// ClickHouse gains nothing from the spool but the decoupling: the rows are replayed into +// the same column builders and sent by the same flush, followed by the same cursor write. +// So the spool is only worth having if it is invisible in the result, which is what this +// pins. +func TestDbProtoClickhouseSpool(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + + var dbDatabase string + clickhouseDSN, _ := setupClickhouseContainer(t, func(ctx context.Context, user, password, database, dsn string, container *clickhouse.ClickHouseContainer) error { + dbDatabase = database + return nil + }) + + run := func(t *testing.T, schemaName string, spooled bool) []*CustomerRow { + t.Helper() + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "alpha")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "beta"), entityCustomer("customer-3", "gamma")), + relationsBlockData(t, "3a", "2025-01-03", entityCustomer("customer-4", "delta")), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-4", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + stateFolder := t.TempDir() + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.DisableAllConstraints(), + UseTransactions: true, + DecodeBatchSize: 1, + Clickhouse: db_proto.SinkerFactoryClickhouse{ + SinkInfoFolder: stateFolder, + CursorFilePath: filepath.Join(stateFolder, "cursor.txt"), + }, + } + if spooled { + options.Spool = &spool.Options{Dir: t.TempDir(), MaxIdle: 100_000_000} + } + + createTestDatabase(t, clickhouseDSN, schemaName) + testDSN := strings.Replace(clickhouseDSN, dbDatabase, schemaName, 1) + + ctx := context.Background() + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options.Defaults())(ctx, testDSN, schemaName, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("clickhouse", testDSN) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "clickhouse").Unsafe() + defer dbx.Close() + + return readRowsBy[CustomerRow](t, dbx, "customers", "customer_id") + } + + direct := run(t, "clickhouse_direct", false) + spooled := run(t, "clickhouse_spooled", true) + + require.NotEmpty(t, direct, "the unspooled run has to write something for the comparison to mean anything") + require.Equal(t, direct, spooled) +} + +// CustomerRow is one row of the customers table, for comparing a spooled run against an +// unspooled one. +type CustomerRow struct { + Meta + CustomerID string `db:"customer_id"` + Name string `db:"name"` +} diff --git a/sink/sql/tests/integration/db_proto_clickhouse_test.go b/sink/sql/tests/integration/db_proto_clickhouse_test.go index 74ecb167b..017675de9 100644 --- a/sink/sql/tests/integration/db_proto_clickhouse_test.go +++ b/sink/sql/tests/integration/db_proto_clickhouse_test.go @@ -17,6 +17,7 @@ import ( pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" sink "github.com/streamingfast/substreams/sink" "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go/modules/clickhouse" @@ -139,9 +140,9 @@ func TestDbProtoClickhouseIntegration(t *testing.T) { options := db_proto.SinkerFactoryOptions{ UseProtoOption: true, - UseConstraints: false, + Constraints: protosql.DisableAllConstraints(), UseTransactions: true, - BlockBatchSize: 1, + DecodeBatchSize: 1, Clickhouse: db_proto.SinkerFactoryClickhouse{ SinkInfoFolder: clickhouseStateFolder, CursorFilePath: filepath.Join(clickhouseStateFolder, "cursor.txt"), diff --git a/sink/sql/tests/integration/db_proto_postgres_block_index_test.go b/sink/sql/tests/integration/db_proto_postgres_block_index_test.go new file mode 100644 index 000000000..3450f929d --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_block_index_test.go @@ -0,0 +1,110 @@ +package tests + +import ( + "context" + "database/sql" + "testing" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresBlockNumberIndex covers the index the sink creates for its own reorg +// path, which is deliberately not part of the constraint pass. +// +// --apply-constraints describes the schema and is the operator's to schedule. This one the +// sink depends on to undo a reorg without sequentially scanning every table, so it goes in +// when the sink starts whatever the constraints say — and concurrently, so a restart onto +// an already-loaded table neither waits for a lock nor takes one. +func TestDbProtoPostgresBlockNumberIndex(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + // validIndexes counts only the ones a query would actually use: an interrupted + // concurrent build leaves an index behind that is there and unusable. + validIndexes := func(schemaName string) []string { + t.Helper() + + var names []string + require.NoError(t, dbx.Select(&names, ` + SELECT cl.relname + FROM pg_index i + JOIN pg_class cl ON cl.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = cl.relnamespace + WHERE n.nspname = $1 AND i.indisvalid AND cl.relname LIKE '%_block_number_idx' + ORDER BY cl.relname`, schemaName)) + + return names + } + + open := func(t *testing.T, schemaName string, constraints protosql.ConstraintPolicy) protosql.Database { + t.Helper() + + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: constraints, + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, schemaName, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + t.Cleanup(func() { database.Close(ctx) }) + + return database + } + + t.Run("created even when every constraint is disabled", func(t *testing.T) { + schemaName := "block_index_no_constraints" + database := open(t, schemaName, protosql.DisableAllConstraints()) + + require.Empty(t, validIndexes(schemaName), "the schema starts without them") + require.NoError(t, database.EnsureBlockNumberIndexes(ctx)) + + created := validIndexes(schemaName) + require.NotEmpty(t, created, "the reorg path deletes by _block_number_ whatever the constraints declare") + require.Contains(t, created, "customers_block_number_idx") + + // Starting again has to be a no-op rather than an "already exists" failure. + require.NoError(t, database.EnsureBlockNumberIndexes(ctx)) + require.Equal(t, created, validIndexes(schemaName)) + }) + + t.Run("left out when the flag asks", func(t *testing.T) { + schemaName := "block_index_disabled" + policy := protosql.DisableAllConstraints() + policy.DisableBlockNumberIndex = true + + database := open(t, schemaName, policy) + + require.NoError(t, database.EnsureBlockNumberIndexes(ctx)) + require.Empty(t, validIndexes(schemaName)) + }) + + t.Run("the constraint pass neither creates nor drops it", func(t *testing.T) { + schemaName := "block_index_not_a_constraint" + database := open(t, schemaName, protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}) + + require.NoError(t, database.ApplyConstraints()) + require.Empty(t, validIndexes(schemaName), "the constraint pass runs in transactions, which a concurrent build cannot") + + require.NoError(t, database.EnsureBlockNumberIndexes(ctx)) + created := validIndexes(schemaName) + require.NotEmpty(t, created) + + require.NoError(t, database.DropConstraints()) + require.Equal(t, created, validIndexes(schemaName), "dropping the constraints leaves the sink's own index alone") + }) +} diff --git a/sink/sql/tests/integration/db_proto_postgres_bytes_test.go b/sink/sql/tests/integration/db_proto_postgres_bytes_test.go index 4eccf1d0a..aba24ca25 100644 --- a/sink/sql/tests/integration/db_proto_postgres_bytes_test.go +++ b/sink/sql/tests/integration/db_proto_postgres_bytes_test.go @@ -110,9 +110,9 @@ func runBytesSinker(t *testing.T, schema string, useConstraints bool, payload [] options := db_proto.SinkerFactoryOptions{ UseProtoOption: true, - UseConstraints: useConstraints, + Constraints: constraintPolicy(useConstraints), UseTransactions: true, - BlockBatchSize: 1, + DecodeBatchSize: 1, Encoding: sqlbytes.EncodingRaw, }.Defaults() options.Encoding = sqlbytes.EncodingRaw diff --git a/sink/sql/tests/integration/db_proto_postgres_completion_test.go b/sink/sql/tests/integration/db_proto_postgres_completion_test.go index 53246f54b..90af6df18 100644 --- a/sink/sql/tests/integration/db_proto_postgres_completion_test.go +++ b/sink/sql/tests/integration/db_proto_postgres_completion_test.go @@ -13,6 +13,7 @@ import ( "github.com/streamingfast/substreams/manifest" sink "github.com/streamingfast/substreams/sink" "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" "github.com/stretchr/testify/require" ) @@ -65,11 +66,11 @@ func TestDbProtoPostgresBoundedRunFlushesOnCompletion(t *testing.T) { options := db_proto.SinkerFactoryOptions{ UseProtoOption: true, - UseConstraints: false, + Constraints: protosql.DisableAllConstraints(), UseTransactions: true, - BlockBatchSize: blockBatchSize, + DecodeBatchSize: blockBatchSize, }.Defaults() - options.BlockBatchSize = blockBatchSize + options.DecodeBatchSize = blockBatchSize createPostgresTestSchema(t, postgresContainer.ConnectionString, schema) diff --git a/sink/sql/tests/integration/db_proto_postgres_constraints_spool_test.go b/sink/sql/tests/integration/db_proto_postgres_constraints_spool_test.go new file mode 100644 index 000000000..b2f5cc1d4 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_constraints_spool_test.go @@ -0,0 +1,90 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoConstraintsAtEndOfRangeWithSpool covers the end of a bounded run that spooled. +// +// The spool has to be drained — the rows it holds are part of the range — but the +// constraints are deliberately left alone. A stop block ends the run without saying the +// backfill is over: a range is routinely one chunk of several, and building them here would +// leave every later chunk loading into a constrained schema, measured at 27.7x. +func TestDbProtoConstraintsAtEndOfRangeWithSpool(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "kept")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "kept")), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-3", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsAuto}, + UseTransactions: true, + DecodeBatchSize: 1, + Spool: &spool.Options{Dir: t.TempDir()}, + }.Defaults() + + testSchema := "constraints_end_of_range_spooled" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + for _, name := range []string{"block_pk", "fk_block", "customers_pk"} { + var count int + require.NoError(t, dbx.Get(&count, ` + SELECT count(*) + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = $1 AND c.conname = $2`, testSchema, name)) + require.Zero(t, count, "reaching a stop block must leave %q to `constraints apply`", name) + } + + var customers int + require.NoError(t, dbx.Get(&customers, fmt.Sprintf(`SELECT count(*) FROM "%s"."customers"`, testSchema))) + require.Equal(t, 2, customers, "the spool is still drained at the end of the range, constraints or not") +} diff --git a/sink/sql/tests/integration/db_proto_postgres_constraints_test.go b/sink/sql/tests/integration/db_proto_postgres_constraints_test.go new file mode 100644 index 000000000..ac497a978 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_constraints_test.go @@ -0,0 +1,176 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresConstraintsBackfill covers what applying constraints has to do on a +// schema that a previous run created without them. +// +// The sink info hash cannot tell those two schemas apart — it is computed over the DDL +// the dialect would emit, constraints included, either way — so turning the flag on has +// to add the missing constraints rather than assume the schema already matches. Running +// it again must then be a no-op rather than an "already exists" failure. +func TestDbProtoPostgresConstraintsBackfill(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + schemaName := "constraints_backfill" + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + setup := func(useConstraints bool) { + t.Helper() + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: constraintPolicy(useConstraints), + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, schemaName, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + require.NoError(t, database.Close(ctx)) + } + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + countConstraints := func(kinds string) int { + t.Helper() + + var count int + query := fmt.Sprintf(` + SELECT count(*) + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = $1 AND c.contype IN (%s)`, kinds) + require.NoError(t, dbx.Get(&count, query, schemaName)) + + return count + } + + constraintExists := func(name string) bool { + t.Helper() + + var count int + require.NoError(t, dbx.Get(&count, ` + SELECT count(*) + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = $1 AND c.conname = $2`, schemaName, name)) + + return count > 0 + } + + // The static DDL gives _sink_info_ and _cursor_ inline primary keys, so the baseline + // is not zero; what the dialect adds on top of it is what this is about. + setup(false) + baseline := countConstraints("'p','u','f'") + require.False(t, constraintExists("block_pk"), "a schema created without constraints must not carry the dialect's own") + + setup(true) + withConstraints := countConstraints("'p','u','f'") + require.Greater(t, withConstraints, baseline, "applying constraints must add them to the existing schema") + require.True(t, constraintExists("block_pk"), "the _blocks_ primary key") + require.True(t, constraintExists("fk_block"), "the foreign key every table has to _blocks_") + + setup(true) + require.Equal(t, withConstraints, countConstraints("'p','u','f'"), "applying the constraints again must not duplicate or fail on them") +} + +// TestDbProtoConstraintsAtHead covers --apply-constraints=auto: the sink loads bare and +// creates the constraints itself once the backfill is over, which for a bounded run is +// the end of the range. +// +// The default is deliberately not this — building them locks every table, which is the +// operator's call to schedule — but when it is asked for it has to actually happen. +// TestDbProtoConstraintsAtHead covers the one moment that says the backfill is over. +// +// The liveness checker is the cursor-based one the run command installs, so the first +// STEP_NEW block turns the stream live — which is what closes the spool and puts the +// constraints on. A stop block does not do this, deliberately: see the spooled +// end-of-range test. +func TestDbProtoConstraintsAtHead(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "kept")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "kept")), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-3", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + // What the run command installs, and what makes a STEP_NEW block read as live. + sink.WithLivenessChecker(sink.NewCursorBasedLivenessChecker())(baseSink) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsAuto}, + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + testSchema := "constraints_at_head" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + for _, name := range []string{"block_pk", "fk_block", "customers_pk"} { + var count int + require.NoError(t, dbx.Get(&count, ` + SELECT count(*) + FROM pg_constraint c + JOIN pg_namespace n ON n.oid = c.connamespace + WHERE n.nspname = $1 AND c.conname = $2`, testSchema, name)) + require.Positive(t, count, "reaching chain HEAD must have created %q", name) + } + + var customers int + require.NoError(t, dbx.Get(&customers, fmt.Sprintf(`SELECT count(*) FROM "%s"."customers"`, testSchema))) + require.Equal(t, 2, customers, "the rows loaded before the constraints are still there") +} diff --git a/sink/sql/tests/integration/db_proto_postgres_decode_workers_test.go b/sink/sql/tests/integration/db_proto_postgres_decode_workers_test.go index 583257da4..22478d202 100644 --- a/sink/sql/tests/integration/db_proto_postgres_decode_workers_test.go +++ b/sink/sql/tests/integration/db_proto_postgres_decode_workers_test.go @@ -13,6 +13,7 @@ import ( "github.com/streamingfast/substreams/manifest" sink "github.com/streamingfast/substreams/sink" "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" "github.com/stretchr/testify/require" ) @@ -140,13 +141,13 @@ func runDecodeWorkersSinker(t *testing.T, schema string, decodeWorkers, blockBat options := db_proto.SinkerFactoryOptions{ UseProtoOption: true, - UseConstraints: false, + Constraints: protosql.DisableAllConstraints(), UseTransactions: true, - BlockBatchSize: blockBatchSize, + DecodeBatchSize: blockBatchSize, DecodeWorkers: decodeWorkers, }.Defaults() options.DecodeWorkers = decodeWorkers - options.BlockBatchSize = blockBatchSize + options.DecodeBatchSize = blockBatchSize createPostgresTestSchema(t, postgresContainer.ConnectionString, schema) diff --git a/sink/sql/tests/integration/db_proto_postgres_live_switch_test.go b/sink/sql/tests/integration/db_proto_postgres_live_switch_test.go new file mode 100644 index 000000000..de23c76a8 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_live_switch_test.go @@ -0,0 +1,242 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresLiveSwitch pins what happens when a buffered backfill reaches the +// chain head. +// +// The local buffer holds rows on disk until a segment fills, which is what a backfill +// wants and the opposite of what a live sink wants: at the head a block should be +// queryable when it arrives. Reaching a live block therefore drains the buffer and +// switches the database to direct inserts, once and for good. +func TestDbProtoPostgresLiveSwitch(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + + // Blocks 1 and 2 are irreversible, so they are backfill and stay in the buffer; block + // 3 arrives undo-able, which is what the cursor-based checker calls live. + irreversible := func(blockIdentifier, blockTime string, entities ...*pbrelations.Entity) *pbsubstreamsrpc.Response { + return blockScopedData(t, blockIdentifier, &pbrelations.Output{Entities: entities}, blockTimepb(t, blockTime), finalBlock(blockIdentifier)) + } + + responses := []interface{}{ + irreversible("1a", "2025-01-01", entityCustomer("customer-1", "backfilled")), + irreversible("2a", "2025-01-02", entityCustomer("customer-2", "backfilled")), + relationsBlockData(t, "3a", "2025-01-03", entityCustomer("customer-3", "live")), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-4", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + sink.WithLivenessChecker(sink.NewCursorBasedLivenessChecker()), + ) + require.NoError(t, err) + + bufferDir := t.TempDir() + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.DisableAllConstraints(), + UseTransactions: true, + // Large enough that nothing would be flushed on batch size alone, so what lands + // in the database is what the switch and the live path put there. + DecodeBatchSize: 100, + Spool: &spool.Options{Dir: bufferDir}, + }.Defaults() + + testSchema := "live_switch" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + ctx := context.Background() + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + var customers int + require.NoError(t, dbx.Get(&customers, fmt.Sprintf(`SELECT count(*) FROM "%s"."customers"`, testSchema))) + require.Equal(t, 3, customers, "every block reaches the database, whether it went through the buffer or straight in") + + // The buffer keeps one directory per schema and removes each segment as it is + // applied, so a drained and closed buffer leaves nothing behind. + segments, err := filepath.Glob(filepath.Join(bufferDir, testSchema, "seg-*")) + require.NoError(t, err) + require.Empty(t, segments, "the buffer must be drained by the switch, not left holding segments") +} + +// TestDbProtoPostgresSwitchToDirectInserts is the switch on its own, where buffering is +// observable: a full run drains at the end whatever happened in the middle, so only a +// mid-run look at the database can tell a buffered write from a direct one. +func TestDbProtoPostgresSwitchToDirectInserts(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + testSchema := "switch_to_direct" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + bufferDir := t.TempDir() + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.DisableAllConstraints(), + UseTransactions: true, + DecodeBatchSize: 1, + Spool: &spool.Options{Dir: bufferDir}, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, testSchema, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + require.NoError(t, database.Open()) + defer database.Close(ctx) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + countCustomers := func() int { + var out int + require.NoError(t, dbx.Get(&out, fmt.Sprintf(`SELECT count(*) FROM "%s"."customers"`, testSchema))) + + return out + } + + write := func(blockNum uint64, customerID string) { + t.Helper() + + require.NoError(t, database.BeginTransaction()) + require.NoError(t, database.InsertBlock(blockNum, fmt.Sprintf("%da", blockNum), fixedBaseTime)) + require.NoError(t, database.Insert("customers", []any{blockNum, fixedBaseTime, customerID, "name"})) + cursor := bstream.Cursor{ + Step: bstream.StepNewIrreversible, + Block: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + HeadBlock: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + LIB: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + } + sinkCursor, err := sink.NewCursor(cursor.ToOpaque()) + require.NoError(t, err) + require.NoError(t, database.StoreCursor(sinkCursor)) + _, err = database.Flush() + require.NoError(t, err) + require.NoError(t, database.CommitTransaction()) + } + + write(1, "customer-1") + require.Zero(t, countCustomers(), "a buffered write must not have reached the database yet") + + require.NoError(t, database.SwitchToDirectInserts(ctx, "stream reached the chain head", true)) + require.Equal(t, 1, countCustomers(), "the switch drains what the buffer was holding") + + write(2, "customer-2") + require.Equal(t, 2, countCustomers(), "after the switch a write is visible as soon as it commits") + + segments, err := filepath.Glob(filepath.Join(bufferDir, testSchema, "seg-*")) + require.NoError(t, err) + require.Empty(t, segments, "the buffer is closed for good, nothing accumulates in it") +} + +// TestDbProtoPostgresClearsSegmentRecordsAtTheChainHead covers the bookkeeping a sink that +// stays live would otherwise accumulate forever. +// +// Every segment applied records a row, and nothing was ever removing them. Reaching the +// head drains the spool and closes it for good, which leaves no directory on disk for a +// record to answer for — so that is where they go. A run that ends instead keeps them, +// having no next restart to grow them. +func TestDbProtoPostgresClearsSegmentRecordsAtTheChainHead(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + testSchema := "segment_records_at_head" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.DisableAllConstraints(), + UseTransactions: true, + DecodeBatchSize: 1, + // One byte of segment, so every write seals and records one. + Spool: &spool.Options{Dir: t.TempDir(), SegmentMaxBytes: 1}, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, testSchema, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + require.NoError(t, database.Open()) + defer database.Close(ctx) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + countIn := func(table string) int { + var out int + require.NoError(t, dbx.Get(&out, fmt.Sprintf(`SELECT count(*) FROM "%s"."%s"`, testSchema, table))) + + return out + } + + write := func(blockNum uint64, customerID string) { + t.Helper() + + require.NoError(t, database.BeginTransaction()) + require.NoError(t, database.InsertBlock(blockNum, fmt.Sprintf("%da", blockNum), fixedBaseTime)) + require.NoError(t, database.Insert("customers", []any{blockNum, fixedBaseTime, customerID, "name"})) + cursor := bstream.Cursor{ + Step: bstream.StepNewIrreversible, + Block: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + HeadBlock: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + LIB: bstream.NewBlockRef(fmt.Sprintf("%da", blockNum), blockNum), + } + sinkCursor, err := sink.NewCursor(cursor.ToOpaque()) + require.NoError(t, err) + require.NoError(t, database.StoreCursor(sinkCursor)) + _, err = database.Flush() + require.NoError(t, err) + require.NoError(t, database.CommitTransaction()) + } + + write(1, "customer-1") + write(2, "customer-2") + require.Positive(t, countIn("_segments_"), "a spooled backfill has to be recording the segments it applies") + + require.NoError(t, database.SwitchToDirectInserts(ctx, "stream reached the chain head", true)) + + require.Zero(t, countIn("_segments_"), "reaching the head clears what the spool recorded") + require.Equal(t, 2, countIn("customers"), "clearing the bookkeeping must not touch the rows") +} diff --git a/sink/sql/tests/integration/db_proto_postgres_missing_constraints_test.go b/sink/sql/tests/integration/db_proto_postgres_missing_constraints_test.go new file mode 100644 index 000000000..05e43b3fa --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_missing_constraints_test.go @@ -0,0 +1,135 @@ +package tests + +import ( + "context" + "fmt" + "testing" + + _ "github.com/lib/pq" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresMissingConstraints covers the startup check. +// +// A run interrupted before the backfill ended, or whose constraint pass was killed +// part-way, leaves a database that answers queries slowly and rejects nothing — and looks +// exactly like a database that is fine. The sink has to be able to say so, which means +// knowing which constraints the policy asks for and which the catalog actually has. +func TestDbProtoPostgresMissingConstraints(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + setup := func(t *testing.T, schemaName string, constraints protosql.ConstraintPolicy) protosql.Database { + t.Helper() + + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: constraints, + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, schemaName, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + t.Cleanup(func() { database.Close(ctx) }) + + return database + } + + t.Run("a schema loaded bare reports every constraint missing", func(t *testing.T) { + database := setup(t, "missing_constraints_bare", protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}) + + missing, err := database.MissingConstraints() + require.NoError(t, err) + require.Subset(t, missing, []string{ + "missing_constraints_bare._blocks_.block_pk", + "missing_constraints_bare.customers.customers_pk", + "missing_constraints_bare.customers.fk_block", + "missing_constraints_bare.orders.fk_block", + }, "each one is named by its relation as well: fk_block is a different constraint on every table") + }) + + t.Run("nothing is missing once they have been created", func(t *testing.T) { + database := setup(t, "missing_constraints_applied", protosql.ConstraintPolicy{Timing: protosql.ConstraintsAlways}) + + missing, err := database.MissingConstraints() + require.NoError(t, err) + require.Empty(t, missing) + }) + + t.Run("nothing the flags leave out is missing", func(t *testing.T) { + database := setup(t, "missing_constraints_disabled", protosql.ConstraintPolicy{ + Timing: protosql.ConstraintsManual, + DisableForeignKeys: true, + DisablePrimaryKeys: []string{protosql.AllTables}, + DisableUniques: []string{protosql.AllTables}, + }) + + missing, err := database.MissingConstraints() + require.NoError(t, err) + require.Empty(t, missing, "nothing the policy leaves out can be missing, and the block number index is not a constraint") + }) +} + +// TestDbProtoConstraintsParallelism covers the pass at every width it can be run at. +// +// Each statement commits on its own whatever the parallelism, so what a killed run +// finished has to still be there — the same property that lets the pass be re-run against +// a schema that is partly constrained already. Running the keys and the foreign keys as +// separate waves is what keeps a foreign key from reaching the server before the key it +// references, which concurrency would otherwise make a race. +func TestDbProtoConstraintsParallelism(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + open := func(t *testing.T, schemaName string, parallelism int) protosql.Database { + t.Helper() + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{ + Timing: protosql.ConstraintsManual, + Parallelism: parallelism, + WorkMem: "64MB", + }, + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + database, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, schemaName, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.NoError(t, err) + t.Cleanup(func() { database.Close(ctx) }) + + return database + } + + for _, parallelism := range []int{1, 3, 10} { + t.Run(fmt.Sprintf("parallelism %d", parallelism), func(t *testing.T) { + schemaName := fmt.Sprintf("constraints_parallelism_%d", parallelism) + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + database := open(t, schemaName, parallelism) + + require.NoError(t, database.ApplyConstraints()) + missing, err := database.MissingConstraints() + require.NoError(t, err) + require.Empty(t, missing) + + // Re-running has to be a no-op rather than an "already exists" failure, which + // is what a resumed pass depends on. + require.NoError(t, database.ApplyConstraints()) + + require.NoError(t, database.DropConstraints()) + missing, err = database.MissingConstraints() + require.NoError(t, err) + require.NotEmpty(t, missing, "dropping has to actually remove them") + }) + } +} diff --git a/sink/sql/tests/integration/db_proto_postgres_segment_cursor_test.go b/sink/sql/tests/integration/db_proto_postgres_segment_cursor_test.go new file mode 100644 index 000000000..9e2d64fd9 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_segment_cursor_test.go @@ -0,0 +1,172 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresSegmentCarriesItsOwnCursor pins the one thing that makes a recorded +// segment trustworthy: the cursor committed with it has to cover the blocks it holds. +// +// A segment sealed before the cursor covering it was recorded carries the previous flush's +// instead, so it commits claiming a range its own cursor stops short of. The next run +// resumes at that cursor and Run's undo deletes every row above it — the segment's own — +// while the record stays behind to answer AlreadyApplied for the segment carrying exactly +// those rows. Recovery then discards that one and replays the segments behind it, moving +// the cursor over a gap it just created. +func TestDbProtoPostgresSegmentCarriesItsOwnCursor(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "alpha")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "beta")), + relationsBlockData(t, "3a", "2025-01-03", entityCustomer("customer-3", "gamma")), + relationsBlockData(t, "4a", "2025-01-04", entityCustomer("customer-4", "delta")), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-5", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}, + UseTransactions: true, + DecodeBatchSize: 1, + // One byte of segment: every flush seals, so each block records a segment of its + // own and the cursor each one carries is visible in the table. + Spool: &spool.Options{Dir: t.TempDir(), SegmentMaxBytes: 1}, + }.Defaults() + + testSchema := "segment_carries_its_own_cursor" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + var recorded []struct { + FirstBlock uint64 `db:"first_block"` + LastBlock uint64 `db:"last_block"` + Cursor string `db:"cursor"` + } + require.NoError(t, dbx.Select(&recorded, fmt.Sprintf( + `SELECT first_block, last_block, cursor FROM "%s"."_segments_" ORDER BY first_block`, testSchema))) + require.NotEmpty(t, recorded, "the run has to have recorded at least one segment") + + for _, segment := range recorded { + cursor, err := sink.NewCursor(segment.Cursor) + require.NoError(t, err) + + require.GreaterOrEqualf(t, cursor.Block().Num(), segment.LastBlock, + "the segment covering blocks %d-%d committed with a cursor at block %d, which does not cover it", + segment.FirstBlock, segment.LastBlock, cursor.Block().Num()) + } +} + +// TestDbProtoPostgresDropsSegmentRecordsPastTheCursor covers the records a database synced +// by an earlier build is still carrying, and anything else that could leave one behind. +// +// Run undoes every row above the stored cursor before a block arrives, so a record +// reaching past it describes rows that are no longer there. Left in place it answers +// AlreadyApplied for the segment carrying exactly those rows. +func TestDbProtoPostgresDropsSegmentRecordsPastTheCursor(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + testSchema := "segment_records_past_the_cursor" + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + openSink := func(t *testing.T) { + t.Helper() + + substreamsClientConfig := setupFakeSubstreamsServer(t, + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "alpha"))) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-2", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}, + UseTransactions: true, + DecodeBatchSize: 1, + Spool: &spool.Options{Dir: t.TempDir(), SegmentMaxBytes: 1}, + }.Defaults() + + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + } + + openSink(t) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + // A record reaching well past anything the run reached, as an interrupted seal under + // the old ordering would have left. + _, err = dbx.Exec(fmt.Sprintf( + `INSERT INTO "%s"."_segments_" (first_block, last_block, cursor) VALUES (900, 999, 'stale')`, testSchema)) + require.NoError(t, err) + + // Opening the sink again is what runs the pass. + openSink(t) + + var remaining int + require.NoError(t, dbx.Get(&remaining, fmt.Sprintf( + `SELECT count(*) FROM "%s"."_segments_" WHERE first_block = 900`, testSchema))) + require.Zero(t, remaining, "the record the stored cursor does not cover must be gone") +} diff --git a/sink/sql/tests/integration/db_proto_postgres_test.go b/sink/sql/tests/integration/db_proto_postgres_test.go index fd56dcf2b..2e3169ce1 100644 --- a/sink/sql/tests/integration/db_proto_postgres_test.go +++ b/sink/sql/tests/integration/db_proto_postgres_test.go @@ -15,6 +15,7 @@ import ( pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" sink "github.com/streamingfast/substreams/sink" "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" "github.com/stretchr/testify/require" ) @@ -92,9 +93,9 @@ func TestDbProtoPostgresIntegration(t *testing.T) { options := db_proto.SinkerFactoryOptions{ UseProtoOption: true, - UseConstraints: false, + Constraints: protosql.DisableAllConstraints(), UseTransactions: true, - BlockBatchSize: 1, + DecodeBatchSize: 1, }.Defaults() sinkerFactory := db_proto.SinkerFactory( diff --git a/sink/sql/tests/integration/db_proto_postgres_undo_relations_test.go b/sink/sql/tests/integration/db_proto_postgres_undo_relations_test.go new file mode 100644 index 000000000..f76609148 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_undo_relations_test.go @@ -0,0 +1,121 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresUndoRelations undoes a reorg over the full relational shape rather +// than a single flat table: nesting (order_extensions and order_items are children of +// orders) and sibling references (orders points at customers, order_items at items). +// +// Those two need opposite things from a delete order. Children have to go before their +// parent, and a referencing table before the table it points at — which is the same rule +// stated twice only if the reference graph is a tree. It is not, so the deletes follow +// the reverse of the schema's topological order. +func TestDbProtoPostgresUndoRelations(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + + tests := []struct { + name string + useConstraints bool + localBuffer bool + }{ + {"without constraints", false, false}, + {"with constraints", true, false}, + {"with the local buffer", false, true}, + {"with constraints and the local buffer", true, true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + blockEntities := func(suffix string) []*pbrelations.Entity { + return []*pbrelations.Entity{ + entityCustomer("customer-"+suffix, "name-"+suffix), + entityItem("item-"+suffix, "item-name-"+suffix, 1.5), + entityOrder("order-"+suffix, "customer-"+suffix, + &pbrelations.OrderExtension{Description: "extension-" + suffix}, + &pbrelations.OrderItem{ItemId: "item-" + suffix, Quantity: 2}, + ), + } + } + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", blockEntities("1")...), + relationsBlockData(t, "2a", "2025-01-02", blockEntities("2")...), + relationsBlockData(t, "3a", "2025-01-03", blockEntities("3")...), + blockUndo(t, "1a"), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-4", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: constraintPolicy(test.useConstraints), + UseTransactions: true, + DecodeBatchSize: 1, + }.Defaults() + + if test.localBuffer { + options.Spool = &spool.Options{Dir: t.TempDir()} + } + + testSchema := "undo_relations_" + strings.ReplaceAll(test.name, " ", "_") + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + ctx := context.Background() + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + count := func(table string) int { + var out int + require.NoError(t, dbx.Get(&out, fmt.Sprintf(`SELECT count(*) FROM "%s"."%s"`, testSchema, table))) + + return out + } + + // Only block 1 survives the undo, and it contributed exactly one row to each. + for _, table := range []string{"_blocks_", "customers", "items", "orders", "order_items", "order_extensions"} { + require.Equal(t, 1, count(table), "table %q keeps only what block 1 wrote", table) + } + }) + } +} diff --git a/sink/sql/tests/integration/db_proto_postgres_undo_test.go b/sink/sql/tests/integration/db_proto_postgres_undo_test.go new file mode 100644 index 000000000..18519aea8 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_undo_test.go @@ -0,0 +1,113 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresUndo covers a reorg in every shape the from-proto sink runs in. +// +// The undo used to be a single DELETE on _blocks_, leaning on `fk_block ... ON DELETE +// CASCADE` to take the entity rows with it. That foreign key only exists with +// constraints, so without them a reorg deleted the block rows and left every +// entity row of those blocks behind — and running without constraints is now the default. +func TestDbProtoPostgresUndo(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + + tests := []struct { + name string + useConstraints bool + localBuffer bool + }{ + {"without constraints", false, false}, + {"with constraints", true, false}, + {"with the local buffer", false, true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + responses := []*pbsubstreamsrpc.Response{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "kept")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "undone")), + relationsBlockData(t, "3a", "2025-01-03", entityCustomer("customer-3", "undone")), + // The fork takes blocks 2 and 3 back. + blockUndo(t, "1a"), + } + + pattern := make([]interface{}, len(responses)) + for i, response := range responses { + pattern[i] = response + } + substreamsClientConfig := setupFakeSubstreamsServer(t, pattern...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-4", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: constraintPolicy(test.useConstraints), + UseTransactions: true, + // One block per batch, so the undone blocks are in the database rather + // than still held in memory when the signal arrives. + DecodeBatchSize: 1, + }.Defaults() + + if test.localBuffer { + options.Spool = &spool.Options{Dir: t.TempDir()} + } + + testSchema := "undo_" + strings.ReplaceAll(strings.ToLower(test.name), " ", "_") + createPostgresTestSchema(t, postgresContainer.ConnectionString, testSchema) + + ctx := context.Background() + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, testSchema, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + count := func(table string) int { + var out int + require.NoError(t, dbx.Get(&out, fmt.Sprintf(`SELECT count(*) FROM "%s"."%s"`, testSchema, table))) + + return out + } + + require.Equal(t, 1, count("_blocks_"), "only the last valid block survives the undo") + require.Equal(t, 1, count("customers"), "the entity rows of the undone blocks must be gone, not orphaned") + }) + } +} diff --git a/sink/sql/tests/integration/db_proto_postgres_write_modes_test.go b/sink/sql/tests/integration/db_proto_postgres_write_modes_test.go new file mode 100644 index 000000000..3e7d71369 --- /dev/null +++ b/sink/sql/tests/integration/db_proto_postgres_write_modes_test.go @@ -0,0 +1,148 @@ +package tests + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/cenkalti/backoff/v4" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + "github.com/streamingfast/bstream" + "github.com/streamingfast/substreams/manifest" + sink "github.com/streamingfast/substreams/sink" + "github.com/streamingfast/substreams/sink/sql/db_proto" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" + "github.com/streamingfast/substreams/sink/sql/db_proto/sql/spool" + pbrelations "github.com/streamingfast/substreams/sink/sql/tests/relations" + "github.com/stretchr/testify/require" +) + +// TestDbProtoPostgresWriteModes runs the same blocks through every write mode and +// requires the resulting tables to be identical. +// +// All three now go through the spool, so what differs is only how a sealed segment is +// pushed: binary COPY files, multi-row INSERTs built from rendered tuples per table, or +// an interleaved log replayed one row at a time. A mode is only a performance choice if +// the rows it produces cannot be told apart, which is what this pins. +func TestDbProtoPostgresWriteModes(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + db, err := sql.Open("postgres", postgresContainer.ConnectionString) + require.NoError(t, err) + dbx := sqlx.NewDb(db, "postgres").Unsafe() + defer dbx.Close() + + type row struct { + CustomerId string `db:"customer_id"` + Name string `db:"name"` + } + + read := func(schemaName string) []row { + t.Helper() + + var rows []row + require.NoError(t, dbx.Select(&rows, fmt.Sprintf(`SELECT customer_id, name FROM "%s"."customers" ORDER BY customer_id`, schemaName))) + + return rows + } + + run := func(t *testing.T, schemaName string, mode protosql.WriteMode) []row { + t.Helper() + + responses := []interface{}{ + relationsBlockData(t, "1a", "2025-01-01", entityCustomer("customer-1", "alpha")), + relationsBlockData(t, "2a", "2025-01-02", entityCustomer("customer-2", "beta"), entityCustomer("customer-3", "gamma")), + // A backslash and a quote: the rendered modes build SQL literals where COPY + // hands pgtype the string as it is, so this is where the two would disagree. + relationsBlockData(t, "3a", "2025-01-03", entityCustomer("customer-4", `delta\path 'quoted'`)), + } + + substreamsClientConfig := setupFakeSubstreamsServer(t, responses...) + substreamsPackage := substreamsTestPackage(pbrelations.File_test_relations_relations_proto, outputMessageDescriptor) + + baseSink, err := sink.New( + sink.SubstreamsModeProduction, + false, + substreamsPackage, + substreamsPackage.Modules.Modules[0], + manifest.ModuleHash{}, + substreamsClientConfig, + logger, + tracer, + sink.WithBlockRange(bstream.MustParseRange("1-4", bstream.WithExclusiveEnd())), + sink.WithRetryBackOff(&backoff.StopBackOff{}), + ) + require.NoError(t, err) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}, + UseTransactions: true, + WriteMode: mode, + DecodeBatchSize: 1, + // A tiny idle window so a run this short seals through the timer rather than + // only at Close, which is the path a stalled stream takes. + Spool: &spool.Options{Dir: t.TempDir(), MaxIdle: 100_000_000}, + }.Defaults() + + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + dbSinker, err := db_proto.SinkerFactory(baseSink, defaultOutputModuleName, outputMessageDescriptor, options)(ctx, postgresContainer.ConnectionString, schemaName, logger, tracer) + require.NoError(t, err) + + require.NoError(t, dbSinker.Run(ctx)) + require.NoError(t, dbSinker.Err()) + + return read(schemaName) + } + + expected := []row{ + {CustomerId: "customer-1", Name: "alpha"}, + {CustomerId: "customer-2", Name: "beta"}, + {CustomerId: "customer-3", Name: "gamma"}, + {CustomerId: "customer-4", Name: `delta\path 'quoted'`}, + } + + for _, test := range []struct { + schema string + mode protosql.WriteMode + }{ + {"write_mode_copy", protosql.WriteModeCopy}, + {"write_mode_batch_insert", protosql.WriteModeBatchInsert}, + {"write_mode_row_insert", protosql.WriteModeRowInsert}, + } { + t.Run(string(test.mode), func(t *testing.T) { + require.Equal(t, expected, run(t, test.schema, test.mode)) + }) + } +} + +// TestDbProtoPostgresWriteModeUnsupported covers the promise that an explicit mode the +// schema cannot support is an error rather than a downgrade. +// +// A cycle in the foreign keys has no table order, so the two table-grouped modes cannot +// keep a parent ahead of its children. Falling back silently is what used to happen, and +// it costs an order of magnitude with only a log line to say so. +func TestDbProtoPostgresWriteModeUnsupported(t *testing.T) { + outputMessageDescriptor := (*pbrelations.Output)(nil).ProtoReflect().Descriptor() + postgresContainer := sharedDbChangesPostgresContainer + ctx := context.Background() + + schemaName := "write_mode_unsupported" + createPostgresTestSchema(t, postgresContainer.ConnectionString, schemaName) + + options := db_proto.SinkerFactoryOptions{ + UseProtoOption: true, + Constraints: protosql.ConstraintPolicy{Timing: protosql.ConstraintsManual}, + UseTransactions: true, + WriteMode: protosql.WriteMode("nonsense"), + DecodeBatchSize: 1, + }.Defaults() + + _, err := db_proto.SetupDatabaseSchema(ctx, postgresContainer.ConnectionString, schemaName, defaultOutputModuleName, outputMessageDescriptor, options, logger, tracer) + require.ErrorContains(t, err, "invalid write mode") +} diff --git a/sink/sql/tests/integration/helpers_test.go b/sink/sql/tests/integration/helpers_test.go index 30c75c016..bbfac29c3 100644 --- a/sink/sql/tests/integration/helpers_test.go +++ b/sink/sql/tests/integration/helpers_test.go @@ -3,6 +3,7 @@ package tests import ( "context" "fmt" + protosql "github.com/streamingfast/substreams/sink/sql/db_proto/sql" "net" "os" "strconv" @@ -10,8 +11,8 @@ import ( "testing" "time" - "github.com/moby/moby/api/types/network" "github.com/jmoiron/sqlx" + "github.com/moby/moby/api/types/network" "github.com/streamingfast/bstream" "github.com/streamingfast/logging" "github.com/streamingfast/logging/zapx" @@ -488,3 +489,14 @@ func readDbChangesRows[T any](t *testing.T, db *sqlx.DB, schema string, table st return readRowsBy[T](t, db, fmt.Sprintf(`"%s"."%s"`, schema, table), "id") } + +// constraintPolicy maps a test's "with constraints" boolean onto a policy: created before +// the load when asked for, none at all otherwise. The sink's own default — everything, +// created once the backfill is done — is exercised separately. +func constraintPolicy(useConstraints bool) protosql.ConstraintPolicy { + if !useConstraints { + return protosql.DisableAllConstraints() + } + + return protosql.ConstraintPolicy{Timing: protosql.ConstraintsAlways} +} diff --git a/tools/devenv/stack.go b/tools/devenv/stack.go index a9a726be0..48cf691d9 100644 --- a/tools/devenv/stack.go +++ b/tools/devenv/stack.go @@ -45,6 +45,10 @@ const BlockType = "sf.acme.type.v1.Block" // DefaultImage is the dummy blockchain image the end-to-end tests are pinned to. const DefaultImage = "ghcr.io/streamingfast/dummy-blockchain:1cea671" +// relayerPort is the firehose-core convention for the relayer's block stream server, which is +// the only port of the container this stack talks to. +const relayerPort = "10014" + // ChainConfig describes the dummy blockchain to run. type ChainConfig struct { // Image defaults to DefaultImage. @@ -118,14 +122,21 @@ func StartDummyBlockchain(ctx context.Context, config ChainConfig) (testcontaine Env: map[string]string{ "DLOG": ".*=debug", }, - ExposedPorts: []string{"10014/tcp"}, + ExposedPorts: []string{relayerPort + "/tcp"}, Mounts: testcontainers.Mounts( testcontainers.BindMount(config.TmpDir, "/app/firehose-data/storage/"), ), // Deliberately not wait.ForListeningPort: that one execs a probe inside the container, // and the exec itself times out while the node is busy writing a large genesis burst. - // The log line plus a dial from the host says the same thing without the exec. - WaitingFor: wait.ForLog("serving gRPC").WithStartupTimeout(config.startupTimeout()), + // + // The line has to name the relayer's own port. A bare "serving gRPC" matches the reader + // node announcing :10010, which it does before the relayer has finished bootstrapping its + // hub — and the dial below proves nothing either, since Docker binds the mapped host port + // when the container starts and accepts connections whether or not anything inside is + // listening. A tier1 started on that gap connects to a relayer that is not serving yet and + // ends up with a live stream carrying no partial blocks at all, which looks like a healthy + // run until something asserts on flash blocks. + WaitingFor: wait.ForLog(`serving gRPC.*`+relayerPort).AsRegexp().WithStartupTimeout(config.startupTimeout()), } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ @@ -406,7 +417,7 @@ func highestMergedBlock(dir string) uint64 { // RelayerEndpoint resolves the host-side address of the container's relayer. func RelayerEndpoint(ctx context.Context, container testcontainers.Container) (string, error) { - port, err := container.MappedPort(ctx, "10014/tcp") + port, err := container.MappedPort(ctx, relayerPort+"/tcp") if err != nil { return "", fmt.Errorf("mapped relayer port: %w", err) } diff --git a/tools/extract-proto.go b/tools/extract-proto.go new file mode 100644 index 000000000..3d6350b64 --- /dev/null +++ b/tools/extract-proto.go @@ -0,0 +1,376 @@ +package tools + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/streamingfast/cli" + "github.com/streamingfast/cli/sflags" + "github.com/streamingfast/substreams" + "github.com/streamingfast/substreams/manifest" + pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1" + "google.golang.org/protobuf/types/descriptorpb" +) + +// schemaProtoPath is where the SQL sink's annotations live, and the import path a proto +// has to use for them. The file is written alongside so the override parses without a +// protobuf toolchain: imports resolve relative to the working directory. +const schemaProtoPath = "sf/substreams/sink/sql/schema/v1/schema.proto" + +// schemaProtoPackage is the proto package that file declares, which is what qualifies the +// extensions: `option (schema.table)`, not the file's directory path. +const schemaProtoPackage = "schema" + +var extractProtoCmd = &cobra.Command{ + Use: "extract-proto [ []]", + Short: "Write a module's output protobuf definition to disk", + Long: cli.Dedent(` + Write the protobuf file defining a module's output message, as the package bundles + it, so it can be edited and fed back to a sink. + + With --sql, the file comes annotated for 'substreams sink postgres' in Relational + Mappings Mode: the schema annotations are imported, every message carries a commented-out + table option, every field a commented-out copy of itself per column option, and the + annotations file itself is written next to it so the result parses as-is. Uncomment + the table option; for a field, a column option lives inside the field's own + brackets, so replace the field with the commented variant that describes it — which + is the primary key, which are unique, which reference another table — and point the + sink at the result: + + substreams tools extract-proto --sql substreams.yaml map_events + substreams sink postgres setup --proto-file-override=./map_events.proto --dsn=... + + Without annotations Relational Mappings Mode derives tables from the message structure + alone: no primary keys, no unique constraints, no foreign keys. Only the index on + _block_number_ is created, that one being the sink's own rather than the schema's. + `), + Args: cobra.RangeArgs(0, 2), + RunE: runExtractProtoE, +} + +func init() { + extractProtoCmd.Flags().Bool("sql", false, "Annotate the output for the SQL sink and write the annotations file beside it") + extractProtoCmd.Flags().String("output-dir", ".", "Directory the files are written to") + + Cmd.AddCommand(extractProtoCmd) +} + +func runExtractProtoE(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + manifestPath := "" + if len(args) > 0 { + manifestPath = args[0] + } + moduleName := "" + if len(args) > 1 { + moduleName = args[1] + } + + reader, err := manifest.NewReader(manifestPath) + if err != nil { + return fmt.Errorf("reading manifest %q: %w", manifestPath, err) + } + + pkgBundle, err := reader.Read() + if err != nil { + return fmt.Errorf("reading manifest %q: %w", manifestPath, err) + } + pkg := pkgBundle.Package + + module, err := resolveOutputModule(pkg, moduleName) + if err != nil { + return err + } + + outputType := strings.TrimPrefix(module.Output.GetType(), "proto:") + file, err := fileDefining(pkg, outputType) + if err != nil { + return err + } + + outputDir := sflags.MustGetString(cmd, "output-dir") + annotate := sflags.MustGetBool(cmd, "sql") + + rendered := renderProtoFile(file, outputType, annotate) + + target := filepath.Join(outputDir, filepath.Base(file.GetName())) + if err := writeFile(target, rendered); err != nil { + return err + } + fmt.Printf("Wrote %s (message %s)\n", target, outputType) + + if annotate { + // The annotations file has to be on disk for the import to resolve: the override + // is parsed with imports looked up from the working directory. Writing the copy + // this binary was built with removes the one step most likely to go wrong. + schemaTarget := filepath.Join(outputDir, schemaProtoPath) + if err := writeFile(schemaTarget, substreams.SQLSchemaProto); err != nil { + return err + } + fmt.Printf("Wrote %s\n", schemaTarget) + + fmt.Printf("\nUncomment the table options, replace a field with the commented variant that describes it, then:\n"+ + " substreams sink postgres setup --proto-file-override=%s --dsn=...\n", target) + } + + return nil +} + +// resolveOutputModule picks the module to extract, inferring it when there is only one map +// module to infer. +func resolveOutputModule(pkg *pbsubstreams.Package, name string) (*pbsubstreams.Module, error) { + var candidates []*pbsubstreams.Module + for _, module := range pkg.Modules.Modules { + if module.GetKindMap() != nil { + candidates = append(candidates, module) + } + } + + if name != "" { + for _, module := range candidates { + if module.Name == name { + return module, nil + } + } + + return nil, fmt.Errorf("no map module named %q in this package", name) + } + + switch len(candidates) { + case 0: + return nil, fmt.Errorf("this package has no map module to extract an output type from") + case 1: + return candidates[0], nil + } + + names := make([]string, len(candidates)) + for i, module := range candidates { + names[i] = module.Name + } + sort.Strings(names) + + return nil, fmt.Errorf("this package has more than one map module, name the one to extract: %s", strings.Join(names, ", ")) +} + +func findFile(pkg *pbsubstreams.Package, name string) *descriptorpb.FileDescriptorProto { + for _, file := range pkg.ProtoFiles { + if file.GetName() == name { + return file + } + } + + return nil +} + +// fileDefining finds the bundled file that declares the output message. +func fileDefining(pkg *pbsubstreams.Package, outputType string) (*descriptorpb.FileDescriptorProto, error) { + for _, file := range pkg.ProtoFiles { + for _, message := range file.MessageType { + if file.GetPackage()+"."+message.GetName() == outputType { + return file, nil + } + } + } + + return nil, fmt.Errorf("the package does not bundle a definition for %q", outputType) +} + +func writeFile(path string, content string) error { + if directory := filepath.Dir(path); directory != "." { + if err := os.MkdirAll(directory, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", directory, err) + } + } + + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + + return nil +} + +// renderProtoFile turns a bundled descriptor back into readable proto source. +// +// The descriptor is what the package carries — comments and formatting are long gone — so +// this reconstructs a file rather than recovering the original. With annotate, every +// message and field gets the SQL sink's options commented out beside it, which is the part +// that saves the reader looking them up. +func renderProtoFile(file *descriptorpb.FileDescriptorProto, outputType string, annotate bool) string { + var out strings.Builder + + out.WriteString("syntax = \"" + file.GetSyntax() + "\";\n\n") + out.WriteString("package " + file.GetPackage() + ";\n\n") + + for _, dependency := range file.Dependency { + out.WriteString("import \"" + dependency + "\";\n") + } + if annotate && !hasDependency(file, schemaProtoPath) { + out.WriteString("import \"" + schemaProtoPath + "\";\n") + } + if len(file.Dependency) > 0 || annotate { + out.WriteString("\n") + } + + for _, message := range file.MessageType { + renderMessage(&out, file, message, outputType, annotate, "") + } + + for _, enum := range file.EnumType { + renderEnum(&out, enum, "") + } + + return out.String() +} + +func renderEnum(out *strings.Builder, enum *descriptorpb.EnumDescriptorProto, indent string) { + out.WriteString(indent + "enum " + enum.GetName() + " {\n") + for _, value := range enum.Value { + fmt.Fprintf(out, "%s %s = %d;\n", indent, value.GetName(), value.GetNumber()) + } + out.WriteString(indent + "}\n\n") +} + +func hasDependency(file *descriptorpb.FileDescriptorProto, name string) bool { + for _, dependency := range file.Dependency { + if dependency == name { + return true + } + } + + return false +} + +func renderMessage(out *strings.Builder, file *descriptorpb.FileDescriptorProto, message *descriptorpb.DescriptorProto, outputType string, annotate bool, indent string) { + fullName := file.GetPackage() + "." + message.GetName() + + out.WriteString(indent + "message " + message.GetName() + " {\n") + + if annotate { + if fullName == outputType { + out.WriteString(indent + " // This is the module's output message. The sink walks it into tables: every\n") + out.WriteString(indent + " // nested message becomes a table of its own, repeated ones a table per element.\n") + } + out.WriteString(indent + " // option (" + schemaProtoPackage + ".table) = { name: \"" + strings.ToLower(message.GetName()) + "\" };\n\n") + } + + for _, field := range message.Field { + declaration := declarationOf(message, field) + + // A map is walked into a table of its own rather than into a column, so none of the + // column options apply to the field declaring it. + if annotate && mapEntryOf(message, field) == nil { + // A field option lives inside the field's own brackets, before the semicolon, + // so it cannot be a comment line of its own the way the table option can. + // These are whole declarations: the one that describes the column replaces the + // plain one under it. + fmt.Fprintf(out, "%s // %s [("+schemaProtoPackage+".field) = { primary_key: true }]; // replaces the line below, one per message\n", indent, declaration) + fmt.Fprintf(out, "%s // %s [("+schemaProtoPackage+".field) = { unique: true }];\n", indent, declaration) + fmt.Fprintf(out, "%s // %s [("+schemaProtoPackage+".field) = { foreign_key: \"other_table.column\" }];\n", indent, declaration) + } + + fmt.Fprintf(out, "%s %s;\n", indent, declaration) + + if annotate { + out.WriteString("\n") + } + } + + // A nested enum is declared by the message and referenced by its fields, so leaving it + // out writes a file naming a type nothing declares — which is the file the command + // tells the operator to feed back through --proto-file-override. + for _, enum := range message.EnumType { + out.WriteString("\n") + renderEnum(out, enum, indent+" ") + } + + for _, nested := range message.NestedType { + // The synthetic entry type of a map field. The field renders as `map`, which + // declares it again, so writing it out here would be a duplicate definition. + if nested.GetOptions().GetMapEntry() { + continue + } + out.WriteString("\n") + renderMessage(out, file, nested, outputType, annotate, indent+" ") + } + + out.WriteString(indent + "}\n\n") +} + +// declarationOf renders a field without its trailing semicolon, so the annotated form and +// the plain one are built from the same string. +func declarationOf(message *descriptorpb.DescriptorProto, field *descriptorpb.FieldDescriptorProto) string { + if entry := mapEntryOf(message, field); entry != nil { + // A map field is carried in the descriptor as a repeated message of a synthetic + // entry type. Rendering what the descriptor says — `repeated Msg.FooEntry foo` — + // names a type the file does not declare, and does not parse. + return fmt.Sprintf("map<%s, %s> %s = %d", typeOf(entryField(entry, 1)), typeOf(entryField(entry, 2)), field.GetName(), field.GetNumber()) + } + + return fmt.Sprintf("%s%s %s = %d", labelOf(field), typeOf(field), field.GetName(), field.GetNumber()) +} + +// mapEntryOf returns the synthetic entry type of a map field, or nil for anything else. +func mapEntryOf(message *descriptorpb.DescriptorProto, field *descriptorpb.FieldDescriptorProto) *descriptorpb.DescriptorProto { + if field.GetType() != descriptorpb.FieldDescriptorProto_TYPE_MESSAGE || field.GetLabel() != descriptorpb.FieldDescriptorProto_LABEL_REPEATED { + return nil + } + + name := field.GetTypeName() + if index := strings.LastIndex(name, "."); index >= 0 { + name = name[index+1:] + } + + for _, nested := range message.NestedType { + if nested.GetName() != name || !nested.GetOptions().GetMapEntry() { + continue + } + if entryField(nested, 1) == nil || entryField(nested, 2) == nil { + return nil + } + + return nested + } + + return nil +} + +// entryField picks the key (1) or the value (2) of a map entry by its field number, which +// is what the encoding fixes, rather than by its position in the descriptor. +func entryField(entry *descriptorpb.DescriptorProto, number int32) *descriptorpb.FieldDescriptorProto { + for _, field := range entry.Field { + if field.GetNumber() == number { + return field + } + } + + return nil +} + +func labelOf(field *descriptorpb.FieldDescriptorProto) string { + if field.GetLabel() == descriptorpb.FieldDescriptorProto_LABEL_REPEATED { + return "repeated " + } + if field.GetProto3Optional() { + return "optional " + } + + return "" +} + +func typeOf(field *descriptorpb.FieldDescriptorProto) string { + if field.TypeName != nil { + return strings.TrimPrefix(field.GetTypeName(), ".") + } + + name := strings.ToLower(strings.TrimPrefix(field.GetType().String(), "TYPE_")) + if name == "" { + return "bytes" + } + + return name +} diff --git a/tools/extract-proto_test.go b/tools/extract-proto_test.go new file mode 100644 index 000000000..288d91c02 --- /dev/null +++ b/tools/extract-proto_test.go @@ -0,0 +1,125 @@ +package tools + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jhump/protoreflect/desc/protoparse" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +// TestRenderProtoFileParsesWithAnnotations is the property that makes the command worth +// having: what it writes has to parse once the operator uncomments an option, against the +// annotations file it wrote beside it. A scaffold that does not compile is worse than no +// scaffold. +func TestRenderProtoFileParsesWithAnnotations(t *testing.T) { + file := &descriptorpb.FileDescriptorProto{ + Name: proto.String("events.proto"), + Package: proto.String("test.output"), + Syntax: proto.String("proto3"), + MessageType: []*descriptorpb.DescriptorProto{ + { + Name: proto.String("Event"), + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("id"), + Number: proto.Int32(1), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), + }, + { + Name: proto.String("amount"), + Number: proto.Int32(2), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_UINT64.Enum(), + }, + { + // A map field, which the descriptor carries as a repeated message + // of a synthetic entry type. + Name: proto.String("balances"), + Number: proto.Int32(3), + Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(), + TypeName: proto.String(".test.output.Event.BalancesEntry"), + }, + { + Name: proto.String("kind"), + Number: proto.Int32(4), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_ENUM.Enum(), + TypeName: proto.String(".test.output.Event.Kind"), + }, + }, + NestedType: []*descriptorpb.DescriptorProto{ + { + Name: proto.String("BalancesEntry"), + Options: &descriptorpb.MessageOptions{MapEntry: proto.Bool(true)}, + Field: []*descriptorpb.FieldDescriptorProto{ + { + Name: proto.String("key"), + Number: proto.Int32(1), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), + }, + { + Name: proto.String("value"), + Number: proto.Int32(2), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_UINT64.Enum(), + }, + }, + }, + }, + EnumType: []*descriptorpb.EnumDescriptorProto{ + { + Name: proto.String("Kind"), + Value: []*descriptorpb.EnumValueDescriptorProto{ + {Name: proto.String("KIND_UNSPECIFIED"), Number: proto.Int32(0)}, + {Name: proto.String("KIND_TRANSFER"), Number: proto.Int32(1)}, + }, + }, + }, + }, + }, + } + + rendered := renderProtoFile(file, "test.output.Event", true) + + require.Contains(t, rendered, `import "`+schemaProtoPath+`"`) + require.Contains(t, rendered, "// option ("+schemaProtoPackage+".table)") + require.Contains(t, rendered, "// string id = 1 [("+schemaProtoPackage+".field) = { primary_key: true }];") + + // A map field renders as a map, not as the entry type it is carried by, and the entry + // type is not declared a second time. + require.Contains(t, rendered, "map balances = 3;") + require.NotContains(t, rendered, "BalancesEntry") + + // A nested enum is declared where it is referenced from. + require.Contains(t, rendered, "enum Kind {") + + // Uncomment what an operator would, then require the result to parse. + rendered = strings.Replace(rendered, ` // option (schema.table) = { name: "event" };`, ` option (schema.table) = { name: "event" };`, 1) + rendered = strings.Replace(rendered, " string id = 1;", " string id = 1 [(schema.field) = { primary_key: true }];", 1) + + directory := t.TempDir() + require.NoError(t, writeFile(filepath.Join(directory, "events.proto"), rendered)) + require.NoError(t, writeFile(filepath.Join(directory, schemaProtoPath), mustReadSchemaProto(t))) + + parser := protoparse.Parser{ImportPaths: []string{directory}} + fds, err := parser.ParseFiles("events.proto") + require.NoError(t, err, "the scaffold has to parse once its options are uncommented") + require.Len(t, fds[0].GetMessageTypes(), 1) +} + +func mustReadSchemaProto(t *testing.T) string { + t.Helper() + + content, err := os.ReadFile(filepath.Join("..", "proto", schemaProtoPath)) + require.NoError(t, err) + + return string(content) +}