testutil: control-plane error contract and rotation-seam tests

docs/testing.md

@@ -168,22 +168,39 @@ upgrades or mirrors.
 
 ### How much of the suite runs on Ministack
 
-Deliberately almost none: exactly one end-to-end test
-([ministack_integration_test.go](../internal/testutil/ministack_integration_test.go))
-covering provision → instance `available` → endpoint discovery →
-`dbconn` connect → PG-major assertion → DDL smoke. Everything else — all
-parser, planner, executor, and connection behavior — runs on the
-data-plane tier against real PostgreSQL. That split is policy, not
-accident: Ministack exists only for the seam where the engine talks to
-AWS APIs, and its share grows only when AWS-facing features land, never
-by moving core-logic tests onto it. Planned growth, in dependency order:
+Deliberately almost none: three tests
+([ministack_integration_test.go](../internal/testutil/ministack_integration_test.go)),
+each pinned to an AWS seam —
+
+- **provision & connect** — provision → instance `available` → endpoint
+  discovery → `dbconn` connect → PG-major assertion → DDL smoke;
+- **control-plane error contract** — unknown identifiers and duplicate
+  creations surface as the AWS SDK's typed RDS faults, matched with
+  `errors.As` (one documented emulator divergence: the duplicate-instance
+  wire code carries a `Fault` suffix real AWS omits, so that case matches
+  by error-code prefix);
+- **password-rotation seam** — `ModifyDBCluster` applies a new master
+  password to the running database; the stale password is refused with
+  SQLSTATE `28P01`, the new one connects.
+
+Everything else — all parser, planner, executor, and connection
+behavior — runs on the data-plane tier against real PostgreSQL. That
+split is policy, not accident: Ministack exists only for the seam where
+the engine talks to AWS APIs, and its share grows only when AWS-facing
+features land, never by moving core-logic tests onto it. Planned growth,
+in dependency order:
 
 - reader/writer topology tests — writer-endpoint targeting with a reader
-  present, endpoint re-discovery after a global-cluster failover — once
-  the engine has endpoint-selection logic to test;
+  present, endpoint re-discovery after a global-cluster failover
+  (metadata-level: every Ministack endpoint resolves to one shared
+  container) — once the engine has endpoint-selection logic to test;
 - Secrets Manager DSN resolution, when that feature lands;
-- RDS IAM-auth token connections (`dbconn.Config.BeforeConnect`), when
-  that feature lands.
+- rotation *recovery* — the engine re-resolving credentials and
+  reconnecting mid-migration — once `pkg/dbconn` grows a
+  credential-refresh hook.
+
+Logical-replication behavior is a data-plane concern and is tested on
+real PostgreSQL, never on this tier.
 
 The tier runs in CI as the `aws-boundary` merge-gate job and inside
 `make test` when Docker is available. It is intentionally **not** part
@@ -201,7 +218,7 @@ the authoritative gate.
 | Verify-full TLS against a live TLS-only server | [pkg/dbconn/tls_integration_test.go](../pkg/dbconn/tls_integration_test.go) |
 | Targeted blocker termination | [pkg/dbconn/dbconn_integration_test.go](../pkg/dbconn/dbconn_integration_test.go) |
 | Test harness self-checks | [internal/testutil](../internal/testutil/postgres_test.go) |
-| RDS control-plane provisioning → endpoint discovery → `dbconn` connect (Ministack) | [internal/testutil/ministack_integration_test.go](../internal/testutil/ministack_integration_test.go) |
+| RDS control-plane provisioning → endpoint discovery → `dbconn` connect, error contract, password-rotation seam (Ministack) | [internal/testutil/ministack_integration_test.go](../internal/testutil/ministack_integration_test.go) |
 | Parse boundary, typed operations, and advisory rewrites | [pkg/statement](../pkg/statement/statement_test.go), [operation tests](../pkg/statement/ops_test.go) |
 | Native / copy-and-swap / refuse classification and safer SQL | [pkg/planner](../pkg/planner/planner_test.go) |
 | Backend routing and copy-and-swap unavailable disposition | [pkg/router](../pkg/router/router_test.go) |

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.27.5
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.5
 	github.com/aws/aws-sdk-go-v2/service/rds v1.124.1
+	github.com/aws/smithy-go v1.27.6
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/moby/moby/api v1.54.2
 	github.com/pganalyze/pg_query_go/v6 v6.2.2
@@ -30,7 +31,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.2 // indirect
-	github.com/aws/smithy-go v1.27.6 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect

internal/testutil/ministack.go

@@ -62,12 +62,48 @@ func ministackImage() string {
 	return "ministackorg/ministack:1.4.13-full"
 }
 
+// AuroraCluster is a provisioned Ministack aurora-postgresql cluster and
+// the control-plane client that owns it. Tests drive further control-plane
+// operations (rotation, duplicate creation, discovery of unknown
+// identifiers) through Client against ClusterID and InstanceID.
+type AuroraCluster struct {
+	// Client is the RDS control-plane client bound to the Ministack gateway.
+	Client *rds.Client
+	// ClusterID is the DBClusterIdentifier of the provisioned cluster.
+	ClusterID string
+	// InstanceID is the DBInstanceIdentifier of the cluster's sole instance.
+	InstanceID string
+
+	dbPort int
+}
+
+// URL returns a connection URL for the cluster's database using the
+// fixture master password the cluster was provisioned with.
+func (c *AuroraCluster) URL() string {
+	return c.URLWithPassword(fixturePassword)
+}
+
+// URLWithPassword returns a connection URL using the given master
+// password, for tests that rotate credentials via ModifyDBCluster.
+//
+// The control-plane endpoint address is container-internal; the URL uses
+// the pinned host-published port instead (see ProvisionAuroraPostgres).
+//
+// sslmode=disable: the sibling database container runs plain PostgreSQL
+// without TLS, and the endpoint is not an *.rds.amazonaws.com hostname,
+// so the production TLS path is out of scope for this tier (it is
+// proven by pkg/dbconn's TLS integration tests).
+func (c *AuroraCluster) URLWithPassword(password string) string {
+	return fmt.Sprintf("postgres://%s:%s@localhost:%d/%s?sslmode=disable",
+		fixtureUser, password, c.dbPort, fixtureDatabase)
+}
+
 // ProvisionAuroraPostgres starts a Ministack container, provisions an
 // aurora-postgresql cluster and instance through the real RDS control-plane
-// API, waits until the instance is available, and returns a connection URL
-// for the cluster's database. The PostgreSQL major follows PG_VERSION: the
-// cluster's database is a real postgres container of that major.
-func ProvisionAuroraPostgres(t *testing.T) string {
+// API, waits until the instance is available, and returns the cluster
+// handle. The PostgreSQL major follows PG_VERSION: the cluster's database
+// is a real postgres container of that major.
+func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster {
 	t.Helper()
 	if os.Getenv("SKIP_INTEGRATION") != "" {
 		t.Skip("SKIP_INTEGRATION set; skipping test that needs Docker")
@@ -174,15 +210,12 @@ func ProvisionAuroraPostgres(t *testing.T) string {
 	require.NotZero(t, aws.ToInt32(clusters.DBClusters[0].Port),
 		"cluster endpoint port must be discoverable")
 
-	// The discovered endpoint address is container-internal; connect via the
-	// pinned host-published port instead (see dbPort above).
-	//
-	// sslmode=disable: the sibling database container runs plain PostgreSQL
-	// without TLS, and the endpoint is not an *.rds.amazonaws.com hostname,
-	// so the production TLS path is out of scope for this tier (it is
-	// proven by pkg/dbconn's TLS integration tests).
-	return fmt.Sprintf("postgres://%s:%s@localhost:%d/%s?sslmode=disable",
-		fixtureUser, fixturePassword, dbPort, fixtureDatabase)
+	return &AuroraCluster{
+		Client:     client,
+		ClusterID:  clusterID,
+		InstanceID: instanceID,
+		dbPort:     dbPort,
+	}
 }
 
 // freePort reserves an ephemeral TCP port and returns it for reuse. The

internal/testutil/ministack_integration_test.go

@@ -2,9 +2,15 @@ package testutil_test
 
 import (
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/rds"
+	"github.com/aws/aws-sdk-go-v2/service/rds/types"
+	"github.com/aws/smithy-go"
+	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -18,10 +24,10 @@ import (
 // through pkg/dbconn (bounded session defaults included), the server runs
 // the requested PostgreSQL major, and DDL executes.
 func TestAuroraControlPlaneProvisionAndConnect(t *testing.T) {
-	url := testutil.ProvisionAuroraPostgres(t)
+	cluster := testutil.ProvisionAuroraPostgres(t)
 
 	pool, err := dbconn.NewPool(t.Context(), dbconn.Config{
-		URL:         url,
+		URL:         cluster.URL(),
 		LockTimeout: 300 * time.Millisecond,
 	})
 	require.NoError(t, err, "connect to provisioned cluster endpoint via dbconn")
@@ -49,3 +55,95 @@ func TestAuroraControlPlaneProvisionAndConnect(t *testing.T) {
 	require.NoError(t, pool.QueryRow(t.Context(), "SELECT to_regclass($1)::oid", schema+".t").Scan(&oid))
 	assert.NotNil(t, oid, "created table must be visible in the catalog")
 }
+
+// TestAuroraControlPlaneErrorContract proves the control-plane error
+// contract the engine's discovery code will rely on: unknown identifiers
+// and duplicate creations surface as the AWS SDK's typed RDS faults,
+// matchable with errors.As — never by message text.
+func TestAuroraControlPlaneErrorContract(t *testing.T) {
+	cluster := testutil.ProvisionAuroraPostgres(t)
+	ctx := t.Context()
+
+	_, err := cluster.Client.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{
+		DBClusterIdentifier: aws.String("pgsprite-does-not-exist"),
+	})
+	var clusterNotFound *types.DBClusterNotFoundFault
+	require.ErrorAs(t, err, &clusterNotFound,
+		"describing an unknown cluster must surface the typed not-found fault")
+
+	_, err = cluster.Client.CreateDBCluster(ctx, &rds.CreateDBClusterInput{
+		DBClusterIdentifier: aws.String(cluster.ClusterID),
+		Engine:              aws.String("aurora-postgresql"),
+		MasterUsername:      aws.String("pgsprite"),
+		MasterUserPassword:  aws.String("test-password-do-not-use"),
+	})
+	var clusterExists *types.DBClusterAlreadyExistsFault
+	require.ErrorAs(t, err, &clusterExists,
+		"creating a duplicate cluster must surface the typed already-exists fault")
+
+	_, err = cluster.Client.CreateDBInstance(ctx, &rds.CreateDBInstanceInput{
+		DBInstanceIdentifier: aws.String(cluster.InstanceID),
+		DBClusterIdentifier:  aws.String(cluster.ClusterID),
+		Engine:               aws.String("aurora-postgresql"),
+		DBInstanceClass:      aws.String("db.t3.medium"),
+	})
+	// Real AWS emits wire code "DBInstanceAlreadyExists" (which the SDK maps
+	// to types.DBInstanceAlreadyExistsFault); Ministack emits
+	// "DBInstanceAlreadyExistsFault", which the SDK leaves as a generic API
+	// error. Match the code prefix so the assertion holds against both the
+	// emulator and a real endpoint — engine code consuming this error must
+	// do the same.
+	var apiErr smithy.APIError
+	require.ErrorAs(t, err, &apiErr,
+		"creating a duplicate instance must surface an RDS API error")
+	assert.True(t, strings.HasPrefix(apiErr.ErrorCode(), "DBInstanceAlreadyExists"),
+		"duplicate instance error code must identify the already-exists fault, got %q", apiErr.ErrorCode())
+}
+
+// TestAuroraControlPlanePasswordRotation proves the rotation seam the
+// credential test theme builds on: ModifyDBCluster applies a new master
+// password to the running database, after which the stale password is
+// refused with SQLSTATE 28P01 (invalid_password) and the new one connects.
+func TestAuroraControlPlanePasswordRotation(t *testing.T) {
+	// Named polling bounds for the rotation to land on the real database.
+	const (
+		rotationDeadline = time.Minute
+		rotationPoll     = time.Second
+	)
+
+	cluster := testutil.ProvisionAuroraPostgres(t)
+	ctx := t.Context()
+
+	const rotatedPassword = "test-password-rotated-do-not-use"
+	_, err := cluster.Client.ModifyDBCluster(ctx, &rds.ModifyDBClusterInput{
+		DBClusterIdentifier: aws.String(cluster.ClusterID),
+		MasterUserPassword:  aws.String(rotatedPassword),
+		ApplyImmediately:    aws.Bool(true),
+	})
+	require.NoError(t, err, "rotate master password via ModifyDBCluster")
+
+	// The rotation must land on the real database, not just the metadata:
+	// poll until the new password opens a connection through dbconn.
+	require.Eventuallyf(t, func() bool {
+		pool, err := dbconn.NewPool(ctx, dbconn.Config{
+			URL:         cluster.URLWithPassword(rotatedPassword),
+			LockTimeout: 300 * time.Millisecond,
+		})
+		if err != nil {
+			return false
+		}
+		pool.Close()
+		return true
+	}, rotationDeadline, rotationPoll,
+		"rotated master password did not become usable within the deadline")
+
+	// The stale password is refused by authentication — the exact failure
+	// a mid-migration connection hits after a production rotation.
+	_, err = dbconn.NewPool(ctx, dbconn.Config{
+		URL:         cluster.URL(),
+		LockTimeout: 300 * time.Millisecond,
+	})
+	var pgErr *pgconn.PgError
+	require.ErrorAs(t, err, &pgErr, "stale password must fail with a server auth error")
+	assert.Equal(t, "28P01", pgErr.Code, "stale password must be refused as invalid_password")
+}