feat(cts): add spec-driven contract test suite for Python, Java, and Rust clients

.github/workflows/contract-tests.yml

@@ -0,0 +1,271 @@
+name: Contract Tests
+
+on:
+  push:
+    branches: [main, "feat/**", "feature/**"]
+    paths:
+      - "docs/src/spec.yaml"
+      - "docs/src/cts-contracts/**"
+      - "ci/**"
+      - "java/**"
+      - "python/**"
+      - "rust/**"
+      - ".github/workflows/contract-tests.yml"
+  pull_request:
+    paths:
+      - "docs/src/spec.yaml"
+      - "docs/src/cts-contracts/**"
+      - "ci/**"
+      - "java/**"
+      - "python/**"
+      - "rust/**"
+      - ".github/workflows/contract-tests.yml"
+  workflow_dispatch:
+    inputs:
+      run_wiremock:
+        description: "Run the legacy WireMock client-conformance matrix"
+        type: boolean
+        default: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # ─────────────────────────────────────────────────────────────
+  # Job 1: Spec validation — guards spec.yaml immutability,
+  #         runs Spectral lint, and checks for breaking changes.
+  # ─────────────────────────────────────────────────────────────
+  spec:
+    name: Spec Validation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0   # required for git diff in verify-spec-untouched
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Verify spec untouched
+        run: make verify-spec-untouched
+
+      - name: Lint spec with Spectral
+        run: |
+          mkdir -p build
+          npx --yes @stoplight/spectral-cli lint docs/src/spec.yaml \
+            --ruleset ci/spectral.yaml \
+            --fail-severity error \
+            --format junit \
+            --output build/spectral-report.xml
+
+      - name: Check breaking changes with oasdiff
+        uses: oasdiff/oasdiff-action/breaking@main
+        with:
+          base: "https://raw.githubusercontent.com/lance-format/lance-namespace/main/docs/src/spec.yaml"
+          revision: "docs/src/spec.yaml"
+          fail-on-diff: true
+
+      - name: Upload Spectral report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: spectral-report
+          path: build/spectral-report.xml
+          if-no-files-found: ignore
+
+  # ─────────────────────────────────────────────────────────────
+  # Job 2: Server conformance (Schemathesis) — TEMPORARILY REMOVED
+  #
+  # Originally this job started the Spring Boot reference server under
+  # the pact profile and ran Schemathesis against it to verify spec
+  # conformance. It has been removed for now because the
+  # `lance-namespace-springboot-server` module currently only contains
+  # OpenAPI-generated API interfaces (NamespaceApi / TableApi /
+  # TransactionApi) — there is no @SpringBootApplication main class,
+  # no controller implementations, and no application.yml, so
+  # `java -jar ...` cannot actually start an HTTP server.
+  #
+  # Re-add this job in a follow-up PR once the springboot-server module
+  # ships a runnable reference server (Main class + stub controllers
+  # honoring spec response codes). The Schemathesis configuration in
+  # ci/schemathesis.toml is kept and already excludes the 3 Arrow IPC
+  # endpoints (CreateTable / InsertIntoTable / MergeInsertIntoTable
+  # using application/vnd.apache.arrow.stream) so it is ready to use
+  # as soon as the server exists.
+  # ─────────────────────────────────────────────────────────────
+
+  # ─────────────────────────────────────────────────────────────
+  # Job 3: Behavioural contract conformance — drives the spec-driven
+  #         CTS YAML bundle through the in-process Rust harness in
+  #         `rust/lance-namespace-cts` against `DirectoryNamespace`.
+  #         This is the primary PR-blocking signal; it covers behaviour
+  #         (state changes, error codes, capability gating) — not just
+  #         HTTP wire shape.  Runs on every push and PR.
+  # ─────────────────────────────────────────────────────────────
+  behavior-conformance:
+    name: Behaviour Conformance (Rust in-process)
+    runs-on: ubuntu-latest
+    needs: spec
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install Python workspace (loader/lint/codegen need pyyaml + jsonschema)
+        run: uv sync --all-packages
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+
+      # `lance-namespace-cts` pulls in `lance-namespace-impls` from the sibling
+      # `lance` repository via cargo `path`, which transitively depends on
+      # `lance-encoding`.  `lance-encoding`'s build script invokes `protoc` to
+      # compile its `.proto` files, so the GitHub-hosted runner needs the
+      # protobuf compiler on PATH before we run `cargo test`.  Locally this
+      # works because contributors typically have `protoc` installed; CI does
+      # not, so we install it explicitly here.
+      - name: Install protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "rust -> rust/target"
+
+      - name: Lint contract bundle (strict)
+        run: uv run python ci/cts/lint_contracts.py --strict
+
+      - name: Verify generated tests are up-to-date
+        run: uv run python ci/cts/gen_contract_tests.py --check
+
+      - name: Run behavioural contract suite
+        run: make test-cts-behavior
+
+      - name: Upload behavioural test logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: contract-test-behavior
+          path: rust/lance-namespace-cts/target/nextest/**/*.xml
+          if-no-files-found: ignore
+
+  # ─────────────────────────────────────────────────────────────
+  # Job 4: Client conformance (legacy WireMock) — matrix over
+  #         java / python / rust.  Each language generates WireMock
+  #         stubs from the spec and runs its thin contract runner.
+  #
+  #         As of P5 (the CI-finalisation phase of the behavioural CTS rollout)
+  #         this job is **opt-in**: it runs on push to `main` only and
+  #         on manual workflow_dispatch with `run_wiremock=true`. It is
+  #         skipped on pull_request events and on push to feature
+  #         branches, to keep PR/dev-branch feedback focused on the
+  #         (much faster) behavioural job above.
+  # ─────────────────────────────────────────────────────────────
+  client-conformance:
+    name: Client Conformance (${{ matrix.lang }})
+    # Opt-in only: WireMock matrix is expensive (java + python + rust),
+    # so we restrict push triggering to `main` and otherwise require an
+    # explicit workflow_dispatch with run_wiremock=true.  PRs never run
+    # this job; the behavioural suite above is the PR-blocking signal.
+    if: >-
+      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
+      (github.event_name == 'workflow_dispatch' && inputs.run_wiremock)
+    runs-on: ubuntu-latest
+    needs: spec
+    strategy:
+      fail-fast: false          # each language fails/passes independently
+      matrix:
+        lang: [java, python, rust]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      # ── Language-specific toolchain setup ──────────────────────
+      - name: Setup Java 17
+        if: matrix.lang == 'java'
+        uses: actions/setup-java@v4
+        with:
+          java-version: "17"
+          distribution: "temurin"
+          cache: "maven"
+
+      - name: Setup Python
+        if: matrix.lang == 'python'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        if: matrix.lang == 'python'
+        uses: astral-sh/setup-uv@v3
+
+      - name: Setup Rust
+        if: matrix.lang == 'rust'
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+
+      # ── Shared: generate WireMock stubs + download jar ─────────
+      - name: Install uv (Python dep for overlay-gen)
+        if: matrix.lang != 'python'
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install Python workspace (overlay-gen needs pyyaml)
+        run: uv sync --all-packages
+
+      - name: Generate WireMock stubs
+        run: make gen-wiremock
+
+      - name: Download WireMock standalone jar
+        run: make build/cts/wiremock-standalone.jar
+
+      # ── Language-specific pre-build ────────────────────────────
+      - name: Build Java clients
+        if: matrix.lang == 'java'
+        run: |
+          cd java
+          ./mvnw -pl lance-namespace-apache-client,lance-namespace-async-client \
+            install -DskipTests --no-transfer-progress -q
+
+      - name: Install Python client
+        if: matrix.lang == 'python'
+        run: uv sync --all-packages
+
+      - name: Cache Rust build
+        if: matrix.lang == 'rust'
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "rust/lance-namespace-reqwest-client"
+
+      # ── Run contract tests ──────────────────────────────────────
+      - name: Run contract tests
+        run: make test-clients-wiremock-${{ matrix.lang }}
+
+      # ── Upload reports ──────────────────────────────────────────
+      - name: Upload test reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: contract-test-${{ matrix.lang }}
+          path: |
+            java/**/target/surefire-reports/*.xml
+            build/reports/*.xml
+          if-no-files-found: ignore

.github/workflows/spec.yml

@@ -52,6 +52,14 @@ jobs:
         run: | 
           make clean
           make gen
+          # Contract tests (CTS) live alongside generated client code and are
+          # wiped by `make clean` (which uses `rm -rf <module>/**`).  Re-run the
+          # WireMock-layer test generator so the working tree matches the
+          # committed sources.  We use `gen-wiremock-tests` rather than the
+          # full `gen-cts` target so this job stays light: we only need the
+          # generated source files to diff against — actually *running* the
+          # tests would also require downloading the WireMock standalone jar.
+          make gen-wiremock-tests
       - name: Check no difference in codegen
         run: |
           output=$(git diff)

AGENTS.md

@@ -144,6 +144,15 @@ You can also run `make <command>-<language>-<module>` inside a language folder t
 - `make gen-rust-reqwest-client`: codegen and lint the Rust reqwest client module
 - `make build-java-springboot-server`: build the Java Spring Boot server module
 
+## Contract Tests
+
+Run the full contract test suite to verify spec compliance:
+```bash
+make test-cts
+```
+
+See [docs/spec-driven-contract-testing.md](docs/spec-driven-contract-testing.md) for details.
+
 ## Documentation
 
 ### Setup

CONTRIBUTING.md

@@ -144,6 +144,91 @@ You can also run `make <command>-<language>-<module>` inside a language folder t
 - `make gen-rust-reqwest-client`: codegen and lint the Rust reqwest client module
 - `make build-java-springboot-server`: build the Java Spring Boot server module
 
+## Contract Tests (CTS)
+
+This repository ships **two** contract test suites; together they form the
+Compatibility Test Suite (CTS):
+
+| Suite | Source of truth | What it verifies | Driver | Default? |
+|---|---|---|---|---|
+| **Behavioural CTS** | `docs/src/cts-contracts/*.yaml` | Operation semantics: state transitions, error codes, capability gating | In-process Rust harness in `rust/lance-namespace-cts` against `DirectoryNamespace` | ✅ runs on every PR via `make test-cts` |
+| **WireMock CTS** | `docs/src/spec.yaml` | HTTP wire shape: paths, methods, status codes, JSON serde | Per-language thin runner (Java / Python / Rust) against a WireMock standalone jar | ⚠️ opt-in via `make test-cts-wiremock`; CI runs it on push to long-lived branches and on manual dispatch only |
+
+Design background and implementation progress notes for the behavioural
+CTS are maintained outside this repository (in the maintainer's working
+notes); the entry points below describe the **stable** authoring
+contract and should be sufficient to add cases without consulting them.
+
+### Authoring a Behavioural Contract Case
+
+1. **Pick the right domain file** under `docs/src/cts-contracts/`:
+   `namespace.yaml` / `table.yaml` / `data.yaml` / `index.yaml` /
+   `tag.yaml` / `transaction.yaml`. Each operation's contracts must
+   live in exactly one file (lint enforces this).
+2. **Write the case** in YAML using the schema in
+   `ci/cts/cts-contracts.schema.json`. A minimal shape is:
+   ```yaml
+   - id: drop_namespace_nonexistent_must_404
+     description: Dropping a non-existent namespace must surface
+                  NamespaceNotFound (4).
+     given:
+       state: empty_catalog
+     when:
+       request:
+         id: ["{{NsMissing}}"]
+     then:
+       error_code: 4
+   ```
+   Capabilities the case requires (e.g. `supports_table_tags`) go in
+   `requires_capabilities`; the harness skips the case at runtime
+   when an implementation does not advertise them.
+3. **Run lint locally**:
+   ```
+   uv run python ci/cts/lint_contracts.py --strict
+   ```
+   `--strict` checks that every `(operation, error_code)` row in
+   `docs/src/namespace/operations/errors.md` is covered by at least
+   one case. CI runs lint in `--strict` mode.
+4. **Regenerate the Rust test sources**:
+   ```
+       make gen-cts-behavior     # rewrites rust/lance-namespace-cts/tests/contracts/
+   ```
+   The generated files **are** checked in (so a fresh checkout
+   `cargo test`s without manual steps); CI verifies they are not stale
+   via `gen_contract_tests.py --check`.
+5. **Run the suite**:
+   ```
+   make test-cts-behavior    # cargo test -p lance-namespace-cts
+   ```
+   `SKIP: missing capabilities: [...]` lines indicate cases that an
+   implementation advertises it does not support — those are
+   informational, not failures.
+
+### Adding a New Operation Field or Capability
+
+- **New request field referenced by a case?** Extend
+  `_render_request_literal` in `ci/cts/gen_contract_tests.py` with
+  the field's optionality and Rust literal shape; mirror the
+  Mustache template `ci/cts/templates/rust_inproc_contract.mustache`
+  if a new request struct needs importing.
+- **New operation?** Register it in `_SUPPORTED_OPS` (also in
+  `gen_contract_tests.py`), add a forwarder method on
+  `ContractCaller` and `InProcessDirectoryCaller` in
+  `rust/lance-namespace-cts/src/caller.rs`, then write the YAML
+  cases. Lint + `--check` will tell you what is missing.
+- **New capability?** Add the ID to
+  `ci/cts/capabilities.directory.txt` (or whichever target's
+  capability file applies) and reference it from
+  `requires_capabilities` on cases.
+
+### When to Use the Legacy WireMock Suite
+
+Run `make test-cts-wiremock` (or `make test-clients-wiremock-{java,python,rust}`)
+when changing **client serialization** or **HTTP wire shape**. It is
+not a substitute for the behavioural suite for namespace
+implementations; behavioural contracts should always be authored
+under `docs/src/cts-contracts/`.
+
 ## Documentation
 
 ### Setup