Add IVF_RQ query bits and format reservations

.github/workflows/ci.yml

@@ -254,7 +254,7 @@ jobs:
           python - <<'PY'
           import io
           import numpy as np
-          from paimon_vindex import VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
+          from paimon_vindex import SearchParams, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
 
           class Input:
               def __init__(self, data):
@@ -275,7 +275,7 @@ jobs:
 
           reader = VectorIndexReader(Input(output.getvalue()))
           reader.optimize_for_search()
-          result_ids, distances = reader.search(data[0], top_k=2, nprobe=2)
+          result_ids, distances = reader.search(data[0], SearchParams(top_k=2, nprobe=2))
           assert result_ids[0] == 1, result_ids
           assert np.isfinite(distances[0]), distances
           reader.close()
@@ -292,7 +292,7 @@ jobs:
           python -c @'
           import io
           import numpy as np
-          from paimon_vindex import VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
+          from paimon_vindex import SearchParams, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
 
           class Input:
               def __init__(self, data):
@@ -313,7 +313,7 @@ jobs:
 
           reader = VectorIndexReader(Input(output.getvalue()))
           reader.optimize_for_search()
-          result_ids, distances = reader.search(data[0], top_k=2, nprobe=2)
+          result_ids, distances = reader.search(data[0], SearchParams(top_k=2, nprobe=2))
           assert result_ids[0] == 1, result_ids
           assert np.isfinite(distances[0]), distances
           reader.close()

README.md

@@ -34,6 +34,7 @@ support multiple index families across Rust, C FFI, Java/JNI, and Python:
 | --- | --- | --- |
 | `IVF_FLAT` | IVF partitioning with uncompressed vectors. | Baseline recall and simple storage. |
 | `IVF_PQ` | IVF with product quantization and optional OPQ rotation. | Compact indexes with fast approximate scans. |
+| `IVF_RQ` | IVF with 1-bit RaBitQ-style rotated residual quantization. | Very compact high-dimensional indexes with low training cost. |
 | `IVF_HNSW_FLAT` | IVF partitioning with an HNSW graph inside each list over raw vectors. | Higher recall within probed IVF lists. |
 | `IVF_HNSW_SQ` | IVF partitioning with per-list HNSW and scalar-quantized vectors. | HNSW-style search with smaller vector storage. |
 
@@ -67,6 +68,14 @@ types:
 - `nprobe`: number of IVF lists to probe.
 - `ef_search`: optional HNSW search breadth for `IVF_HNSW_FLAT` and
   `IVF_HNSW_SQ`. A value of `0` uses the default.
+- `rq.query_bits` / `query_bits`: optional `IVF_RQ` query quantization bits.
+  Supported values are `0`, `4`, and `8`; `0` keeps the default float-query
+  byte-LUT path.
+
+`IVF_RQ` currently writes 1-bit codes by default. Its v1 on-disk header already
+records `num_bits`, `rotation_type`, `factor_layout`, and optional-section
+flags for future multi-bit codes, but current readers intentionally reject
+reserved multi-bit payloads until the corresponding scanner is implemented.
 
 Readers also expose an optional search warm-up API. Call
 `optimize_for_search` in Rust, C++, and Python,
@@ -75,8 +84,8 @@ Readers also expose an optional search warm-up API. Call
 searches. The call builds in-memory search caches and does not change the
 serialized index format or search results. Currently `IVF_PQ` builds residual
 L2 precomputed tables for repeated PQ searches, `IVF_HNSW_SQ` builds SQ decode
-LUTs for filtered-search SQ scan and fallback paths, and other index types
-preload metadata. The `IVF_HNSW_SQ` LUTs are not expected to speed up the
+LUTs for filtered-search SQ scan and fallback paths, and `IVF_RQ` plus other
+index types preload metadata. The `IVF_HNSW_SQ` LUTs are not expected to speed up the
 normal unfiltered HNSW graph-search path.
 
 ### Rust
@@ -111,6 +120,14 @@ let mut reader = VectorIndexReader::open(file)?;
 reader.optimize_for_search()?;
 let params = VectorSearchParams::with_ef_search(10, 16, 80);
 let (ids, distances) = reader.search(&query, params)?;
+
+let rq_params = VectorSearchParams {
+    top_k: 10,
+    nprobe: 16,
+    ef_search: 0,
+    query_bits: 4,
+};
+let (ids, distances) = reader.search(&query, rq_params)?;
 ```
 
 Other Rust configs follow the same shape:
@@ -169,8 +186,14 @@ paimon_vindex_reader_optimize_for_search(reader);
 
 int64_t ids[10];
 float distances[10];
+PaimonVindexSearchParams search_params = {
+    .top_k = 10,
+    .nprobe = 16,
+    .ef_search = 80,
+    .query_bits = 0,
+};
 paimon_vindex_reader_search(
-    reader, query, 10, 16, 80, ids, distances, 10);
+    reader, query, search_params, ids, distances, 10);
 paimon_vindex_reader_free(reader);
 ```
 
@@ -203,7 +226,7 @@ writer.write_index(output_file);
 paimon::vindex::Reader reader(input_file);
 auto metadata = reader.metadata();
 reader.optimize_for_search();
-auto result = reader.search(query.data(), 10, 16, 80);
+auto result = reader.search(query.data(), paimon::vindex::SearchParams{10, 16, 80});
 ```
 
 ### Java/JNI
@@ -216,6 +239,7 @@ import java.util.Map;
 import org.apache.paimon.index.vector.VectorIndexInput;
 import org.apache.paimon.index.vector.VectorIndexMetadata;
 import org.apache.paimon.index.vector.VectorIndexReader;
+import org.apache.paimon.index.vector.VectorSearchParams;
 import org.apache.paimon.index.vector.VectorIndexTrainer;
 import org.apache.paimon.index.vector.VectorIndexTraining;
 import org.apache.paimon.index.vector.VectorSearchResult;
@@ -261,7 +285,7 @@ try (VectorIndexTrainer trainer = VectorIndexTrainer.create(options)) {
 try (VectorIndexReader reader = new VectorIndexReader(vectorIndexInput)) {
     VectorIndexMetadata metadata = reader.metadata();
     reader.optimizeForSearch();
-    VectorSearchResult result = reader.search(query, 10, 16, 80);
+    VectorSearchResult result = reader.search(query, new VectorSearchParams(10, 16, 80, 0));
 }
 ```
 
@@ -272,7 +296,7 @@ and validates the options when the trainer is created.
 ### Python
 
 ```python
-from paimon_vindex import VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
+from paimon_vindex import SearchParams, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter
 
 
 class VectorIndexInput:
@@ -299,7 +323,8 @@ writer.write(output)
 
 reader = VectorIndexReader(VectorIndexInput(index_bytes))
 reader.optimize_for_search()
-ids, distances = reader.search(query, top_k=10, nprobe=16, ef_search=80)
+ids, distances = reader.search(query, SearchParams(top_k=10, nprobe=16, ef_search=80))
+ids, distances = reader.search(query, SearchParams(top_k=10, nprobe=16, query_bits=4))
 ```
 
 The Python package is pure Python and uses `ctypes` to load
@@ -332,7 +357,7 @@ Row IDs must be non-negative to map directly into `RoaringTreemap`'s `u64` domai
 ## ANN Benchmark
 
 The core crate includes an ANN-style benchmark for comparing Paimon's
-`IVF_PQ`, `IVF_HNSW_FLAT`, and `IVF_HNSW_SQ` implementations. It reports build
+`IVF_PQ`, `IVF_RQ`, `IVF_HNSW_FLAT`, and `IVF_HNSW_SQ` implementations. It reports build
 time, reader open/load time, first-query latency, batch query throughput, and
 serialized index size:
 
@@ -345,6 +370,7 @@ The benchmark is configured with environment variables:
 ```bash
 ANN_N=100000 ANN_NQ=1000 ANN_D=128 ANN_K=10 ANN_NLIST=256 ANN_NPROBE=16 \
 ANN_PQ_M=16 ANN_HNSW_M=20 ANN_HNSW_EF_CONSTRUCTION=150 ANN_HNSW_EF_SEARCH=80 \
+ANN_RQ_QUERY_BITS=0 \
 cargo bench -p paimon-vindex-core --bench ann_bench -- --nocapture
 ```
 

STORAGE_FORMAT.md

@@ -50,7 +50,7 @@ of the compatibility contract.
 
 ### Delta-Varint IDs
 
-IVF-PQ and IVF-FLAT v1 sort each non-empty list by signed row id before writing.
+IVF-PQ, IVF-FLAT, and IVF-RQ v1 sort each non-empty list by signed row id before writing.
 The first id is stored as `base_id: i64`. The id stream then stores one unsigned
 LEB128 varint per id, including the first id's zero delta. Each delta is computed
 with wrapping unsigned subtraction from the previous signed id. Readers reject a
@@ -162,6 +162,72 @@ For each non-empty list payload:
 | `id_bytes` | bytes | delta-varint ids |
 | `vectors` | `count * d` `f32` | raw stored vectors |
 
+## IVF-RQ v1
+
+Magic: `IVRQ` (`0x49565251`). Version: `1`. Header size: 64 bytes.
+
+| Offset | Size | Type | Field |
+| ---: | ---: | --- | --- |
+| 0 | 4 | `u32` | magic |
+| 4 | 4 | `u32` | version |
+| 8 | 4 | `i32` | dimension `d` |
+| 12 | 4 | `i32` | IVF list count `nlist` |
+| 16 | 4 | `u32` | metric (`0=L2`, `1=InnerProduct`, `2=Cosine`) |
+| 20 | 4 | `u32` | flags |
+| 24 | 8 | `i64` | total vector count |
+| 32 | 8 | `u64` | deterministic rotation seed |
+| 40 | 4 | `u32` | deterministic rotation rounds |
+| 44 | 4 | `i32` | bytes per primary sign-code plane, `ceil(d / 8)` |
+| 48 | 4 | `u32` | RQ `num_bits`; currently `1`, reserved values `2`, `4`, and `8` |
+| 52 | 4 | `u32` | `rotation_type`; currently `1` for deterministic Kac rotation |
+| 56 | 4 | `u32` | `factor_layout`; currently `1` for three RaBitQ `f32` factors |
+| 60 | 4 | `u32` | RQ format flags |
+
+Flags:
+
+| Bit | Meaning |
+| ---: | --- |
+| 0 | delta-varint ids are used; required in v1 |
+
+RQ format flags:
+
+| Bit | Meaning |
+| ---: | --- |
+| 0 | optional `ex_codes` section is present |
+| 1 | optional `error_factor` section is present |
+
+Sections after the header:
+
+1. IVF coarse centroids: `nlist * d` `f32` values.
+2. Offset table: `nlist` entries of `(offset: i64, count: i32, id_bytes_len: i32)`.
+3. List payloads.
+
+For each non-empty list payload:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `base_id` | `i64` | first sorted row id |
+| `id_bytes_len` | `i32` | byte length of encoded id stream |
+| `id_bytes` | bytes | delta-varint ids |
+| `codes` | `count * ceil(d / 8)` bytes | primary RaBitQ sign-code plane over rotated residuals |
+| `ex_codes` | optional bytes | reserved for additional code planes when `num_bits > 1`; present when RQ format flag bit 0 is set |
+| `factors` | `count * 3` `f32` | per-vector `(residual_norm_sqr, vector_norm_sqr, dp_multiplier)` correction factors |
+| `error_factor` | optional `count` `f32` | reserved per-vector refinement factor; present when RQ format flag bit 1 is set |
+
+The deterministic rotation is derived from `(d, rotation_seed, rotation_rounds)`
+and applied to both indexed residuals and query residuals. The rotation parameters
+are stored in the header so independently written files can use the same
+distance estimator after being read by any v1 reader.
+
+Current writers emit `num_bits=1`, `rotation_type=1`, `factor_layout=1`, and no
+optional RQ sections. Current readers validate these format fields and reject
+reserved multi-bit or optional-section encodings until the corresponding scanner
+is implemented.
+
+`query_bits` is a search-time IVF-RQ parameter and is not serialized. `0` uses
+the default float-query byte-LUT estimator; `4` and `8` quantize the rotated
+query residual into sign and magnitude bit planes for bitwise/popcount scanning.
+
 ## IVF-HNSW-FLAT v1
 
 Magic: `IHFL` (`0x4948464C`). Version: `1`. Header size: 64 bytes.

c/test_vindex.c

@@ -50,7 +50,7 @@ struct MemBuffer {
 };
 
 enum {
-    ROUNDTRIP_DIMENSION = 2,
+    ROUNDTRIP_DIMENSION = 8,
     ROUNDTRIP_NLIST = 4,
     ROUNDTRIP_PER_LIST = 128,
     ROUNDTRIP_VECTOR_COUNT = ROUNDTRIP_NLIST * ROUNDTRIP_PER_LIST,
@@ -156,12 +156,20 @@ static void fill_roundtrip_data(float *data, int64_t *ids) {
         size_t cluster = i / ROUNDTRIP_PER_LIST;
         size_t local = i % ROUNDTRIP_PER_LIST;
         float center = (float)cluster * 20.0f;
-        data[i * ROUNDTRIP_DIMENSION] = center + (float)(local % 16) * 0.001f;
-        data[i * ROUNDTRIP_DIMENSION + 1] = center + (float)(local / 16) * 0.001f;
+        for (size_t dim = 0; dim < ROUNDTRIP_DIMENSION; dim++) {
+            data[i * ROUNDTRIP_DIMENSION + dim] =
+                center + (float)dim * 0.01f + (float)(local % 16) * 0.001f;
+        }
         ids[i] = cluster_base_id(cluster) + (int64_t)local;
     }
 }
 
+static void fill_query(float *query, float center) {
+    for (size_t dim = 0; dim < ROUNDTRIP_DIMENSION; dim++) {
+        query[dim] = center + (float)dim * 0.01f;
+    }
+}
+
 static void assert_id_in_cluster(int64_t id, size_t cluster) {
     int64_t base = cluster_base_id(cluster);
     ASSERT_TRUE(id >= base);
@@ -186,7 +194,7 @@ static void run_roundtrip(
     if (paimon_vindex_trainer_dimension(trainer, &dimension) != 0) {
         fail_ffi("trainer dimension failed");
     }
-    ASSERT_EQ_I64(dimension, 2);
+    ASSERT_EQ_I64(dimension, ROUNDTRIP_DIMENSION);
 
     float *data = (float *)malloc(sizeof(float) * ROUNDTRIP_VECTOR_COUNT * ROUNDTRIP_DIMENSION);
     int64_t *ids = (int64_t *)malloc(sizeof(int64_t) * ROUNDTRIP_VECTOR_COUNT);
@@ -240,7 +248,7 @@ static void run_roundtrip(
     }
     ASSERT_EQ_I64(metadata.index_type, expected_index_type);
     ASSERT_EQ_I64(metadata.metric, PAIMON_VINDEX_METRIC_L2);
-    ASSERT_EQ_I64(metadata.dimension, 2);
+    ASSERT_EQ_I64(metadata.dimension, ROUNDTRIP_DIMENSION);
     ASSERT_EQ_I64(metadata.nlist, 4);
     ASSERT_EQ_I64(metadata.total_vectors, ROUNDTRIP_VECTOR_COUNT);
     ASSERT_EQ_I64(metadata.pq_m, expected_pq_m);
@@ -250,25 +258,48 @@ static void run_roundtrip(
         fail_ffi("reader optimize_for_search failed");
     }
 
-    const float query[] = {0.0f, 0.0f};
+    float query[ROUNDTRIP_DIMENSION];
+    fill_query(query, 0.0f);
     int64_t result_ids[2] = {0};
     float result_distances[2] = {0};
+    struct PaimonVindexSearchParams search_params = {2, 4, 16, 0};
     if (paimon_vindex_reader_search(
-            reader, query, 2, 4, 16, result_ids, result_distances, 2) != 0) {
+            reader, query, search_params, result_ids, result_distances, 2) != 0) {
         fail_ffi("reader search failed");
     }
     assert_id_in_cluster(result_ids[0], 0);
     ASSERT_TRUE(isfinite(result_distances[0]));
+    if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) {
+        search_params.query_bits = 4;
+        if (paimon_vindex_reader_search(
+                reader, query, search_params, result_ids, result_distances, 2) != 0) {
+            fail_ffi("reader search with query bits failed");
+        }
+        assert_id_in_cluster(result_ids[0], 0);
+        ASSERT_TRUE(isfinite(result_distances[0]));
+    }
 
-    const float queries[] = {0.0f, 0.0f, 20.0f, 20.0f};
+    float queries[2 * ROUNDTRIP_DIMENSION];
+    fill_query(queries, 0.0f);
+    fill_query(queries + ROUNDTRIP_DIMENSION, 20.0f);
     int64_t batch_ids[2] = {0};
     float batch_distances[2] = {0};
+    struct PaimonVindexSearchParams batch_params = {1, 4, 16, 0};
     if (paimon_vindex_reader_search_batch(
-            reader, queries, 2, 1, 4, 16, batch_ids, batch_distances, 2) != 0) {
+            reader, queries, 2, batch_params, batch_ids, batch_distances, 2) != 0) {
         fail_ffi("reader search batch failed");
     }
     assert_id_in_cluster(batch_ids[0], 0);
     assert_id_in_cluster(batch_ids[1], 1);
+    if (expected_index_type == PAIMON_VINDEX_INDEX_TYPE_IVF_RQ) {
+        batch_params.query_bits = 8;
+        if (paimon_vindex_reader_search_batch(
+                reader, queries, 2, batch_params, batch_ids, batch_distances, 2) != 0) {
+            fail_ffi("reader search batch with query bits failed");
+        }
+        assert_id_in_cluster(batch_ids[0], 0);
+        assert_id_in_cluster(batch_ids[1], 1);
+    }
 
     paimon_vindex_reader_free(reader);
     free(buf.data);
@@ -353,7 +384,7 @@ static void test_input_read_callback_error_propagates(void) {
 
 static void test_supported_index_roundtrips(void) {
     const char *flat_keys[] = {"index.type", "dimension", "nlist", "metric"};
-    const char *flat_values[] = {"ivf_flat", "2", "4", "l2"};
+    const char *flat_values[] = {"ivf_flat", "8", "4", "l2"};
     run_roundtrip(
         "ivf_flat_roundtrip",
         flat_keys,
@@ -364,18 +395,29 @@ static void test_supported_index_roundtrips(void) {
         0);
 
     const char *pq_keys[] = {"index.type", "dimension", "nlist", "metric", "pq.m"};
-    const char *pq_values[] = {"ivf_pq", "2", "4", "l2", "1"};
+    const char *pq_values[] = {"ivf_pq", "8", "4", "l2", "4"};
     run_roundtrip(
         "ivf_pq_roundtrip",
         pq_keys,
         pq_values,
         5,
         PAIMON_VINDEX_INDEX_TYPE_IVF_PQ,
-        1,
+        4,
+        0);
+
+    const char *rq_keys[] = {"index.type", "dimension", "nlist", "metric"};
+    const char *rq_values[] = {"ivf_rq", "8", "4", "l2"};
+    run_roundtrip(
+        "ivf_rq_roundtrip",
+        rq_keys,
+        rq_values,
+        4,
+        PAIMON_VINDEX_INDEX_TYPE_IVF_RQ,
+        0,
         0);
 
     const char *hnsw_flat_keys[] = {"index.type", "dimension", "nlist", "metric", "hnsw.m"};
-    const char *hnsw_flat_values[] = {"ivf_hnsw_flat", "2", "4", "l2", "4"};
+    const char *hnsw_flat_values[] = {"ivf_hnsw_flat", "8", "4", "l2", "4"};
     run_roundtrip(
         "ivf_hnsw_flat_roundtrip",
         hnsw_flat_keys,
@@ -386,7 +428,7 @@ static void test_supported_index_roundtrips(void) {
         4);
 
     const char *hnsw_sq_keys[] = {"index.type", "dimension", "nlist", "metric", "hnsw.m"};
-    const char *hnsw_sq_values[] = {"ivf_hnsw_sq", "2", "4", "l2", "4"};
+    const char *hnsw_sq_values[] = {"ivf_hnsw_sq", "8", "4", "l2", "4"};
     run_roundtrip(
         "ivf_hnsw_sq_roundtrip",
         hnsw_sq_keys,