From 3602ef0a8047ef222ad994a316d69ffec26e98bd Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 29 Oct 2026 15:58:16 -1000 Subject: [PATCH] Add support for Lucene 10.3 This PR adds support for Lucene 10.3. Notes for the reviewers: - The fern/pages/lucene_api directory is generated from sources and can be completely ignored. - The majority of the actually interesting changes are in the java/cuvs-lucene/compat directory, which adds version-specific implementations. The way it works: each incompatible change is captured in an abstract base class that is then inherited from in the shared code. For example, KnnFloatVectorQuery introduced incompatible changes in 10.3, so there are now two abstract CompatKnnFloatVectorQuery classes, for 10.2 and 10.3, that take care of the differences, and GPUKnnFloatVectorQuery in the shared code adds all the version-independent logic. This approach is repeated for every incompatible change. - java/cuvs-lucene/compat/README.md explains the details. - Most of the code changes are mechanical, with a one exceptions: in 10.3, Lucene made KnnFloatVectorQuery collection "optimistic" (apache/lucene#14226), which depending on segment sizes, now breaks GPU search with k close to 1024. So all changes around MAX_CAGRA_K are a temporary workaround for that issue to make tests pass. Relates to #2702 --- .pre-commit-config.yaml | 1 + fern/docs.yml | 8 + fern/pages/installation/java.md | 7 +- fern/pages/lucene_api/index.md | 6 +- ...idia-cuvs-lucene-cuvs2510gpusearchcodec.md | 6 +- ...ia-cuvs-lucene-cuvs2510gpuvectorsreader.md | 64 +++-- ...e-api-com-nvidia-cuvs-lucene-cuvscodecs.md | 8 +- ...idia-cuvs-lucene-gpuknnfloatvectorquery.md | 6 +- ...vs-lucene-lucene101acceleratedhnswcodec.md | 4 +- ...e103acceleratedhnswbinaryquantizedcodec.md | 47 ++++ ...vs-lucene-lucene103acceleratedhnswcodec.md | 48 ++++ ...e103acceleratedhnswscalarquantizedcodec.md | 47 ++++ ...cuvs-lucene-lucene103cuvsgpusearchcodec.md | 48 ++++ ...ceneacceleratedhnswbinaryquantizedcodec.md | 4 +- ...ceneacceleratedhnswscalarquantizedcodec.md | 4 +- fern/pages/user_guide/lucene.md | 19 +- fern/scripts/generate_api_reference.py | 18 +- java/cuvs-lucene/MAINTAINING.md | 232 ++++++++++++++++++ java/cuvs-lucene/README.md | 43 +++- java/cuvs-lucene/compat/README.md | 39 +++ .../lucene/CompatKnnFloatVectorQuery.java | 67 +++++ .../lucene/CompatKnnFloatVectorQuery.java | 110 +++++++++ .../cuvs/lucene/CompatKnnVectorsReader.java | 81 ++++++ .../cuvs/lucene/CompatKnnVectorsReader.java | 95 +++++++ .../nvidia/cuvs/lucene/TestLuceneCompat.java | 37 +++ .../nvidia/cuvs/lucene/TestLuceneCompat.java | 41 ++++ java/cuvs-lucene/lucene-10.2/pom.xml | 4 + .../com/nvidia/cuvs/lucene/LuceneCompat.java | 12 + java/cuvs-lucene/lucene-10.3/pom.xml | 63 +++++ .../com/nvidia/cuvs/lucene/LuceneCompat.java | 91 +++++++ .../services/org.apache.lucene.codecs.Codec | 12 + .../org.apache.lucene.codecs.KnnVectorsFormat | 7 + ...03AcceleratedHNSWBinaryQuantizedCodec.java | 39 +++ .../lucene/Lucene103AcceleratedHNSWCodec.java | 39 +++ ...03AcceleratedHNSWScalarQuantizedCodec.java | 39 +++ .../lucene/Lucene103CuVSGPUSearchCodec.java | 40 +++ java/cuvs-lucene/pom.xml | 19 +- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 7 +- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 39 ++- .../com/nvidia/cuvs/lucene/CuVSCodecs.java | 10 +- .../nvidia/cuvs/lucene/CuVSFilterCodec.java | 104 +++++++- .../cuvs/lucene/GPUKnnFloatVectorQuery.java | 37 +-- .../lucene/Lucene101AcceleratedHNSWCodec.java | 7 +- ...neAcceleratedHNSWBinaryQuantizedCodec.java | 7 +- ...neAcceleratedHNSWScalarQuantizedCodec.java | 7 +- .../nvidia/cuvs/lucene/TestBackCompat.java | 136 ++++++++-- .../cuvs/lucene/TestBackCompatIndices.java | 227 ++++++++++++++--- .../lucene/TestCompatKnnFloatVectorQuery.java | 181 ++++++++++++++ .../cuvs/lucene/TestCuVSDeletedDocuments.java | 9 +- .../cuvs/lucene/TestCuVSVectorsFormat.java | 3 +- .../cuvs/lucene/TestGPUSearchLargeTopK.java | 72 ++++++ ...tLucene99AcceleratedHNSWVectorsFormat.java | 3 +- .../lucene/TestPerSegmentGPUFilterSearch.java | 2 +- .../lucene/TestQuantizedVectorsFormats.java | 2 +- .../src/test/resources/backcompat/README.md | 7 + 55 files changed, 2139 insertions(+), 176 deletions(-) create mode 100644 fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec.md create mode 100644 fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec.md create mode 100644 fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec.md create mode 100644 fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec.md create mode 100644 java/cuvs-lucene/MAINTAINING.md create mode 100644 java/cuvs-lucene/compat/README.md create mode 100644 java/cuvs-lucene/compat/knn-float-vector-query-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java create mode 100644 java/cuvs-lucene/compat/knn-float-vector-query-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java create mode 100644 java/cuvs-lucene/compat/knn-vectors-reader-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java create mode 100644 java/cuvs-lucene/compat/knn-vectors-reader-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java create mode 100644 java/cuvs-lucene/compat/leaf-reader-102-bits/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java create mode 100644 java/cuvs-lucene/compat/leaf-reader-103-acceptdocs/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java create mode 100644 java/cuvs-lucene/lucene-10.3/pom.xml create mode 100644 java/cuvs-lucene/lucene-10.3/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java create mode 100644 java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec create mode 100644 java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat create mode 100644 java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java create mode 100644 java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java create mode 100644 java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java create mode 100644 java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCompatKnnFloatVectorQuery.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchLargeTopK.java diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f22a135fea..0c6685cab4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,6 +69,7 @@ repos: ^cpp/include/.*[.](h|hpp|cuh)$| ^python/cuvs/cuvs/.*[.](py|pyx|pxd)$| ^java/cuvs-(java|lucene)/src/main/.*[.]java$| + ^java/cuvs-lucene/lucene-[^/]+/src/since/.*[.]java$| ^rust/cuvs/src/.*[.]rs$| ^go/.*[.]go$| ^fern/scripts/generate_api_reference[.]py$| diff --git a/fern/docs.yml b/fern/docs.yml index 53bd96bc4d..8c30b9f25d 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -615,6 +615,14 @@ navigation: path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md" - page: "Lucene101AcceleratedHNSWCodec" path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md" + - page: "Lucene103AcceleratedHNSWBinaryQuantizedCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec.md" + - page: "Lucene103AcceleratedHNSWCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec.md" + - page: "Lucene103AcceleratedHNSWScalarQuantizedCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec.md" + - page: "Lucene103CuVSGPUSearchCodec" + path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec.md" - page: "Lucene99AcceleratedHNSWVectorsFormat" path: "./pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat.md" - page: "Lucene99AcceleratedHNSWVectorsWriter" diff --git a/fern/pages/installation/java.md b/fern/pages/installation/java.md index cfad822f1b..a1b189bfee 100644 --- a/fern/pages/installation/java.md +++ b/fern/pages/installation/java.md @@ -69,7 +69,7 @@ java/panama-bindings/generate-bindings.sh `cuvs-lucene` provides Apache Lucene codecs that offload vector index build, and optionally search, to the GPU. It is published as a separate artifact that builds on the Java API described above. For usage guidance, see the [Lucene Integration](/user-guide/lucene) guide. -`cuvs-lucene` supports Lucene 10.2 and requires Maven 3.10 or newer. It compiles and runs on JDK 21, the minimum of Lucene 10, but uses the GPU only on a JDK 22 or newer runtime: on JDK 21 the accelerated HNSW codecs build graphs on the CPU and the GPU search codec is unavailable. It inherits the NVIDIA cuVS [CUDA GPU requirements](/installation#cuda-gpu-requirements). +`cuvs-lucene` supports Lucene 10.2 and 10.3 and requires Maven 3.10 or newer. It compiles and runs on JDK 21, the minimum of Lucene 10, but uses the GPU only on a JDK 22 or newer runtime: on JDK 21 the accelerated HNSW codecs build graphs on the CPU and the GPU search codec is unavailable. It inherits the NVIDIA cuVS [CUDA GPU requirements](/installation#cuda-gpu-requirements). The three accelerated HNSW codecs (plain, scalar-quantized and binary-quantized) fall back to CPU index construction when no GPU or native library is available, so an application using them works on GPU and non-GPU hosts alike. The GPU search codec requires a working cuVS installation. See [Lucene Integration](/user-guide/lucene) for details. @@ -80,13 +80,14 @@ Lucene changes its codec APIs between minor releases, so `cuvs-lucene` is publis | Lucene | Artifact | | --- | --- | | 10.2.x | `cuvs-lucene-10.2` | +| 10.3.x | `cuvs-lucene-10.3` | -For Lucene 10.2, add the following dependency to your `pom.xml`: +For example, for Lucene 10.3, add the following dependency to your `pom.xml`: ```xml com.nvidia.cuvs.lucene - cuvs-lucene-10.2 + cuvs-lucene-10.3 26.12.0 ``` diff --git a/fern/pages/lucene_api/index.md b/fern/pages/lucene_api/index.md index 62fe249776..f0ca37543f 100644 --- a/fern/pages/lucene_api/index.md +++ b/fern/pages/lucene_api/index.md @@ -1,6 +1,6 @@ # Lucene API Documentation -These pages are generated from the Java source files in `java/cuvs-lucene/src/main`. +These pages are generated from the Java source files in `java/cuvs-lucene/src/main` and `java/cuvs-lucene/lucene-*/src/since`. For an introduction to the codecs, configuration, and tuning, see the [Lucene Integration](/user-guide/lucene) guide. @@ -27,6 +27,10 @@ For an introduction to the codecs, configuration, and tuning, see the [Lucene In - [GPUBuiltHnswGraph](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpubuilthnswgraph) - [GPUKnnFloatVectorQuery](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery) - [Lucene101AcceleratedHNSWCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec) +- [Lucene103AcceleratedHNSWBinaryQuantizedCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec) +- [Lucene103AcceleratedHNSWCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec) +- [Lucene103AcceleratedHNSWScalarQuantizedCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec) +- [Lucene103CuVSGPUSearchCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec) - [Lucene99AcceleratedHNSWVectorsFormat](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorsformat) - [Lucene99AcceleratedHNSWVectorsWriter](/api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene99acceleratedhnswvectorswriter) - [LuceneAcceleratedHNSWBinaryQuantizedCodec](/api-reference/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec) diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md index 4f0b9f11b2..e1f8c932fc 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpusearchcodec.md @@ -13,8 +13,8 @@ public class CuVS2510GPUSearchCodec extends CuVSFilterCodec cuVS based codec for GPU based vector search that enables both - indexing and search on the GPU. cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. -This codec wraps `Lucene101Codec`, the default codec of Lucene 10.2. `CuVSCodecs#gpuSearch` -returns the codec of this family for the Lucene release in use. +This codec wraps `Lucene101Codec`, the default codec of Lucene 10.2. On later Lucene +releases it can only read existing indexes; use `CuVSCodecs#gpuSearch` to write. ## Public Members @@ -97,6 +97,6 @@ Initialize a named codec with explicit delegate, GPU search, and filter-cache pa | `params` | GPU index and search parameters | | `filterCacheConfig` | filter-bitset-cache configuration | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:70`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:71`_ _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md index 8ff135ea25..71b1c316a3 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvs2510gpuvectorsreader.md @@ -35,7 +35,7 @@ Initializes the `CuVS2510GPUVectorsReader`, checks and loads the index. | --- | --- | | `IOException` | I/O exception | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:91`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:95`_ ### CuVS2510GPUVectorsReader @@ -59,7 +59,7 @@ Initializes the reader with the cache owned by its vectors format. | --- | --- | | `IOException` | I/O exception | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:104`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:108`_ ### readSimilarityFunction @@ -85,7 +85,7 @@ an instance of VectorSimilarityFunction | --- | --- | | `IOException` | | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:230`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:235`_ ### readVectorEncoding @@ -111,7 +111,7 @@ the vector encoding | --- | --- | | `IOException` | | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:245`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:250`_ ### getFieldEntry @@ -132,7 +132,7 @@ cuVS index for it. the meta information for the field, or `null` -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:308`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:313`_ ### openCagraIndexForMerge @@ -165,7 +165,7 @@ a freshly loaded CAGRA index, or `null` if this segment has none for the field | --- | --- | | `IOException` | I/O exception | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:327`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:332`_ ### lazilyLoadedIndexCount @@ -176,7 +176,7 @@ int lazilyLoadedIndexCount() Returns the number of indexes a merge-opened reader has loaded so far. Package-private so that tests can check that loading is per field. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:404`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:409`_ ### close @@ -186,7 +186,7 @@ _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVecto Closes the resources. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:445`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:450`_ ### checkIntegrity @@ -196,7 +196,7 @@ _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVecto Checks consistency of this reader. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:475`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:480`_ ### getFloatVectorValues @@ -206,7 +206,7 @@ _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVecto Returns the FloatVectorValues for the given field. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:484`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:489`_ ### getByteVectorValues @@ -218,29 +218,29 @@ Returns the ByteVectorValues for the given field. This is not supported. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:494`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:499`_ -### search +### doSearch ```java -@Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException +@Override protected void doSearch(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException ``` Returns the k nearest neighbor documents using cuVS's CAGRA or brute force algorithm for this field, to the given vector. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:518`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:523`_ -### search +### doSearch ```java -@Override public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException +@Override protected void doSearch(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException ``` Return the k nearest neighbor documents as determined by comparison of their vector values for this field, to the given vector. This is not supported. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:679`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:691`_ ### readEntry @@ -268,7 +268,23 @@ an instance of FieldEntry | --- | --- | | `IOException` | I/O Exceptions | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:707`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:719`_ + +### hasCagraIndex + +```java +boolean hasCagraIndex(String field) +``` + +Whether this segment has a CAGRA index for the given field. Unlike `#getCagraIndexForField`, this never loads an index onto the GPU. + +**Parameters** + +| Name | Description | +| --- | --- | +| `field` | the vector field name | + +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:770`_ ### getCagraIndexForField @@ -299,7 +315,7 @@ the CAGRA index, or `null` | --- | --- | | `IOException` | I/O exception | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:764`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:789`_ ### getFilterBitsetCache @@ -309,7 +325,7 @@ FilterBitsetCache getFilterBitsetCache() Returns the filter cache owned by the vectors format that created this reader. -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:773`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:798`_ ### getFieldInfos @@ -323,7 +339,7 @@ Gets the instance of FieldInfos. the instance of FieldInfos -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:782`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:807`_ ### getCuvsIndexes @@ -337,7 +353,7 @@ Gets the map of `GPUIndex` objects. the map of GPU index objects, or `null` if this reader was opened for a merge -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:791`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:816`_ ### getFieldEntries @@ -351,6 +367,6 @@ Gets the map of FieldEntry objects that hold the meta information for the field. the map of FieldEntry objects -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:800`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:825`_ -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:63`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java:61`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvscodecs.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvscodecs.md index 2030651654..b4bef7b9ac 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvscodecs.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-cuvscodecs.md @@ -13,10 +13,10 @@ public final class CuVSCodecs Creates the cuvs-lucene codecs and vectors formats that write indexes on the Lucene release this artifact is built for. -Each cuvs-lucene codec wraps the default codec of one Lucene release, for example `Lucene101AcceleratedHNSWCodec` wraps Lucene 10.2's `Lucene101Codec`, so the codec to write -with changes along with Lucene. Creating codecs through this class instead of naming them keeps -application code unchanged when it moves to another Lucene release and the matching cuvs-lucene -artifact. +Each cuvs-lucene codec wraps the default codec of one Lucene release, so the codec to write +with changes along with Lucene: `Lucene101AcceleratedHNSWCodec` on Lucene 10.2, `Lucene103AcceleratedHNSWCodec` on 10.3. The older codecs stay available for reading existing +indexes. Creating codecs through this class instead of naming them keeps application code +unchanged when it moves to another Lucene release and the matching cuvs-lucene artifact. Every method throws `IllegalStateException` if the Lucene release in use is not the one this cuvs-lucene artifact is built for. diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md index f8909e5f77..7e62d4710e 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery.md @@ -62,7 +62,7 @@ and max_iterations auto-selected (0). | `iTopK` | CAGRA itopk_size parameter | | `searchWidth` | CAGRA search_width parameter | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:141`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:140`_ ### GPUKnnFloatVectorQuery @@ -86,6 +86,6 @@ Initializes `GPUKnnFloatVectorQuery`. | `maxIterations` | nonnegative CAGRA max_iterations (0 = auto) | | `searchAlgo` | CAGRA search algorithm | -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:159`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:158`_ -_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:82`_ +_Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java:81`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md index a998925387..39c77f6df1 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene101acceleratedhnswcodec.md @@ -14,8 +14,8 @@ A codec that enables GPU-based accelerated HNSW capability and can be used to accelerated indexing using GPUs and search using CPUs. Fallbacks to CPU based indexing when used on a machine without a GPU and/or cuVS. -This codec wraps `Lucene101Codec`, the default codec of Lucene 10.2. `CuVSCodecs#acceleratedHNSW` -returns the codec of this family for the Lucene release in use. +This codec wraps `Lucene101Codec`, the default codec of Lucene 10.2. On later Lucene +releases it can only read existing indexes; use `CuVSCodecs#acceleratedHNSW` to write. ## Public Members diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec.md new file mode 100644 index 0000000000..d8f51a5f9c --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec.md @@ -0,0 +1,47 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswbinaryquantizedcodec +--- + +# Lucene103AcceleratedHNSWBinaryQuantizedCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene103AcceleratedHNSWBinaryQuantizedCodec extends CuVSFilterCodec +``` + +A codec that builds HNSW graphs on the GPU over binary-quantized vectors, and writes them in +Lucene's HNSW format so that they are searched on the CPU. See `LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat`. + +This codec wraps `Lucene103Codec`, the default codec of Lucene 10.3. Once a later Lucene +release replaces that default, this codec can only read existing indexes; use `CuVSCodecs#acceleratedHNSWBinaryQuantized` to write. + +## Public Members + +### Lucene103AcceleratedHNSWBinaryQuantizedCodec + +```java +public Lucene103AcceleratedHNSWBinaryQuantizedCodec() +``` + +Creates the codec with default parameters. + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java:22`_ + +### Lucene103AcceleratedHNSWBinaryQuantizedCodec + +```java +public Lucene103AcceleratedHNSWBinaryQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Creates the codec. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | the index build parameters | + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java:31`_ + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec.md new file mode 100644 index 0000000000..b4e83b6470 --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec.md @@ -0,0 +1,48 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswcodec +--- + +# Lucene103AcceleratedHNSWCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene103AcceleratedHNSWCodec extends CuVSFilterCodec +``` + +A codec that builds HNSW graphs on the GPU, and writes them in Lucene's HNSW format so that they +are searched on the CPU. Falls back to building the graph on the CPU when no GPU or cuVS is +available. + +This codec wraps `Lucene103Codec`, the default codec of Lucene 10.3. Once a later Lucene +release replaces that default, this codec can only read existing indexes; use `CuVSCodecs#acceleratedHNSW` to write. + +## Public Members + +### Lucene103AcceleratedHNSWCodec + +```java +public Lucene103AcceleratedHNSWCodec() +``` + +Creates the codec with default parameters. + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java:22`_ + +### Lucene103AcceleratedHNSWCodec + +```java +public Lucene103AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Creates the codec. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | the index build parameters | + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java:31`_ + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec.md new file mode 100644 index 0000000000..a02980356d --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec.md @@ -0,0 +1,47 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103acceleratedhnswscalarquantizedcodec +--- + +# Lucene103AcceleratedHNSWScalarQuantizedCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene103AcceleratedHNSWScalarQuantizedCodec extends CuVSFilterCodec +``` + +A codec that builds HNSW graphs on the GPU over scalar-quantized vectors, and writes them in +Lucene's HNSW format so that they are searched on the CPU. See `LuceneAcceleratedHNSWScalarQuantizedVectorsFormat`. + +This codec wraps `Lucene103Codec`, the default codec of Lucene 10.3. Once a later Lucene +release replaces that default, this codec can only read existing indexes; use `CuVSCodecs#acceleratedHNSWScalarQuantized` to write. + +## Public Members + +### Lucene103AcceleratedHNSWScalarQuantizedCodec + +```java +public Lucene103AcceleratedHNSWScalarQuantizedCodec() +``` + +Creates the codec with default parameters. + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java:22`_ + +### Lucene103AcceleratedHNSWScalarQuantizedCodec + +```java +public Lucene103AcceleratedHNSWScalarQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) +``` + +Creates the codec. + +**Parameters** + +| Name | Description | +| --- | --- | +| `acceleratedHNSWParams` | the index build parameters | + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java:31`_ + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec.md new file mode 100644 index 0000000000..dc48a8148e --- /dev/null +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec.md @@ -0,0 +1,48 @@ +--- +slug: api-reference/lucene-api-com-nvidia-cuvs-lucene-lucene103cuvsgpusearchcodec +--- + +# Lucene103CuVSGPUSearchCodec + +_Java package: `com.nvidia.cuvs.lucene`_ + +```java +public class Lucene103CuVSGPUSearchCodec extends CuVSFilterCodec +``` + +A codec that both builds and searches CAGRA indexes on the GPU. cuVS serialization formats are in +experimental phase and hence backward compatibility cannot be guaranteed. + +This codec wraps `Lucene103Codec`, the default codec of Lucene 10.3. Once a later Lucene +release replaces that default, this codec can only read existing indexes; use `CuVSCodecs#gpuSearch` to write. + +## Public Members + +### Lucene103CuVSGPUSearchCodec + +```java +public Lucene103CuVSGPUSearchCodec() +``` + +Creates the codec with default parameters. + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java:21`_ + +### Lucene103CuVSGPUSearchCodec + +```java +public Lucene103CuVSGPUSearchCodec( GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) +``` + +Creates the codec. + +**Parameters** + +| Name | Description | +| --- | --- | +| `params` | GPU index and search parameters | +| `filterCacheConfig` | filter-bitset-cache configuration | + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java:31`_ + +_Source: `java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java:17`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md index 3ed7890ae3..ab262f5502 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswbinaryquantizedcodec.md @@ -12,7 +12,7 @@ public class LuceneAcceleratedHNSWBinaryQuantizedCodec extends CuVSFilterCodec CuVS based codec for GPU based vector search -This codec, named `Lucene101AcceleratedHNSWBinaryQuantizedCodec`, wraps `Lucene101Codec`, the default codec of Lucene 10.2. `CuVSCodecs#acceleratedHNSWBinaryQuantized` returns the codec of -this family for the Lucene release in use. +This codec, named `Lucene101AcceleratedHNSWBinaryQuantizedCodec`, wraps `Lucene101Codec`, the default codec of Lucene 10.2. On later Lucene releases it can only read +existing indexes; use `CuVSCodecs#acceleratedHNSWBinaryQuantized` to write. _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java:18`_ diff --git a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md index ce57feb558..df63419123 100644 --- a/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md +++ b/fern/pages/lucene_api/lucene-api-com-nvidia-cuvs-lucene-luceneacceleratedhnswscalarquantizedcodec.md @@ -12,7 +12,7 @@ public class LuceneAcceleratedHNSWScalarQuantizedCodec extends CuVSFilterCodec CuVS based codec for GPU based vector search -This codec, named `Lucene101AcceleratedHNSWScalarQuantizedCodec`, wraps `Lucene101Codec`, the default codec of Lucene 10.2. `CuVSCodecs#acceleratedHNSWScalarQuantized` returns the codec of -this family for the Lucene release in use. +This codec, named `Lucene101AcceleratedHNSWScalarQuantizedCodec`, wraps `Lucene101Codec`, the default codec of Lucene 10.2. On later Lucene releases it can only read +existing indexes; use `CuVSCodecs#acceleratedHNSWScalarQuantized` to write. _Source: `java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java:18`_ diff --git a/fern/pages/user_guide/lucene.md b/fern/pages/user_guide/lucene.md index fef52faceb..fddaf5d424 100644 --- a/fern/pages/user_guide/lucene.md +++ b/fern/pages/user_guide/lucene.md @@ -6,7 +6,7 @@ slug: user-guide/lucene NVIDIA cuVS Lucene (`cuvs-lucene`) is a pluggable [KnnVectorsFormat](https://lucene.apache.org/core/10_2_0/core/org/apache/lucene/codecs/KnnVectorsFormat.html) that offloads vector index build, and optionally search, from the CPU to NVIDIA GPUs. Because it plugs in through a standard Lucene codec, an existing Lucene application adopts it by setting a codec on its `IndexWriterConfig`; the rest of the indexing and query code is unchanged. -`cuvs-lucene` supports Lucene 10.2, with one artifact per Lucene minor release, and is built on the [NVIDIA cuVS Java APIs](/api-reference/java-api-documentation). For installation and build instructions, see [cuVS Lucene](/installation/java#cuvs-lucene) in the Java installation guide. For class-level details, see the [Lucene API Documentation](/api-reference/lucene-api-documentation). +`cuvs-lucene` supports Lucene 10.2 and 10.3, with one artifact per Lucene minor release, and is built on the [NVIDIA cuVS Java APIs](/api-reference/java-api-documentation). For installation and build instructions, see [cuVS Lucene](/installation/java#cuvs-lucene) in the Java installation guide. For class-level details, see the [Lucene API Documentation](/api-reference/lucene-api-documentation). ## When To Use It @@ -39,9 +39,18 @@ The GPU search codec has no such fallback. Its on-disk format has no CPU reader, ## Lucene Versions and Upgrades -Lucene changes its codec APIs between minor releases, so `cuvs-lucene` is published as one artifact per Lucene minor release, such as `cuvs-lucene-10.2`; see [cuVS Lucene](/installation/java#cuvs-lucene). Use the one that matches your application's Lucene version. Against another Lucene minor release, its codecs fail when they are used; see [Troubleshooting](#troubleshooting). +Lucene changes its codec APIs between minor releases, so `cuvs-lucene` is published as one artifact per Lucene minor release, `cuvs-lucene-10.2` and `cuvs-lucene-10.3`; see [cuVS Lucene](/installation/java#cuvs-lucene). Use the one that matches your application's Lucene version. Against another Lucene minor release, its codecs fail when they are used; see [Troubleshooting](#troubleshooting). -Each codec also wraps the default codec of one Lucene release, and Lucene records the codec's name in every segment it writes. On Lucene 10.2 these are `Lucene101AcceleratedHNSWCodec`, `Lucene101AcceleratedHNSWScalarQuantizedCodec`, `Lucene101AcceleratedHNSWBinaryQuantizedCodec` and `CuVS2510GPUSearchCodec`, all wrapping `Lucene101Codec`. `CuVSCodecs` returns the codecs for the Lucene release in use, so code that creates its codecs through it does not change when the codec names change with a later Lucene release. +Each codec also wraps the default codec of one Lucene release, and Lucene records the codec's name in every segment it writes. The codec that writes therefore changes with Lucene: + +| Lucene | Accelerated HNSW codec | GPU search codec | +| --- | --- | --- | +| 10.2 | `Lucene101AcceleratedHNSWCodec` | `CuVS2510GPUSearchCodec` | +| 10.3 | `Lucene103AcceleratedHNSWCodec` | `Lucene103CuVSGPUSearchCodec` | + +The quantized codecs follow the same pattern, for example `Lucene103AcceleratedHNSWScalarQuantizedCodec`; on Lucene 10.2 they are `Lucene101AcceleratedHNSWScalarQuantizedCodec` and `Lucene101AcceleratedHNSWBinaryQuantizedCodec`. `CuVSCodecs` always returns the codecs that write on the Lucene release in use, so code that creates its codecs through it does not change when the application moves to another Lucene release. + +To upgrade Lucene, switch to the matching `cuvs-lucene` artifact. Every artifact also includes the codecs of the earlier Lucene releases it supports, so existing segments stay readable. Those codecs are read-only, like Lucene's own backward codecs: new segments, including those produced by merges, are written with the current codec. Writing with an older one throws `UnsupportedOperationException`, and since Lucene treats that as a fatal indexing error, it also closes the `IndexWriter` and loses the documents added since the last commit. Configuration that names a codec, rather than creating it through `CuVSCodecs`, has to be updated to the new name before the upgrade. Releases before 26.12 published a single `cuvs-lucene` artifact, built for Lucene 10.2. Its coordinates are relocated to `cuvs-lucene-10.2`; see [cuVS Lucene](/installation/java#cuvs-lucene). @@ -134,7 +143,7 @@ This program is a condensed version of `AcceleratedHnswExample`. That example an ## Searching on the GPU -The GPU search codec, from `CuVSCodecs.gpuSearch`, runs both index build and search on the GPU. Configure it with [`GPUSearchParams`](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams). A stock `KnnFloatVectorQuery` already searches on the GPU against this codec, so existing query code keeps working; [`GPUKnnFloatVectorQuery`](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery) adds control over the CAGRA search parameters and lets cuVS serve all segments in a single request, which is usually faster. This program is the GPU counterpart of the quickstart, with a filter applied to the search: +The GPU search codec, from `CuVSCodecs.gpuSearch`, runs both index build and search on the GPU. Configure it with [`GPUSearchParams`](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpusearchparams). A stock `KnnFloatVectorQuery` already searches on the GPU against this codec, so existing query code keeps working; [`GPUKnnFloatVectorQuery`](/api-reference/lucene-api-com-nvidia-cuvs-lucene-gpuknnfloatvectorquery) adds control over the CAGRA search parameters and lets cuVS serve all segments in a single request, which is usually faster. With the default index type, which builds only a CAGRA index, a search returns at most 1024 hits from each segment. This program is the GPU counterpart of the quickstart, with a filter applied to the search: ```java import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; @@ -285,6 +294,8 @@ Install the handle on each search thread before that thread runs its first searc **`IllegalStateException: cuvs-lucene-10.X is built for Lucene 10.X.x, but Lucene 10.Y.z is in use`.** The `cuvs-lucene` artifact on the classpath was built for another Lucene minor release than the one the application uses. Lucene itself keeps working, and so do indexes that do not use the cuvs-lucene codecs; getting a codec from `CuVSCodecs`, or reading or writing an index that uses a cuvs-lucene codec, fails with this message. A codec created with its constructor fails only once it is used. The same problem is logged at `SEVERE` the first time Lucene looks up its codecs, usually at startup. Applications that would rather refuse to start can call `CuVSCodecs.checkLuceneVersion()` during startup, which throws this exception. Replace the artifact with the one the message names, if it exists; see [Lucene Versions and Upgrades](#lucene-versions-and-upgrades). +**`UnsupportedOperationException: The ... codec can only read indexes`.** The application writes with the codec of an earlier Lucene release, which the current release can only read. Lucene closes the `IndexWriter` on this error, so documents added since the last commit are lost. Create the codec through `CuVSCodecs`, or switch to the codec for the current release, and open a new `IndexWriter` with it. + **Native libraries do not match.** A missing or mismatched native cuVS library is caught during resource creation and logged at `WARNING` rather than thrown, so it surfaces as one of the two symptoms above rather than as a load error. Make sure the installed native libraries match the `cuvs-lucene` version; see [cuVS Lucene](/installation/java#cuvs-lucene). **Errors about native access, or indexing never uses the GPU.** `cuvs-lucene` is compiled for Java 21, but `cuvs-java` reaches the GPU through the Panama FFI APIs, which need a JDK 22 or newer runtime on Linux (`amd64` or `aarch64`). On JDK 21, cuVS reports that it is not supported: the accelerated HNSW codecs build graphs on the CPU, and the GPU search codec throws `UnsupportedOperationException`. Run the JVM with `--enable-native-access=ALL-UNNAMED` to suppress native-access warnings, or `--enable-native-access=com.nvidia.cuvs` if you put `cuvs-java` on the module path. diff --git a/fern/scripts/generate_api_reference.py b/fern/scripts/generate_api_reference.py index 22542ca044..e7c9216ea3 100755 --- a/fern/scripts/generate_api_reference.py +++ b/fern/scripts/generate_api_reference.py @@ -34,6 +34,9 @@ ] LUCENE_SOURCE_DIRS = [ REPO_DIR / "java" / "cuvs-lucene" / "src" / "main" / "java", + # Classes that first appear in a later Lucene release (such as the + # Lucene103* codecs) live in that module's lucene-10.x/src/since/java. + *(REPO_DIR / "java" / "cuvs-lucene").glob("lucene-*/src/since/java"), ] API_NAV_SECTIONS = [ ("C API Documentation", "c_api", "c-api-documentation", "C API", "c-api"), @@ -146,6 +149,13 @@ "CuVSFilterCodec", } ) +# Package-private ``cuvs-lucene`` bases that adapt a Lucene class to the Lucene +# release a module is built for (java/cuvs-lucene/compat). Readers cannot see +# them, so the pages name the Lucene class each one extends instead. +JAVA_HIDDEN_SUPERTYPES = { + "CompatKnnFloatVectorQuery": "KnnFloatVectorQuery", + "CompatKnnVectorsReader": "KnnVectorsReader", +} JAVA_SUPERTYPE_RE = re.compile(r"\b(?:extends|implements)\s+(?P\w+)") PUBLIC_JAVA_TYPE_RE = re.compile( r"\bpublic\s+(?:abstract\s+|final\s+|sealed\s+|non-sealed\s+)?" @@ -3019,7 +3029,7 @@ def generate_lucene_api_pages() -> None: title="Lucene API Documentation", intro=( "These pages are generated from the Java source files in " - "`java/cuvs-lucene/src/main`.\n\n" + "`java/cuvs-lucene/src/main` and `java/cuvs-lucene/lucene-*/src/since`.\n\n" "For an introduction to the codecs, configuration, and tuning, see the " "[Lucene Integration](/user-guide/lucene) guide." ), @@ -3154,6 +3164,12 @@ def parse_java_class(path: Path) -> JavaClass | None: if not type_match: return None signature, class_line = java_signature_at(text, type_match.start()) + signature = re.sub( + r"\b(extends\s+)(\w+)", + lambda m: m.group(1) + + JAVA_HIDDEN_SUPERTYPES.get(m.group(2), m.group(2)), + signature, + ) doc = parse_javadoc(comment_before(text, type_match.start())) klass = JavaClass( package=package, diff --git a/java/cuvs-lucene/MAINTAINING.md b/java/cuvs-lucene/MAINTAINING.md new file mode 100644 index 0000000000..fba100847f --- /dev/null +++ b/java/cuvs-lucene/MAINTAINING.md @@ -0,0 +1,232 @@ +# Maintaining cuvs-lucene across Lucene releases + +`cuvs-lucene` is published as one artifact per supported Lucene minor release (`cuvs-lucene-10.2`, +`cuvs-lucene-10.3`), all built from the same sources. This page lists exactly what to change when Lucene +publishes a new patch release (for example 10.3.3) or a new minor release (for example 10.4.0). For how +the layout works, see [Supporting several Lucene releases](README.md#supporting-several-lucene-releases). + +All commands run from `java/cuvs-lucene/` unless stated otherwise, with `libcuvs` built and `cuvs-java` +installed in the local Maven repository (`./build.sh libcuvs java` from the repository root). + +## A new patch release (example: 10.3.3) + +Every artifact accepts any patch release of its minor (the version check compares only `major.minor`), +so users can already run `cuvs-lucene-10.3` on Lucene 10.3.3. Moving the build to the new patch release +makes the artifact depend on, and be tested against, it. + +1. In `lucene-10.3/pom.xml`, set `10.3.3`. +2. Run the tests of that module: `mvn -pl lucene-10.3 verify` (add `-am` if the parent is not installed). +3. If something no longer compiles or a test fails, Lucene broke its API or behaviour in a patch + release, which is rare. Fix it in that module's `LuceneCompat` (`lucene-10.3/src/main/java`), or + in a new `compat/` variant used only by that module (see step 2 of the minor release below), never + in the shared `src/`. A module targets one patch release at a time: if the fix cannot work with + both 10.3.2 and 10.3.3, raise the minimum in the documentation. + +Nothing else needs to change: no back-compat indexes, no codecs, no CI, no docs. + +## A new minor release (example: 10.4.0) + +Work through the steps in order. Steps 1–3 always apply; the others depend on what the release changed. + +### 1. Create the module + +1. Copy the newest module: `cp -r lucene-10.3 lucene-10.4`, then `rm -rf lucene-10.4/target + lucene-10.4/src/since`. The `src/since/java` of the copied module belongs to that module only; + later modules compile it from there (below), and step 4 adds the new module's own if it needs one. +2. In `lucene-10.4/pom.xml`: + - change `artifactId` and `name` to `cuvs-lucene-10.4`, and the version in `description`; + - set `10.4.0`; + - keep the `add-since-sources` list, but name the `src/since/java` directory of **every** earlier + module that has one instead of its own (today `../lucene-10.3/src/since/java`); step 4 adds its + own if it gets one; + - keep the `cuvs.lucene.compat.*` properties: the new release starts with the same variant of each + Lucene API as the previous one (see [compat/README.md](compat/README.md)). +3. In the parent `pom.xml`, add `lucene-10.4`. +4. In `lucene-10.4/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java`, set `LUCENE_MINOR = 4` + and the Lucene version in the javadoc. Keep `LUCENE_MAJOR`/`LUCENE_MINOR` compile-time constants: + the version check reads them without loading `LuceneCompat`. + +`ci/release/update-version.sh`, `build.sh` and the CI scripts and workflows (`java/cuvs-lucene/*/target/`) +all match `lucene-*`. `ci/test_lucene.sh --min-jdk` tests the newest `lucene-*` module, so it moves to +the new module by itself. + +### 2. Make it compile + +```bash +mvn -pl lucene-10.4 -am test-compile +``` + +Fix every error without changing what the other modules compile: + +- calls whose signature or class location changed (a class moved to `org.apache.lucene.backward_codecs`, + a constructor lost or gained parameters, an enum moved to another class): in + `lucene-10.4/src/main/java/.../LuceneCompat.java`; +- a Lucene class cuvs-lucene adapts in `compat/` changed (for example `KnnVectorsReader`): copy the + variant the module uses to `compat/-104-` (for example + `compat/knn-vectors-reader-104-`), change it there, and point the module's property + (`cuvs.lucene.compat.knnVectorsReader`) at it. Do not edit a variant other modules still use; +- a Lucene class with no variants yet changed (for example `KnnVectorsWriter`): move its adapter from + the shared `src/` into two new variants, `-102-` for the existing modules and + `-104-`, add a `cuvs.lucene.compat.` property to every module and a matching + `` line to the `add-shared-sources` execution of the parent `pom.xml` (see + [compat/README.md](compat/README.md)); +- test-only calls, such as `LeafReader.searchNearestVectors`: the same, with `TestLuceneCompat` in the + `leaf-reader-*` variants (their `src/test/java`). + +The shared `src/` must keep compiling against every supported release. If a shared class needs an API +that differs between releases, add a method to **every** module's `LuceneCompat` and call that instead. +Never use reflection, and never read version constants such as `VERSION_CURRENT` from Lucene: they +change meaning between releases. + +### 3. Look for changes the compiler cannot see + +A method Lucene adds with a default implementation compiles fine but may be wrong for cuvs-lucene (10.3's +`KnnVectorsReader.getOffHeapByteSize` is an example). Compare the overridable API of every Lucene class +cuvs-lucene extends, and of their superclasses, between the previous and the new release. `javap` lists +only the members a class declares itself, so the list names the superclasses too: + +```bash +M=~/.m2/repository/org/apache/lucene/lucene-core +for cls in org.apache.lucene.codecs.FilterCodec org.apache.lucene.codecs.KnnVectorsFormat \ + org.apache.lucene.codecs.KnnVectorsReader org.apache.lucene.codecs.KnnVectorsWriter \ + org.apache.lucene.codecs.KnnFieldVectorsWriter org.apache.lucene.search.KnnFloatVectorQuery \ + org.apache.lucene.search.AbstractKnnVectorQuery org.apache.lucene.search.TopKnnCollector \ + org.apache.lucene.search.AbstractKnnCollector org.apache.lucene.search.KnnCollector \ + org.apache.lucene.search.Query \ + org.apache.lucene.search.Weight org.apache.lucene.search.Scorer \ + org.apache.lucene.search.ScorerSupplier org.apache.lucene.search.DocIdSetIterator \ + 'org.apache.lucene.util.hnsw.HnswGraph' 'org.apache.lucene.util.hnsw.HnswGraph$NodesIterator'; do + diff <(javap -protected -cp $M/10.3.2/lucene-core-10.3.2.jar "$cls" | grep -Ev ' (final|static) ' | sort) \ + <(javap -protected -cp $M/10.4.0/lucene-core-10.4.0.jar "$cls" | grep -Ev ' (final|static) ' | sort) \ + | grep '^[<>]' && echo "^^^ $cls" +done +``` + +For each new or changed method, decide whether Lucene's default is right for the cuvs-lucene subclass. +If not, override it in the `Compat*` class of the variant the new release uses, creating a new variant +if older releases share that one (see how `compat/knn-vectors-reader-103-acceptdocs` overrides +`getOffHeapByteSize`). To refresh the class list, run `javap` on the classes in +`lucene-10.4/target/classes/com/nvidia/cuvs/lucene` and collect their `org.apache.lucene` supertypes, +all the way up. + +Behaviour can also change without any signature changing. Read the source of the Lucene methods that call +into cuvs-lucene and compare it with the previous release; the most important is +`AbstractKnnVectorQuery.rewrite`, which decides how each segment is searched. For example, Lucene 10.3 +started expecting more than `k` hits from a filtered segment search, and throwing away a search that +returns fewer; `CompatKnnFloatVectorQuery.perLeafTopK` follows its rule, and has to follow it again if +it changes. The 10.3 variant also tells filtered segment searches from unfiltered ones by the visit limit +`rewrite` passes to `approximateSearch` (`Integer.MAX_VALUE` when it applies no filter), so check that +this still holds. + +Also read Lucene's `CHANGES.txt` for the release, looking for: + +- changes to `Lucene99HnswVectorsFormat`'s on-disk format. The GPU writers write its `.vex`/`.vem` files + themselves (`AcceleratedHNSWUtils`) with `VERSION_START`; the new release's reader must still accept + that version. If Lucene replaced `Lucene99HnswVectorsFormat` as its HNSW format, plan a new + cuvs-lucene format; that is feature work, not part of this checklist; +- vector formats moved to `backward_codecs` (they can then only be read, see step 4); +- changes to `BaseKnnVectorsFormatTestCase` (see step 5). + +### 4. If Lucene's default codec changed + +Check which codec the release writes by default: + +```bash +unzip -p ~/.m2/repository/org/apache/lucene/lucene-core/10.4.0/lucene-core-10.4.0.jar \ + META-INF/services/org.apache.lucene.codecs.Codec +``` + +If it is still `Lucene103Codec`, skip this step: the `Lucene103*` codecs keep writing on 10.4. +If it is new, for example `Lucene104Codec`: + +1. Create `lucene-10.4/src/since/java/com/nvidia/cuvs/lucene/` with the four codecs of the new + generation, copied from `lucene-10.3/src/since/java` and renamed: `Lucene104AcceleratedHNSWCodec`, + `Lucene104AcceleratedHNSWScalarQuantizedCodec`, `Lucene104AcceleratedHNSWBinaryQuantizedCodec`, + `Lucene104CuVSGPUSearchCodec`. Each passes generation `104` and `() -> LuceneCompat.lucene104Codec()` + (a lambda, not a method reference) to the `CuVSFilterCodec` constructor. The codec name + is written into every segment, so never reuse or rename one once released. +2. Add `src/since/java` to the `add-since-sources` list in `lucene-10.4/pom.xml` (and, later, in every + module after it). +3. In `lucene-10.4/.../LuceneCompat.java`: + - add `lucene104Codec()` returning `new org.apache.lucene.codecs.lucene104.Lucene104Codec()`; + - point `lucene103Codec()` at `org.apache.lucene.backward_codecs.lucene103.Lucene103Codec`, where + Lucene moves the old default codec; + - set `CURRENT_CODEC_GENERATION = 104`. The new codecs pass `104` as their generation to the + `CuVSFilterCodec` constructor, and the `Lucene103*` codecs, which pass `103`, become read-only; + - make `acceleratedHNSWCodec`, `acceleratedHNSWScalarQuantizedCodec`, + `acceleratedHNSWBinaryQuantizedCodec` and `gpuSearchCodec` return the `Lucene104*` codecs. +4. Register the four new codecs in `lucene-10.4/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec`, + after the existing ones. List only cuvs-lucene classes in the services files: a Lucene class there + breaks every codec or format lookup in the JVM once a later release moves it. +5. Update the list of codecs per Lucene release in the class javadoc of `CuVSCodecs`. + +If a vector format cuvs-lucene writes moved to `backward_codecs`, the codecs using it can no longer +write on the new release: add a new format next to the codecs of step 1, register it in +`META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat`, and have the new codecs and the matching +`LuceneCompat` format method use it. + +### 5. Make the tests pass + +```bash +mvn -pl lucene-10.4 -am verify +``` + +- New abstract methods or new tests in Lucene's `BaseKnnVectorsFormatTestCase` are handled once, in + `src/test/java/.../BaseCuVSKnnVectorsFormatTestCase.java`, which every format test extends, and without + `@Override`, so that they still compile against older releases. +- A test call whose signature changed goes through `TestLuceneCompat`, in a `compat/` variant (step 2). +- `TestBackCompat` fails if a codec from `CuVSCodecs` is not registered, or if an older codec can still + write. + +### 6. Run everything + +```bash +./build.sh --run-java-tests --build-java-examples +``` + +This builds and tests every module, installs them, and builds the examples. It also checks the upgrade +from every earlier supported release: `TestBackCompatIndices` in each module writes indexes with its +codecs to `target/backcompat`, and reads and upgrades those that the modules for earlier releases wrote +there before it. Nothing needs to be committed for that. + +### 7. Update the documentation + +| File | Change | +| --- | --- | +| `README.md` (this directory) | the supported range and the artifact list; the codec paragraph if step 4 applied | +| `../../fern/pages/installation/java.md` | "supports Lucene 10.2 through 10.X", the artifact table | +| `../../fern/pages/user_guide/lucene.md` | the supported range, the artifact list, the codec table if step 4 applied | + +The API reference pages in `fern/pages/lucene_api` are generated: the `fern-api-reference` pre-commit +hook picks up new classes in `lucene-*/src/since/java` by itself. Run `pre-commit run --all-files` last. + +## After a cuvs-lucene release + +Add the indexes written by the published jars to the back-compat tests, so that later changes cannot +silently make indexes written by that release unreadable. On a host with a GPU and the matching native +libraries: + +```bash +src/test/backcompat/generate-released-indices.sh +``` + +This currently handles releases published as the single `cuvs-lucene` artifact (up to 26.10); releases +from 26.12 on are published per Lucene release, so the script needs extending to pick one of those +artifacts. Record the new zips in `src/test/resources/backcompat/README.md`. + +## Dropping a minor release + +1. Remove its `` from the parent `pom.xml` and delete its `lucene-10.X` directory. If it has a + `src/since/java`, move that directory to the oldest remaining module that lists it, and update the + `add-since-sources` lists of the later modules. +2. Delete every `compat/` variant no remaining module uses (`grep -h cuvs.lucene.compat lucene-*/pom.xml`). + If all remaining modules use the same variant of an API, move its classes into the shared `src/`, + delete the variant directory and the property, and remove its `` line from the parent + `pom.xml`. Do the same for any `LuceneCompat` method that is now identical in every module. +3. Keep its codecs and its `src/test/resources/backcompat/releases/*/10.X/` indexes as long as later + releases must read indexes written with it. +4. Update the documentation listed in step 7 above. +5. When dropping 10.2, the relocation in `relocation/pom.xml` would point to an artifact that is no longer + published: remove the `relocation` module (and stop publishing it) at that point. The deprecated + `LuceneProvider` in `lucene-10.2/src/main/java`, kept only for callers of the old artifact, goes + with the module. diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index 651a30b0e7..4d44c0fcdb 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -13,7 +13,7 @@ This is a project for using [cuVS](https://github.com/NVIDIA/cuvs), NVIDIA's GPU ## What is cuvs-lucene? -`cuvs-lucene` provides a pluggable [KnnVectorsFormat](https://lucene.apache.org/core/10_2_0/core/org/apache/lucene/codecs/KnnVectorsFormat.html) that uses cuVS to offload vector index build — and optionally search — to NVIDIA GPUs. Because it plugs in through a standard Lucene codec, existing Lucene applications can take advantage of GPU acceleration with minimal code changes. It supports Lucene 10.2. +`cuvs-lucene` provides a pluggable [KnnVectorsFormat](https://lucene.apache.org/core/10_2_0/core/org/apache/lucene/codecs/KnnVectorsFormat.html) that uses cuVS to offload vector index build — and optionally search — to NVIDIA GPUs. Because it plugs in through a standard Lucene codec, existing Lucene applications can take advantage of GPU acceleration with minimal code changes. It supports Lucene 10.2 and 10.3. Four codecs are currently provided, created through `CuVSCodecs`: @@ -22,9 +22,11 @@ Four codecs are currently provided, created through `CuVSCodecs`: - `CuVSCodecs.acceleratedHNSWBinaryQuantized()` — graph built over binary-quantized vectors. - `CuVSCodecs.gpuSearch()` — GPU-accelerated build and GPU search. Requires a GPU. -Each codec wraps the default codec of one Lucene release, for example `Lucene101AcceleratedHNSWCodec` wraps -`Lucene101Codec`, the default codec of Lucene 10.2. Creating the codecs through `CuVSCodecs` rather than by class -name keeps application code unchanged when the codec to use changes with a later Lucene release. +Each codec wraps the default codec of one Lucene release, so the codec that writes changes with Lucene: for +example `Lucene101AcceleratedHNSWCodec` on Lucene 10.2 and `Lucene103AcceleratedHNSWCodec` on 10.3. Codecs of +earlier releases stay available so that existing indexes can still be read after an upgrade, but they refuse to +write. Creating the codecs through `CuVSCodecs` rather than by class name keeps application code unchanged when +the codec to use changes with a later Lucene release. For guidance on choosing between them, configuring builds, and tuning GPU resources, see the [Lucene Integration](https://docs.nvidia.com/cuvs/user-guide/lucene) guide. @@ -43,13 +45,13 @@ For guidance on choosing between them, configuring builds, and tuning GPU resour ### Maven Lucene changes its codec APIs between minor releases, so `cuvs-lucene` is published once per supported Lucene -minor release, as `cuvs-lucene-10.X`. Pick the one matching the Lucene version your application uses. For -Lucene 10.2, add the following dependency to your `pom.xml`: +minor release, as `cuvs-lucene-10.2` and `cuvs-lucene-10.3`. Pick the one matching the Lucene version your +application uses. For Lucene 10.3, add the following dependency to your `pom.xml`: ```xml com.nvidia.cuvs.lucene - cuvs-lucene-10.2 + cuvs-lucene-10.3 26.12.0 ``` @@ -170,23 +172,40 @@ java/cuvs-lucene/ src/main/java shared code: everything that compiles against every supported Lucene release src/test/java shared tests src/test/resources/backcompat indexes written by each release, see below + compat//src the adapter for one Lucene class as it looks from one release on, shared by + the releases that have it, such as knn-vectors-reader-102-bits and + knn-vectors-reader-103-acceptdocs; see compat/README.md lucene-10.X/ - pom.xml the Lucene 10.X dependencies + pom.xml the Lucene 10.X dependencies, and which compat/ variant of each API it uses src/main/java LuceneCompat: the calls that differ on Lucene 10.X src/main/resources the META-INF/services files: the codecs and formats Lucene 10.X can load + src/since/java code first needed on Lucene 10.X, such as the codecs wrapping its default + codec; compiled by this module and every later one relocation/pom.xml relocates the pre-26.12 cuvs-lucene coordinates to cuvs-lucene-10.2 ``` Each module compiles the shared sources against its own Lucene release, so the compiler checks every release. -The shared code never refers to a Lucene API that differs between releases. Instead it goes through -`LuceneCompat`, for calls whose signature or class location differs, such as the codec each release writes with. +The shared code never refers to a Lucene API that differs between releases. Instead it goes through: + +- `LuceneCompat`, for calls whose signature or class location differs, such as the codecs of earlier releases, + which move to Lucene's backward codecs. +- `CompatKnnVectorsReader` and `CompatKnnFloatVectorQuery`, which override the Lucene methods whose signatures + changed and forward to a method with the same signature on every release. They live in `compat/`, one + directory per variant, and each module picks its variants in its `pom.xml`. Do not look up Lucene classes or constants by reflection. It hides API changes from the compiler, and constants such as `Lucene99HnswVectorsFormat.VERSION_CURRENT` change meaning between releases. `TestBackCompatIndices` reads indexes written by released cuvs-lucene versions, stored as zips in -`src/test/resources/backcompat`, and checks that every module can still read, search and upgrade them. `TestBackCompat` checks that the codecs from -`CuVSCodecs` are registered. +`src/test/resources/backcompat`, and checks that every module can still read, search and upgrade them. It also +writes indexes with each module into `target/backcompat`, and checks that every later module tested in the same +build can read, search and upgrade those. `TestBackCompat` checks that the codecs from `CuVSCodecs` are +registered, and that only they write. + +### Adding a Lucene release + +[MAINTAINING.md](MAINTAINING.md) lists exactly what to change when Lucene publishes a new patch or minor +release, and how to drop support for one. ## References diff --git a/java/cuvs-lucene/compat/README.md b/java/cuvs-lucene/compat/README.md new file mode 100644 index 0000000000..b2f6768f73 --- /dev/null +++ b/java/cuvs-lucene/compat/README.md @@ -0,0 +1,39 @@ +# Lucene API variants + +Some Lucene classes that cuvs-lucene extends or calls changed between Lucene releases. The code adapting +to such a class differs between releases, but is the same in all the releases that share one version of +the class, so it lives here once instead of being copied into every `lucene-10.X` module. + +Each directory is one **variant**: the adapter for one Lucene class, as that class looks from one release +on. Directories are named `--`: + +- `` is the Lucene class being adapted. There is one `cuvs.lucene.compat.` property per + class, and each module's `pom.xml` sets it to the variant its Lucene release needs. The parent `pom.xml` + adds the chosen directories to the module's sources. +- `` is the first supported Lucene release with that version of the class (`102` is the oldest + supported release, so `102` variants describe how the class looked before any change we adapt to). + Names never change afterwards, even when older releases are dropped. +- `` says what that release changed, so that variants can be told apart without opening them. + +| Property | Variant | Used by | What changed | +| --- | --- | --- | --- | +| `cuvs.lucene.compat.knnVectorsReader` | `knn-vectors-reader-102-bits` | 10.2 | `search` takes the accepted documents as `Bits` | +| | `knn-vectors-reader-103-acceptdocs` | 10.3 | `search` takes `AcceptDocs`; readers also report `getOffHeapByteSize`; merge methods throw `IOException` | +| `cuvs.lucene.compat.knnFloatVectorQuery` | `knn-float-vector-query-102-bits` | 10.2 | `approximateSearch` takes `Bits` | +| | `knn-float-vector-query-103-acceptdocs` | 10.3 | `approximateSearch` takes `AcceptDocs`; filtered segment searches must return `perLeafTopK` hits, often more than `k` | +| `cuvs.lucene.compat.leafReader` (tests only) | `leaf-reader-102-bits` | 10.2 | `searchNearestVectors` takes `Bits` | +| | `leaf-reader-103-acceptdocs` | 10.3 | `searchNearestVectors` takes `AcceptDocs` | + +Code that differs in (almost) every release, such as which Lucene classes moved to `backward_codecs`, +stays in each module's own `LuceneCompat`. + +- **A new release changes a class that already has variants:** copy the variant the previous release uses + to `--`, change it there, and point the new module's property at it. Never + edit a variant that another module still uses. +- **A new release changes a class with no variants yet:** move the adapter out of the shared `src/` into + two variants, `-102-` and `--`, add a + `cuvs.lucene.compat.` property to every module, and a matching `` line to the + `add-shared-sources` (or `add-shared-test-sources`) execution of the parent `pom.xml`. +- **A release is dropped:** delete every variant no remaining module uses. When all remaining modules use + the same variant of a class, move its code into the shared `src/`, delete the variant and the property, + and remove its `` line from the parent `pom.xml`. diff --git a/java/cuvs-lucene/compat/knn-float-vector-query-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java b/java/cuvs-lucene/compat/knn-float-vector-query-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java new file mode 100644 index 0000000000..cfa741b89b --- /dev/null +++ b/java/cuvs-lucene/compat/knn-float-vector-query-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.knn.KnnCollectorManager; +import org.apache.lucene.util.Bits; + +/** + * Base class of the GPU query, {@code GPUKnnFloatVectorQuery}, that hides a signature change in + * {@link KnnFloatVectorQuery}. Lucene 10.2 passes the documents a segment search may return to + * {@code approximateSearch} as {@link Bits}; Lucene 10.3 changed that parameter to + * {@code AcceptDocs}. Each variant of this class implements the {@code approximateSearch} of its + * Lucene release with the collector from {@code newPerLeafCollector}, so the shared query only + * provides that collector. + * + *

Each variant also tells the shared query how many hits Lucene expects from a segment search, + * {@link #perLeafTopK}: Lucene 10.2 expects {@code k}, Lucene 10.3 more. + */ +abstract class CompatKnnFloatVectorQuery extends KnnFloatVectorQuery { + + CompatKnnFloatVectorQuery(String field, float[] target, int k, Query filter) { + super(field, target, k, filter); + LuceneVersionGuard.ensureCompatible(); + } + + /** + * Returns the collector for searching one segment. It may hold fewer hits than asked for, but + * never fewer than the query's {@code k}. + * + * @param context the segment + * @param k the number of hits to collect + * @param visitedLimit the visit budget + */ + abstract KnnCollector newPerLeafCollector(LeafReaderContext context, int k, int visitedLimit) + throws IOException; + + /** + * Returns how many hits Lucene expects from searching a segment with a filter. Lucene 10.2 searches + * a segment exactly when the filter accepts at most this many documents in it, and it does so too + * when an approximate search returns fewer hits. On Lucene 10.2 this is {@code k}. + * + * @param context the segment + */ + int perLeafTopK(LeafReaderContext context) { + return k; + } + + @Override + protected TopDocs approximateSearch( + LeafReaderContext context, + Bits acceptDocs, + int visitedLimit, + KnnCollectorManager knnCollectorManager) + throws IOException { + KnnCollector results = newPerLeafCollector(context, k, visitedLimit); + context.reader().searchNearestVectors(field, getTargetCopy(), results, acceptDocs); + return results.topDocs(); + } +} diff --git a/java/cuvs-lucene/compat/knn-float-vector-query-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java b/java/cuvs-lucene/compat/knn-float-vector-query-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java new file mode 100644 index 0000000000..12486baf02 --- /dev/null +++ b/java/cuvs-lucene/compat/knn-float-vector-query-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnFloatVectorQuery.java @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.search.TopKnnCollector; +import org.apache.lucene.search.knn.KnnCollectorManager; + +/** + * Base class of the GPU query, {@code GPUKnnFloatVectorQuery}, that hides a signature change in + * {@link KnnFloatVectorQuery}. Since Lucene 10.3, the documents a segment search may return come + * to {@code approximateSearch} as {@link AcceptDocs}, where Lucene 10.2 passed {@code Bits}. Each + * variant of this class implements the {@code approximateSearch} of its Lucene release with the + * collector from {@code newPerLeafCollector}, so the shared query only provides that collector. + * + *

Each variant also tells the shared query how many hits Lucene expects from a segment search, + * {@link #perLeafTopK}: Lucene 10.2 expects {@code k}, Lucene 10.3 more. + * + *

This variant also makes the collector manager non-optimistic, so that Lucene does not repeat + * a segment's search; see {@link #getKnnCollectorManager}. + */ +abstract class CompatKnnFloatVectorQuery extends KnnFloatVectorQuery { + + CompatKnnFloatVectorQuery(String field, float[] target, int k, Query filter) { + super(field, target, k, filter); + LuceneVersionGuard.ensureCompatible(); + } + + // AbstractKnnVectorQuery.LAMBDA, which is private. + private static final int LAMBDA = 16; + + /** + * Returns the collector for searching one segment. It may hold fewer hits than asked for, but + * never fewer than the query's {@code k}. + * + * @param context the segment + * @param k the number of hits to collect + * @param visitedLimit the visit budget + */ + abstract KnnCollector newPerLeafCollector(LeafReaderContext context, int k, int visitedLimit) + throws IOException; + + /** + * Returns how many hits Lucene expects from searching a segment with a filter. Since Lucene 10.3, + * it searches a segment exactly when the filter accepts at most this many documents in it, and + * it does so too when an approximate search returns fewer hits, throwing that search's results + * away. + * + *

This is the segment's expected share of the top {@code k} hits plus a margin, computed like + * Lucene's private {@code AbstractKnnVectorQuery.perLeafTopKCalculation}, so it is often more + * than {@code k}. + * + * @param context the segment + */ + int perLeafTopK(LeafReaderContext context) { + float leafProportion = + context.parent == null + ? 1f + : context.reader().maxDoc() / (float) context.parent.reader().maxDoc(); + return (int) + Math.max( + 1, k * leafProportion + LAMBDA * Math.sqrt(k * leafProportion * (1 - leafProportion))); + } + + /** + * Returns a manager that is not optimistic. Since Lucene 10.3, {@code rewrite} searches again, + * expecting more hits, every segment whose hits all beat the global top-k when the manager is + * optimistic. {@link #approximateSearch} does not use the manager, so that second search would + * only repeat the first one. + */ + @Override + protected KnnCollectorManager getKnnCollectorManager(int k, IndexSearcher searcher) { + // A lambda does not override isOptimistic(), which defaults to false. approximateSearch never + // calls it, so it builds no collector until then. + return (visitedLimit, searchStrategy, context) -> + new TopKnnCollector(k, visitedLimit, searchStrategy); + } + + @Override + protected TopDocs approximateSearch( + LeafReaderContext context, + AcceptDocs acceptDocs, + int visitedLimit, + KnnCollectorManager knnCollectorManager) + throws IOException { + // Lucene keeps the hits of a filtered search only if there are perLeafTopK of them. It passes + // no visit limit when it applies no filter, including one rewritten to match all documents, + // and then keeps whatever the search returns. + boolean filtered = visitedLimit != Integer.MAX_VALUE; + int leafK = filtered ? Math.max(k, perLeafTopK(context)) : k; + KnnCollector results = newPerLeafCollector(context, leafK, visitedLimit); + if (results.k() < leafK) { + // The collector cannot hold as many hits as Lucene needs, so Lucene would discard them and + // search the segment exactly anyway: skip the approximate search and let it. + return TopDocsCollector.EMPTY_TOPDOCS; + } + context.reader().searchNearestVectors(field, getTargetCopy(), results, acceptDocs); + return results.topDocs(); + } +} diff --git a/java/cuvs-lucene/compat/knn-vectors-reader-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java b/java/cuvs-lucene/compat/knn-vectors-reader-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java new file mode 100644 index 0000000000..1e280ab04a --- /dev/null +++ b/java/cuvs-lucene/compat/knn-vectors-reader-102-bits/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; + +/** + * Base class of the GPU search reader, {@code CuVS2510GPUVectorsReader}, that hides a signature + * change in {@link KnnVectorsReader}. Lucene 10.2 passes the documents a search may return as + * {@link Bits}; Lucene 10.3 changed that parameter to {@code AcceptDocs}. Each variant of this + * class overrides the {@code search} methods of its Lucene release and forwards them to + * {@code doSearch}, whose signature is the same on every release, so the shared reader implements + * {@code doSearch} once. + * + *

It also lets the raw vectors reader prepare for a merge, whose methods throw no checked + * exception on Lucene 10.2 but do on 10.3. + */ +abstract class CompatKnnVectorsReader extends KnnVectorsReader { + + private final FlatVectorsReader rawVectorsReader; + + /** + * @param rawVectorsReader reads the raw vectors of the fields + */ + CompatKnnVectorsReader(FlatVectorsReader rawVectorsReader) { + this.rawVectorsReader = rawVectorsReader; + } + + /** + * Has the raw vectors reader read sequentially, as merges read every vector in order, and + * returns this reader. + */ + @Override + public KnnVectorsReader getMergeInstance() { + // The subclass reads the raw vectors through the reader it was created with, so this relies on + // the raw vectors reader preparing itself for the merge and returning itself, as + // Lucene99FlatVectorsReader does, rather than returning another instance. + rawVectorsReader.getMergeInstance(); + return this; + } + + /** Has the raw vectors reader go back to the read pattern of searches. */ + @Override + public void finishMerge() throws IOException { + rawVectorsReader.finishMerge(); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) + throws IOException { + doSearch(field, target, knnCollector, acceptDocs); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) + throws IOException { + doSearch(field, target, knnCollector, acceptDocs); + } + + /** + * Implements {@link #search(String, float[], KnnCollector, Bits)}. + * + * @param acceptDocs the documents that may be returned, or {@code null} if all may be + */ + protected abstract void doSearch( + String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException; + + /** + * Implements {@link #search(String, byte[], KnnCollector, Bits)}. + * + * @param acceptDocs the documents that may be returned, or {@code null} if all may be + */ + protected abstract void doSearch( + String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException; +} diff --git a/java/cuvs-lucene/compat/knn-vectors-reader-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java b/java/cuvs-lucene/compat/knn-vectors-reader-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java new file mode 100644 index 0000000000..d9484868be --- /dev/null +++ b/java/cuvs-lucene/compat/knn-vectors-reader-103-acceptdocs/src/main/java/com/nvidia/cuvs/lucene/CompatKnnVectorsReader.java @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import java.util.Map; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; + +/** + * Base class of the GPU search reader, {@code CuVS2510GPUVectorsReader}, that hides a signature + * change in {@link KnnVectorsReader}. Since Lucene 10.3, the documents a search may return come as + * {@link AcceptDocs}, where Lucene 10.2 passed {@code Bits}. Each variant of this class overrides + * the {@code search} methods of its Lucene release and forwards them to {@code doSearch}, whose + * signature is the same on every release, so the shared reader implements {@code doSearch} once. + * + *

This variant converts with {@code AcceptDocs.bits()}. It also reports the reader's off-heap + * size, which Lucene 10.3 started asking for, and lets the raw vectors reader prepare for a merge, + * whose methods throw {@link IOException} since Lucene 10.3. + */ +abstract class CompatKnnVectorsReader extends KnnVectorsReader { + + private final FlatVectorsReader rawVectorsReader; + + /** + * @param rawVectorsReader reads the raw vectors of the fields + */ + CompatKnnVectorsReader(FlatVectorsReader rawVectorsReader) { + this.rawVectorsReader = rawVectorsReader; + } + + /** + * Reports only the raw vectors. Lucene asks for the memory that searches need to avoid page + * faults; searches read the raw vectors from their file, but search the cuVS indexes on the GPU. + * The file holding the cuVS indexes is only read to load them onto the GPU, so it is left out, + * even though it stays open. + */ + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return rawVectorsReader.getOffHeapByteSize(fieldInfo); + } + + /** + * Has the raw vectors reader read sequentially, as merges read every vector in order, and + * returns this reader. + */ + @Override + public KnnVectorsReader getMergeInstance() throws IOException { + // The subclass reads the raw vectors through the reader it was created with, so this relies on + // the raw vectors reader preparing itself for the merge and returning itself, as + // Lucene99FlatVectorsReader does, rather than returning another instance. + rawVectorsReader.getMergeInstance(); + return this; + } + + /** Has the raw vectors reader go back to the read pattern of searches. */ + @Override + public void finishMerge() throws IOException { + rawVectorsReader.finishMerge(); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) + throws IOException { + doSearch(field, target, knnCollector, acceptDocs == null ? null : acceptDocs.bits()); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) + throws IOException { + doSearch(field, target, knnCollector, acceptDocs == null ? null : acceptDocs.bits()); + } + + /** + * Implements {@link #search(String, float[], KnnCollector, AcceptDocs)}. + * + * @param acceptDocs the documents that may be returned, or {@code null} if all may be + */ + protected abstract void doSearch( + String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException; + + /** + * Implements {@link #search(String, byte[], KnnCollector, AcceptDocs)}. + * + * @param acceptDocs the documents that may be returned, or {@code null} if all may be + */ + protected abstract void doSearch( + String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException; +} diff --git a/java/cuvs-lucene/compat/leaf-reader-102-bits/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java b/java/cuvs-lucene/compat/leaf-reader-102-bits/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java new file mode 100644 index 0000000000..ba897c6718 --- /dev/null +++ b/java/cuvs-lucene/compat/leaf-reader-102-bits/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Bits; + +/** The Lucene test calls that differ between the Lucene releases cuvs-lucene supports. */ +final class TestLuceneCompat { + + private TestLuceneCompat() {} + + /** + * Calls {@link LeafReader#searchNearestVectors} with the accepted documents given as {@link + * Bits}, which is what Lucene 10.2 takes. + */ + static void searchNearestVectors( + LeafReader reader, String field, float[] target, KnnCollector collector, Bits acceptDocs) + throws IOException { + reader.searchNearestVectors(field, target, collector, acceptDocs); + } + + /** + * Calls {@link LeafReader#searchNearestVectors} with the accepted documents given as {@link + * Bits}, {@code null} meaning all, which is what Lucene 10.2 takes. + */ + static TopDocs searchNearestVectors( + LeafReader reader, String field, float[] target, int k, Bits acceptDocs, int visitedLimit) + throws IOException { + return reader.searchNearestVectors(field, target, k, acceptDocs, visitedLimit); + } +} diff --git a/java/cuvs-lucene/compat/leaf-reader-103-acceptdocs/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java b/java/cuvs-lucene/compat/leaf-reader-103-acceptdocs/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java new file mode 100644 index 0000000000..dcd6544f4d --- /dev/null +++ b/java/cuvs-lucene/compat/leaf-reader-103-acceptdocs/src/test/java/com/nvidia/cuvs/lucene/TestLuceneCompat.java @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Bits; + +/** The Lucene test calls that differ between the Lucene releases cuvs-lucene supports. */ +final class TestLuceneCompat { + + private TestLuceneCompat() {} + + /** + * Calls {@link LeafReader#searchNearestVectors} with the accepted documents given as {@link + * Bits}. Since Lucene 10.3 it takes them as {@link AcceptDocs}. + */ + static void searchNearestVectors( + LeafReader reader, String field, float[] target, KnnCollector collector, Bits acceptDocs) + throws IOException { + reader.searchNearestVectors( + field, target, collector, AcceptDocs.fromLiveDocs(acceptDocs, reader.maxDoc())); + } + + /** + * Calls {@link LeafReader#searchNearestVectors} with the accepted documents given as {@link + * Bits}, {@code null} meaning all. Since Lucene 10.3 it takes them as {@link AcceptDocs}, which + * must not be {@code null}. + */ + static TopDocs searchNearestVectors( + LeafReader reader, String field, float[] target, int k, Bits acceptDocs, int visitedLimit) + throws IOException { + return reader.searchNearestVectors( + field, target, k, AcceptDocs.fromLiveDocs(acceptDocs, reader.maxDoc()), visitedLimit); + } +} diff --git a/java/cuvs-lucene/lucene-10.2/pom.xml b/java/cuvs-lucene/lucene-10.2/pom.xml index 01814ab005..e06686828f 100644 --- a/java/cuvs-lucene/lucene-10.2/pom.xml +++ b/java/cuvs-lucene/lucene-10.2/pom.xml @@ -22,6 +22,10 @@ SPDX-License-Identifier: Apache-2.0 10.2.0 + + knn-vectors-reader-102-bits + knn-float-vector-query-102-bits + leaf-reader-102-bits diff --git a/java/cuvs-lucene/lucene-10.2/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java b/java/cuvs-lucene/lucene-10.2/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java index 0ef3b2fa11..756d9f4d0a 100644 --- a/java/cuvs-lucene/lucene-10.2/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java +++ b/java/cuvs-lucene/lucene-10.2/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java @@ -9,6 +9,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.ReadAdvice; /** * The Lucene calls that differ between the Lucene releases cuvs-lucene supports, for Lucene 10.2. @@ -24,8 +26,18 @@ final class LuceneCompat { static final int LUCENE_MAJOR = 10; static final int LUCENE_MINOR = 2; + // The codec generation that writes on this Lucene release: the version number of the Lucene + // codec that codecs of this generation wrap (Lucene101Codec). Codecs of other generations are + // read-only. Also a compile-time constant, for the same reason. + static final int CURRENT_CODEC_GENERATION = 101; + private LuceneCompat() {} + /** Returns {@code context} with a hint that the file will be read sequentially. */ + static IOContext sequentialReadContext(IOContext context) { + return context.withReadAdvice(ReadAdvice.SEQUENTIAL); + } + /** Returns Lucene 10.2's {@code Lucene101Codec}. */ static Codec lucene101Codec() { return new Lucene101Codec(); diff --git a/java/cuvs-lucene/lucene-10.3/pom.xml b/java/cuvs-lucene/lucene-10.3/pom.xml new file mode 100644 index 0000000000..70ceceaf4d --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.nvidia.cuvs.lucene + cuvs-lucene-parent + + 26.12.0 + + cuvs-lucene-10.3 + cuvs-lucene-10.3 + jar + + + Pluggable codecs for Apache Lucene 10.3.x that enable GPU-based accelerated vector search. + + + + 10.3.2 + + knn-vectors-reader-103-acceptdocs + knn-float-vector-query-103-acceptdocs + leaf-reader-103-acceptdocs + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + + add-since-sources + generate-sources + + add-source + + + + src/since/java + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + diff --git a/java/cuvs-lucene/lucene-10.3/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java b/java/cuvs-lucene/lucene-10.3/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java new file mode 100644 index 0000000000..75e074b76d --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/main/java/com/nvidia/cuvs/lucene/LuceneCompat.java @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import org.apache.lucene.backward_codecs.lucene101.Lucene101Codec; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene103.Lucene103Codec; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.store.DataAccessHint; +import org.apache.lucene.store.IOContext; + +/** + * The Lucene calls that differ between the Lucene releases cuvs-lucene supports, for Lucene 10.3. + * + *

Every lucene-X.Y module has its own copy of this class, compiled against that release. The + * shared sources go through it instead of calling those parts of Lucene directly. + */ +final class LuceneCompat { + + // The Lucene release this module is built for. Both must stay compile-time constants: javac + // copies them into the classes that read them, so LuceneVersionGuard can check the Lucene version + // without loading this class, which does not link against other Lucene releases. + static final int LUCENE_MAJOR = 10; + static final int LUCENE_MINOR = 3; + + // The codec generation that writes on this Lucene release: the version number of the Lucene + // codec that codecs of this generation wrap (Lucene103Codec). Codecs of other generations are + // read-only. Also a compile-time constant, for the same reason. + static final int CURRENT_CODEC_GENERATION = 103; + + private LuceneCompat() {} + + /** Returns {@code context} with a hint that the file will be read sequentially. */ + static IOContext sequentialReadContext(IOContext context) { + return context.withHints(DataAccessHint.SEQUENTIAL); + } + + /** Returns Lucene 10.2's {@code Lucene101Codec}, which Lucene 10.3 can only read. */ + static Codec lucene101Codec() { + return new Lucene101Codec(); + } + + /** Returns Lucene 10.3's {@code Lucene103Codec}. */ + static Codec lucene103Codec() { + return new Lucene103Codec(); + } + + /** Returns the flat format that stores the vectors of the scalar-quantized codecs. */ + static FlatVectorsFormat lucene99ScalarQuantizedFlatFormat() { + return new Lucene99ScalarQuantizedVectorsFormat(); + } + + // The codecs for this Lucene release, returned by CuVSCodecs. + + static Codec acceleratedHNSWCodec(AcceleratedHNSWParams params) { + return new Lucene103AcceleratedHNSWCodec(params); + } + + static KnnVectorsFormat acceleratedHNSWFormat(AcceleratedHNSWParams params) { + return new Lucene99AcceleratedHNSWVectorsFormat(params); + } + + static Codec acceleratedHNSWScalarQuantizedCodec(AcceleratedHNSWParams params) { + return new Lucene103AcceleratedHNSWScalarQuantizedCodec(params); + } + + static KnnVectorsFormat acceleratedHNSWScalarQuantizedFormat(AcceleratedHNSWParams params) { + return new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat(params); + } + + static Codec acceleratedHNSWBinaryQuantizedCodec(AcceleratedHNSWParams params) { + return new Lucene103AcceleratedHNSWBinaryQuantizedCodec(params); + } + + static KnnVectorsFormat acceleratedHNSWBinaryQuantizedFormat(AcceleratedHNSWParams params) { + return new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat(params); + } + + static Codec gpuSearchCodec(GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) { + return new Lucene103CuVSGPUSearchCodec(params, filterCacheConfig); + } + + static KnnVectorsFormat gpuSearchFormat( + GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) { + return new CuVS2510GPUVectorsFormat(params, filterCacheConfig); + } +} diff --git a/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000000..e503916f23 --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Every codec generation this Lucene release can read. Only the newest one can write. +com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec +com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedCodec +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedCodec +com.nvidia.cuvs.lucene.Lucene103AcceleratedHNSWCodec +com.nvidia.cuvs.lucene.Lucene103CuVSGPUSearchCodec +com.nvidia.cuvs.lucene.Lucene103AcceleratedHNSWBinaryQuantizedCodec +com.nvidia.cuvs.lucene.Lucene103AcceleratedHNSWScalarQuantizedCodec diff --git a/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat new file mode 100644 index 0000000000..1fa8442e48 --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat +com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat diff --git a/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java new file mode 100644 index 0000000000..ff01041d8a --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWBinaryQuantizedCodec.java @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +/** + * A codec that builds HNSW graphs on the GPU over binary-quantized vectors, and writes them in + * Lucene's HNSW format so that they are searched on the CPU. See {@link + * LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat}. + * + *

This codec wraps {@code Lucene103Codec}, the default codec of Lucene 10.3. Once a later Lucene + * release replaces that default, this codec can only read existing indexes; use {@link + * CuVSCodecs#acceleratedHNSWBinaryQuantized} to write. + * + * @since 26.12 + */ +public class Lucene103AcceleratedHNSWBinaryQuantizedCodec extends CuVSFilterCodec { + + private static final String NAME = "Lucene103AcceleratedHNSWBinaryQuantizedCodec"; + + /** Creates the codec with default parameters. */ + public Lucene103AcceleratedHNSWBinaryQuantizedCodec() { + this(new AcceleratedHNSWParams.Builder().build()); + } + + /** + * Creates the codec. + * + * @param acceleratedHNSWParams the index build parameters + */ + public Lucene103AcceleratedHNSWBinaryQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) { + super( + NAME, + 103, + () -> LuceneCompat.lucene103Codec(), + () -> new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat(acceleratedHNSWParams)); + } +} diff --git a/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java new file mode 100644 index 0000000000..55cfff3f04 --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWCodec.java @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +/** + * A codec that builds HNSW graphs on the GPU, and writes them in Lucene's HNSW format so that they + * are searched on the CPU. Falls back to building the graph on the CPU when no GPU or cuVS is + * available. + * + *

This codec wraps {@code Lucene103Codec}, the default codec of Lucene 10.3. Once a later Lucene + * release replaces that default, this codec can only read existing indexes; use {@link + * CuVSCodecs#acceleratedHNSW} to write. + * + * @since 26.12 + */ +public class Lucene103AcceleratedHNSWCodec extends CuVSFilterCodec { + + private static final String NAME = "Lucene103AcceleratedHNSWCodec"; + + /** Creates the codec with default parameters. */ + public Lucene103AcceleratedHNSWCodec() { + this(new AcceleratedHNSWParams.Builder().build()); + } + + /** + * Creates the codec. + * + * @param acceleratedHNSWParams the index build parameters + */ + public Lucene103AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) { + super( + NAME, + 103, + () -> LuceneCompat.lucene103Codec(), + () -> new Lucene99AcceleratedHNSWVectorsFormat(acceleratedHNSWParams)); + } +} diff --git a/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java new file mode 100644 index 0000000000..7b0aa83040 --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103AcceleratedHNSWScalarQuantizedCodec.java @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +/** + * A codec that builds HNSW graphs on the GPU over scalar-quantized vectors, and writes them in + * Lucene's HNSW format so that they are searched on the CPU. See {@link + * LuceneAcceleratedHNSWScalarQuantizedVectorsFormat}. + * + *

This codec wraps {@code Lucene103Codec}, the default codec of Lucene 10.3. Once a later Lucene + * release replaces that default, this codec can only read existing indexes; use {@link + * CuVSCodecs#acceleratedHNSWScalarQuantized} to write. + * + * @since 26.12 + */ +public class Lucene103AcceleratedHNSWScalarQuantizedCodec extends CuVSFilterCodec { + + private static final String NAME = "Lucene103AcceleratedHNSWScalarQuantizedCodec"; + + /** Creates the codec with default parameters. */ + public Lucene103AcceleratedHNSWScalarQuantizedCodec() { + this(new AcceleratedHNSWParams.Builder().build()); + } + + /** + * Creates the codec. + * + * @param acceleratedHNSWParams the index build parameters + */ + public Lucene103AcceleratedHNSWScalarQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) { + super( + NAME, + 103, + () -> LuceneCompat.lucene103Codec(), + () -> new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat(acceleratedHNSWParams)); + } +} diff --git a/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java new file mode 100644 index 0000000000..c3e5bd08e6 --- /dev/null +++ b/java/cuvs-lucene/lucene-10.3/src/since/java/com/nvidia/cuvs/lucene/Lucene103CuVSGPUSearchCodec.java @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +/** + * A codec that both builds and searches CAGRA indexes on the GPU. cuVS serialization formats are in + * experimental phase and hence backward compatibility cannot be guaranteed. + * + *

This codec wraps {@code Lucene103Codec}, the default codec of Lucene 10.3. Once a later Lucene + * release replaces that default, this codec can only read existing indexes; use {@link + * CuVSCodecs#gpuSearch} to write. + * + * @since 26.12 + */ +public class Lucene103CuVSGPUSearchCodec extends CuVSFilterCodec { + + private static final String NAME = "Lucene103CuVSGPUSearchCodec"; + + /** Creates the codec with default parameters. */ + public Lucene103CuVSGPUSearchCodec() { + this(new GPUSearchParams.Builder().build(), FilterBitsetCacheConfig.DEFAULT); + } + + /** + * Creates the codec. + * + * @param params GPU index and search parameters + * @param filterCacheConfig filter-bitset-cache configuration + */ + public Lucene103CuVSGPUSearchCodec( + GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) { + super( + NAME, + 103, + () -> LuceneCompat.lucene103Codec(), + () -> new CuVS2510GPUVectorsFormat(params, filterCacheConfig)); + } +} diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 74aa89b429..b5a767f5ae 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 This project provides pluggable codecs for Apache Lucene to enable GPU-based accelerated vector search. Lucene changes its codec APIs between minor releases, so there is one artifact per supported Lucene - minor version (cuvs-lucene-10.2 for Lucene 10.2), all built from the same sources. + minor version (cuvs-lucene-10.2, cuvs-lucene-10.3, ...), all built from the same sources. https://rapids.ai @@ -60,6 +60,7 @@ SPDX-License-Identifier: Apache-2.0 --> lucene-10.2 + lucene-10.3 relocation @@ -76,6 +77,12 @@ SPDX-License-Identifier: Apache-2.0 10.2.0 ${project.basedir}/../src + + ${project.basedir}/../compat @@ -208,6 +215,9 @@ SPDX-License-Identifier: Apache-2.0 ${cuvs.lucene.shared.dir}/main/java + + ${cuvs.lucene.compat.dir}/${cuvs.lucene.compat.knnVectorsReader}/src/main/java + ${cuvs.lucene.compat.dir}/${cuvs.lucene.compat.knnFloatVectorQuery}/src/main/java @@ -234,6 +244,7 @@ SPDX-License-Identifier: Apache-2.0 ${cuvs.lucene.shared.dir}/test/java + ${cuvs.lucene.compat.dir}/${cuvs.lucene.compat.leafReader}/src/test/java @@ -322,6 +333,11 @@ SPDX-License-Identifier: Apache-2.0 0 ${cuvs.lucene.shared.dir}/main/resources/logging.properties + + ${project.basedir}/../target/backcompat ${project.build.directory}/cufile.log @@ -423,6 +439,7 @@ SPDX-License-Identifier: Apache-2.0 src/**/*.java + compat/**/*.java 1.27.0 diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 8a16c10ea8..5b4e2c06c7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -10,8 +10,8 @@ * cuVS based codec for GPU based vector search that enables both - indexing and search on the GPU. * cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. * - *

This codec wraps {@code Lucene101Codec}, the default codec of Lucene 10.2. {@link CuVSCodecs#gpuSearch} - * returns the codec of this family for the Lucene release in use. + *

This codec wraps {@code Lucene101Codec}, the default codec of Lucene 10.2. On later Lucene + * releases it can only read existing indexes; use {@link CuVSCodecs#gpuSearch} to write. * * @since 25.10 */ @@ -56,7 +56,8 @@ public CuVS2510GPUSearchCodec(GPUSearchParams params) { public CuVS2510GPUSearchCodec(GPUSearchParams params, FilterBitsetCacheConfig filterCacheConfig) { super( NAME, - delegate(NAME, () -> LuceneCompat.lucene101Codec()), + 101, + () -> LuceneCompat.lucene101Codec(), () -> new CuVS2510GPUVectorsFormat(params, filterCacheConfig)); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index a9c77a47fa..4ab3f17465 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -30,7 +30,6 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; import org.apache.lucene.index.ByteVectorValues; @@ -50,7 +49,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.ReadAdvice; import org.apache.lucene.util.Bits; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.hnsw.IntToIntFunction; @@ -60,7 +58,13 @@ * * @since 25.10 */ -public class CuVS2510GPUVectorsReader extends KnnVectorsReader { +public class CuVS2510GPUVectorsReader extends CompatKnnVectorsReader { + + /** + * Largest number of hits searched for with CAGRA in one segment. Beyond it the brute force index + * is searched instead if the segment has one, and otherwise CAGRA returns only this many. + */ + static final int MAX_CAGRA_K = 1024; private static final List VECTOR_SIMILARITY_FUNCTIONS = Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; @@ -105,6 +109,7 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe CuVS2510GPUVectorsReader( SegmentReadState state, FlatVectorsReader flatReader, FilterBitsetCache filterBitsetCache) throws IOException { + super(flatReader); this.flatVectorsReader = flatReader; this.filterBitsetCache = filterBitsetCache; this.fieldInfos = state.fieldInfos; @@ -131,7 +136,7 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe } finally { CodecUtil.checkFooter(meta, priorException); } - var ioContext = state.context.withReadAdvice(ReadAdvice.SEQUENTIAL); + var ioContext = LuceneCompat.sequentialReadContext(state.context); cuvsIndexInput = openCuVSInput(state, versionMeta, ioContext); /* * Do not load indexes on the GPU when this reader is opened for a merge, which reads them @@ -517,7 +522,7 @@ private static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFu * Returns the k nearest neighbor documents using cuVS's CAGRA or brute force algorithm for this field, to the given vector. */ @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) + protected void doSearch(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { var fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); if (fieldEntry.count() == 0 || knnCollector.k() == 0) { @@ -568,7 +573,14 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits try { List> searchResult = null; - if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { + // Lucene's own query may also ask for more than MAX_CAGRA_K hits, even for a k below it: on + // Lucene 10.3, as many as it expects from this segment. Without a brute force index, CAGRA + // returns as many as it can. That is enough for any k up to MAX_CAGRA_K; a filtered search + // that needs more is redone exactly by Lucene, but an unfiltered one gets at most + // MAX_CAGRA_K hits from this segment, a documented limit of CAGRA-only indexes. + if (cuvsIndex.getCagraIndex() != null + && (knnCollector.k() <= MAX_CAGRA_K || cuvsIndex.getBruteforceIndex() == null)) { + topK = Math.min(topK, MAX_CAGRA_K); CagraSearchParams searchParams; if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; @@ -678,7 +690,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits * This is not supported. */ @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) + protected void doSearch(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { throw new UnsupportedOperationException("Byte vectors are not currently supported"); } @@ -750,6 +762,19 @@ private static void checkVersion(int versionMeta, int versionVectorData, IndexIn } } + /** + * Whether this segment has a CAGRA index for the given field. Unlike {@link + * #getCagraIndexForField}, this never loads an index onto the GPU. + * + * @param field the vector field name + */ + boolean hasCagraIndex(String field) { + FieldInfo info = fieldInfos.fieldInfo(field); + if (info == null) return false; + FieldEntry fieldEntry = fields.get(info.number); + return fieldEntry != null && fieldEntry.cagraIndexLength() > 0; + } + /** * Returns the {@link CagraIndex} for the given field, or {@code null} if this segment has none * (e.g., when the field is missing or the segment only holds a brute force index). diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodecs.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodecs.java index ac0811d893..2af7c9e927 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodecs.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodecs.java @@ -11,11 +11,11 @@ * Creates the cuvs-lucene codecs and vectors formats that write indexes on the Lucene release this * artifact is built for. * - *

Each cuvs-lucene codec wraps the default codec of one Lucene release, for example {@code - * Lucene101AcceleratedHNSWCodec} wraps Lucene 10.2's {@code Lucene101Codec}, so the codec to write - * with changes along with Lucene. Creating codecs through this class instead of naming them keeps - * application code unchanged when it moves to another Lucene release and the matching cuvs-lucene - * artifact. + *

Each cuvs-lucene codec wraps the default codec of one Lucene release, so the codec to write + * with changes along with Lucene: {@code Lucene101AcceleratedHNSWCodec} on Lucene 10.2, {@code + * Lucene103AcceleratedHNSWCodec} on 10.3. The older codecs stay available for reading existing + * indexes. Creating codecs through this class instead of naming them keeps application code + * unchanged when it moves to another Lucene release and the matching cuvs-lucene artifact. * *

Every method throws {@link IllegalStateException} if the Lucene release in use is not the one * this cuvs-lucene artifact is built for. diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFilterCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFilterCodec.java index d837d67cc1..f149655879 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFilterCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFilterCodec.java @@ -5,6 +5,7 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.LibraryException; +import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.logging.Level; @@ -33,8 +34,12 @@ * {@link KnnVectorsFormat}. * *

Each cuvs-lucene codec wraps the default codec of one Lucene release, for example - * {@code Lucene101Codec} for Lucene 10.2. Use {@link CuVSCodecs} to get the codecs for the Lucene - * release in use. + * {@code Lucene101Codec} for Lucene 10.2; that codec's version number is the cuvs-lucene codec's + * generation. Once a newer Lucene release replaces that default codec, Lucene moves the + * old one to its backward codecs, which can read but not write. A cuvs-lucene codec of that + * generation is kept so that existing indexes stay readable, and like the codec it wraps it refuses + * to write: only the generation {@code LuceneCompat.CURRENT_CODEC_GENERATION} names writes. Use + * {@link CuVSCodecs} to get the codecs that can write on the Lucene release in use. */ abstract class CuVSFilterCodec extends FilterCodec { @@ -42,18 +47,44 @@ abstract class CuVSFilterCodec extends FilterCodec { // Every registered codec runs into the same problem, so it is logged only once. private static final AtomicBoolean unavailableLogged = new AtomicBoolean(); + private final boolean readOnly; private KnnVectorsFormat format; /** - * Creates a codec. + * Creates one of cuvs-lucene's own codecs, which wraps the default codec of a Lucene release and + * can only write on the Lucene release where that codec is still the default. * * @param name the codec's name - * @param delegate the codec to use for everything but vectors; for the codecs of this module, - * the result of {@link #delegate(String, Supplier)} + * @param generation the version number of the Lucene codec it wraps, such as 101 for + * {@code Lucene101Codec} + * @param delegate creates that Lucene codec, as a lambda; see {@link #delegate(String, Supplier)} + * @param format creates the {@link KnnVectorsFormat} to use for vectors + */ + protected CuVSFilterCodec( + String name, int generation, Supplier delegate, Supplier format) { + this( + name, + delegate(name, delegate), + generation != LuceneCompat.CURRENT_CODEC_GENERATION, + format); + } + + /** + * Creates a codec with a delegate chosen by the caller, which is never made read-only: whether + * the delegate can write is the caller's responsibility. + * + * @param name the codec's name + * @param delegate the codec to use for everything but vectors * @param format creates the {@link KnnVectorsFormat} to use for vectors */ protected CuVSFilterCodec(String name, Codec delegate, Supplier format) { + this(name, delegate, false, format); + } + + private CuVSFilterCodec( + String name, Codec delegate, boolean readOnly, Supplier format) { super(name, delegate); + this.readOnly = readOnly; try { setKnnVectorsFormat(format.get()); } catch (LibraryException | LinkageError e) { @@ -135,13 +166,31 @@ public KnnVectorsFormat knnVectorsFormat() { return format; } + /** Whether this codec can only read on the Lucene release in use. */ + boolean isReadOnly() { + return readOnly; + } + + /** Whether {@code format} is a format that a read-only codec wrapped so that it refuses to write. */ + static boolean isReadOnlyFormat(KnnVectorsFormat format) { + return format instanceof ReadOnlyFormat; + } + /** - * Sets the {@link KnnVectorsFormat} this codec uses for vectors. + * Sets the {@link KnnVectorsFormat} this codec uses for vectors. On a codec that can only read on + * the Lucene release in use, the format is wrapped so that it refuses to write. * * @param format the {@link KnnVectorsFormat} to set */ public void setKnnVectorsFormat(KnnVectorsFormat format) { - this.format = format; + // A format read back from the knnVectorsFormat() of a read-only codec, this one or another, is + // unwrapped first, so that whether it writes depends only on this codec. + if (format instanceof ReadOnlyFormat readOnlyFormat) { + format = readOnlyFormat.in; + } + // An unavailable format already refuses to write, and says why. + boolean wrap = readOnly && format != null && format instanceof UnavailableFormat == false; + this.format = wrap ? new ReadOnlyFormat(getName(), format) : format; } /** @@ -255,4 +304,45 @@ public int getMaxDimensions(String fieldName) { throw unavailable(); } } + + /** Reads through the wrapped format, and refuses to write. */ + private static final class ReadOnlyFormat extends KnnVectorsFormat { + + private final String codecName; + private final KnnVectorsFormat in; + + ReadOnlyFormat(String codecName, KnnVectorsFormat in) { + super(in.getName()); + this.codecName = codecName; + this.in = in; + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + // On another Lucene release than this artifact is built for, the artifact to use is the more + // useful message. + LuceneVersionGuard.ensureCompatible(); + throw new UnsupportedOperationException( + "The " + + codecName + + " codec can only read indexes on Lucene " + + Version.LATEST + + ". Use the codecs from CuVSCodecs to write."); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return in.fieldsReader(state); + } + + @Override + public int getMaxDimensions(String fieldName) { + return in.getMaxDimensions(fieldName); + } + + @Override + public String toString() { + return "ReadOnly(" + in + ")"; + } + } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index 4e29d80c9f..7538349ffb 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -40,9 +41,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; -import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; @@ -79,7 +78,7 @@ * * @since 25.10 */ -public class GPUKnnFloatVectorQuery extends KnnFloatVectorQuery { +public class GPUKnnFloatVectorQuery extends CompatKnnFloatVectorQuery { /** Smallest supported CAGRA intermediate-result count. */ public static final int MIN_ITOPK = 1; @@ -168,7 +167,6 @@ public GPUKnnFloatVectorQuery( int maxIterations, CagraSearchParams.SearchAlgo searchAlgo) { super(field, target, k, filter); - LuceneVersionGuard.ensureCompatible(); validateSearchParameters(iTopK, searchWidth, k, threadBlockSize, maxIterations, searchAlgo); this.iTopK = iTopK; this.searchWidth = searchWidth; @@ -392,17 +390,26 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException { // ------------------------------------------------------------------------- @Override - protected TopDocs approximateSearch( - LeafReaderContext context, - Bits acceptDocs, - int visitedLimit, - KnnCollectorManager knnCollectorManager) - throws IOException { - GPUPerLeafCuVSKnnCollector results = - new GPUPerLeafCuVSKnnCollector( - k, visitedLimit, iTopK, searchWidth, threadBlockSize, maxIterations, searchAlgo); - context.reader().searchNearestVectors(field, getTargetCopy(), results, acceptDocs); - return results.topDocs(); + KnnCollector newPerLeafCollector(LeafReaderContext context, int leafK, int visitedLimit) { + int collectorK = leafK; + CuVS2510GPUVectorsReader gpuReader = unwrapGpuReader(context, field); + // Not getCagraIndexForField, which would load the index of a reader opened for a merge onto the + // GPU even if the search then finds nothing to match. + if (gpuReader != null && gpuReader.hasCagraIndex(field)) { + // Lucene may ask for more hits than this query's k, but CAGRA returns at most + // CuVS2510GPUVectorsReader.MAX_CAGRA_K, and SINGLE_CTA at most MAX_SINGLE_CTA_ITOPK (the + // constructor checks the query's own k against the latter only). Capped, the collector holds + // fewer hits than Lucene asked for, and Lucene searches a filtered segment exactly instead, + // which is slower but still correct. The query's own k is never reduced. Other segments, + // such as those searched with Lucene's HNSW, have no such limit. + int limit = + searchAlgo == CagraSearchParams.SearchAlgo.SINGLE_CTA + ? MAX_SINGLE_CTA_ITOPK + : CuVS2510GPUVectorsReader.MAX_CAGRA_K; + collectorK = Math.max(k, Math.min(leafK, limit)); + } + return new GPUPerLeafCuVSKnnCollector( + collectorK, visitedLimit, iTopK, searchWidth, threadBlockSize, maxIterations, searchAlgo); } // ------------------------------------------------------------------------- diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 860118c3b3..1340cf5c3f 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -11,8 +11,8 @@ * to accelerated indexing using GPUs and search using CPUs. Fallbacks to CPU * based indexing when used on a machine without a GPU and/or cuVS. * - *

This codec wraps {@code Lucene101Codec}, the default codec of Lucene 10.2. {@link CuVSCodecs#acceleratedHNSW} - * returns the codec of this family for the Lucene release in use. + *

This codec wraps {@code Lucene101Codec}, the default codec of Lucene 10.2. On later Lucene + * releases it can only read existing indexes; use {@link CuVSCodecs#acceleratedHNSW} to write. * * @since 25.10 */ @@ -49,7 +49,8 @@ public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { public Lucene101AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) { super( NAME, - delegate(NAME, () -> LuceneCompat.lucene101Codec()), + 101, + () -> LuceneCompat.lucene101Codec(), () -> new Lucene99AcceleratedHNSWVectorsFormat(acceleratedHNSWParams)); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java index 2549b17049..218f88a17c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java @@ -10,8 +10,8 @@ * CuVS based codec for GPU based vector search * *

This codec, named {@code Lucene101AcceleratedHNSWBinaryQuantizedCodec}, wraps {@code - * Lucene101Codec}, the default codec of Lucene 10.2. {@link CuVSCodecs#acceleratedHNSWBinaryQuantized} returns the codec of - * this family for the Lucene release in use. + * Lucene101Codec}, the default codec of Lucene 10.2. On later Lucene releases it can only read + * existing indexes; use {@link CuVSCodecs#acceleratedHNSWBinaryQuantized} to write. * * @since 26.02 */ @@ -35,7 +35,8 @@ public LuceneAcceleratedHNSWBinaryQuantizedCodec(String name, Codec delegate) { public LuceneAcceleratedHNSWBinaryQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) { super( NAME, - delegate(NAME, () -> LuceneCompat.lucene101Codec()), + 101, + () -> LuceneCompat.lucene101Codec(), () -> new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat(acceleratedHNSWParams)); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java index bd278c4719..4b88660f4d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java @@ -10,8 +10,8 @@ * CuVS based codec for GPU based vector search * *

This codec, named {@code Lucene101AcceleratedHNSWScalarQuantizedCodec}, wraps {@code - * Lucene101Codec}, the default codec of Lucene 10.2. {@link CuVSCodecs#acceleratedHNSWScalarQuantized} returns the codec of - * this family for the Lucene release in use. + * Lucene101Codec}, the default codec of Lucene 10.2. On later Lucene releases it can only read + * existing indexes; use {@link CuVSCodecs#acceleratedHNSWScalarQuantized} to write. * * @since 26.02 */ @@ -35,7 +35,8 @@ public LuceneAcceleratedHNSWScalarQuantizedCodec(String name, Codec delegate) { public LuceneAcceleratedHNSWScalarQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) { super( NAME, - delegate(NAME, () -> LuceneCompat.lucene101Codec()), + 101, + () -> LuceneCompat.lucene101Codec(), () -> new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat(acceleratedHNSWParams)); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java index 9f09fc94ac..5bb4bd1340 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java @@ -5,6 +5,7 @@ package com.nvidia.cuvs.lucene; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; @@ -12,21 +13,27 @@ import static org.junit.Assert.assertTrue; import com.nvidia.cuvs.LibraryException; +import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.util.Version; import org.junit.Test; /** - * Tests that the codecs from {@link CuVSCodecs} are registered, and that a cuvs-lucene artifact - * on another Lucene release than it is built for fails only when its codecs are used. + * Tests how the codecs of successive Lucene releases coexist: only the codecs from {@link + * CuVSCodecs} write, and every older codec this Lucene release can read is registered, read-only. + * Also tests that a cuvs-lucene artifact on another Lucene release than it is built for fails only + * when its codecs are used. * * @since 25.12 */ public class TestBackCompat { - private static List codecs() { + private static List writableCodecs() { return List.of( CuVSCodecs.acceleratedHNSW(), CuVSCodecs.acceleratedHNSWScalarQuantized(), @@ -34,14 +41,93 @@ private static List codecs() { CuVSCodecs.gpuSearch()); } + private static List registeredCodecs() { + List codecs = new ArrayList<>(); + for (String name : Codec.availableCodecs()) { + if (Codec.forName(name) instanceof CuVSFilterCodec codec) { + codecs.add(codec); + } + } + return codecs; + } + @Test - public void testCodecsAreRegistered() { - for (Codec codec : codecs()) { + public void testWritableCodecsAreRegistered() { + for (Codec codec : writableCodecs()) { Codec registered = Codec.forName(codec.getName()); assertSame(codec.getName(), codec.getClass(), registered.getClass()); } } + @Test + public void testOlderCodecsAreReadOnly() { + Set writable = + writableCodecs().stream().map(Codec::getName).collect(Collectors.toSet()); + List registered = registeredCodecs(); + // One codec per family and Lucene codec generation this release can read. + assertEquals(0, registered.size() % writable.size()); + for (CuVSFilterCodec codec : registered) { + KnnVectorsFormat format = codec.knnVectorsFormat(); + assertNotNull(codec.getName(), format); + assertEquals( + codec.getName(), writable.contains(codec.getName()) == false, codec.isReadOnly()); + if (writable.contains(codec.getName())) { + continue; + } + UnsupportedOperationException e = + assertThrows(UnsupportedOperationException.class, () -> format.fieldsWriter(null)); + assertTrue(e.getMessage(), e.getMessage().contains(codec.getName())); + } + } + + @Test + public void testCallerChosenDelegateIsNeverReadOnly() { + // Lucene101Codec is a read-only backward codec from Lucene 10.3 on, but a codec built with a + // delegate chosen by the caller leaves that to the caller instead of guessing from the class. + // Created by class on purpose: CuVSCodecs cannot pass a delegate, and this constructor is what + // is under test. + Codec codec = new Lucene101AcceleratedHNSWCodec("CustomCodec", Codec.forName("Lucene101")); + assertTrue( + codec.knnVectorsFormat().toString(), + codec.knnVectorsFormat() instanceof Lucene99AcceleratedHNSWVectorsFormat); + } + + @Test + public void testReadOnlyFormatIsNotWrappedTwice() { + for (CuVSFilterCodec codec : registeredCodecs()) { + KnnVectorsFormat previous = codec.knnVectorsFormat(); + try { + codec.setKnnVectorsFormat(previous); + // Handing back the format a read-only codec returned must not wrap it once more. + KnnVectorsFormat format = codec.knnVectorsFormat(); + assertEquals( + format.toString(), + CuVSFilterCodec.isReadOnlyFormat(previous), + CuVSFilterCodec.isReadOnlyFormat(format)); + assertFalse(format.toString(), format.toString().startsWith("ReadOnly(ReadOnly(")); + } finally { + // The registered codecs are shared by the whole JVM. + codec.setKnnVectorsFormat(previous); + } + } + } + + @Test + public void testWritableCodecWritesWithFormatOfReadOnlyCodec() { + for (CuVSFilterCodec old : registeredCodecs()) { + if (old.isReadOnly() == false) { + continue; + } + // Created through CuVSCodecs, not looked up, so that the registered codec stays unchanged. + CuVSFilterCodec current = (CuVSFilterCodec) CuVSCodecs.acceleratedHNSW(); + assertFalse(current.getName(), current.isReadOnly()); + current.setKnnVectorsFormat(old.knnVectorsFormat()); + assertFalse( + old.getName() + " made " + current.getName() + " read-only", + CuVSFilterCodec.isReadOnlyFormat(current.knnVectorsFormat())); + } + } + @Test public void testLuceneFormatsStillResolve() { // Listing a Lucene format in this module's services file used to break every lookup once a @@ -98,28 +184,34 @@ public void testUnavailableFormat() { private static void checkUnavailableFormat(Throwable cause) { boolean logged = CuVSFilterCodec.setUnavailableLoggedForTesting(true); try { - CuVSFilterCodec codec = - new CuVSFilterCodec( - "TestCodec", - Codec.getDefault(), - () -> { - if (cause instanceof Error error) { - throw error; - } - throw (RuntimeException) cause; - }) {}; - KnnVectorsFormat format = codec.knnVectorsFormat(); - IllegalStateException e = - assertThrows(IllegalStateException.class, () -> format.fieldsWriter(null)); - assertTrue(e.getMessage(), e.getMessage().contains(cause.getMessage())); - assertThrows(IllegalStateException.class, () -> format.fieldsReader(null)); - // Everything but vectors still works. - assertNotNull(codec.postingsFormat()); + Supplier format = + () -> { + if (cause instanceof Error error) { + throw error; + } + throw (RuntimeException) cause; + }; + checkUnavailableFormat( + new CuVSFilterCodec("TestCodec", Codec.getDefault(), format) {}, cause); + // A codec of an older generation, which can only read, still reports why its format is + // unavailable rather than that it is read-only. + checkUnavailableFormat( + new CuVSFilterCodec("TestCodec", 0, () -> Codec.getDefault(), format) {}, cause); } finally { CuVSFilterCodec.setUnavailableLoggedForTesting(logged); } } + private static void checkUnavailableFormat(CuVSFilterCodec codec, Throwable cause) { + KnnVectorsFormat format = codec.knnVectorsFormat(); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> format.fieldsWriter(null)); + assertTrue(e.getMessage(), e.getMessage().contains(cause.getMessage())); + assertThrows(IllegalStateException.class, () -> format.fieldsReader(null)); + // Everything but vectors still works. + assertNotNull(codec.postingsFormat()); + } + @Test public void testLuceneVersionMatches() { // Loading the compatibility layer checks that this is the Lucene release it was built for. diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompatIndices.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompatIndices.java index 583c2a1678..ab1ea443f2 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompatIndices.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompatIndices.java @@ -4,14 +4,18 @@ */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -20,6 +24,7 @@ import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; @@ -34,6 +39,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.KnnVectorValues; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.StoredFields; @@ -41,6 +47,7 @@ import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.util.Version; @@ -57,15 +64,26 @@ * the cuvs-lucene vectors format its vector field uses. Each module reads the ones written on its own Lucene * release and all earlier ones. See the {@code README.md} next to them for how to add more. * + *

Released versions only cover the Lucene releases they supported, so each module also writes + * indexes with the codecs from {@link CuVSCodecs} to {@code

//.zip} + * in the directory the {@value #DIR_PROPERTY} system property names, and reads and upgrades the + * ones that modules for earlier Lucene releases wrote there. The parent pom points every module at + * the same directory, and Maven builds the modules from the oldest Lucene release up, so one {@code + * mvn test} checks every upgrade from one supported Lucene release to a later one with the code + * being tested. + * *

Reading and upgrading need no GPU, so those tests also run where cuVS is not available. */ @SuppressSysoutChecks(bugUrl = "") public class TestBackCompatIndices extends LuceneTestCase { private static final String RESOURCE_DIR = "/backcompat"; + private static final String DIR_PROPERTY = "tests.cuvs.backCompatDir"; // The path of an index below RESOURCE_DIR; see the class javadoc. private static final Pattern INDEX_PATH = Pattern.compile("releases/[\\d.]+/(\\d+)\\.(\\d+)/(\\w+)\\.zip"); + // The path of an index below DIR_PROPERTY, written by a module of this build. + private static final Pattern MODULE_INDEX_PATH = Pattern.compile("(\\d+)\\.(\\d+)/(\\w+)\\.zip"); private static final String ID_FIELD = "id"; private static final String VECTOR_FIELD = "vector"; private static final int NUM_DOCS = 200; @@ -75,21 +93,7 @@ public class TestBackCompatIndices extends LuceneTestCase { public void testReadIndices() throws Exception { for (Path zip : indicesReadableByThisRelease()) { - Matcher matcher = indexPath(zip); - Path indexDir = createTempDir(zip.getFileName().toString()); - unzip(zip, indexDir); - try (Directory dir = newFSDirectory(indexDir)) { - String name = matcher.group(3); - if (KnnVectorsFormat.availableKnnVectorsFormats().contains(name)) { - // Lucene's own codec with a cuvs-lucene format per field: check the field's format. - checkIndex(dir, null, name, NUM_DOCS); - } else { - // A cuvs-lucene codec: check the codec recorded in every segment. - checkIndex(dir, name, null, NUM_DOCS); - } - } catch (AssertionError | Exception e) { - throw new AssertionError("Failed to read " + matcher.group(), e); - } + readIndex(zip, indexPath(zip)); } } @@ -100,30 +104,85 @@ public void testReadIndices() throws Exception { */ public void testUpgradeIndices() throws Exception { for (Path zip : indicesReadableByThisRelease()) { - Matcher matcher = indexPath(zip); - Codec codec = currentCodecFor(matcher.group(3)); - Path indexDir = createTempDir(zip.getFileName().toString()); - unzip(zip, indexDir); - try (Directory dir = newFSDirectory(indexDir)) { - IndexWriterConfig config = - new IndexWriterConfig() - .setCodec(codec) - .setOpenMode(IndexWriterConfig.OpenMode.APPEND) - .setUseCompoundFile(false); - try (IndexWriter writer = new IndexWriter(dir, config)) { - Random random = new Random(7); - for (int i = NUM_DOCS; i < NUM_DOCS + UPGRADE_DOCS; i++) { - writer.addDocument(document(i, randomVector(random))); - } - writer.forceMerge(1); + upgradeIndex(zip, indexPath(zip)); + } + } + + /** Writes the indexes that the modules for later Lucene releases read. */ + public void testWriteIndicesForLaterReleases() throws Exception { + // Written on the GPU, like the indexes of the applications this test stands for. + assumeTrue("cuVS not supported", isSupported()); + Path dir = moduleIndexDir(); + assumeTrue("set -D" + DIR_PROPERTY + " to write the indexes", dir != null); + Path targetDir = dir.resolve(Version.LATEST.major + "." + Version.LATEST.minor); + Files.createDirectories(targetDir); + for (Codec codec : codecsToWrite()) { + Path indexDir = createTempDir(codec.getName()); + writeIndex(indexDir, codec); + zip(indexDir, targetDir.resolve(codec.getName() + ".zip")); + } + } + + public void testReadIndicesFromEarlierReleases() throws Exception { + for (Path zip : indicesFromEarlierModules()) { + readIndex(zip, moduleIndexPath(zip)); + } + } + + public void testUpgradeIndicesFromEarlierReleases() throws Exception { + for (Path zip : indicesFromEarlierModules()) { + upgradeIndex(zip, moduleIndexPath(zip)); + } + } + + private void readIndex(Path zip, Matcher matcher) throws IOException { + Path indexDir = createTempDir(zip.getFileName().toString()); + unzip(zip, indexDir); + try (Directory dir = newFSDirectory(indexDir)) { + String name = matcher.group(3); + if (KnnVectorsFormat.availableKnnVectorsFormats().contains(name)) { + // Lucene's own codec with a cuvs-lucene format per field: check the field's format. + checkIndex(dir, null, name, NUM_DOCS); + } else { + // A cuvs-lucene codec: check the codec recorded in every segment. + checkIndex(dir, name, null, NUM_DOCS); + } + } catch (AssertionError | Exception e) { + throw new AssertionError("Failed to read " + zip, e); + } + } + + private void upgradeIndex(Path zip, Matcher matcher) throws IOException { + Codec codec = currentCodecFor(matcher.group(3)); + Path indexDir = createTempDir(zip.getFileName().toString()); + unzip(zip, indexDir); + try (Directory dir = newFSDirectory(indexDir)) { + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(codec) + .setOpenMode(IndexWriterConfig.OpenMode.APPEND) + .setUseCompoundFile(false); + try (IndexWriter writer = new IndexWriter(dir, config)) { + Random random = new Random(7); + for (int i = NUM_DOCS; i < NUM_DOCS + UPGRADE_DOCS; i++) { + writer.addDocument(document(i, randomVector(random))); } - checkIndex(dir, codec.getName(), null, NUM_DOCS + UPGRADE_DOCS); - } catch (AssertionError | Exception e) { - throw new AssertionError("Failed to upgrade " + matcher.group(), e); + writer.forceMerge(1); } + checkIndex(dir, codec.getName(), null, NUM_DOCS + UPGRADE_DOCS); + } catch (AssertionError | Exception e) { + throw new AssertionError("Failed to upgrade " + zip, e); } } + /** The codecs that write the indexes; the GPU search codec's format makes no such promise. */ + private static List codecsToWrite() { + return List.of( + CuVSCodecs.acceleratedHNSW(), + CuVSCodecs.acceleratedHNSWScalarQuantized(), + CuVSCodecs.acceleratedHNSWBinaryQuantized()); + } + /** Returns the codec that writes on this Lucene release in the family of the given codec. */ private static Codec currentCodecFor(String codecName) { if (codecName.contains("ScalarQuantized")) { @@ -149,16 +208,65 @@ private static Matcher indexPath(Path zip) throws URISyntaxException { return matcher; } + /** Returns the directory {@link #DIR_PROPERTY} names, or {@code null} if it is not set. */ + private static Path moduleIndexDir() { + String dir = System.getProperty(DIR_PROPERTY); + return dir == null || dir.isEmpty() ? null : Path.of(dir); + } + + /** Matches the path of an index zip below {@link #moduleIndexDir()}. */ + private static Matcher moduleIndexPath(Path zip) { + String path = moduleIndexDir().relativize(zip).toString().replace('\\', '/'); + Matcher matcher = MODULE_INDEX_PATH.matcher(path); + assertTrue("unexpected back-compat index path: " + path, matcher.matches()); + return matcher; + } + + /** + * Compares the Lucene release an index path records ({@code major.minor} in groups 1 and 2) with + * the one in use: negative if it is earlier, 0 if it is the same, positive if it is later. + */ + private static int compareToLuceneInUse(Matcher indexPath) { + int major = Integer.parseInt(indexPath.group(1)); + int minor = Integer.parseInt(indexPath.group(2)); + return major != Version.LATEST.major + ? Integer.compare(major, Version.LATEST.major) + : Integer.compare(minor, Version.LATEST.minor); + } + + /** + * Returns the indexes the modules for earlier Lucene releases wrote with {@link + * #testWriteIndicesForLaterReleases}, and skips the test if there are none. + */ + private static List indicesFromEarlierModules() throws IOException { + Path dir = moduleIndexDir(); + assumeTrue("set -D" + DIR_PROPERTY + " to read the indexes of earlier releases", dir != null); + List zips = new ArrayList<>(); + if (Files.isDirectory(dir)) { + try (Stream files = Files.walk(dir)) { + for (Path f : files.filter(f -> f.toString().endsWith(".zip")).sorted().toList()) { + if (compareToLuceneInUse(moduleIndexPath(f)) < 0) { + zips.add(f); + } + } + } + } + // The module for the oldest supported release has none to read, and a module tested on its + // own has none unless an earlier build wrote them. + assumeFalse( + "no indexes written by the modules for earlier Lucene releases in " + + dir + + "; test those modules in the same build to write them", + zips.isEmpty()); + return zips; + } + private static List indicesReadableByThisRelease() throws IOException, URISyntaxException { List zips = new ArrayList<>(); try (Stream files = Files.walk(resourceDir())) { for (Path f : files.filter(f -> f.toString().endsWith(".zip")).sorted().toList()) { // Every zip has to be where the tests expect it, or a misplaced one would go unread. - Matcher matcher = indexPath(f); - int major = Integer.parseInt(matcher.group(1)); - int minor = Integer.parseInt(matcher.group(2)); - if (major < Version.LATEST.major - || (major == Version.LATEST.major && minor <= Version.LATEST.minor)) { + if (compareToLuceneInUse(indexPath(f)) <= 0) { zips.add(f); } } @@ -167,6 +275,23 @@ private static List indicesReadableByThisRelease() throws IOException, URI return zips; } + /** Writes the same documents as {@code GenerateReleasedIndices#writeIndex}. */ + private static void writeIndex(Path indexDir, Codec codec) throws IOException { + Random random = new Random(42); + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(codec) + .setUseCompoundFile(false) + .setMergePolicy(NoMergePolicy.INSTANCE); + try (Directory dir = FSDirectory.open(indexDir); + IndexWriter writer = new IndexWriter(dir, config)) { + for (int i = 0; i < NUM_DOCS; i++) { + writer.addDocument(document(i, randomVector(random))); + } + writer.commit(); + } + } + private static float[] randomVector(Random random) { float[] vector = new float[DIMENSIONS]; for (int d = 0; d < DIMENSIONS; d++) { @@ -240,4 +365,28 @@ private static void unzip(Path zip, Path targetDir) throws IOException { } } } + + /** + * Zips the files of the index's latest commit, leaving out anything else in the directory. The + * zip is written under a temporary name and then renamed, so that a module reading the directory + * meanwhile, as one built in parallel can, never sees a partly written zip. + */ + private static void zip(Path indexDir, Path zip) throws IOException { + List files; + try (Directory dir = FSDirectory.open(indexDir)) { + files = SegmentInfos.readLatestCommit(dir).files(true).stream().sorted().toList(); + } + Path tmp = zip.resolveSibling(zip.getFileName() + ".tmp"); + try (OutputStream out = Files.newOutputStream(tmp); + ZipOutputStream zipOut = new ZipOutputStream(out)) { + for (String file : files) { + zipOut.putNextEntry(new ZipEntry(file)); + try (InputStream in = Files.newInputStream(indexDir.resolve(file))) { + in.transferTo(zipOut); + } + zipOut.closeEntry(); + } + } + Files.move(tmp, zip, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCompatKnnFloatVectorQuery.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCompatKnnFloatVectorQuery.java new file mode 100644 index 0000000000..66409f6943 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCompatKnnFloatVectorQuery.java @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.QueryTimeout; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopKnnCollector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.junit.Test; + +/** Tests the per-segment search of {@link CompatKnnFloatVectorQuery}, which needs no GPU. */ +public class TestCompatKnnFloatVectorQuery extends LuceneTestCase { + + private static final String FIELD = "f"; + private static final String TAG_FIELD = "tag"; + // A document's position in its segment, for filters accepting a given number per segment. + private static final String POSITION_FIELD = "position"; + + /** + * Each segment must be searched once. Since Lucene 10.3, an optimistic collector manager makes + * {@code rewrite} search again every segment whose hits all beat the global top-k, expecting a + * wider second search; the query ignores the manager, so that search would only repeat the first, + * which for the GPU query is a second identical CAGRA search. + */ + @Test + public void testSearchesEachSegmentOnce() throws Exception { + int k = 5; + try (Directory dir = newDirectory()) { + IndexWriterConfig cfg = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter w = new IndexWriter(dir, cfg)) { + // One segment holds every nearest neighbor of the query, the other only distant vectors, + // so all of the first segment's hits make the global top-k. + addDocs(w, 0f, 20); + w.commit(); + addDocs(w, 1000f, 20); + w.commit(); + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + CountingQuery query = new CountingQuery(new float[] {0f, 0f}, k, null); + assertEquals(k, new IndexSearcher(reader).search(query, k).scoreDocs.length); + assertEquals("segment searches", reader.leaves().size(), query.searches.get()); + } + } + } + + /** + * A filtered segment search must return as many hits as Lucene expects from it. Since Lucene 10.3 + * that is often more than k, and Lucene throws away a search that returns fewer and searches the + * segment exactly instead, which for the GPU query wastes the GPU search. + */ + @Test + public void testFilteredSearchResultsAreKept() throws Exception { + int k = 5; + try (Directory dir = newDirectory()) { + IndexWriterConfig cfg = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter w = new IndexWriter(dir, cfg)) { + // Two equal segments, each with far more accepted documents than perLeafTopK. + addDocs(w, 0f, 200); + w.commit(); + addDocs(w, 1000f, 200); + w.commit(); + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + // A filter Lucene cannot rewrite away, unlike MatchAllDocsQuery, that accepts every + // document. + Query filter = new TermQuery(new Term(TAG_FIELD, "y")); + CountingQuery query = new CountingQuery(new float[] {0f, 0f}, k, filter); + assertTrue( + "perLeafTopK " + query.perLeafTopK(reader.leaves().get(0)), + query.perLeafTopK(reader.leaves().get(0)) < 200); + assertEquals(k, new IndexSearcher(reader).search(query, k).scoreDocs.length); + assertEquals("segment searches", reader.leaves().size(), query.searches.get()); + assertEquals("exact searches", 0, query.exactSearches.get()); + } + } + } + + /** + * {@link CompatKnnFloatVectorQuery#perLeafTopK} must be the boundary Lucene itself applies: a + * segment with that many accepted documents is searched exactly, one with one more + * approximately. Lucene computes it in a private method, so this catches a release that changes + * it. + */ + @Test + public void testPerLeafTopKIsLucenesExactSearchBoundary() throws Exception { + int k = 5; + try (Directory dir = newDirectory()) { + IndexWriterConfig cfg = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter w = new IndexWriter(dir, cfg)) { + addDocs(w, 0f, 200); + w.commit(); + addDocs(w, 1000f, 200); + w.commit(); + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(2, reader.leaves().size()); + float[] target = new float[] {0f, 0f}; + // Both segments are the same size, so they have the same perLeafTopK. + int perLeafTopK = new CountingQuery(target, k, null).perLeafTopK(reader.leaves().get(0)); + IndexSearcher searcher = new IndexSearcher(reader); + + CountingQuery atBoundary = + new CountingQuery( + target, k, IntPoint.newRangeQuery(POSITION_FIELD, 0, perLeafTopK - 1)); + searcher.search(atBoundary, k); + assertEquals("approximate searches at " + perLeafTopK, 0, atBoundary.searches.get()); + assertEquals("exact searches at " + perLeafTopK, 2, atBoundary.exactSearches.get()); + + CountingQuery aboveBoundary = + new CountingQuery(target, k, IntPoint.newRangeQuery(POSITION_FIELD, 0, perLeafTopK)); + searcher.search(aboveBoundary, k); + assertEquals( + "approximate searches at " + (perLeafTopK + 1), 2, aboveBoundary.searches.get()); + } + } + } + + private static void addDocs(IndexWriter w, float offset, int count) throws Exception { + for (int i = 0; i < count; i++) { + Document doc = new Document(); + doc.add( + new KnnFloatVectorField( + FIELD, new float[] {offset + i, offset + i}, VectorSimilarityFunction.EUCLIDEAN)); + doc.add(new StringField(TAG_FIELD, "y", Field.Store.NO)); + doc.add(new IntPoint(POSITION_FIELD, i)); + w.addDocument(doc); + } + } + + /** Counts the per-segment approximate and exact searches. */ + private static final class CountingQuery extends CompatKnnFloatVectorQuery { + + final AtomicInteger searches = new AtomicInteger(); + final AtomicInteger exactSearches = new AtomicInteger(); + + CountingQuery(float[] target, int k, Query filter) { + super(FIELD, target, k, filter); + } + + @Override + KnnCollector newPerLeafCollector(LeafReaderContext context, int k, int visitedLimit) { + searches.incrementAndGet(); + return new TopKnnCollector(k, visitedLimit); + } + + @Override + protected TopDocs exactSearch( + LeafReaderContext context, DocIdSetIterator acceptIterator, QueryTimeout queryTimeout) + throws IOException { + exactSearches.incrementAndGet(); + return super.exactSearch(context, acceptIterator, queryTimeout); + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index e2acf61e25..246e480d00 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -429,9 +429,12 @@ public void testSearchWithAcceptDocsMatchingNoVectors() throws IOException { float[] queryVector = generateRandomVector(dimensions, random); for (LeafReaderContext ctx : reader.leaves()) { TopKnnCollector collector = new TopKnnCollector(topK, Integer.MAX_VALUE); - ctx.reader() - .searchNearestVectors( - "vector", queryVector, collector, new Bits.MatchNoBits(ctx.reader().maxDoc())); + TestLuceneCompat.searchNearestVectors( + ctx.reader(), + "vector", + queryVector, + collector, + new Bits.MatchNoBits(ctx.reader().maxDoc())); assertEquals( "An empty accepted-ordinal set should collect nothing", 0, diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index 9cb4ed6153..687dc16e83 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -91,7 +91,8 @@ public void testTwoVectorFieldsPerDoc() throws Exception { assertVectorsKeepTheirDocuments(r, "f2", f2); // opportunistically check boundary condition - search with a 0 topK - var topDocs = r.searchNearestVectors("f1", randomVector(384), 0, null, 10); + var topDocs = + TestLuceneCompat.searchNearestVectors(r, "f1", randomVector(384), 0, null, 10); assertEquals(0, topDocs.scoreDocs.length); assertEquals(0, topDocs.totalHits.value()); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchLargeTopK.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchLargeTopK.java new file mode 100644 index 0000000000..e2dbb74f19 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchLargeTopK.java @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.Test; + +/** + * Searches a GPU index that has no brute force index, the default, for more hits than {@link + * CuVS2510GPUVectorsReader#MAX_CAGRA_K}. Since Lucene 10.3, Lucene's own {@link + * KnnFloatVectorQuery} asks a segment for as many hits as it expects from it, which for a segment + * holding most of the index exceeds {@code k}, and so can exceed that limit even when {@code k} + * does not. + */ +@SuppressSysoutChecks(bugUrl = "") +public class TestGPUSearchLargeTopK extends LuceneTestCase { + + private static final String VECTOR_FIELD = "vectors"; + private static final int DIMENSIONS = 8; + + @Test + public void largeSegmentWithLargeTopKIsSearched() throws Exception { + assumeTrue("cuVS not supported", isSupported()); + // A segment holding 90% of the documents: with k = 1000, Lucene 10.3 asks it for about 1052. + int k = 1000; + int[] segmentSizes = {9000, 1000}; + try (Directory dir = newDirectory(new ByteBuffersDirectory())) { + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(CuVSCodecs.gpuSearch()) + .setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter writer = new IndexWriter(dir, config)) { + for (int size : segmentSizes) { + for (float[] vector : generateDataset(random(), size, DIMENSIONS)) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField(VECTOR_FIELD, vector, EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + } + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertEquals(segmentSizes.length, reader.leaves().size()); + float[] query = generateDataset(random(), 1, DIMENSIONS)[0]; + int hits = + new IndexSearcher(reader) + .search(new KnnFloatVectorQuery(VECTOR_FIELD, query, k), k) + .scoreDocs + .length; + assertEquals(k, hits); + } + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java index bb5d5b2299..aa5f7b800e 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java @@ -91,7 +91,8 @@ public void testTwoVectorFieldsPerDoc() throws Exception { assertVectorsKeepTheirDocuments(r, "f2", f2); // opportunistically check boundary condition - search with a 0 topK - var topDocs = r.searchNearestVectors("f1", randomVector(384), 0, null, 10); + var topDocs = + TestLuceneCompat.searchNearestVectors(r, "f1", randomVector(384), 0, null, 10); assertEquals(0, topDocs.scoreDocs.length); assertEquals(0, topDocs.totalHits.value()); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestPerSegmentGPUFilterSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestPerSegmentGPUFilterSearch.java index cf91052157..89ba3989a4 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestPerSegmentGPUFilterSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestPerSegmentGPUFilterSearch.java @@ -77,7 +77,7 @@ public void testSelectiveFilterDoesNotLeakTrailingOrdinals() throws Exception { } for (float[] q : queries) { TopKnnCollector collector = new TopKnnCollector(topK, Integer.MAX_VALUE); - leaf.searchNearestVectors(VECTOR_FIELD, q, collector, acceptDocs); + TestLuceneCompat.searchNearestVectors(leaf, VECTOR_FIELD, q, collector, acceptDocs); for (ScoreDoc hit : collector.topDocs().scoreDocs) { assertTrue( "per-segment search returned doc " + hit.doc + " outside filter category " + c, diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java index 5053ad13a9..d28446ef11 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java @@ -152,7 +152,7 @@ public void testCosineSimilarity() throws Exception { assertEquals(R, values.size()); float[] queryVector = randomVector(D); - var topDocs = r.searchNearestVectors(F, queryVector, 2, null, 10); + var topDocs = TestLuceneCompat.searchNearestVectors(r, F, queryVector, 2, null, 10); assertTrue("Should return at least one result", topDocs.scoreDocs.length > 0); assertTrue("Scores should be non-negative", topDocs.scoreDocs[0].score >= 0); } diff --git a/java/cuvs-lucene/src/test/resources/backcompat/README.md b/java/cuvs-lucene/src/test/resources/backcompat/README.md index 1f1d1538d2..cbaba12576 100644 --- a/java/cuvs-lucene/src/test/resources/backcompat/README.md +++ b/java/cuvs-lucene/src/test/resources/backcompat/README.md @@ -12,6 +12,13 @@ releases///.zip codec, the cuvs-lucene vectors format its vector field uses. `TestBackCompatIndices` fails on a zip placed anywhere else, so that it can't go unread. +Released versions only cover the Lucene releases they supported. To also check upgrades between Lucene +releases that no released version supports yet, `TestBackCompatIndices` in every module writes the same +documents with its own codecs to `target/backcompat//.zip` (the parent `pom.xml` +sets the directory), and reads and upgrades the indexes that the modules for earlier Lucene releases wrote +there. Maven tests the modules from the oldest Lucene release up, so `mvn test` checks every such upgrade +with the code being tested. Nothing is committed for these. + ## Written by released cuvs-lucene versions `releases//`: written by the jars of a cuvs-lucene release published to Maven Central, with